mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
feat(home): migrate optimization workflows and evaluation budgets
This commit is contained in:
parent
860f67b555
commit
9d80f4b14e
27 changed files with 3121 additions and 125 deletions
|
|
@ -164,6 +164,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
|
|||
`home.evaluation` Component. It still only prepares; it does not run evaluations.
|
||||
Each evaluation controls `sampling` and `record_history` separately.
|
||||
|
||||
The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
|
||||
advisory time between completed formal evaluations. It defaults to 150 minutes;
|
||||
set a positive number to change it or null to disable timed reminders.
|
||||
|
||||
The top-level `max_validations` sets the optimizer's formal-evaluation budget
|
||||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
After preparation, run local one-shot optimization with the same configuration:
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home.py --optimize
|
||||
# Use the same --config and --workspace-dir overrides as preparation, if supplied.
|
||||
python benchmark/beam/run_home.py --optimize --instruction "Improve retrieval quality"
|
||||
```
|
||||
|
||||
This starts a fresh application, applies both budgets and the sampling policy,
|
||||
runs `home_optimize`, and closes the application. It does not repeat preparation.
|
||||
Python callers can use `await optimize(config_path, workspace_dir=...)` from the
|
||||
same module. The default `run()` and CLI behavior remain preparation only.
|
||||
|
||||
Python callers can use the same lifecycle:
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -154,4 +154,21 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
|
|||
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
|
||||
和 `record_history`。
|
||||
|
||||
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
|
||||
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
|
||||
|
||||
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home.py --optimize
|
||||
```
|
||||
|
||||
若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
|
||||
可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
|
||||
应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
|
||||
不加 `--optimize` 时仍只准备。
|
||||
|
||||
默认选择 BEAM **100K 全部 case**。
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
|
||||
# Preparation by default; --optimize applies these settings to the prepared workspace.
|
||||
# Paths are relative to the ReMe repository root, not the cwd.
|
||||
workspace_dir: benchmark/beam/workspaces/home-100K
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_validations: 10
|
||||
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
|
|
@ -18,8 +23,6 @@ preparation:
|
|||
source_ref: HEAD # Committed source only; resolved and saved as a full SHA.
|
||||
config: benchmark.yaml
|
||||
plugins: [beam]
|
||||
candidate:
|
||||
paths: [plugins/beam/src]
|
||||
debug:
|
||||
enabled: true
|
||||
sessions_per_case: 3
|
||||
|
|
|
|||
|
|
@ -1,22 +1,145 @@
|
|||
"""Prepare the beam Home workspace, dataset and candidate code; do not evaluate."""
|
||||
"""Prepare the BEAM Home workspace, or optimize it with --optimize after preparation."""
|
||||
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
import yaml
|
||||
|
||||
# Support direct execution from any working directory as well as module imports.
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
|
||||
# pylint: disable=wrong-import-position
|
||||
from reme_home.preparation.spec import PreparationSpec # noqa: E402
|
||||
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
||||
from reme_home.sampling import SamplingPolicy # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
|
||||
|
||||
|
||||
class RunHomeConfig(BaseModel):
|
||||
"""Runner settings kept separate from Home's public preparation contract."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
workspace_dir: str = Field(min_length=1)
|
||||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_validations: int = Field(default=10, ge=1)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Use repository-relative paths consistently regardless of the caller's cwd."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Resolve source paths and validate explicit Home preparation parameters."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
config = RunHomeConfig.model_validate(raw)
|
||||
config.preparation.setdefault("candidate", {"paths": ["plugins/beam/src"]})
|
||||
spec = PreparationSpec.model_validate(config.preparation)
|
||||
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
|
||||
if spec.benchmark.evidence_source is not None:
|
||||
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
|
||||
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
|
||||
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
|
||||
return workspace, spec.model_dump(mode="json"), config
|
||||
|
||||
|
||||
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
|
||||
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
|
||||
return workspace, spec
|
||||
|
||||
|
||||
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
|
||||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={
|
||||
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
|
||||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
"max_validations": config.max_validations,
|
||||
},
|
||||
},
|
||||
},
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
if optimize_workspace:
|
||||
arguments = {} if instruction is None else {"instruction": instruction}
|
||||
response = await app.run_job("home_optimize", **arguments)
|
||||
else:
|
||||
response = await app.run_job("home_prepare", spec=spec)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
return {"workspace_dir": str(workspace), **response.answer}
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Prepare once without running evaluations or optimization."""
|
||||
return await _run(config_path, workspace_dir=workspace_dir)
|
||||
|
||||
|
||||
async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
|
||||
"""Optimize an already prepared workspace using this config's actual model budgets."""
|
||||
return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
|
||||
|
||||
|
||||
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Prepare through the same Home Job as the CLI, then close the application."""
|
||||
return await prepare(config_path, workspace_dir=workspace_dir)
|
||||
|
||||
|
||||
def main(default_config: Path = DEFAULT_CONFIG) -> None:
|
||||
"""Run preparation or explicit local one-shot optimization with the benchmark config."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
|
||||
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
|
||||
parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
|
||||
parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
|
||||
args = parser.parse_args()
|
||||
if args.instruction is not None and not args.optimize:
|
||||
parser.error("--instruction requires --optimize")
|
||||
if args.optimize:
|
||||
result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return
|
||||
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"workspace_dir": result["workspace_dir"],
|
||||
"case_count": result["dataset"]["case_count"],
|
||||
"query_count": result["dataset"]["query_count"],
|
||||
"source_commit": result["code"]["source_commit"],
|
||||
"initial_commit": result["code"]["initial_commit"],
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(DEFAULT_CONFIG)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
"""Shared preparation-only entry point for benchmark Home experiments."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
import yaml
|
||||
from reme_home.preparation.spec import PreparationSpec
|
||||
from reme_home.judge import JUDGE_PLUGINS
|
||||
from reme_home.sampling import SamplingPolicy
|
||||
|
||||
from reme.application import Application
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RunHomeConfig(BaseModel):
|
||||
"""Runner settings kept separate from Home's public preparation contract."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
workspace_dir: str = Field(min_length=1)
|
||||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Use repository-relative paths consistently regardless of the caller's cwd."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Resolve source paths and validate explicit Home preparation parameters."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
config = RunHomeConfig.model_validate(raw)
|
||||
spec = PreparationSpec.model_validate(config.preparation)
|
||||
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
|
||||
if spec.benchmark.evidence_source is not None:
|
||||
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
|
||||
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
|
||||
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
|
||||
return workspace, spec.model_dump(mode="json"), config.sampling_policy
|
||||
|
||||
|
||||
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
|
||||
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
|
||||
return workspace, spec
|
||||
|
||||
|
||||
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Start Home, prepare once, and close all resources without evaluating cases."""
|
||||
workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
response = await app.run_job("home_prepare", spec=spec)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
return {"workspace_dir": str(workspace), **response.answer}
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
def main(default_config: Path) -> None:
|
||||
"""Run a benchmark's default preparation from the command line."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
|
||||
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"workspace_dir": result["workspace_dir"],
|
||||
"case_count": result["dataset"]["case_count"],
|
||||
"query_count": result["dataset"]["query_count"],
|
||||
"source_commit": result["code"]["source_commit"],
|
||||
"initial_commit": result["code"]["initial_commit"],
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
)
|
||||
|
|
@ -137,6 +137,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
|
|||
`home.evaluation` Component. It still only prepares; it does not run evaluations.
|
||||
Each evaluation controls `sampling` and `record_history` separately.
|
||||
|
||||
The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
|
||||
advisory time between completed formal evaluations. It defaults to 150 minutes;
|
||||
set a positive number to change it or null to disable timed reminders.
|
||||
|
||||
The top-level `max_validations` sets the optimizer's formal-evaluation budget
|
||||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
After preparation, run local one-shot optimization with the same configuration:
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home.py --optimize
|
||||
# Use the same --config and --workspace-dir overrides as preparation, if supplied.
|
||||
python benchmark/longmemeval/run_home.py --optimize --instruction "Improve retrieval quality"
|
||||
```
|
||||
|
||||
This starts a fresh application, applies both budgets and the sampling policy,
|
||||
runs `home_optimize`, and closes the application. It does not repeat preparation.
|
||||
Python callers can use `await optimize(config_path, workspace_dir=...)` from the
|
||||
same module. The default `run()` and CLI behavior remain preparation only.
|
||||
|
||||
Python callers can use the same lifecycle:
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -125,6 +125,23 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
|
|||
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
|
||||
和 `record_history`。
|
||||
|
||||
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
|
||||
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
|
||||
|
||||
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home.py --optimize
|
||||
```
|
||||
|
||||
若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
|
||||
可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
|
||||
应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
|
||||
不加 `--optimize` 时仍只准备。
|
||||
|
||||
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
|
||||
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
|
||||
运行时无需读取分组文件,group_6–group_10 保持为留出集。
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
|
||||
# Preparation by default; --optimize applies these settings to the prepared workspace.
|
||||
# Paths are relative to the ReMe repository root, not the cwd.
|
||||
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
|
||||
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_validations: 10
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
|
|
|
|||
|
|
@ -1,22 +1,145 @@
|
|||
"""Prepare the longmemeval Home workspace, dataset and candidate code; do not evaluate."""
|
||||
"""Prepare the LongMemEval Home workspace, or optimize it with --optimize after preparation."""
|
||||
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
import yaml
|
||||
|
||||
# Support direct execution from any working directory as well as module imports.
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
|
||||
# pylint: disable=wrong-import-position
|
||||
from reme_home.preparation.spec import PreparationSpec # noqa: E402
|
||||
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
|
||||
from reme_home.sampling import SamplingPolicy # noqa: E402
|
||||
|
||||
from reme.application import Application # noqa: E402
|
||||
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
|
||||
|
||||
|
||||
class RunHomeConfig(BaseModel):
|
||||
"""Runner settings kept separate from Home's public preparation contract."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
workspace_dir: str = Field(min_length=1)
|
||||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_validations: int = Field(default=10, ge=1)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Use repository-relative paths consistently regardless of the caller's cwd."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Resolve source paths and validate explicit Home preparation parameters."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
config = RunHomeConfig.model_validate(raw)
|
||||
config.preparation.setdefault("candidate", {"paths": ["plugins/lme/src"]})
|
||||
spec = PreparationSpec.model_validate(config.preparation)
|
||||
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
|
||||
if spec.benchmark.evidence_source is not None:
|
||||
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
|
||||
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
|
||||
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
|
||||
return workspace, spec.model_dump(mode="json"), config
|
||||
|
||||
|
||||
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
|
||||
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
|
||||
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
|
||||
return workspace, spec
|
||||
|
||||
|
||||
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
|
||||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={
|
||||
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
|
||||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
"max_validations": config.max_validations,
|
||||
},
|
||||
},
|
||||
},
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
if optimize_workspace:
|
||||
arguments = {} if instruction is None else {"instruction": instruction}
|
||||
response = await app.run_job("home_optimize", **arguments)
|
||||
else:
|
||||
response = await app.run_job("home_prepare", spec=spec)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
return {"workspace_dir": str(workspace), **response.answer}
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Prepare once without running evaluations or optimization."""
|
||||
return await _run(config_path, workspace_dir=workspace_dir)
|
||||
|
||||
|
||||
async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
|
||||
"""Optimize an already prepared workspace using this config's actual model budgets."""
|
||||
return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
|
||||
|
||||
|
||||
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
|
||||
"""Prepare through the same Home Job as the CLI, then close the application."""
|
||||
return await prepare(config_path, workspace_dir=workspace_dir)
|
||||
|
||||
|
||||
def main(default_config: Path = DEFAULT_CONFIG) -> None:
|
||||
"""Run preparation or explicit local one-shot optimization with the benchmark config."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
|
||||
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
|
||||
parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
|
||||
parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
|
||||
args = parser.parse_args()
|
||||
if args.instruction is not None and not args.optimize:
|
||||
parser.error("--instruction requires --optimize")
|
||||
if args.optimize:
|
||||
result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return
|
||||
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"workspace_dir": result["workspace_dir"],
|
||||
"case_count": result["dataset"]["case_count"],
|
||||
"query_count": result["dataset"]["query_count"],
|
||||
"source_commit": result["code"]["source_commit"],
|
||||
"initial_commit": result["code"]["initial_commit"],
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(DEFAULT_CONFIG)
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -838,3 +838,150 @@ reme home_case_study evaluation_path=evaluations/candidate/abcdef0/1 'case_ids=[
|
|||
Calls with `_allowed_paths` restrictions fail closed, as other Home artifact
|
||||
Jobs do. Structured inspection tools run in the host with their existing
|
||||
containment checks; only Bash/Python subprocesses run inside the OS sandbox.
|
||||
|
||||
## Diagnostic and optimizer agents
|
||||
|
||||
`home_diagnose` and `home_optimize` provide the minimal Meta-ReMe workflow using
|
||||
Home's existing Jobs. Enable Home with a benchmark judge and prepare the
|
||||
workspace first. Start the configured service in one terminal:
|
||||
|
||||
```bash
|
||||
reme start config=/absolute/path/to/home.yaml
|
||||
```
|
||||
|
||||
Call from another terminal:
|
||||
|
||||
```bash
|
||||
reme home_diagnose 'evaluation_paths=["evaluations/init/abcdef0/1"]' 'instruction=Explain the lowest-scoring cases'
|
||||
reme home_optimize 'instruction=Improve evidence retrieval while reducing query tokens'
|
||||
```
|
||||
|
||||
Diagnosis requires explicit, unique result directories, including failed runs
|
||||
with saved evidence. Each call creates a fresh diagnostic agent, delegates deep
|
||||
case investigation to `home_case_study`, and returns ranked, evidence-supported
|
||||
causes without prescribing fixes. An optimizer invocation automatically injects
|
||||
its available caller messages and compression summary; the model cannot supply
|
||||
or override that trajectory. Standalone calls record an empty trajectory.
|
||||
Reports and trajectories are saved under `home/logs/diagnostic-reports/` and
|
||||
`home/logs/diagnostic-trajectories/` with their evidence paths and session IDs.
|
||||
|
||||
Optimization reuses a completed full initial baseline covering every prepared
|
||||
training case and query, with no infrastructure errors. Otherwise it runs that
|
||||
baseline first, requiring clean `init` to be checked out. It never switches away
|
||||
from an existing candidate to repair a missing baseline. Baseline observations
|
||||
feed sampling history. Candidate execution failures retain Home's existing
|
||||
failure thresholds; low scores are valid evidence. Infrastructure failures stop
|
||||
baseline startup and are distinguished from candidate defects during iteration.
|
||||
|
||||
The optimizer diagnoses, edits declared `candidate_paths`, commits through
|
||||
managed Git Jobs, runs debug until execution passes, then evaluates using the
|
||||
host's adaptive sampling policy. Formal evaluations always rebuild memory.
|
||||
The agent cannot change execution settings, selections, sampling mode, or
|
||||
construction reuse through its evaluation tool; only `debug` is model-selected.
|
||||
Normal `home_evaluate` callers retain their existing interface and replay support.
|
||||
Unchanged candidates and infrastructure retries follow the existing repetition
|
||||
rules. There is no admission gate, architecture agent,
|
||||
construction-equivalence reviewer, or query-only optimizer policy in this workflow.
|
||||
|
||||
Both agents share the case-study OS sandbox implementation. All file tools,
|
||||
Bash and Python use it, including file metadata and Edit/Write checks. Evidence,
|
||||
fixed runtime and Git metadata are read-only. Each call receives a private
|
||||
scratch directory under `home/tmp/`; only the optimizer additionally receives
|
||||
writes to declared candidate roots. Network is denied. Diagnostic writes are
|
||||
limited to scratch. Host Jobs alone write reports, evaluation artifacts and Git
|
||||
metadata. Sandbox startup is probed before model calls and never falls back to
|
||||
unrestricted execution. Declared write roots must exist and remain real paths.
|
||||
Only one optimizer can run per Home workspace; cancellation releases its lock
|
||||
after nested tool cleanup. Separate diagnostic calls and case studies may run
|
||||
concurrently with fresh state.
|
||||
|
||||
The final optimizer response must identify `branch_name`, full `commit_sha`,
|
||||
`evaluation_path` and `report`. The host verifies a clean checkout at that exact
|
||||
branch/commit and a matching completed, scored, end-to-end evaluation without
|
||||
infrastructure failures. Sampled formal evidence qualifies; debug or query-only
|
||||
evidence does not. Keeping the initial baseline is valid. Invalid final claims
|
||||
fail while preserving changes and the raw report in
|
||||
`home/logs/optimization-reports/`. The host verifies provenance rather than
|
||||
ranking candidates by a single average score.
|
||||
|
||||
Dedicated `as_llm` and `agent_wrapper` components are named `home_optimizer` and
|
||||
`home_diagnostic`; override them through ordinary application configuration.
|
||||
Both default to `qwen3.8-max`, with context sizes 160,000 and 128,000, respectively.
|
||||
The optimizer has 3,000 iterations and a 150-minute advisory validation-cycle timer;
|
||||
configure `components.agent_wrapper.home_optimizer.validation_cycle_time_budget_minutes`
|
||||
(null disables timed reminders). The previous `time_budget_minutes` setting is a
|
||||
fallback when the cycle setting is absent. Diagnosis retains 80 iterations and
|
||||
a 60-minute whole-invocation timer. Neither timer enforces a time limit.
|
||||
|
||||
The optimizer follows dev_meta b53048e4: before reasoning it checks 50%, 75%, 90%,
|
||||
then 100% and each additional 10% of the cycle budget. Reminders include elapsed
|
||||
time, budget percentage, completed context compressions and guidance to finish
|
||||
editing, commit, pass the smoke test and begin validation. Temporary and durable
|
||||
reminders alternate, beginning with temporary. The cycle includes nested diagnosis,
|
||||
debug and evaluation time. Only a new counted formal evaluation returning
|
||||
`success` or `completed` resets the clock, thresholds, alternation and compression
|
||||
count; scores and candidate failures do not control reset. As in dev_meta, a
|
||||
completed result can reset even when its case results contain infrastructure errors;
|
||||
an incomplete failure, rejected call, cancellation, debug or pre-sandbox failure
|
||||
cannot. The initial full baseline precedes the first optimizer cycle and is excluded
|
||||
from its formal-validation numbering. A durable hint records the completed
|
||||
validation number and commit prefix, expires that cycle's old timer reminders and
|
||||
announces the next cycle with the remaining formal-validation count.
|
||||
Persisted old reminders are retained as in dev_meta.
|
||||
Hints use AgentScope's current reply message so accumulated token usage is retained.
|
||||
Evaluation tools queue transitions in request state; the optimizer injects them
|
||||
before the next reasoning call, after all tool results have been written. Thus a
|
||||
transition never separates an API tool call from its matching result.
|
||||
|
||||
`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching
|
||||
dev_meta. `get_validation_budget` reports consumed and remaining slots, the search
|
||||
phase and next-candidate guidance. A non-init formal run consumes one slot when
|
||||
its first sandbox initializes successfully, including runs that later fail or are
|
||||
cancelled. Initial baselines, debug, rejected calls and pre-start failures are free.
|
||||
Accounting reuses persistent started-run evidence and survives optimizer restarts;
|
||||
removing the rebuildable sampling cache does not restore the budget. The evaluator
|
||||
checks the limit while holding the formal-validation lock, before starting another
|
||||
run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects
|
||||
from existing evidence. The first `max_validations // 2` starts favor structural
|
||||
exploration; later starts favor evidence-driven refinement. This guidance is advisory.
|
||||
Ordinary `home_evaluate` calls remain unbounded unless the host supplies
|
||||
`max_validations`; all recorded non-init formal starts in that workspace count.
|
||||
|
||||
Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage.
|
||||
The optimizer follows dev_meta's compression recovery policy. Recoverable model
|
||||
or summary-output failures leave context intact below the emergency ratio
|
||||
`(trigger_ratio + 1) / 2`. At or above that ratio (90% for an 80% trigger), old
|
||||
messages are atomically saved under `logs/context-offloads/`; recent context and
|
||||
the existing summary are retained, with a reminder pointing to the readable JSON
|
||||
file. This fallback makes no additional model call and counts as one completed
|
||||
compression. Cancellation, invalid configuration and uncompressible system prompts
|
||||
still propagate during normal operation. Diagnostic and case-study agents retain
|
||||
their existing compression behavior. Bounded draft notebook tools are not included.
|
||||
|
||||
Host-only execution settings belong in the optimizer Step; model, iteration and
|
||||
timer settings belong in its wrapper. For example:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
home_optimize:
|
||||
steps:
|
||||
- backend: home_optimize_step
|
||||
agent_wrapper: home_optimizer
|
||||
evaluation_options:
|
||||
concurrency: 4
|
||||
timeout_seconds: 7200
|
||||
max_construction_failure_rate: 0.03
|
||||
# memory_limit_mib: 0
|
||||
components:
|
||||
agent_wrapper:
|
||||
home_optimizer:
|
||||
react_config:
|
||||
max_iters: 3000
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
max_validations: 10
|
||||
```
|
||||
|
||||
`evaluation_options` also accepts `base_image`, `fail_fast` and
|
||||
`max_artifact_bytes`. Omit `fail_fast` to retain Home's debug/scored defaults.
|
||||
Restricted `_allowed_paths` calls fail closed. No real model or Docker benchmark
|
||||
is required by the isolated workflow tests.
|
||||
|
|
|
|||
|
|
@ -116,7 +116,8 @@ def _format_minutes(minutes: float) -> str:
|
|||
"""Format a timer duration compactly."""
|
||||
|
||||
if minutes < 1:
|
||||
return f"{max(0, round(minutes * 60))} seconds"
|
||||
seconds = max(0, round(minutes * 60))
|
||||
return f"{seconds} {'second' if seconds == 1 else 'seconds'}"
|
||||
rounded = round(minutes, 1)
|
||||
value = int(rounded) if rounded.is_integer() else rounded
|
||||
unit = "minute" if value == 1 else "minutes"
|
||||
|
|
|
|||
126
plugins/home/src/reme_home/agent/compression.py
Normal file
126
plugins/home/src/reme_home/agent/compression.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""Meta-ReMe compression recovery adapted to AgentScope's native context pipeline."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from agentscope.exception import ToolJSONDecodeError
|
||||
from agentscope.message import TextBlock
|
||||
import jsonschema
|
||||
import openai
|
||||
from pydantic import ValidationError
|
||||
|
||||
from ..evaluation.artifacts import contained, write_json
|
||||
from .bounded import BoundedReActAgent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RecoverableCompressionAgent(BoundedReActAgent):
|
||||
"""Skip recoverable failures until emergency offload is needed.
|
||||
|
||||
Override the native implementation so compression middleware, successful-change
|
||||
counters, finalization and cancellation remain owned by their existing layers.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, fallback_directory=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fallback_directory = Path(fallback_directory) if fallback_directory is not None else None
|
||||
|
||||
async def _compress_context_impl(self, context_config=None, instructions=None):
|
||||
"""Keep the old trigger-derived emergency boundary without silent truncation."""
|
||||
config = context_config or self.context_config
|
||||
model_input = await self._prepare_model_input()
|
||||
context_ratio = await self.model.count_tokens(**model_input) / self.model.context_size
|
||||
if context_ratio < config.trigger_ratio:
|
||||
return
|
||||
# The SDK's default truncation would hide the failure and discard old
|
||||
# evidence before Home can decide whether emergency offload is needed.
|
||||
config = config.model_copy(update={"compression_fallback_to_truncation": False})
|
||||
try:
|
||||
await super()._compress_context_impl(context_config=config, instructions=instructions)
|
||||
except Exception as exc:
|
||||
if not self._is_recoverable_compression_error(exc):
|
||||
raise
|
||||
emergency_ratio = (config.trigger_ratio + 1.0) / 2.0
|
||||
if context_ratio < emergency_ratio:
|
||||
logger.warning(
|
||||
"Context compression failed at %.1f%%, below the %.1f%% emergency threshold; skipping: %s: %s",
|
||||
context_ratio * 100,
|
||||
emergency_ratio * 100,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return
|
||||
logger.warning(
|
||||
"Context compression failed at %.1f%%; using local offload at the %.1f%% emergency threshold: %s: %s",
|
||||
context_ratio * 100,
|
||||
emergency_ratio * 100,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
await self._fallback_compress_context(exc, context_ratio, config)
|
||||
|
||||
@staticmethod
|
||||
def _is_recoverable_compression_error(exc):
|
||||
"""Match dev_meta's model/output failures, excluding uncompressible prompts."""
|
||||
if isinstance(exc, RuntimeError):
|
||||
return not str(exc).startswith("The system prompt")
|
||||
return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError))
|
||||
|
||||
async def _fallback_compress_context(self, cause, context_ratio, config):
|
||||
"""Persist older messages before replacing state; never call a second model."""
|
||||
if self.fallback_directory is None:
|
||||
raise RuntimeError("Optimizer fallback_directory is required for local context compression") from cause
|
||||
model_input = await self._prepare_model_input()
|
||||
old_messages, recent_messages = await self._split_context_for_compression(
|
||||
config.reserve_ratio * self.model.context_size,
|
||||
model_input.get("tools", []),
|
||||
)
|
||||
if not old_messages:
|
||||
raise RuntimeError(
|
||||
"Local fallback compression cannot remove any context while preserving the configured reserve",
|
||||
) from cause
|
||||
|
||||
async def apply_change():
|
||||
contained(self.fallback_directory.parent, self.fallback_directory, directory=True)
|
||||
path = contained(self.fallback_directory, f"context-{uuid4().hex}.json")
|
||||
write_json(
|
||||
path,
|
||||
{
|
||||
"session_id": self.state.session_id,
|
||||
"context_ratio": context_ratio,
|
||||
"trigger_ratio": config.trigger_ratio,
|
||||
"emergency_ratio": (config.trigger_ratio + 1.0) / 2.0,
|
||||
"failure": {"type": type(cause).__name__, "message": str(cause)},
|
||||
"messages": [message.model_dump(mode="json", exclude_none=False) for message in old_messages],
|
||||
},
|
||||
)
|
||||
notice = (
|
||||
"<system-reminder>Model-based context compression failed near the context limit. "
|
||||
f"Older context was offloaded without semantic summarization to '{path}'. "
|
||||
"The recent context is preserved; use Read on that file if older experiments, decisions, "
|
||||
"validation results, or tool outputs are needed.</system-reminder>"
|
||||
)
|
||||
await self._clear_unreserved_read_cache(recent_messages)
|
||||
# Recent SDK versions also carry current-reply usage through compression.
|
||||
usage = self._get_reply_usage()
|
||||
summary = self.state.summary
|
||||
self.state.summary = (
|
||||
"\n".join(part for part in (summary, notice) if part)
|
||||
if isinstance(summary, str)
|
||||
else [*summary, TextBlock(text=notice)]
|
||||
)
|
||||
self.state.context = recent_messages
|
||||
if usage is not None and self._get_reply_usage() is None:
|
||||
self.state.append_context(self.name, [])
|
||||
self.state.context[-1].usage = usage
|
||||
logger.warning("Local compression offloaded %d message(s) to %s", len(old_messages), path)
|
||||
|
||||
task = asyncio.create_task(apply_change())
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
await task
|
||||
raise
|
||||
159
plugins/home/src/reme_home/agent/diagnostic.py
Normal file
159
plugins/home/src/reme_home/agent/diagnostic.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
"""Diagnose saved Home evaluations with an isolated evidence-driven agent."""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
||||
from agentscope.state import AgentState
|
||||
from agentscope.tool import FunctionTool
|
||||
from pydantic import Field, field_validator
|
||||
|
||||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from ..evaluation.artifacts import contained, read_json, write_json
|
||||
from ..evaluation.component import blocking
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace
|
||||
from .workflow_common import required_job, save_record, workflow_reply
|
||||
from .wrapper import BoundedAgentWrapper
|
||||
|
||||
|
||||
class DiagnosticInput(SpecModel):
|
||||
"""Require explicit evaluation evidence; caller state is never model-selected."""
|
||||
|
||||
evaluation_paths: list[str] = Field(min_length=1)
|
||||
instruction: str = Field(default="Identify and rank the evidence-supported root causes.", min_length=1)
|
||||
|
||||
@field_validator("evaluation_paths")
|
||||
@classmethod
|
||||
def validate_paths(cls, values):
|
||||
"""Reject empty and duplicate input references before creating an agent."""
|
||||
if any(not value.strip() for value in values) or len(set(values)) != len(values):
|
||||
raise ValueError("evaluation_paths must contain unique nonempty paths")
|
||||
return values
|
||||
|
||||
|
||||
def evaluation_paths(home, values):
|
||||
"""Require explicit, unique local runs with actual evidence, including failed runs."""
|
||||
if not values:
|
||||
raise ValueError("evaluation_paths must identify at least one run")
|
||||
paths = []
|
||||
root = contained(home, "evaluations", directory=True)
|
||||
for value in values:
|
||||
path = contained(home, value, directory=True)
|
||||
if not path.is_relative_to(root) or path == root:
|
||||
raise ValueError("evaluation_paths must be inside home/evaluations")
|
||||
read_json(contained(path, "manifest.json"))
|
||||
evidence = [path / "summary.json", path / "failure.json"]
|
||||
evidence += list(path.glob("cases/*/memory_construction/result.json"))
|
||||
evidence += list(path.glob("cases/*/queries/result.json"))
|
||||
if not any(contained(path, item).is_file() for item in evidence):
|
||||
raise ValueError(f"Evaluation contains no results: {path}")
|
||||
paths.append(str(path))
|
||||
if len(paths) != len(set(paths)):
|
||||
raise ValueError("evaluation_paths must be unique")
|
||||
return paths
|
||||
|
||||
|
||||
def trajectory(state):
|
||||
"""Serialize only caller messages and compression summary, never credentials or permissions."""
|
||||
if state is not None and not isinstance(state, AgentState):
|
||||
raise TypeError("Injected caller state must be an AgentState")
|
||||
if state is None:
|
||||
return {"messages": [], "summary": None}
|
||||
summary = state.summary
|
||||
if isinstance(summary, list):
|
||||
summary = [block.model_dump(mode="json") for block in summary]
|
||||
return {
|
||||
"session_id": state.session_id,
|
||||
"messages": [message.model_dump(mode="json") for message in state.context],
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
|
||||
def diagnostic_tool(wrapper):
|
||||
"""Inject the optimizer's real trajectory into a fresh diagnostic invocation."""
|
||||
job = required_job(wrapper, "home_diagnose")
|
||||
|
||||
async def diagnose(
|
||||
evaluation_paths: list[str], # pylint: disable=redefined-outer-name
|
||||
instruction: str = "Identify and rank the evidence-supported root causes.",
|
||||
**kwargs,
|
||||
):
|
||||
if set(kwargs) - {"_agent_state"}:
|
||||
raise ValueError("Caller trajectory is host-owned and cannot be supplied by the model")
|
||||
response = await job(
|
||||
instruction=instruction,
|
||||
evaluation_paths=evaluation_paths,
|
||||
caller_trajectory=trajectory(kwargs.get("_agent_state")),
|
||||
)
|
||||
return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
|
||||
|
||||
tool = FunctionTool(
|
||||
diagnose,
|
||||
name="home_diagnose",
|
||||
description=job.description,
|
||||
is_concurrency_safe=False,
|
||||
is_state_injected=True,
|
||||
)
|
||||
tool.input_schema = copy.deepcopy(job.parameters)
|
||||
return tool
|
||||
|
||||
|
||||
class HomeDiagnoseStep(BaseStep):
|
||||
"""Aggregate explicit evaluation evidence through a read-only diagnostic agent."""
|
||||
|
||||
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
|
||||
agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
|
||||
|
||||
async def execute(self):
|
||||
"""Validate evidence, save the host trajectory and persist a provenance-bearing report."""
|
||||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home diagnosis requires access to its managed evidence")
|
||||
if not self.workspace.is_started or not self.agent_wrapper.is_started:
|
||||
raise RuntimeError("Start the Home workspace and diagnostic wrapper first")
|
||||
spec = DiagnosticInput.model_validate(
|
||||
{key: self.context[key] for key in DiagnosticInput.model_fields if key in self.context},
|
||||
)
|
||||
home = self.workspace.home_path
|
||||
paths = await blocking(evaluation_paths, home, spec.evaluation_paths)
|
||||
caller = self.context.get("caller_trajectory", trajectory(None))
|
||||
if not isinstance(caller, dict):
|
||||
raise ValueError("Caller trajectory must be a host-supplied object")
|
||||
source = await blocking(save_record, home, "diagnostic-trajectories", caller)
|
||||
payload = {
|
||||
"status": "running",
|
||||
"instruction": spec.instruction,
|
||||
"evaluation_paths": paths,
|
||||
"trajectory_path": source,
|
||||
}
|
||||
report_path = await blocking(save_record, home, "diagnostic-reports", payload)
|
||||
self.context.response.metadata.update(diagnostic_report_path=report_path, trajectory_path=source)
|
||||
try:
|
||||
result = await workflow_reply(
|
||||
self.agent_wrapper,
|
||||
home,
|
||||
"diagnostic",
|
||||
f"Diagnostic request:\n{spec.instruction}\n\nComplete available caller trajectory: {source}\n"
|
||||
f"Explicit evaluation paths: {json.dumps(paths, ensure_ascii=False)}",
|
||||
job_names=("home_case_study",),
|
||||
memory_limit_mib=self.kwargs.get("memory_limit_mib"),
|
||||
)
|
||||
if not isinstance(result.get("result"), str) or not result["result"].strip():
|
||||
raise ValueError("Diagnostic agent returned no report")
|
||||
payload.update(
|
||||
status="completed",
|
||||
diagnosis=result["result"],
|
||||
session_id=result.get("session_id"),
|
||||
usage=result.get("usage"),
|
||||
)
|
||||
self.context.response.answer = {**payload, "diagnostic_report_path": report_path}
|
||||
self.context.response.metadata.update(
|
||||
{key: result[key] for key in ("session_id", "usage") if key in result},
|
||||
)
|
||||
except BaseException as exc:
|
||||
payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
await blocking(write_json, contained(home, report_path), payload)
|
||||
return self.context.response
|
||||
32
plugins/home/src/reme_home/agent/diagnostic.yaml
Normal file
32
plugins/home/src/reme_home/agent/diagnostic.yaml
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
system_prompt: |-
|
||||
You are Home's read-only code diagnostician, migrated from Meta-ReMe.
|
||||
Identify and rank evidence-supported mechanisms that caused evaluation failures or limited candidate quality.
|
||||
The candidate repository is {code_repository}; guides are under {docs}; private scratch is {scratch}.
|
||||
The caller provides explicit evaluation paths and its available complete trajectory, including any compression summary.
|
||||
|
||||
Produce one concise report organized as Priority scope, Failure clusters, Ranked shared root causes, Outliers,
|
||||
and Missing evidence. State which low-scoring cases and question types you prioritized and what you deferred.
|
||||
For each cause, name affected cases and queries, lifecycle stage, causal chain, supporting and contradicting
|
||||
evidence, confidence, and the smallest missing observation. Separate fixed-runtime limitations.
|
||||
|
||||
Read architecture-guide.md and validation-artifact-guide.md when needed. Confirm each run's exact commit,
|
||||
status, score contract and coverage. Compare runs only where contracts and case/query coverage permit it.
|
||||
Prioritize the lowest-scoring cases and question types; you need not explain every failure.
|
||||
Use home_case_study batches early as the default for deep case investigation, even for one case. Pass one
|
||||
evaluation_path, case_ids and a shared focus; do not put case-specific instructions in the shared focus.
|
||||
Use its reports to identify shared causes, then inspect only remaining cross-case or code-level evidence.
|
||||
Direct investigation is appropriate for quick lookups, aggregate comparisons and specific evidence checks.
|
||||
|
||||
Classify infrastructure, candidate-execution, answer-quality and judging failures separately. Trace extraction,
|
||||
persistence, overwrite, indexing, retrieval, ranking, context presentation or reasoning as evidence requires.
|
||||
Treat partial_success construction as requiring repair. Do not replace persisted runtime failure statuses.
|
||||
Original sessions and derived memory are both evidence; absence from summaries does not prove unavailability.
|
||||
Prefer independent evidence_session_ids; inferred evidence is a lead, not proof. Explicitly state uncertainty.
|
||||
Use home_list_validation_results and home_inspect_validation to locate evidence. Pin revision to the evaluated
|
||||
full commit when using home_list_code_tree, home_search_code and home_read_code_file. The current checkout may
|
||||
differ. Read artifacts and runtime docs with Read, Grep, Glob, AutoBash or python_execute, using absolute paths.
|
||||
Keep stderr visible; denied reads are not proof of missing content.
|
||||
|
||||
The OS sandbox allows evidence reads and writes only to private scratch. You may create disposable scripts
|
||||
and analysis files there. Do not modify candidate code, datasets, evaluations, reports or Git metadata.
|
||||
Do not run ReMe, evaluations or network operations. Do not implement or prescribe fixes; report supported causes.
|
||||
513
plugins/home/src/reme_home/agent/optimizer.py
Normal file
513
plugins/home/src/reme_home/agent/optimizer.py
Normal file
|
|
@ -0,0 +1,513 @@
|
|||
"""Optimize Home candidates and verify their baseline and final evaluation evidence."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from agentscope.message import HintBlock
|
||||
from agentscope.state import AgentState
|
||||
from agentscope.tool import Edit, FunctionTool
|
||||
from pydantic import Field
|
||||
|
||||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from ..code.repository import validate_repository, require_clean, validate_tree, status
|
||||
from ..evaluation.artifacts import contained, read_json, write_json
|
||||
from ..evaluation.component import blocking
|
||||
from ..evaluation.inputs import EvaluationSpec, load_inputs
|
||||
from ..evaluation.history import read_state
|
||||
from ..evaluation.budget import validate_max_validations, validation_budget_status
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from .diagnostic import diagnostic_tool, evaluation_paths
|
||||
from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity
|
||||
from .compression import RecoverableCompressionAgent
|
||||
from .workflow_common import required_job, save_record, workflow_reply
|
||||
from .wrapper import CaseStudyAgentWrapper
|
||||
|
||||
MUTATION_JOBS = (
|
||||
"home_create_candidate_branch",
|
||||
"home_switch_candidate_branch",
|
||||
"home_merge_candidate_branch",
|
||||
"home_commit_candidate",
|
||||
)
|
||||
PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions"
|
||||
|
||||
|
||||
class ValidationCycleTimer:
|
||||
"""Emit escalating reminders as a scored-validation cycle ages."""
|
||||
|
||||
def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None:
|
||||
if (
|
||||
not isinstance(time_budget_minutes, (int, float))
|
||||
or isinstance(time_budget_minutes, bool)
|
||||
or not math.isfinite(time_budget_minutes)
|
||||
or time_budget_minutes <= 0
|
||||
):
|
||||
raise ValueError("time_budget_minutes must be a positive number")
|
||||
self.time_budget_minutes = float(time_budget_minutes)
|
||||
self._clock = clock
|
||||
self._started_at = self._clock()
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Start a fresh cycle after a scored validation finishes."""
|
||||
|
||||
self._started_at = self._clock()
|
||||
self._last_reminded_ratio = 0.0
|
||||
self._next_reminder_is_persistent = False
|
||||
self.context_compression_count = 0
|
||||
|
||||
def record_context_compression(self) -> None:
|
||||
"""Record one completed compression since the last scored validation."""
|
||||
|
||||
self.context_compression_count += 1
|
||||
|
||||
def next_reminder(self) -> TimeBudgetReminder | None:
|
||||
"""Return one reminder for the latest newly reached deadline."""
|
||||
|
||||
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
|
||||
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
|
||||
reached_ratio: float | None = None
|
||||
if elapsed_ratio >= 1.0:
|
||||
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
|
||||
else:
|
||||
for threshold in (0.9, 0.75, 0.5):
|
||||
if elapsed_ratio >= threshold:
|
||||
reached_ratio = threshold
|
||||
break
|
||||
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
|
||||
return None
|
||||
self._last_reminded_ratio = reached_ratio
|
||||
persistent = self._next_reminder_is_persistent
|
||||
self._next_reminder_is_persistent = not persistent
|
||||
elapsed = _format_minutes(elapsed_minutes)
|
||||
percentage = round(reached_ratio * 100)
|
||||
timing = (
|
||||
f"The optimizer agent run has used {elapsed} since the last scored validation "
|
||||
f"({percentage}% of its {_format_minutes(self.time_budget_minutes)} soft time budget). "
|
||||
)
|
||||
compression = (
|
||||
f"It has completed {self.context_compression_count} context compressions since the last "
|
||||
"scored validation. "
|
||||
)
|
||||
if elapsed_ratio >= 1.0:
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
|
||||
"validation as soon as possible.</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
)
|
||||
remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes)
|
||||
return TimeBudgetReminder(
|
||||
content=(
|
||||
f"<system-reminder>{timing}{compression}"
|
||||
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
|
||||
"</system-reminder>"
|
||||
),
|
||||
persistent=persistent,
|
||||
)
|
||||
|
||||
|
||||
class OptimizerAgent(RecoverableCompressionAgent):
|
||||
"""Add Meta-ReMe's per-validation reminders to Home's bounded agent."""
|
||||
|
||||
def __init__(self, *args, validation_cycle_timer=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.validation_cycle_timer = validation_cycle_timer
|
||||
|
||||
async def _reasoning(self, *args, **kwargs):
|
||||
"""Deliver hints after tool results without splitting the SDK's reply message."""
|
||||
transitions = self.state.middle_context.pop(PENDING_CYCLE_TRANSITIONS, [])
|
||||
for content in transitions:
|
||||
self.state.append_context(
|
||||
self.name,
|
||||
[HintBlock(source="meta_reme_validation_cycle_transition", hint=content)],
|
||||
)
|
||||
timer = self.validation_cycle_timer
|
||||
reminder = timer.next_reminder() if timer is not None else None
|
||||
message = None
|
||||
if reminder is not None:
|
||||
message = HintBlock(source="meta_reme_validation_timer", hint=reminder.content)
|
||||
self.state.append_context(self.name, [message])
|
||||
try:
|
||||
async for item in super()._reasoning(*args, **kwargs):
|
||||
yield item
|
||||
finally:
|
||||
if reminder is not None and not reminder.persistent:
|
||||
_remove_hint_by_identity(self.state.context, message)
|
||||
|
||||
async def compress_context(self, *args, **kwargs):
|
||||
"""Count completed context replacements in the current validation cycle."""
|
||||
summary_before, context_before = self.state.summary, self.state.context
|
||||
await super().compress_context(*args, **kwargs)
|
||||
if self.validation_cycle_timer is not None and (
|
||||
self.state.summary is not summary_before or self.state.context is not context_before
|
||||
):
|
||||
self.validation_cycle_timer.record_context_compression()
|
||||
|
||||
|
||||
class OptimizerAgentWrapper(CaseStudyAgentWrapper):
|
||||
"""Keep each optimizer's tools and cycle timer bound to one reply invocation."""
|
||||
|
||||
def create_cycle_timer(self, options=None):
|
||||
"""Use the legacy time setting as a fallback; explicit null disables reminders."""
|
||||
options = self._merged_kwargs(options or {})
|
||||
budget = options.get("validation_cycle_time_budget_minutes", options.get("time_budget_minutes"))
|
||||
return None if budget is None else ValidationCycleTimer(budget)
|
||||
|
||||
def validation_limit(self):
|
||||
"""Keep the formal count limit host-owned and independent of elapsed time."""
|
||||
return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10))
|
||||
|
||||
def _create_agent(self, *, options, **kwargs):
|
||||
timer = (
|
||||
options["_validation_cycle_timer"]
|
||||
if "_validation_cycle_timer" in options
|
||||
else self.create_cycle_timer(options)
|
||||
)
|
||||
return OptimizerAgent(
|
||||
validation_cycle_timer=timer,
|
||||
fallback_directory=options.get("_context_fallback_directory", self.session_path / "context-offloads"),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def validation_cycle_transition(manifest, validation_number, remaining):
|
||||
"""Expire old reminders and announce the next cycle with its remaining budget."""
|
||||
commit = manifest["commit_sha"][:7]
|
||||
return HintBlock(
|
||||
source="meta_reme_validation_cycle_transition",
|
||||
hint=(
|
||||
f"<system-reminder>Formal validation {validation_number} completed successfully for commit {commit}. "
|
||||
f"The previous cycle timer has been reset, and all timer reminders for cycle {validation_number} "
|
||||
f"are now expired. Remaining formal validations: {remaining}. Start cycle {validation_number + 1}."
|
||||
"</system-reminder>"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def completed_validation(home, result, started_before):
|
||||
"""Match dev_meta's new-count plus completed-status boundary, independent of scores."""
|
||||
if not isinstance(result, dict) or result.get("status") not in {"success", "completed"}:
|
||||
return None
|
||||
run = contained(home, result["result_dir"], directory=True)
|
||||
key = run.relative_to(contained(home, "evaluations", directory=True)).as_posix()
|
||||
started = read_state(home)["started_validations"]
|
||||
# The initial full baseline precedes optimizer creation in dev_meta and
|
||||
# never consumes its validation budget. Home records it in sampling history.
|
||||
if key in started_before or key not in started or key.startswith("init/"):
|
||||
return None
|
||||
manifest = read_json(contained(run, "manifest.json"))
|
||||
if (
|
||||
manifest.get("status") != "completed"
|
||||
or manifest.get("mode") != "scored"
|
||||
or manifest.get("dataset_kind") != "training-set"
|
||||
or not manifest.get("sandbox_started_at")
|
||||
):
|
||||
return None
|
||||
return manifest, sum(not name.startswith("init/") for name in started)
|
||||
|
||||
|
||||
class OptimizationInput(SpecModel):
|
||||
"""The host owns runtime settings; the caller supplies only the optimization focus."""
|
||||
|
||||
instruction: str = Field(default="Improve the candidate using the completed baseline evaluation.", min_length=1)
|
||||
|
||||
|
||||
class OptimizationResult(SpecModel):
|
||||
"""An agent claim that must be checked against Git and saved formal evidence."""
|
||||
|
||||
branch_name: str = Field(min_length=1)
|
||||
commit_sha: str = Field(pattern="^[0-9a-f]{40}$")
|
||||
evaluation_path: str = Field(min_length=1)
|
||||
report: str = Field(min_length=1)
|
||||
|
||||
|
||||
def parse_result(text):
|
||||
"""Accept a JSON object, optionally enclosed in one Markdown JSON fence."""
|
||||
text = text.strip()
|
||||
if text.startswith("```") and text.endswith("```"):
|
||||
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
|
||||
return OptimizationResult.model_validate(json.loads(text))
|
||||
|
||||
|
||||
EVALUATION_OPTIONS = {
|
||||
"concurrency",
|
||||
"timeout_seconds",
|
||||
"base_image",
|
||||
"fail_fast",
|
||||
"max_construction_failure_rate",
|
||||
"max_artifact_bytes",
|
||||
}
|
||||
|
||||
|
||||
def evaluation_options(raw):
|
||||
"""Validate host-only execution settings without exposing selection or replay."""
|
||||
if not isinstance(raw, dict) or set(raw) - EVALUATION_OPTIONS:
|
||||
raise ValueError("evaluation_options may contain only Home execution settings")
|
||||
return EvaluationSpec.model_validate(raw).model_dump(exclude_unset=True)
|
||||
|
||||
|
||||
def verify_evaluation(home, path, commit, *, baseline=False):
|
||||
"""Require completed end-to-end evidence with no infrastructure failures."""
|
||||
run = Path(evaluation_paths(home, [str(path)])[0])
|
||||
manifest = read_json(contained(run, "manifest.json"))
|
||||
summary = read_json(contained(run, "summary.json"))
|
||||
required = {
|
||||
"commit_sha": commit,
|
||||
"status": "completed",
|
||||
"success": True,
|
||||
"mode": "scored",
|
||||
"dataset_kind": "training-set",
|
||||
"execution_mode": "full_validation",
|
||||
}
|
||||
if any(manifest.get(key) != value for key, value in required.items()):
|
||||
raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
|
||||
if (
|
||||
not manifest.get("sandbox_started_at")
|
||||
or not summary.get("case_count")
|
||||
or summary.get("scoring_enabled") is not True
|
||||
or any(summary.get(key, {}).get("infra_error") != 0 for key in ("case_status_counts", "query_status_counts"))
|
||||
):
|
||||
raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
|
||||
_record, dataset, cases = load_inputs(home, EvaluationSpec())
|
||||
if manifest.get("fingerprints", {}).get("dataset") != dataset.normalized_fingerprint:
|
||||
raise ValueError("Evaluation dataset no longer matches the prepared dataset")
|
||||
selected = manifest.get("case_ids", [])
|
||||
queries = manifest.get("query_ids", {})
|
||||
expected = {case.case_id: {q.query_id for q in case.queries} for case in cases}
|
||||
if (
|
||||
not selected
|
||||
or len(set(selected)) != len(selected)
|
||||
or set(selected) - expected.keys()
|
||||
or set(queries) != set(selected)
|
||||
or any(not ids or len(set(ids)) != len(ids) or set(ids) - expected[key] for key, ids in queries.items())
|
||||
):
|
||||
raise ValueError("Evaluation has an invalid case/query selection")
|
||||
if baseline and (set(selected) != set(expected) or any(set(queries[key]) != ids for key, ids in expected.items())):
|
||||
raise ValueError("Initial baseline must cover every prepared training case and query")
|
||||
rows = summary.get("cases", {})
|
||||
if (
|
||||
set(rows) != set(selected)
|
||||
or summary["case_count"] != len(selected)
|
||||
or summary.get("expected_query_count") != sum(len(ids) for ids in queries.values())
|
||||
):
|
||||
raise ValueError("Evaluation summary has incomplete case coverage")
|
||||
# Failed construction can legitimately omit answers; the recorded candidate
|
||||
# failure is evidence, not an infrastructure failure or a missing result.
|
||||
for key, row in rows.items():
|
||||
if set(row.get("queries", {})) != set(queries[key]) and row.get("construction", {}).get("status") != "failure":
|
||||
raise ValueError(f"Evaluation summary has incomplete query coverage: {key}")
|
||||
return {"evaluation_path": str(run), "manifest": manifest, "summary": summary}
|
||||
|
||||
|
||||
def baseline_state(home):
|
||||
"""Inspect a clean repository and locate a usable initial full evaluation."""
|
||||
_validate_directory(home, home / "metadata")
|
||||
with _initialization_lock(home / "metadata/workspace.lock"):
|
||||
repo, record = validate_repository(home)
|
||||
require_clean(repo)
|
||||
validate_tree(repo, record, "HEAD")
|
||||
initial = record["code"]["initial_commit"]
|
||||
current = status(repo, record)
|
||||
for manifest in sorted((home / "evaluations/init").glob("*/*/manifest.json"), reverse=True):
|
||||
try:
|
||||
evidence = verify_evaluation(home, manifest.parent, initial, baseline=True)
|
||||
return current, evidence
|
||||
except (ValueError, OSError, KeyError, TypeError):
|
||||
continue
|
||||
if current["current_branch"] != "init" or current["commit_sha"] != initial:
|
||||
raise ValueError("A complete initial baseline is missing; check out clean init before home_optimize")
|
||||
return current, None
|
||||
|
||||
|
||||
def verify_final(home, result):
|
||||
"""Check the actual checkout and evidence; never repair or discard agent edits."""
|
||||
_validate_directory(home, home / "metadata")
|
||||
with _initialization_lock(home / "metadata/workspace.lock"):
|
||||
repo, record = validate_repository(home)
|
||||
require_clean(repo)
|
||||
validate_tree(repo, record, "HEAD")
|
||||
current = status(repo, record)
|
||||
if current["current_branch"] != result.branch_name or current["commit_sha"] != result.commit_sha:
|
||||
raise ValueError("Final branch/commit does not match the checked-out HEAD")
|
||||
evidence = verify_evaluation(home, result.evaluation_path, result.commit_sha)
|
||||
if evidence["manifest"]["branch_name"] != result.branch_name:
|
||||
raise ValueError("Final evaluation belongs to a different branch")
|
||||
return evidence["evaluation_path"]
|
||||
|
||||
|
||||
def validation_budget_tool(home, max_validations):
|
||||
"""Expose dev_meta's read-only count, phase and next-candidate guidance."""
|
||||
validate_max_validations(max_validations)
|
||||
|
||||
async def get_validation_budget():
|
||||
"""Return started formal validations, remaining count and optimization guidance."""
|
||||
return await blocking(validation_budget_status, home, max_validations)
|
||||
|
||||
return FunctionTool(
|
||||
get_validation_budget,
|
||||
name="get_validation_budget",
|
||||
description=(
|
||||
"Report how many formal optimizer validations have started at least one sandbox, how many remain, "
|
||||
"and whether the next candidate should emphasize structural code changes or evidence-driven refinement. "
|
||||
"Debug-case smoke validations, the initial baseline and failures before sandbox startup are excluded."
|
||||
),
|
||||
is_read_only=True,
|
||||
)
|
||||
|
||||
|
||||
def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
|
||||
"""Only debug mode is model-selected; all formal runs rebuild construction."""
|
||||
job = required_job(wrapper, "home_evaluate")
|
||||
validate_max_validations(max_validations)
|
||||
|
||||
async def evaluate(debug: bool = False, **kwargs):
|
||||
"""Run debug after committing; after debug passes, run a sampled formal evaluation.
|
||||
|
||||
Debug checks execution, not answer quality. Inspect infrastructure failures
|
||||
separately; retries keep the existing commit and selection recovery rules.
|
||||
"""
|
||||
if not isinstance(debug, bool):
|
||||
raise ValueError("debug must be boolean")
|
||||
if set(kwargs) - {"_agent_state"}:
|
||||
raise ValueError("Optimizer state is host-owned and cannot be supplied by the model")
|
||||
state = kwargs.get("_agent_state")
|
||||
if not isinstance(state, AgentState):
|
||||
raise TypeError("Cycle reminders require the host-injected AgentState")
|
||||
started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set()
|
||||
response = await job(
|
||||
**options,
|
||||
**({"max_validations": max_validations} if not debug else {}),
|
||||
debug=debug,
|
||||
sampling=not debug,
|
||||
record_history=True,
|
||||
case_ids=None,
|
||||
query_ids=None,
|
||||
reuse_construction_from=None,
|
||||
)
|
||||
if not debug:
|
||||
budget = await blocking(validation_budget_status, home, max_validations)
|
||||
if isinstance(response.answer, dict):
|
||||
response.answer = {**response.answer, "validation_budget": budget}
|
||||
completed = await blocking(completed_validation, home, response.answer, started_before)
|
||||
if completed is not None:
|
||||
manifest, validation_number = completed
|
||||
if timer is not None:
|
||||
timer.reset()
|
||||
transition = validation_cycle_transition(manifest, validation_number, budget["remaining_validations"])
|
||||
state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint)
|
||||
return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
|
||||
|
||||
tool = FunctionTool(evaluate, name="home_evaluate", is_concurrency_safe=False, is_state_injected=True)
|
||||
tool.input_schema = {
|
||||
"type": "object",
|
||||
"properties": {"debug": {"type": "boolean", "default": False}},
|
||||
"additionalProperties": False,
|
||||
}
|
||||
return tool
|
||||
|
||||
|
||||
class HomeOptimizeStep(BaseStep):
|
||||
"""Require a full initial baseline, run the coding agent, and verify its final claim."""
|
||||
|
||||
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
|
||||
agent_wrapper: OptimizerAgentWrapper = Ref(OptimizerAgentWrapper, "agent_wrapper")
|
||||
|
||||
async def execute(self):
|
||||
"""One optimizer per workspace; never hold the evaluation/Git lock across model calls."""
|
||||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home optimization requires access to its managed workspace")
|
||||
if not self.workspace.is_started or not self.agent_wrapper.is_started:
|
||||
raise RuntimeError("Start the Home workspace and optimizer wrapper first")
|
||||
spec = OptimizationInput.model_validate(
|
||||
{key: self.context[key] for key in OptimizationInput.model_fields if key in self.context},
|
||||
)
|
||||
options = evaluation_options(self.kwargs.get("evaluation_options", {}))
|
||||
self.agent_wrapper.validation_limit()
|
||||
home = self.workspace.home_path
|
||||
_validate_directory(home, self.workspace.state_path)
|
||||
with _initialization_lock(self.workspace.state_path / "optimization.lock"):
|
||||
_current, baseline = await blocking(baseline_state, home)
|
||||
if baseline is None:
|
||||
response = await required_job(self.agent_wrapper, "home_evaluate")(
|
||||
**options,
|
||||
debug=False,
|
||||
sampling=False,
|
||||
record_history=True,
|
||||
case_ids=None,
|
||||
query_ids=None,
|
||||
reuse_construction_from=None,
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(f"Initial baseline failed: {response.answer}")
|
||||
current, _ = await blocking(baseline_state, home)
|
||||
baseline = await blocking(
|
||||
verify_evaluation,
|
||||
home,
|
||||
response.answer["result_dir"],
|
||||
current["commit_sha"],
|
||||
baseline=True,
|
||||
)
|
||||
return await self._optimize(home, spec, baseline, options)
|
||||
|
||||
async def _optimize(self, home, spec, baseline, options):
|
||||
payload = {"status": "running", "instruction": spec.instruction, "baseline_path": baseline["evaluation_path"]}
|
||||
report_path = await blocking(save_record, home, "optimization-reports", payload)
|
||||
self.context.response.metadata["optimization_report_path"] = report_path
|
||||
try:
|
||||
timer = self.agent_wrapper.create_cycle_timer()
|
||||
max_validations = self.agent_wrapper.validation_limit()
|
||||
budget = await blocking(validation_budget_status, home, max_validations)
|
||||
reply = await workflow_reply(
|
||||
self.agent_wrapper,
|
||||
home,
|
||||
"optimizer",
|
||||
f"Begin optimization.\n{spec.instruction}\n\n"
|
||||
f"Completed full initial baseline: {baseline['evaluation_path']}\n"
|
||||
f"Formal validation budget: {json.dumps(budget)}\n"
|
||||
"Return exactly one JSON object with branch_name, commit_sha (full SHA), evaluation_path and report. "
|
||||
"Leave that evaluated branch and commit checked out with no uncommitted changes.",
|
||||
tool_factory=lambda backend: [
|
||||
Edit(backend=backend),
|
||||
diagnostic_tool(self.agent_wrapper),
|
||||
validation_budget_tool(home, max_validations),
|
||||
evaluation_tool(
|
||||
self.agent_wrapper,
|
||||
options,
|
||||
home=home,
|
||||
timer=timer,
|
||||
max_validations=max_validations,
|
||||
),
|
||||
],
|
||||
job_names=MUTATION_JOBS,
|
||||
agent_options={
|
||||
"_validation_cycle_timer": timer,
|
||||
"_context_fallback_directory": contained(home, "logs/context-offloads", directory=True),
|
||||
},
|
||||
memory_limit_mib=self.kwargs.get("memory_limit_mib"),
|
||||
)
|
||||
payload.update(
|
||||
agent_report=reply.get("result"),
|
||||
session_id=reply.get("session_id"),
|
||||
usage=reply.get("usage"),
|
||||
)
|
||||
result = parse_result(reply["result"])
|
||||
evidence_path = await blocking(verify_final, home, result)
|
||||
payload.update(result.model_dump(), evaluation_path=evidence_path, status="completed")
|
||||
self.context.response.answer = {**payload, "optimization_report_path": report_path}
|
||||
self.context.response.metadata.update({key: reply[key] for key in ("session_id", "usage") if key in reply})
|
||||
except BaseException as exc:
|
||||
payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
await blocking(write_json, contained(home, report_path), payload)
|
||||
return self.context.response
|
||||
60
plugins/home/src/reme_home/agent/optimizer.yaml
Normal file
60
plugins/home/src/reme_home/agent/optimizer.yaml
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
system_prompt: |-
|
||||
You are Home's primary optimization agent, migrated from Meta-ReMe. Execute the optimization loop;
|
||||
your output is the best formally evaluated candidate, with its branch and exact commit checked out.
|
||||
ReMe is local-first and file-native. User-owned memory is the source of truth; indexes and caches are rebuildable.
|
||||
Improve general memory mechanisms, robustness, recoverability and transparent behavior. Never introduce
|
||||
benchmark-specific shortcuts, held-out answer leakage or case-specific rules. Fixed runtime is host-owned.
|
||||
Candidate source is {code_repository}; guides are under {docs}; disposable scratch is {scratch}.
|
||||
|
||||
Optimize answer quality by supplying precise evidence at lower query-time token cost. Score improvement is
|
||||
more important than query token reduction. Every proposed change should aim for both. A candidate branch
|
||||
represents one coherent, falsifiable hypothesis. Candidate paths are the only editable source roots.
|
||||
|
||||
Repeat this loop:
|
||||
1. Start from the completed full baseline supplied by the host or the current candidate's completed formal
|
||||
evaluation. Inventory evaluations and branches, commits and coverage before editing.
|
||||
2. Diagnose before editing. Strongly prefer home_diagnose with the actual evaluation_paths; it automatically
|
||||
receives your available full trajectory. Prioritize low-scoring cases and question types, identify at least
|
||||
two distinct supported causes when evidence permits, estimate impact, coverage and confidence, and rank
|
||||
directions. Direct investigation is appropriate when evidence is already narrow and clear. Treat partial
|
||||
construction as a defect. Distinguish infrastructure errors from candidate behavior; do not patch candidate
|
||||
code to compensate for infrastructure outages. Defer unrelated, redundant and low-impact investigation.
|
||||
3. Choose lineage before editing. Continue the current direction on its branch; fork a promising branch for
|
||||
a distinct direction with home_create_candidate_branch. Never edit init. Implement the highest-impact
|
||||
supported direction, or two or three compatible changes supporting one coherent claim. Inspect diffs before
|
||||
merging. Create, switch, merge and commit exclusively through the managed Home Jobs; keep evaluated history.
|
||||
Preserve a branch at each evaluated commit you may want to select finally: before advancing such a branch,
|
||||
fork a new candidate. Managed switching selects existing branch heads, not arbitrary historical commits.
|
||||
4. Commit before testing, then call home_evaluate(debug=true). Repair concrete execution failures, commit and
|
||||
repeat until debug passes. Debug establishes execution only; it has no answer-quality score.
|
||||
5. Call home_evaluate(debug=false) on clean committed code after debug passes. Formal evaluation uses host-owned
|
||||
adaptive sampling and always rebuilds memory before answering. Inspect the resulting evidence before the
|
||||
next iteration. Candidate failures and low scores require a code change; infrastructure failures may retry
|
||||
the same commit using the host's frozen previous selection. Empty commits do not bypass repetition checks.
|
||||
|
||||
Call get_validation_budget before planning the next candidate. The host limits formal validations; one run
|
||||
consumes one slot when its first sandbox starts successfully, even if it later fails or is cancelled. Debug,
|
||||
initial baseline and pre-sandbox failures do not consume slots. The budget persists across optimizer invocations.
|
||||
During structural_exploration prefer substantial mechanisms or coherent architecture changes; during
|
||||
evidence_driven_refinement prefer evidence-driven parameters, prompts or scoped changes. When exhausted,
|
||||
select the best existing evaluated branch without another formal validation. There is no candidate admission
|
||||
gate; do not call architecture or construction-equivalence tools. Use the configured iteration cap and advisory time
|
||||
reminders to finish a useful search; time reminders do not terminate execution. The cycle clock includes diagnosis,
|
||||
editing, debug and formal evaluation. Only a newly counted, completed formal evaluation resets it and its context
|
||||
compression count; debug, rejected calls and incomplete evaluations do not. A cycle transition expires earlier
|
||||
timer reminders and announces the next cycle. You may stop once the evidence
|
||||
supports a final selection. Do not keep evaluating without a concrete improvement hypothesis.
|
||||
Compare scores on common cases and queries, weigh coverage, difficulty, regressions and lineage, and do not
|
||||
rank sampled candidates by aggregate score alone. Prefer quality, stability, query cost and simplicity.
|
||||
Do not primarily improve scores by increasing top-k, iteration limits, token budgets or output length.
|
||||
Investigate better extraction, persistence, routing, retrieval, fusion, deduplication, reranking and context selection.
|
||||
|
||||
Read the runtime guides when needed. Tools can inspect memory checkpoints, evidence and commit-pinned code.
|
||||
Use revision explicitly when examining code associated with an evaluation. Shell and Python run within the same
|
||||
OS file sandbox as Read/Edit/Write; only declared candidate roots and scratch are writable. Network is denied.
|
||||
Never write Git metadata, runtime, datasets, evaluation artifacts or reports. Run ReMe only through home_evaluate.
|
||||
|
||||
Finish with a clean checkout of the best evaluated branch and full commit. Choosing the baseline is valid if
|
||||
improvements are unsupported. Return exactly one JSON object with branch_name, commit_sha, evaluation_path,
|
||||
and report. report must explain the chosen evidence, comparisons, regressions, costs and limitations. The host
|
||||
will reject a mismatched checkout or missing, incomplete, debug, query-only or infrastructure-failed evidence.
|
||||
230
plugins/home/src/reme_home/agent/workflow_common.py
Normal file
230
plugins/home/src/reme_home/agent/workflow_common.py
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
"""Shared sandbox, tools and invocation helpers for Home workflow agents."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from agentscope.tool import BackendBase, DirEntry, Glob, Grep, Read, Toolkit, Write
|
||||
import yaml
|
||||
|
||||
from ..auto_bash import AutoBash
|
||||
from ..code.repository import validate_repository
|
||||
from ..evaluation.artifacts import contained, write_json
|
||||
from ..evaluation.component import blocking
|
||||
from .case_study import python_tool
|
||||
from .sandbox import create_case_study_sandbox
|
||||
|
||||
INSPECTION_JOBS = (
|
||||
"home_list_validation_results",
|
||||
"home_inspect_validation",
|
||||
"home_trace_query_evidence",
|
||||
"home_inspect_memory_history",
|
||||
"home_compare_memory",
|
||||
"home_code_status",
|
||||
"home_list_candidate_branches",
|
||||
"home_code_diff",
|
||||
"home_list_code_tree",
|
||||
"home_read_code_file",
|
||||
"home_search_code",
|
||||
)
|
||||
|
||||
|
||||
def required_job(wrapper, name):
|
||||
"""Do not silently bypass a disabled or unstarted Home Job."""
|
||||
job = wrapper.app_context.jobs[name]
|
||||
if not job.enabled or not job.is_started:
|
||||
raise RuntimeError(f"Workflow requires a started Job: {name}")
|
||||
return job
|
||||
|
||||
|
||||
def create_toolkit(wrapper, backend, scope, *, extra_tools=(), job_names=()):
|
||||
"""All builtin tools share the OS sandbox; privileged mutations use managed Jobs."""
|
||||
tools = [
|
||||
Read(backend=backend),
|
||||
Glob(backend=backend),
|
||||
Grep(backend=backend),
|
||||
Write(backend=backend),
|
||||
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=False),
|
||||
python_tool(backend, scope["scratch"]),
|
||||
]
|
||||
tools[-1].description = (
|
||||
tools[-1]
|
||||
.description.replace(
|
||||
"No host path is writable.",
|
||||
"Only the host-granted scratch and candidate paths are writable.",
|
||||
)
|
||||
.replace("case directory", "scratch directory")
|
||||
)
|
||||
tools[-2].description = (
|
||||
"Run shell commands inside the OS file sandbox; network is denied. Use absolute allowed paths. "
|
||||
"Only private scratch and, for the optimizer, declared candidate paths are writable. "
|
||||
"Git metadata is read-only: commit, switch, create and merge only through Home Jobs. "
|
||||
"Run ReMe evaluations only through home_evaluate, never through shell or Python. "
|
||||
"Keep stderr visible and use set -o pipefail for pipelines. timeout is in milliseconds."
|
||||
)
|
||||
tools.extend(extra_tools)
|
||||
tools.extend(
|
||||
wrapper._make_tool(required_job(wrapper, name)) # pylint: disable=protected-access
|
||||
for name in (*INSPECTION_JOBS, *job_names)
|
||||
)
|
||||
for tool in tools:
|
||||
tool.is_concurrency_safe = False
|
||||
return Toolkit(tools=tools)
|
||||
|
||||
|
||||
async def workflow_reply(
|
||||
wrapper,
|
||||
home,
|
||||
role,
|
||||
task,
|
||||
*,
|
||||
tool_factory=None,
|
||||
job_names=(),
|
||||
agent_options=None,
|
||||
memory_limit_mib=None,
|
||||
):
|
||||
"""Use fresh state, the dedicated configured model and sandboxed tools for every call."""
|
||||
scope = await blocking(prepare_scope, home, role)
|
||||
backend = WorkflowSandbox(scope, memory_limit_mib)
|
||||
probe = await backend.exec_shell(
|
||||
[sys.executable, "-I", "-B", "-c", "print('home workflow sandbox')"],
|
||||
timeout=10,
|
||||
)
|
||||
if not probe.ok():
|
||||
raise RuntimeError("Home workflow sandbox unavailable: " + probe.stderr.decode(errors="replace"))
|
||||
prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8"))
|
||||
return await wrapper.reply(
|
||||
task,
|
||||
system_prompt=prompt["system_prompt"].format(**scope),
|
||||
toolkit=create_toolkit(
|
||||
wrapper,
|
||||
backend,
|
||||
scope,
|
||||
extra_tools=tool_factory(backend) if tool_factory is not None else (),
|
||||
job_names=job_names,
|
||||
),
|
||||
builtin_tools=[],
|
||||
job_tools=[],
|
||||
skills=[],
|
||||
resume=None,
|
||||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
permission_mode="bypass",
|
||||
**(agent_options or {}),
|
||||
)
|
||||
|
||||
|
||||
def prepare_scope(home, role):
|
||||
"""Grant evidence reads and private scratch writes, plus declared optimizer roots."""
|
||||
repo, record = validate_repository(home)
|
||||
scratch = contained(home, f"tmp/{role}-{uuid4().hex}", directory=True)
|
||||
scratch.mkdir()
|
||||
roots = [
|
||||
contained(home, name, directory=True)
|
||||
for name in ("code", "datasets", "docs", "evaluations", "logs", "sampling")
|
||||
]
|
||||
roots.append(contained(home, "metadata/preparation.json"))
|
||||
writable = [scratch]
|
||||
if role == "optimizer":
|
||||
for value in record["code"]["candidate_paths"]:
|
||||
path = contained(repo, value)
|
||||
if ".git" in path.relative_to(repo).parts or path == repo:
|
||||
raise ValueError("Optimizer cannot write Git metadata or the repository root")
|
||||
writable.append(path)
|
||||
return {
|
||||
"home": str(home),
|
||||
"code_repository": str(repo),
|
||||
"scratch": str(scratch),
|
||||
"docs": str(home / "docs"),
|
||||
"read_only_paths": [str(path) for path in roots],
|
||||
"writable_paths": [str(path) for path in writable],
|
||||
}
|
||||
|
||||
|
||||
def save_record(home, category, payload):
|
||||
"""Write a uniquely named host-owned provenance record, outside agent write grants."""
|
||||
path = contained(home, f"logs/{category}/{uuid4().hex}.json")
|
||||
write_json(path, payload)
|
||||
return str(path)
|
||||
|
||||
|
||||
_INSPECT = """\
|
||||
import json, pathlib, sys
|
||||
path = pathlib.Path(sys.argv[2])
|
||||
def entry(p):
|
||||
s = p.stat()
|
||||
return dict(name=p.name, is_dir=p.is_dir(), size_bytes=None if p.is_dir() else s.st_size, mtime=s.st_mtime)
|
||||
if sys.argv[1] == 'stat':
|
||||
print(json.dumps(entry(path)))
|
||||
else:
|
||||
result = []
|
||||
for p in path.iterdir():
|
||||
try:
|
||||
result.append(entry(p))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
print(json.dumps(result))
|
||||
"""
|
||||
|
||||
|
||||
class WorkflowSandbox(BackendBase):
|
||||
"""Route every builtin I/O primitive through Home's OS sandbox, including metadata.
|
||||
|
||||
Rebuild the backend before each operation: managed branch switches can
|
||||
replace an allowed file's inode. Core validation still checks paths and
|
||||
identities for each subprocess; no operation falls back to local I/O.
|
||||
"""
|
||||
|
||||
def __init__(self, scope, memory_limit_mib=None):
|
||||
self.scope = scope
|
||||
self.memory_limit_mib = memory_limit_mib
|
||||
|
||||
def _backend(self):
|
||||
return create_case_study_sandbox(
|
||||
cwd=self.scope["scratch"],
|
||||
read_only_paths=self.scope["read_only_paths"],
|
||||
writable_paths=self.scope["writable_paths"],
|
||||
memory_limit_mib=self.memory_limit_mib,
|
||||
)
|
||||
|
||||
async def getcwd(self):
|
||||
"""Use private scratch as the default, never the host process directory."""
|
||||
return self.scope["scratch"]
|
||||
|
||||
async def exec_shell(self, command, *, cwd=None, timeout=None):
|
||||
"""Keep command execution, environment and process cleanup inside isolation."""
|
||||
return await self._backend().exec_shell(command, cwd=cwd, timeout=timeout)
|
||||
|
||||
async def read_file(self, path):
|
||||
"""Use core sandboxed file reads, not LocalBackend's host reads."""
|
||||
return await self._backend().backend.read_file(path)
|
||||
|
||||
async def write_file(self, path, data):
|
||||
"""Use core sandboxed writes, including parent creation and stdin transport."""
|
||||
await self._backend().backend.write_file(path, data)
|
||||
|
||||
async def _inspect(self, operation, path):
|
||||
result = await self.exec_shell([sys.executable, "-I", "-B", "-c", _INSPECT, operation, path], timeout=10)
|
||||
if not result.ok():
|
||||
raise OSError(result.stderr.decode(errors="replace"))
|
||||
return json.loads(result.stdout)
|
||||
|
||||
async def stat(self, path):
|
||||
"""Use portable sandboxed metadata on Linux and macOS."""
|
||||
return DirEntry(**await self._inspect("stat", path))
|
||||
|
||||
async def scandir(self, path):
|
||||
"""Avoid GNU find assumptions and host filesystem metadata access."""
|
||||
return [DirEntry(**item) for item in await self._inspect("scandir", path)]
|
||||
|
||||
async def stat_mtime(self, path):
|
||||
"""Preserve the builtin Edit/Write optimistic modification checks."""
|
||||
return (await self.stat(path)).mtime
|
||||
|
||||
async def list_dir(self, path, *, recursive=False):
|
||||
"""Use the portable metadata implementation for immediate children."""
|
||||
if recursive:
|
||||
return await super().list_dir(path, recursive=True)
|
||||
return [entry.name for entry in await self.scandir(path)]
|
||||
43
plugins/home/src/reme_home/evaluation/budget.py
Normal file
43
plugins/home/src/reme_home/evaluation/budget.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""Formal optimizer budgets derived from Home's durable sandbox-start evidence."""
|
||||
|
||||
from .history import read_state
|
||||
|
||||
|
||||
def validate_max_validations(value):
|
||||
"""Accept only a positive host-supplied integer, matching dev_meta."""
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise ValueError("max_validations must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def validation_budget_status(home, max_validations):
|
||||
"""Report started formal runs, independently of completion, score or sampler success."""
|
||||
validate_max_validations(max_validations)
|
||||
used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"])
|
||||
remaining = max(0, max_validations - used)
|
||||
gated = max_validations // 2
|
||||
if not remaining:
|
||||
phase = "exhausted"
|
||||
guidance = (
|
||||
"The formal validation budget is exhausted; select among existing evidence without another validation."
|
||||
)
|
||||
elif used < gated:
|
||||
phase = "structural_exploration"
|
||||
guidance = (
|
||||
"Prefer a substantial code-level mechanism or coherent plugin architecture change. "
|
||||
"Do not spend an early validation on prompt-only or single-parameter tuning."
|
||||
)
|
||||
else:
|
||||
phase = "evidence_driven_refinement"
|
||||
guidance = (
|
||||
"Prefer evidence-driven parameter tuning, fine-grained prompt refinement, or a small scoped code change. "
|
||||
"Avoid a new broad architecture unless existing validation evidence clearly requires it."
|
||||
)
|
||||
return {
|
||||
"used_validations": used,
|
||||
"max_validations": max_validations,
|
||||
"remaining_validations": remaining,
|
||||
"gated_validation_count": gated,
|
||||
"optimization_phase": phase,
|
||||
"next_validation_guidance": guidance,
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ from ..judge import HomeJudge
|
|||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from ..sampling import SamplingPolicy, create_sampling_plan
|
||||
from .history import HomeEvaluationHistory
|
||||
from .budget import validation_budget_status
|
||||
from .repetition import previous_candidate_result
|
||||
|
||||
|
||||
|
|
@ -85,7 +86,25 @@ class HomeEvaluation(BaseComponent):
|
|||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
|
||||
with lock:
|
||||
return await self._evaluate(spec)
|
||||
budgeted = spec.max_validations is not None and not spec.debug
|
||||
if budgeted:
|
||||
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations)
|
||||
if budget["remaining_validations"] < 1:
|
||||
return {
|
||||
"status": "rejected",
|
||||
"error_code": "VALIDATION_BUDGET_EXHAUSTED",
|
||||
"error": "Optimizer validation budget is exhausted; no sandbox was started.",
|
||||
"validation_budget": budget,
|
||||
"result_dir": None,
|
||||
}
|
||||
result = await self._evaluate(spec)
|
||||
if budgeted:
|
||||
result["validation_budget"] = await blocking(
|
||||
validation_budget_status,
|
||||
self.workspace.home_path,
|
||||
spec.max_validations,
|
||||
)
|
||||
return result
|
||||
|
||||
def _prepare(self, spec, environment):
|
||||
home = self.workspace.home_path
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ class EvaluationSpec(SpecModel):
|
|||
debug: bool = False
|
||||
sampling: bool = False
|
||||
record_history: bool = True
|
||||
max_validations: int | None = Field(default=None, ge=1)
|
||||
sampling_policy: SamplingPolicy | None = None
|
||||
concurrency: int = Field(default=4, ge=1)
|
||||
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
|
||||
|
|
@ -67,6 +68,8 @@ class EvaluationSpec(SpecModel):
|
|||
@model_validator(mode="after")
|
||||
def resolve_mode(self):
|
||||
"""Debug stops on failure by default and never reuses scored evidence."""
|
||||
if self.max_validations is not None and not self.debug and not self.record_history:
|
||||
raise ValueError("Budgeted formal evaluations require record_history=true")
|
||||
if self.debug and self.reuse_construction_from:
|
||||
raise ValueError("Construction reuse requires training-set")
|
||||
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):
|
||||
|
|
|
|||
|
|
@ -39,6 +39,9 @@ class HomeEvaluateStep(BaseStep):
|
|||
self.context.response.metadata["result_dir"] = str(error.path)
|
||||
return self.context.response
|
||||
self.context.response.answer = result
|
||||
if result.get("status") == "rejected":
|
||||
self.context.response.success = False
|
||||
return self.context.response
|
||||
# Candidate failures are evaluation observations; infrastructure failures
|
||||
# must remain visible to ordinary Job callers checking Response.success.
|
||||
self.context.response.success = result["case_status_counts"]["infra_error"] == 0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
backends:
|
||||
home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper
|
||||
home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep
|
||||
home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep
|
||||
home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
|
||||
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
|
||||
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
|
||||
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep
|
||||
|
|
@ -20,6 +24,30 @@ backends:
|
|||
application_defaults:
|
||||
components:
|
||||
as_llm:
|
||||
home_optimizer:
|
||||
backend: ${LLM_BACKEND:-openai}
|
||||
model: qwen3.8-max
|
||||
stream: false
|
||||
context_size: 160000
|
||||
max_retries: 3
|
||||
credential:
|
||||
api_key: ${LLM_API_KEY:-}
|
||||
base_url: ${LLM_BASE_URL:-}
|
||||
parameters:
|
||||
max_tokens: 65536
|
||||
thinking_enable: false
|
||||
home_diagnostic:
|
||||
backend: ${LLM_BACKEND:-openai}
|
||||
model: qwen3.8-max
|
||||
stream: false
|
||||
context_size: 128000
|
||||
max_retries: 3
|
||||
credential:
|
||||
api_key: ${LLM_API_KEY:-}
|
||||
base_url: ${LLM_BASE_URL:-}
|
||||
parameters:
|
||||
max_tokens: 65536
|
||||
thinking_enable: false
|
||||
home_case_study:
|
||||
backend: ${LLM_BACKEND:-openai}
|
||||
model: qwen3.7-max
|
||||
|
|
@ -33,6 +61,22 @@ application_defaults:
|
|||
max_tokens: 65536
|
||||
thinking_enable: false
|
||||
agent_wrapper:
|
||||
home_optimizer:
|
||||
backend: home_optimizer_agent
|
||||
max_validations: 10
|
||||
as_llm: home_optimizer
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
context_config:
|
||||
trigger_ratio: 0.85
|
||||
reserve_ratio: 0.15
|
||||
react_config:
|
||||
max_iters: 3000
|
||||
home_diagnostic:
|
||||
backend: home_workflow_agent
|
||||
as_llm: home_diagnostic
|
||||
time_budget_minutes: 60
|
||||
react_config:
|
||||
max_iters: 80
|
||||
home_case_study:
|
||||
backend: home_case_study_agent
|
||||
as_llm: home_case_study
|
||||
|
|
@ -64,6 +108,41 @@ application_defaults:
|
|||
default:
|
||||
backend: home_workspace
|
||||
jobs:
|
||||
home_diagnose:
|
||||
backend: base
|
||||
description: "Diagnose explicit evaluation runs with a fresh read-only agent; aggregate case studies and return evidence-supported root causes without prescribing fixes"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [evaluation_paths]
|
||||
properties:
|
||||
evaluation_paths:
|
||||
type: array
|
||||
minItems: 1
|
||||
uniqueItems: true
|
||||
items: {type: string, minLength: 1}
|
||||
description: "Explicit evaluation result directories, absolute or relative to home/"
|
||||
instruction:
|
||||
type: string
|
||||
minLength: 1
|
||||
description: "Diagnostic question or suspected regression"
|
||||
steps:
|
||||
- backend: home_diagnose_step
|
||||
agent_wrapper: home_diagnostic
|
||||
home_optimize:
|
||||
backend: base
|
||||
description: "Check or run the full initial baseline, then diagnose, edit, commit, debug and evaluate candidates; verify the final clean checkout against saved formal evidence"
|
||||
parameters:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
instruction:
|
||||
type: string
|
||||
minLength: 1
|
||||
description: "Optimization focus; runtime, selection and sandbox settings are host-owned"
|
||||
steps:
|
||||
- backend: home_optimize_step
|
||||
agent_wrapper: home_optimizer
|
||||
home_case_study:
|
||||
backend: base
|
||||
description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order"
|
||||
|
|
@ -227,6 +306,10 @@ application_defaults:
|
|||
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
|
||||
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
|
||||
max_artifact_bytes: {type: integer, minimum: 1}
|
||||
max_validations:
|
||||
type: [integer, "null"]
|
||||
minimum: 1
|
||||
description: "Optional host formal-evaluation budget; optimizer injects its configured limit"
|
||||
steps:
|
||||
- backend: home_evaluate_step
|
||||
home_prepare:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
"""Benchmark Home defaults and lifecycle wiring, without models or dataset writes."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
|
@ -10,7 +12,8 @@ import yaml
|
|||
|
||||
from reme.schema import Response
|
||||
|
||||
from benchmark import home_runner
|
||||
from benchmark.beam import run_home as beam_runner
|
||||
from benchmark.longmemeval import run_home as lme_runner
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
|
@ -21,7 +24,7 @@ def test_preserved_lme_split_and_evidence():
|
|||
assert hashlib.sha256((directory / "home_evidence_sessions.jsonl").read_bytes()).hexdigest() == (
|
||||
"35655c51210a96b595910923576fa97b93c81abd82e3cfcda801180007c23a7d"
|
||||
)
|
||||
workspace, spec = home_runner.load_config(directory / "config_home.yaml")
|
||||
workspace, spec = lme_runner.load_config(directory / "config_home.yaml")
|
||||
selected = spec["benchmark"]["case_ids"]
|
||||
assert len(selected) == len(set(selected)) == 250
|
||||
# Hash the original first-five-group sequence to preserve both IDs and order.
|
||||
|
|
@ -36,7 +39,7 @@ def test_preserved_lme_split_and_evidence():
|
|||
def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
|
||||
"""Paths and selection do not depend on where Python was launched."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
workspace, spec = home_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
|
||||
workspace, spec = beam_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
|
||||
assert spec["benchmark"]["variant"] == "100K"
|
||||
assert spec["benchmark"]["all_cases"] is True
|
||||
assert spec["benchmark"]["case_ids"] is None
|
||||
|
|
@ -47,6 +50,18 @@ def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
|
|||
assert workspace == ROOT / "benchmark/beam/workspaces/home-100K"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
def test_runner_preserves_explicit_candidate_paths(tmp_path, benchmark):
|
||||
"""Runner defaults preserve custom scopes from existing configurations."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config["preparation"]["candidate"] = {"paths": ["reme/steps"]}
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
_, spec = module.load_config(path)
|
||||
assert spec["candidate"]["paths"] == ["reme/steps"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, benchmark):
|
||||
|
|
@ -60,7 +75,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(home_runner, "Application", factory)
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
result = await module.run(workspace_dir=str(tmp_path / "experiment"))
|
||||
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
|
||||
assert observed["service"] == {"backend": "cli"}
|
||||
|
|
@ -72,6 +87,10 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
"ewma_alpha": 0.3,
|
||||
"uniform_exploration": 0.2,
|
||||
}
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"validation_cycle_time_budget_minutes": 150,
|
||||
"max_validations": 10,
|
||||
}
|
||||
application.start.assert_awaited_once()
|
||||
application.close.assert_awaited_once()
|
||||
assert application.run_job.await_count == 1
|
||||
|
|
@ -80,9 +99,11 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch, benchmark):
|
||||
"""Partial policies merge typed defaults and never leak into preparation inputs."""
|
||||
config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
|
|
@ -94,8 +115,8 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
|
|||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(home_runner, "Application", factory)
|
||||
await home_runner.prepare(path)
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
await module.prepare(path)
|
||||
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
|
||||
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
|
||||
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
|
||||
|
|
@ -103,22 +124,193 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
|
|||
config["sampling_policy"] = invalid
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with pytest.raises(ValueError):
|
||||
await home_runner.prepare(path)
|
||||
await module.prepare(path)
|
||||
assert application.start.await_count == 1
|
||||
assert application.run_job.await_count == 1
|
||||
assert application.run_job.call_args.args == ("home_prepare",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("budget", ["omitted", 45.5, None])
|
||||
async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, benchmark, budget):
|
||||
"""Both local runners apply the default, custom duration and explicit disable value."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
if budget == "omitted":
|
||||
config.pop("validation_cycle_time_budget_minutes", None)
|
||||
else:
|
||||
config["validation_cycle_time_budget_minutes"] = budget
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(answer={})
|
||||
observed = {}
|
||||
|
||||
def factory(**kwargs):
|
||||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
await module.run(path)
|
||||
expected = 150 if budget == "omitted" else budget
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"validation_cycle_time_budget_minutes": expected,
|
||||
"max_validations": 10,
|
||||
}
|
||||
assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"]
|
||||
application.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("budget", [0, -1, True, "150", float("nan"), float("inf")])
|
||||
async def test_runner_rejects_invalid_cycle_budget_before_start(tmp_path, monkeypatch, benchmark, budget):
|
||||
"""Reject nonpositive, nonnumeric and nonfinite durations without creating an application."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config["validation_cycle_time_budget_minutes"] = budget
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
created = []
|
||||
monkeypatch.setattr(module, "Application", lambda **kwargs: created.append(kwargs))
|
||||
with pytest.raises(ValueError):
|
||||
await module.run(path)
|
||||
assert not created
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("failure", ["startup", "preparation"])
|
||||
async def test_runner_propagates_failure_and_closes(monkeypatch, failure):
|
||||
async def test_runner_propagates_failure_and_closes(monkeypatch, failure, benchmark):
|
||||
"""A failed preparation is never printed or returned as success."""
|
||||
application = AsyncMock()
|
||||
if failure == "startup":
|
||||
application.start.side_effect = RuntimeError("startup failed")
|
||||
else:
|
||||
application.run_job.return_value = Response(success=False, answer="already prepared")
|
||||
monkeypatch.setattr(home_runner, "Application", lambda **kwargs: application)
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
monkeypatch.setattr(module, "Application", lambda **kwargs: application)
|
||||
with pytest.raises(RuntimeError):
|
||||
await home_runner.prepare(ROOT / "benchmark/beam/config_home.yaml")
|
||||
await module.prepare(module.DEFAULT_CONFIG)
|
||||
application.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("limit", ["omitted", 3, 0, -1, True, "10", 1.5, None])
|
||||
async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchmark, limit):
|
||||
"""Validate host count budgets and pass them only to the optimizer wrapper."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
if limit == "omitted":
|
||||
config.pop("max_validations", None)
|
||||
else:
|
||||
config["max_validations"] = limit
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(answer={})
|
||||
observed = {}
|
||||
|
||||
def factory(**kwargs):
|
||||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
if limit in ("omitted", 3):
|
||||
await module.prepare(path)
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == (
|
||||
10 if limit == "omitted" else limit
|
||||
)
|
||||
assert "max_validations" not in application.run_job.call_args.kwargs["spec"]
|
||||
else:
|
||||
with pytest.raises(ValueError):
|
||||
await module.prepare(path)
|
||||
assert not observed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("minutes", [30, None])
|
||||
async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_path, monkeypatch, benchmark, minutes):
|
||||
"""A separate optimization launch consumes the same count, timer and sampling settings."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes)
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(answer={"status": "completed"})
|
||||
observed = {}
|
||||
|
||||
def factory(**kwargs):
|
||||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval")
|
||||
assert result["status"] == "completed"
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"max_validations": 3,
|
||||
"validation_cycle_time_budget_minutes": minutes,
|
||||
}
|
||||
application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval")
|
||||
application.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
async def test_optimizer_failure_closes_application(monkeypatch, benchmark):
|
||||
"""Failed optimization retains the normal cleanup and error contract."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(success=False, answer="optimization failed")
|
||||
monkeypatch.setattr(module, "Application", lambda **kwargs: application)
|
||||
with pytest.raises(RuntimeError, match="optimization failed"):
|
||||
await module.optimize()
|
||||
application.run_job.assert_awaited_once_with("home_optimize")
|
||||
application.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
def test_optimizer_cli_forwards_configuration_and_focus(tmp_path, monkeypatch, capsys, benchmark):
|
||||
"""The explicit CLI mode reaches optimization and preserves the preparation default."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = tmp_path / "home.yaml"
|
||||
optimize = AsyncMock(return_value={"status": "completed"})
|
||||
prepare = AsyncMock()
|
||||
monkeypatch.setattr(module, "optimize", optimize)
|
||||
monkeypatch.setattr(module, "prepare", prepare)
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"run_home.py",
|
||||
"--optimize",
|
||||
"--config",
|
||||
str(config),
|
||||
"--workspace-dir",
|
||||
str(tmp_path / "existing"),
|
||||
"--instruction",
|
||||
"Improve retrieval",
|
||||
],
|
||||
)
|
||||
module.main()
|
||||
optimize.assert_awaited_once_with(
|
||||
config,
|
||||
workspace_dir=str(tmp_path / "existing"),
|
||||
instruction="Improve retrieval",
|
||||
)
|
||||
prepare.assert_not_called()
|
||||
assert json.loads(capsys.readouterr().out) == {"status": "completed"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
def test_cli_rejects_optimization_focus_in_preparation_mode(monkeypatch, benchmark):
|
||||
"""Do not silently discard an instruction when the caller forgot --optimize."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
monkeypatch.setattr(sys, "argv", ["run_home.py", "--instruction", "Improve retrieval"])
|
||||
with pytest.raises(SystemExit) as error:
|
||||
module.main()
|
||||
assert error.value.code == 2
|
||||
|
|
|
|||
215
plugins/home/tests/test_optimizer_compression.py
Normal file
215
plugins/home/tests/test_optimizer_compression.py
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
"""Compression recovery parity with dev_meta, without model or filesystem services."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
|
||||
# pylint: disable=redefined-outer-name,unused-argument
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from agentscope.agent import Agent, ContextConfig
|
||||
from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage
|
||||
from agentscope.state import AgentState
|
||||
import jsonschema
|
||||
import openai
|
||||
import pytest
|
||||
from reme_home.agent import compression, optimizer
|
||||
|
||||
|
||||
class FailingCompressionModel:
|
||||
def __init__(self, ratio=0.9, failure=None):
|
||||
self.context_size = 90 / ratio
|
||||
self.failure = failure or RuntimeError("Failed to generate structured output for model.")
|
||||
self.calls = 0
|
||||
|
||||
async def count_tokens(self, messages, tools=None):
|
||||
return sum(10 for message in messages if (message.get_text_content() or "").startswith("message-"))
|
||||
|
||||
async def generate_structured_output(self, **kwargs):
|
||||
self.calls += 1
|
||||
raise self.failure
|
||||
|
||||
|
||||
def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing summary"):
|
||||
return optimizer.OptimizerAgent(
|
||||
name="optimizer",
|
||||
system_prompt="system",
|
||||
model=FailingCompressionModel(ratio, failure),
|
||||
state=AgentState(
|
||||
summary=summary,
|
||||
context=[UserMsg(name="user", content=f"message-{index}") for index in range(9)],
|
||||
),
|
||||
context_config=ContextConfig(trigger_ratio=trigger, reserve_ratio=0.2),
|
||||
fallback_directory=tmp_path / "offloads",
|
||||
validation_cycle_timer=optimizer.ValidationCycleTimer(150),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("trigger", "ratio", "expected"),
|
||||
[(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)],
|
||||
)
|
||||
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected):
|
||||
agent = make_agent(tmp_path, ratio, trigger)
|
||||
context_before, summary_before = agent.state.context, agent.state.summary
|
||||
await agent.compress_context()
|
||||
assert agent.validation_cycle_timer.context_compression_count == int(expected)
|
||||
assert len(list(tmp_path.rglob("context-*.json"))) == int(expected)
|
||||
assert agent.model.calls == int(ratio >= trigger)
|
||||
assert agent.context_config.compression_fallback_to_truncation # The shared config was not mutated.
|
||||
if not expected:
|
||||
assert agent.state.context is context_before
|
||||
assert agent.state.summary is summary_before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]])
|
||||
@pytest.mark.parametrize("reserve_ratio", [0.2, 0])
|
||||
async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio):
|
||||
agent = make_agent(tmp_path, summary=summary)
|
||||
agent.context_config.reserve_ratio = reserve_ratio
|
||||
agent.state.context[-1] = AssistantMsg(
|
||||
id=agent.state.reply_id,
|
||||
name=agent.name,
|
||||
content="message-8",
|
||||
usage=Usage(input_tokens=100, output_tokens=10),
|
||||
)
|
||||
original = [msg.model_dump(mode="json", exclude_none=False) for msg in agent.state.context]
|
||||
await agent.compress_context()
|
||||
(path,) = tmp_path.rglob("context-*.json")
|
||||
data = json.loads(path.read_text())
|
||||
assert data["messages"] == original[:7] if reserve_ratio else data["messages"] == original
|
||||
if reserve_ratio:
|
||||
assert [msg.get_text_content() for msg in agent.state.context] == ["message-7", "message-8"]
|
||||
else:
|
||||
assert len(agent.state.context) == 1
|
||||
assert agent.state.context[0].content == [] # SDK usage carrier after full offload.
|
||||
assert data["session_id"] == agent.state.session_id
|
||||
assert data["context_ratio"] == 0.9
|
||||
assert data["emergency_ratio"] == 0.9
|
||||
assert data["failure"]["type"] == "RuntimeError"
|
||||
text = agent.state.summary
|
||||
if isinstance(text, list):
|
||||
text = "\n".join(block.text for block in text)
|
||||
assert "existing summary" in text
|
||||
assert "without semantic summarization" in text
|
||||
assert str(path) in text
|
||||
assert agent._get_reply_usage().input_tokens == 100
|
||||
assert not list(path.parent.glob(".result-*"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"failure",
|
||||
[RuntimeError("The system prompt exceeds threshold"), ValueError("bad configuration"), asyncio.CancelledError()],
|
||||
)
|
||||
async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failure):
|
||||
agent = make_agent(tmp_path, failure=failure)
|
||||
original = agent.state.context
|
||||
with pytest.raises(type(failure)):
|
||||
await agent.compress_context()
|
||||
assert agent.state.context is original
|
||||
assert not list(tmp_path.rglob("*.json"))
|
||||
assert agent.validation_cycle_timer.context_compression_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("failure", "expected"),
|
||||
[
|
||||
(RuntimeError("provider failed"), True),
|
||||
(RuntimeError("The system prompt exceeds threshold"), False),
|
||||
(jsonschema.ValidationError("invalid summary"), True),
|
||||
(openai.OpenAIError("provider unavailable"), True),
|
||||
(ValueError("programming error"), False),
|
||||
(OSError("disk full"), False),
|
||||
],
|
||||
)
|
||||
def test_recoverable_error_classifier(failure, expected):
|
||||
assert compression.RecoverableCompressionAgent._is_recoverable_compression_error(failure) is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_override_config_and_successful_compression_use_native_pipeline(tmp_path, monkeypatch):
|
||||
agent = make_agent(tmp_path, ratio=0.85)
|
||||
config = ContextConfig(trigger_ratio=0.65, reserve_ratio=0.1)
|
||||
calls = []
|
||||
|
||||
async def compress(self, context_config=None, instructions=None):
|
||||
calls.append((context_config, instructions))
|
||||
self.state.summary = "semantic summary"
|
||||
self.state.context = []
|
||||
|
||||
monkeypatch.setattr(Agent, "_compress_context_impl", compress)
|
||||
await agent.compress_context(context_config=config)
|
||||
assert calls[0][0].trigger_ratio == 0.65
|
||||
assert calls[0][0].reserve_ratio == 0.1
|
||||
assert not calls[0][0].compression_fallback_to_truncation
|
||||
assert config.compression_fallback_to_truncation
|
||||
assert agent.state.summary == "semantic summary"
|
||||
assert agent.validation_cycle_timer.context_compression_count == 1
|
||||
assert not list(tmp_path.rglob("*.json"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_offload_write_failure_keeps_context_and_does_not_count(tmp_path, monkeypatch):
|
||||
agent = make_agent(tmp_path)
|
||||
original = agent.state.context
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr(compression, "write_json", fail)
|
||||
with pytest.raises(OSError, match="disk full"):
|
||||
await agent.compress_context()
|
||||
assert agent.state.context is original
|
||||
assert agent.state.summary == "existing summary"
|
||||
assert agent.validation_cycle_timer.context_compression_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_during_apply_finishes_consistent_offload_then_propagates(tmp_path, monkeypatch):
|
||||
agent = make_agent(tmp_path)
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def clear_cache(messages):
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
monkeypatch.setattr(agent, "_clear_unreserved_read_cache", clear_cache)
|
||||
task = asyncio.create_task(agent.compress_context())
|
||||
await started.wait()
|
||||
task.cancel()
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert len(list(tmp_path.rglob("context-*.json"))) == 1
|
||||
assert "offloaded" in agent.state.summary
|
||||
assert len(agent.state.context) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_directory_or_unsplittable_context_fails_explicitly(tmp_path, monkeypatch):
|
||||
agent = make_agent(tmp_path)
|
||||
agent.fallback_directory = None
|
||||
with pytest.raises(RuntimeError, match="fallback_directory"):
|
||||
await agent.compress_context()
|
||||
agent.fallback_directory = tmp_path / "offloads"
|
||||
|
||||
async def no_old_messages(*args):
|
||||
return [], agent.state.context
|
||||
|
||||
monkeypatch.setattr(agent, "_split_context_for_compression", no_old_messages)
|
||||
with pytest.raises(RuntimeError, match="cannot remove any context"):
|
||||
await agent.compress_context()
|
||||
assert not list(tmp_path.rglob("*.json"))
|
||||
|
||||
|
||||
def test_wrapper_uses_host_offload_directory(tmp_path):
|
||||
wrapper = optimizer.OptimizerAgentWrapper(name="optimizer")
|
||||
agent = wrapper._create_agent(
|
||||
options={"_context_fallback_directory": tmp_path / "logs/context-offloads"},
|
||||
name="optimizer",
|
||||
system_prompt="system",
|
||||
model=FailingCompressionModel(),
|
||||
)
|
||||
assert agent.fallback_directory == tmp_path / "logs/context-offloads"
|
||||
297
plugins/home/tests/test_optimizer_cycle.py
Normal file
297
plugins/home/tests/test_optimizer_cycle.py
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
"""Meta-ReMe cycle timing parity with fake clocks and isolated evaluation evidence."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
|
||||
# pylint: disable=redefined-outer-name,unused-argument
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from agentscope.agent import ModelConfig, ReActConfig
|
||||
from agentscope.message import HintBlock, TextBlock, UserMsg
|
||||
from agentscope.model import ChatResponse, ChatUsage
|
||||
from agentscope.state import AgentState
|
||||
from reme_home.agent import optimizer
|
||||
from reme_home.agent.bounded import BoundedReActAgent
|
||||
from reme_home.evaluation.artifacts import write_json
|
||||
from test_bounded_agent import RecordingModel
|
||||
|
||||
|
||||
def make_agent(timer, model=None):
|
||||
return optimizer.OptimizerAgent(
|
||||
name="optimizer",
|
||||
system_prompt="Improve general memory mechanisms.",
|
||||
model=model or RecordingModel(),
|
||||
validation_cycle_timer=timer,
|
||||
model_config=ModelConfig(max_retries=0),
|
||||
react_config=ReActConfig(max_iters=100),
|
||||
)
|
||||
|
||||
|
||||
def cycle_hints(messages, source="meta_reme_validation_timer"):
|
||||
return [
|
||||
block
|
||||
for message in messages
|
||||
for block in message.content
|
||||
if isinstance(block, HintBlock) and block.source == source
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
|
||||
def test_cycle_rejects_invalid_budget(budget):
|
||||
with pytest.raises(ValueError, match="positive"):
|
||||
optimizer.ValidationCycleTimer(budget)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cycle_injects_dev_meta_reminders_and_resets():
|
||||
now = [0.0]
|
||||
timer = optimizer.ValidationCycleTimer(100, clock=lambda: now[0])
|
||||
|
||||
class SnapshotModel(RecordingModel):
|
||||
async def __call__(self, **kwargs):
|
||||
return await super().__call__(**copy.deepcopy(kwargs))
|
||||
|
||||
agent = make_agent(timer, SnapshotModel())
|
||||
timer.record_context_compression()
|
||||
timer.record_context_compression()
|
||||
delivered = []
|
||||
for minute in (49, 50, 74, 75, 90, 100, 109, 110, 120):
|
||||
now[0] = minute * 60
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
reminders = [block.hint for block in cycle_hints(agent.model.calls[-1]["messages"])]
|
||||
if minute in (50, 75, 90, 100, 110, 120):
|
||||
delivered.append(reminders[-1])
|
||||
assert timer.next_reminder() is None
|
||||
for percentage, text in zip((50, 75, 90, 100, 110, 120), delivered):
|
||||
assert f"({percentage}% of its 100 minutes soft time budget)" in text
|
||||
assert "2 context compressions since the last scored validation" in text
|
||||
assert "start validation within 50 minutes" in delivered[0]
|
||||
assert "start validation within 10 minutes" in delivered[2]
|
||||
assert "starting validation as soon as possible" in delivered[3]
|
||||
persistent = cycle_hints(agent.state.context)
|
||||
assert len(persistent) == 3
|
||||
for percentage, message in zip((75, 100, 120), persistent):
|
||||
assert f"({percentage}%" in message.hint
|
||||
timer.reset()
|
||||
assert timer.context_compression_count == 0
|
||||
now[0] += 60
|
||||
assert timer.next_reminder() is None
|
||||
# Reset keeps historical persistent messages and restarts alternation.
|
||||
assert cycle_hints(agent.state.context) == persistent
|
||||
now[0] = 170 * 60
|
||||
assert timer.next_reminder().persistent is False
|
||||
|
||||
|
||||
def test_cycle_skips_old_thresholds_and_formats_seconds():
|
||||
now = [0.0]
|
||||
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
|
||||
now[0] = 59
|
||||
reminder = timer.next_reminder()
|
||||
assert "(90%" in reminder.content
|
||||
assert "within 1 second." in reminder.content
|
||||
now[0] = 101
|
||||
assert "(160%" in timer.next_reminder().content
|
||||
assert timer.next_reminder() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", [RuntimeError("model unavailable"), asyncio.CancelledError()])
|
||||
async def test_cycle_ephemeral_cleanup_on_error_and_cancellation(monkeypatch, failure):
|
||||
now = [0.0]
|
||||
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
|
||||
agent = make_agent(timer)
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise failure
|
||||
yield # pylint: disable=unreachable
|
||||
|
||||
monkeypatch.setattr(BoundedReActAgent, "_reasoning", fail)
|
||||
now[0] = 30
|
||||
with pytest.raises(type(failure)):
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
assert not cycle_hints(agent.state.context)
|
||||
now[0] = 45
|
||||
with pytest.raises(type(failure)):
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
assert len(cycle_hints(agent.state.context)) == 1 # The second reminder is persistent.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cycle_counts_only_completed_context_replacements(monkeypatch):
|
||||
timer = optimizer.ValidationCycleTimer(1)
|
||||
agent = make_agent(timer)
|
||||
|
||||
async def unchanged(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(BoundedReActAgent, "compress_context", unchanged)
|
||||
await agent.compress_context()
|
||||
assert timer.context_compression_count == 0
|
||||
|
||||
async def compressed(self):
|
||||
self.state.summary = "Retained previous evidence."
|
||||
self.state.context = []
|
||||
|
||||
monkeypatch.setattr(BoundedReActAgent, "compress_context", compressed)
|
||||
await agent.compress_context()
|
||||
assert timer.context_compression_count == 1
|
||||
|
||||
async def failed(self):
|
||||
raise RuntimeError("compression unavailable")
|
||||
|
||||
monkeypatch.setattr(BoundedReActAgent, "compress_context", failed)
|
||||
with pytest.raises(RuntimeError, match="unavailable"):
|
||||
await agent.compress_context()
|
||||
assert timer.context_compression_count == 1
|
||||
|
||||
|
||||
def write_history(home, started):
|
||||
write_json(
|
||||
home / "sampling/state.json",
|
||||
{
|
||||
"schema_version": 2,
|
||||
"round": len(started),
|
||||
"successful_round": 0,
|
||||
"started_validations": {key: index for index, key in enumerate(started, 1)},
|
||||
"processed_validations": [],
|
||||
"cases": {},
|
||||
"queries": {},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("timed", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"outcome,reset",
|
||||
[
|
||||
("completed", True),
|
||||
("success", True),
|
||||
("candidate_failure", True),
|
||||
("completed_infra_error", True),
|
||||
("debug", False),
|
||||
("rejected", False),
|
||||
("failed", False),
|
||||
("partial", False),
|
||||
("not_started", False),
|
||||
("preexisting", False),
|
||||
("cancelled", False),
|
||||
],
|
||||
)
|
||||
async def test_evaluation_reset_boundary_matches_dev_meta(tmp_path, outcome, reset, timed):
|
||||
home = tmp_path
|
||||
previous = ["init/initial/1", "candidate/previous/1"]
|
||||
key = "candidate/current/1"
|
||||
if outcome == "preexisting":
|
||||
previous.append(key)
|
||||
write_history(home, previous)
|
||||
now = [0.0]
|
||||
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
|
||||
timer.record_context_compression()
|
||||
now[0] = 45
|
||||
timer.next_reminder() # First reminder is ephemeral.
|
||||
state = AgentState()
|
||||
old = UserMsg(name="meta_reme_validation_timer", content="Previous persistent reminder.")
|
||||
state.context.append(old)
|
||||
|
||||
class EvaluationJob:
|
||||
enabled = is_started = True
|
||||
|
||||
async def __call__(self, **kwargs):
|
||||
assert kwargs["debug"] is (outcome == "debug")
|
||||
# Tool execution time remains part of the previous cycle until completion.
|
||||
now[0] = 90
|
||||
if outcome == "cancelled":
|
||||
raise asyncio.CancelledError()
|
||||
if outcome not in {"debug", "rejected", "not_started", "preexisting"}:
|
||||
write_history(home, [*previous, key])
|
||||
manifest = {
|
||||
"commit_sha": "1b2cdcc" + "0" * 33,
|
||||
"status": "failed" if outcome in {"failed", "partial"} else "completed",
|
||||
"mode": "debug" if outcome == "debug" else "scored",
|
||||
"dataset_kind": "training-set",
|
||||
"sandbox_started_at": None if outcome == "not_started" else "2026-10-31T00:00:00Z",
|
||||
}
|
||||
run = home / "evaluations" / key
|
||||
write_json(run / "manifest.json", manifest)
|
||||
status = outcome if outcome in {"success", "rejected", "failed", "partial"} else "completed"
|
||||
return SimpleNamespace(
|
||||
success=outcome not in {"completed_infra_error", "rejected", "failed", "partial"},
|
||||
answer={"status": status, "result_dir": str(run), "mean_query_score": 0.0},
|
||||
metadata={},
|
||||
)
|
||||
|
||||
wrapper = SimpleNamespace(app_context=SimpleNamespace(jobs={"home_evaluate": EvaluationJob()}))
|
||||
tool = optimizer.evaluation_tool(wrapper, {}, home=home, timer=timer if timed else None)
|
||||
assert set(tool.input_schema["properties"]) == {"debug"}
|
||||
if outcome == "cancelled":
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await tool._func(debug=False, _agent_state=state)
|
||||
else:
|
||||
await tool._func(debug=outcome == "debug", _agent_state=state)
|
||||
assert len(state.context) == 1 # The tool cannot insert anything before its result is written.
|
||||
pending = state.middle_context.get(optimizer.PENDING_CYCLE_TRANSITIONS, [])
|
||||
assert len(pending) == int(reset)
|
||||
agent = make_agent(None)
|
||||
agent.state = state
|
||||
_ = [item async for item in agent._reasoning()]
|
||||
transitions = cycle_hints(state.context, "meta_reme_validation_cycle_transition")
|
||||
assert len(transitions) == int(reset)
|
||||
assert state.context[0] is old
|
||||
if reset:
|
||||
assert transitions[0].hint == (
|
||||
"<system-reminder>Formal validation 2 completed successfully for commit 1b2cdcc. "
|
||||
"The previous cycle timer has been reset, and all timer reminders for cycle 2 are now expired. "
|
||||
"Remaining formal validations: 8. Start cycle 3.</system-reminder>"
|
||||
)
|
||||
if reset and timed:
|
||||
assert timer.context_compression_count == 0
|
||||
assert timer.next_reminder() is None
|
||||
now[0] += 30
|
||||
assert timer.next_reminder().persistent is False
|
||||
else:
|
||||
assert timer.context_compression_count == 1
|
||||
assert "(150%" in timer.next_reminder().content
|
||||
|
||||
|
||||
def test_wrapper_owns_fresh_cycle_timers_and_supports_disabling():
|
||||
wrapper = optimizer.OptimizerAgentWrapper(name="optimizer", validation_cycle_time_budget_minutes=150)
|
||||
first, second = wrapper.create_cycle_timer(), wrapper.create_cycle_timer()
|
||||
assert first is not second
|
||||
assert first.time_budget_minutes == 150
|
||||
first.record_context_compression()
|
||||
assert second.context_compression_count == 0
|
||||
assert wrapper.create_cycle_timer({"validation_cycle_time_budget_minutes": None}) is None
|
||||
legacy = optimizer.OptimizerAgentWrapper(name="optimizer", time_budget_minutes=75)
|
||||
assert legacy.create_cycle_timer().time_budget_minutes == 75
|
||||
assert legacy.create_cycle_timer({"validation_cycle_time_budget_minutes": 30}).time_budget_minutes == 30
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_temporary_and_persistent_reminders_preserve_total_reply_usage():
|
||||
now = [0.0]
|
||||
|
||||
class AdvancingModel(RecordingModel):
|
||||
async def __call__(self, **kwargs):
|
||||
result = await super().__call__(**kwargs)
|
||||
now[0] = 30 if len(self.calls) == 1 else 45
|
||||
return result
|
||||
|
||||
model = AdvancingModel(
|
||||
final=ChatResponse(
|
||||
content=[TextBlock(text="final report")],
|
||||
is_last=True,
|
||||
usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
|
||||
),
|
||||
)
|
||||
agent = make_agent(optimizer.ValidationCycleTimer(1, clock=lambda: now[0]), model)
|
||||
agent.react_config.max_iters = 2
|
||||
reply = await agent.reply(UserMsg(name="user", content="Investigate"))
|
||||
assert len(model.calls) == 3
|
||||
assert reply.usage.input_tokens == 6
|
||||
assert reply.usage.output_tokens == 9
|
||||
retained = cycle_hints(agent.state.context)
|
||||
assert len(retained) == 1
|
||||
assert "(75%" in retained[0].hint
|
||||
142
plugins/home/tests/test_validation_budget.py
Normal file
142
plugins/home/tests/test_validation_budget.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
"""Formal budget accounting through real evaluation Jobs and fake Docker boundaries."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument,protected-access
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from reme_home.agent.optimizer import OptimizerAgentWrapper, validation_budget_tool
|
||||
from reme_home.evaluation import component as evaluation
|
||||
from reme_home.evaluation.artifacts import write_json
|
||||
from reme_home.evaluation.budget import validation_budget_status
|
||||
from reme_home.evaluation.inputs import EvaluationSpec
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
commit_candidate,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
|
||||
|
||||
def seed_history(home, keys):
|
||||
write_json(
|
||||
home / "sampling/state.json",
|
||||
{
|
||||
"schema_version": 2,
|
||||
"round": len(keys),
|
||||
"successful_round": 0,
|
||||
"cases": {},
|
||||
"queries": {},
|
||||
"started_validations": {key: index for index, key in enumerate(keys, 1)},
|
||||
"processed_validations": [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"used,phase,remaining",
|
||||
[
|
||||
(0, "structural_exploration", 10),
|
||||
(4, "structural_exploration", 6),
|
||||
(5, "evidence_driven_refinement", 5),
|
||||
(9, "evidence_driven_refinement", 1),
|
||||
(10, "exhausted", 0),
|
||||
(11, "exhausted", 0),
|
||||
],
|
||||
)
|
||||
def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining):
|
||||
seed_history(tmp_path, ["init/first/1", *(f"candidate/hash/{index}" for index in range(used))])
|
||||
result = validation_budget_status(tmp_path, 10)
|
||||
assert result["used_validations"] == used
|
||||
assert result["remaining_validations"] == remaining
|
||||
assert result["optimization_phase"] == phase
|
||||
assert result["gated_validation_count"] == 5
|
||||
assert result["next_validation_guidance"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None])
|
||||
def test_invalid_optimizer_budgets_rejected(invalid):
|
||||
wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid)
|
||||
with pytest.raises(ValueError, match="positive integer"):
|
||||
wrapper.validation_limit()
|
||||
if invalid is not None:
|
||||
with pytest.raises(ValueError):
|
||||
EvaluationSpec(max_validations=invalid)
|
||||
|
||||
|
||||
def test_budgeted_evaluation_cannot_disable_started_history():
|
||||
with pytest.raises(ValueError, match="record_history"):
|
||||
EvaluationSpec(max_validations=10, record_history=False)
|
||||
assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug
|
||||
assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path):
|
||||
seed_history(tmp_path, ["init/base/1", "candidate/hash/1"])
|
||||
before = (tmp_path / "sampling/state.json").read_bytes()
|
||||
tool = validation_budget_tool(tmp_path, 3)
|
||||
assert tool.name == "get_validation_budget"
|
||||
result = await tool._func()
|
||||
assert result["remaining_validations"] == 2
|
||||
assert result["optimization_phase"] == "evidence_driven_refinement"
|
||||
assert (tmp_path / "sampling/state.json").read_bytes() == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, prepared):
|
||||
baseline = await app.run_job("home_evaluate")
|
||||
assert baseline.success
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 0
|
||||
commit_candidate(prepared)
|
||||
result = await app.run_job("home_evaluate", max_validations=1)
|
||||
assert result.success, result.answer
|
||||
assert result.answer["validation_budget"]["remaining_validations"] == 0
|
||||
count = len(FakeFactory.instances)
|
||||
# A fresh status lookup and cache rebuild both retain the charge.
|
||||
(prepared / "sampling/state.json").unlink()
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 1
|
||||
commit_candidate(prepared)
|
||||
rejected = await app.run_job("home_evaluate", max_validations=1)
|
||||
assert not rejected.success
|
||||
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
|
||||
assert rejected.answer["result_dir"] is None
|
||||
assert len(FakeFactory.instances) == count
|
||||
debug = await app.run_job("home_evaluate", debug=True, max_validations=1)
|
||||
assert debug.success, debug.answer
|
||||
assert "validation_budget" not in debug.answer
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outcome", ["candidate_failure", "infra_failure", "cancelled", "prestart_failure"])
|
||||
async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, monkeypatch, outcome):
|
||||
commit_candidate(prepared)
|
||||
if outcome == "candidate_failure":
|
||||
monkeypatch.setattr(FakeFactory, "fail_build", "1")
|
||||
elif outcome == "infra_failure":
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", True)
|
||||
elif outcome == "cancelled":
|
||||
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
|
||||
else:
|
||||
|
||||
class PrestartFailureFactory(FakeFactory):
|
||||
async def create_cases(self, ids, **kwargs):
|
||||
raise RuntimeError("sandbox initialization failed")
|
||||
|
||||
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory)
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1))
|
||||
if outcome == "cancelled":
|
||||
while not FakeFactory.instances:
|
||||
await asyncio.sleep(0)
|
||||
await FakeFactory.instances[-1].entered.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
else:
|
||||
await task
|
||||
expected = int(outcome != "prestart_failure")
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == expected
|
||||
(prepared / "sampling/state.json").unlink(missing_ok=True)
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == expected
|
||||
496
plugins/home/tests/test_workflows.py
Normal file
496
plugins/home/tests/test_workflows.py
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
"""Minimal Home agents exercise real Jobs, Git and AS tools with fake model/Docker boundaries."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
|
||||
# pylint: disable=unused-argument,protected-access
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from agentscope.formatter import OpenAIChatFormatter
|
||||
from agentscope.message import HintBlock, TextBlock, ToolCallBlock
|
||||
from agentscope.model import ChatResponse, ChatUsage
|
||||
from agentscope.tool import LocalBackend
|
||||
from jsonschema import Draft202012Validator
|
||||
from reme_home.agent import optimizer as optimizer_module, diagnostic as diagnostic_module, workflow_common, case_study
|
||||
from reme_home.evaluation.artifacts import read_json, write_json
|
||||
from reme_home.preparation.code import git
|
||||
from reme_home.workspace import _initialization_lock
|
||||
|
||||
from test_bounded_agent import RecordingModel
|
||||
from test_case_study import RecordingSandbox
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
FakeFactory,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
)
|
||||
|
||||
|
||||
class ScriptedModel(RecordingModel):
|
||||
def __init__(self, calls=(), final="Supported diagnostic report."):
|
||||
super().__init__()
|
||||
self.script = list(calls)
|
||||
self.report = final
|
||||
|
||||
async def __call__(self, **kwargs):
|
||||
index = len(self.calls)
|
||||
self.calls.append(kwargs)
|
||||
if index < len(self.script):
|
||||
name, arguments = self.script[index]
|
||||
arguments = arguments() if callable(arguments) else arguments
|
||||
content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))]
|
||||
else:
|
||||
final = self.report() if callable(self.report) else self.report
|
||||
content = [TextBlock(text=final)]
|
||||
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sandbox(monkeypatch):
|
||||
policies = []
|
||||
|
||||
def create(**kwargs):
|
||||
# Mock only subprocess isolation. Writes are restricted by the real
|
||||
# sandbox in the dedicated OS tests below, not by this local fake.
|
||||
policies.append(kwargs)
|
||||
local = LocalBackend()
|
||||
return SimpleNamespace(backend=local, exec_shell=local.exec_shell)
|
||||
|
||||
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", create)
|
||||
monkeypatch.setattr(case_study, "create_case_study_sandbox", RecordingSandbox)
|
||||
return policies
|
||||
|
||||
|
||||
def set_model(app, name, model):
|
||||
wrapper = app.context.components["agent_wrapper"][name]
|
||||
wrapper.as_llm = SimpleNamespace(model=model)
|
||||
return wrapper
|
||||
|
||||
|
||||
async def baseline(app):
|
||||
result = await app.run_job("home_evaluate")
|
||||
assert result.success, result.answer
|
||||
return result.answer["result_dir"]
|
||||
|
||||
|
||||
def final_report(home, branch="init", path=None):
|
||||
repo = home / "code/repo"
|
||||
if path is None:
|
||||
path = sorted((home / "evaluations" / branch).glob("*/*/summary.json"))[-1].parent
|
||||
return json.dumps(
|
||||
{
|
||||
"branch_name": branch,
|
||||
"commit_sha": git(repo, "rev-parse", "HEAD"),
|
||||
"evaluation_path": str(path),
|
||||
"report": "Selected using matching formal evidence; sampled coverage limits confidence.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepared, sandbox, monkeypatch):
|
||||
home = prepared
|
||||
now, timers = [0.0], []
|
||||
timer_class = optimizer_module.ValidationCycleTimer
|
||||
|
||||
def create_timer(budget):
|
||||
timer = timer_class(budget, clock=lambda: now[0])
|
||||
timers.append(timer)
|
||||
return timer
|
||||
|
||||
class TimedModel(ScriptedModel):
|
||||
async def __call__(self, **kwargs):
|
||||
formatted = await OpenAIChatFormatter().format(kwargs["messages"])
|
||||
pending = set()
|
||||
for message in formatted:
|
||||
if pending:
|
||||
assert message["role"] == "tool", formatted
|
||||
pending.remove(message["tool_call_id"])
|
||||
else:
|
||||
assert message["role"] != "tool", formatted
|
||||
pending.update(call["id"] for call in message.get("tool_calls", []))
|
||||
assert not pending
|
||||
response = await super().__call__(**kwargs)
|
||||
now[0] += 60 * 60
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(optimizer_module, "ValidationCycleTimer", create_timer)
|
||||
target = home / "code/repo/plugins/beam/src/reme_beam/__init__.py"
|
||||
baseline_path = await baseline(app)
|
||||
diagnostic = ScriptedModel(
|
||||
[("home_case_study", {"evaluation_path": baseline_path, "case_ids": ["1"], "focus": "Trace wrong answers"})],
|
||||
)
|
||||
set_model(app, "home_diagnostic", diagnostic)
|
||||
set_model(app, "home_case_study", ScriptedModel())
|
||||
optimizer = TimedModel(
|
||||
[
|
||||
("get_validation_budget", {}),
|
||||
("home_diagnose", {"evaluation_paths": [baseline_path]}),
|
||||
("home_create_candidate_branch", {"branch_name": "mechanism"}),
|
||||
("Read", {"file_path": str(target)}),
|
||||
("Edit", {"file_path": str(target), "old_string": "value = 1", "new_string": "value = 2"}),
|
||||
("home_commit_candidate", {"message": "Change mechanism"}),
|
||||
("home_evaluate", {"debug": True}),
|
||||
("home_evaluate", {"debug": False}),
|
||||
],
|
||||
final=lambda: final_report(home, "mechanism"),
|
||||
)
|
||||
set_model(app, "home_optimizer", optimizer)
|
||||
response = await app.run_job("home_optimize", instruction="Improve general retrieval")
|
||||
assert response.success, response.answer
|
||||
assert target.read_text() == "value = 2\n"
|
||||
assert response.answer["branch_name"] == "mechanism"
|
||||
assert response.answer["status"] == "completed"
|
||||
transitions = [
|
||||
block.hint
|
||||
for message in optimizer.calls[-1]["messages"]
|
||||
for block in message.content
|
||||
if isinstance(block, HintBlock) and block.source == "meta_reme_validation_cycle_transition"
|
||||
]
|
||||
assert len(transitions) == 1
|
||||
assert "Formal validation 1 completed successfully" in transitions[0]
|
||||
assert "Remaining formal validations: 9. Start cycle 2." in transitions[0]
|
||||
assert len(timers) == 1
|
||||
assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call.
|
||||
assert any(
|
||||
isinstance(block, HintBlock) and block.source == "meta_reme_validation_timer"
|
||||
for message in optimizer.calls[-1]["messages"]
|
||||
for block in message.content
|
||||
)
|
||||
assert response.metadata["usage"]["input_tokens"] == 2 * len(optimizer.calls)
|
||||
assert response.metadata["usage"]["output_tokens"] == 3 * len(optimizer.calls)
|
||||
assert "sampled coverage" in response.answer["report"]
|
||||
assert read_json(Path(response.metadata["optimization_report_path"]))["status"] == "completed"
|
||||
assert len(diagnostic.calls) == 2
|
||||
source = next((home / "logs/diagnostic-trajectories").glob("*.json"))
|
||||
trace = read_json(source)
|
||||
assert trace["session_id"] and any("Improve general retrieval" in str(msg) for msg in trace["messages"])
|
||||
assert "permission_context" not in trace
|
||||
schemas = {item["function"]["name"]: item["function"]["parameters"] for item in optimizer.calls[0]["tools"]}
|
||||
assert set(schemas["home_evaluate"]["properties"]) == {"debug"}
|
||||
assert "caller_trajectory" not in schemas["home_diagnose"]["properties"]
|
||||
assert "get_validation_budget" in schemas
|
||||
assert not any("equivalence" in name for name in schemas)
|
||||
for manifest_path in (home / "evaluations").glob("*/*/*/manifest.json"):
|
||||
manifest = read_json(manifest_path)
|
||||
assert manifest["execution_mode"] == "full_validation"
|
||||
assert any(read_json(path)["mode"] == "debug" for path in (home / "evaluations").glob("*/*/*/manifest.json"))
|
||||
diagnostic_schemas = {item["function"]["name"] for item in diagnostic.calls[0]["tools"]}
|
||||
assert "home_case_study" in diagnostic_schemas
|
||||
assert not set(optimizer_module.MUTATION_JOBS) & diagnostic_schemas
|
||||
assert "Edit" not in diagnostic_schemas and "home_evaluate" not in diagnostic_schemas
|
||||
assert "Write" in diagnostic_schemas # Private scratch only.
|
||||
assert any(str(target.parent.parent) in policy["writable_paths"] for policy in sandbox)
|
||||
assert all(str(home / "code/repo") not in policy["writable_paths"] for policy in sandbox)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
|
||||
app,
|
||||
prepared,
|
||||
inputs,
|
||||
sandbox,
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
benchmark,
|
||||
):
|
||||
baseline_path = await baseline(app)
|
||||
await app.close()
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = tmp_path / "home.yaml"
|
||||
config.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"workspace_dir": str(prepared.parent),
|
||||
"preparation": inputs[0],
|
||||
"max_validations": 3,
|
||||
"validation_cycle_time_budget_minutes": 30,
|
||||
},
|
||||
),
|
||||
)
|
||||
application_class = type(app)
|
||||
agents = []
|
||||
model = ScriptedModel([("get_validation_budget", {})], final=lambda: final_report(prepared, path=baseline_path))
|
||||
|
||||
def factory(**kwargs):
|
||||
application = application_class(**kwargs, log_to_console=False)
|
||||
original_start = application.start
|
||||
|
||||
async def start():
|
||||
await original_start()
|
||||
wrapper = set_model(application, "home_optimizer", model)
|
||||
original_create = wrapper._create_agent
|
||||
|
||||
def create(**options):
|
||||
agent = original_create(**options)
|
||||
agents.append(agent)
|
||||
return agent
|
||||
|
||||
monkeypatch.setattr(wrapper, "_create_agent", create)
|
||||
|
||||
monkeypatch.setattr(application, "start", start)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
result = await module.optimize(config, instruction="Keep the best existing candidate")
|
||||
assert result["status"] == "completed"
|
||||
assert result["branch_name"] == "init"
|
||||
assert len(agents) == 1
|
||||
assert agents[0].validation_cycle_timer.time_budget_minutes == 30
|
||||
task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"])
|
||||
assert '"max_validations": 3' in task
|
||||
assert '"remaining_validations": 3' in task
|
||||
assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox):
|
||||
model = ScriptedModel(final=lambda: final_report(prepared))
|
||||
set_model(app, "home_optimizer", model)
|
||||
response = await app.run_job("home_optimize")
|
||||
assert response.success, response.answer
|
||||
assert len(FakeFactory.instances) == 1
|
||||
assert response.answer["branch_name"] == "init"
|
||||
run = response.answer["baseline_path"]
|
||||
manifest = read_json(Path(run) / "manifest.json")
|
||||
assert set(manifest["case_ids"]) == {"1", "2"} and not manifest["options"]["sampling"]
|
||||
assert manifest["options"]["record_history"]
|
||||
set_model(app, "home_optimizer", ScriptedModel(final=lambda: final_report(prepared)))
|
||||
second = await app.run_job("home_optimize")
|
||||
assert second.success, second.answer
|
||||
assert len(FakeFactory.instances) == 1
|
||||
assert second.answer["session_id"] != response.answer["session_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_diagnosis_fresh_state_and_persisted_provenance(app, prepared, sandbox):
|
||||
path = await baseline(app)
|
||||
model = ScriptedModel()
|
||||
set_model(app, "home_diagnostic", model)
|
||||
first = await app.run_job("home_diagnose", evaluation_paths=[path])
|
||||
second = await app.run_job("home_diagnose", evaluation_paths=[path], instruction="Compare failure clusters")
|
||||
assert first.success and second.success
|
||||
assert first.answer["session_id"] != second.answer["session_id"]
|
||||
assert first.answer["trajectory_path"] != second.answer["trajectory_path"]
|
||||
assert first.answer["evaluation_paths"] == [path]
|
||||
record = read_json(Path(first.answer["diagnostic_report_path"]))
|
||||
assert record["diagnosis"] == "Supported diagnostic report."
|
||||
assert read_json(Path(first.answer["trajectory_path"])) == {"messages": [], "summary": None}
|
||||
assert all(
|
||||
len(policy["writable_paths"]) == 1 and "/tmp/diagnostic-" in policy["writable_paths"][0] for policy in sandbox
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["infra", "partial", "debug", "replay", "wrong_commit"])
|
||||
async def test_bad_baseline_rejected_before_optimizer(app, prepared, sandbox, change):
|
||||
path = Path(await baseline(app))
|
||||
manifest = read_json(path / "manifest.json")
|
||||
summary = read_json(path / "summary.json")
|
||||
if change == "infra":
|
||||
summary["case_status_counts"]["infra_error"] = 1
|
||||
elif change == "partial":
|
||||
manifest["case_ids"] = ["1"]
|
||||
manifest["query_ids"] = {"1": manifest["query_ids"]["1"]}
|
||||
elif change == "debug":
|
||||
manifest["mode"] = "debug"
|
||||
elif change == "replay":
|
||||
manifest["execution_mode"] = "query_only_replay"
|
||||
else:
|
||||
manifest["commit_sha"] = "0" * 40
|
||||
write_json(path / "manifest.json", manifest)
|
||||
write_json(path / "summary.json", summary)
|
||||
# Missing valid baseline must not silently switch away from a candidate.
|
||||
response = await app.run_job("home_create_candidate_branch", branch_name="existing")
|
||||
assert response.success
|
||||
model = ScriptedModel()
|
||||
set_model(app, "home_optimizer", model)
|
||||
response = await app.run_job("home_optimize")
|
||||
assert not response.success and "baseline is missing" in str(response.answer)
|
||||
assert not model.calls and not sandbox
|
||||
assert git(prepared / "code/repo", "branch", "--show-current") == "existing"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["dirty", "branch", "commit", "path", "infra", "debug", "replay", "invalid_json"])
|
||||
async def test_final_claim_rejected_without_discarding_changes(app, prepared, sandbox, change):
|
||||
path = Path(await baseline(app))
|
||||
target = prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py"
|
||||
|
||||
def finish():
|
||||
claim = json.loads(final_report(prepared, path=path))
|
||||
if change == "dirty":
|
||||
target.write_text("preserve unfinished changes\n")
|
||||
elif change == "branch":
|
||||
claim["branch_name"] = "invented"
|
||||
elif change == "commit":
|
||||
claim["commit_sha"] = "0" * 40
|
||||
elif change == "path":
|
||||
claim["evaluation_path"] = str(prepared / "tmp")
|
||||
elif change == "infra":
|
||||
summary = read_json(path / "summary.json")
|
||||
summary["query_status_counts"]["infra_error"] = 1
|
||||
write_json(path / "summary.json", summary)
|
||||
elif change in {"debug", "replay"}:
|
||||
manifest = read_json(path / "manifest.json")
|
||||
manifest["mode" if change == "debug" else "execution_mode"] = (
|
||||
"debug" if change == "debug" else "query_only_replay"
|
||||
)
|
||||
write_json(path / "manifest.json", manifest)
|
||||
else:
|
||||
return "not a structured final claim"
|
||||
return json.dumps(claim)
|
||||
|
||||
set_model(app, "home_optimizer", ScriptedModel(final=finish))
|
||||
response = await app.run_job("home_optimize")
|
||||
assert not response.success
|
||||
record = read_json(Path(response.metadata["optimization_report_path"]))
|
||||
assert record["status"] == "failed" and record["error"]
|
||||
if change == "dirty":
|
||||
assert target.read_text() == "preserve unfinished changes\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infrastructure_baseline_failure_never_calls_optimizer(app, prepared, sandbox, monkeypatch):
|
||||
async def broken(*args, **kwargs):
|
||||
raise RuntimeError("Docker unavailable")
|
||||
|
||||
monkeypatch.setattr(FakeFactory, "create_cases", broken)
|
||||
model = ScriptedModel()
|
||||
set_model(app, "home_optimizer", model)
|
||||
response = await app.run_job("home_optimize")
|
||||
assert not response.success and not model.calls
|
||||
assert "baseline failed" in str(response.answer)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_scope_schema_and_injected_trajectory(app, prepared, sandbox):
|
||||
path = await baseline(app)
|
||||
wrapper = set_model(app, "home_optimizer", ScriptedModel())
|
||||
tool = diagnostic_module.diagnostic_tool(wrapper)
|
||||
with pytest.raises(ValueError, match="host-owned"):
|
||||
await tool._func(evaluation_paths=[path], caller_trajectory={"spoof": True})
|
||||
for values in ([str(prepared / "tmp")], [path, path], [], ["../escape"]):
|
||||
result = await app.run_job("home_diagnose", evaluation_paths=values)
|
||||
assert not result.success
|
||||
result = await app.run_job("home_diagnose", evaluation_paths=[path], _allowed_paths=[path])
|
||||
assert not result.success and not sandbox
|
||||
for name in ("home_optimize", "home_diagnose"):
|
||||
Draft202012Validator.check_schema(app.context.jobs[name].parameters)
|
||||
assert "caller_trajectory" not in app.context.jobs["home_diagnose"].parameters["properties"]
|
||||
with pytest.raises(ValueError, match="execution settings"):
|
||||
optimizer_module.evaluation_options({"reuse_construction_from": path})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_optimizer_lock_and_cancellation_cleanup(app, prepared, sandbox, monkeypatch):
|
||||
await baseline(app)
|
||||
entered, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def blocked(*args, **kwargs):
|
||||
entered.set()
|
||||
await release.wait()
|
||||
|
||||
monkeypatch.setattr(optimizer_module, "workflow_reply", blocked)
|
||||
running = asyncio.create_task(app.run_job("home_optimize"))
|
||||
await asyncio.wait_for(entered.wait(), 5)
|
||||
second = await app.run_job("home_optimize")
|
||||
assert not second.success and "already in progress" in str(second.answer)
|
||||
running.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await running
|
||||
reports = list((prepared / "logs/optimization-reports").glob("*.json"))
|
||||
assert len(reports) == 1 and read_json(reports[0])["status"] == "cancelled"
|
||||
with _initialization_lock(prepared / "metadata/optimization.lock"):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_link_rejected_before_opening_any_lock(app, prepared, tmp_path):
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
state = prepared / "metadata"
|
||||
saved = prepared / "saved-metadata"
|
||||
state.rename(saved)
|
||||
state.symlink_to(outside, target_is_directory=True)
|
||||
try:
|
||||
result = await app.run_job("home_optimize")
|
||||
assert not result.success and "real directory" in str(result.answer)
|
||||
with pytest.raises(ValueError, match="real directory"):
|
||||
optimizer_module.baseline_state(prepared)
|
||||
with pytest.raises(ValueError, match="real directory"):
|
||||
optimizer_module.verify_final(prepared, None)
|
||||
assert not list(outside.iterdir())
|
||||
finally:
|
||||
state.unlink()
|
||||
saved.rename(state)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_workflow_sandbox_enforces_read_write_and_git_boundaries(tmp_path):
|
||||
home = tmp_path / "home"
|
||||
repo = home / "repo"
|
||||
candidate = repo / "src"
|
||||
scratch = home / "scratch"
|
||||
evidence = home / "evidence"
|
||||
for directory in (candidate, repo / ".git", scratch, evidence):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
target = candidate / "module.py"
|
||||
target.write_text("old\n")
|
||||
secret = tmp_path / "outside.txt"
|
||||
secret.write_text("outside")
|
||||
(evidence / "result.json").write_text("evidence")
|
||||
for role in ("optimizer", "diagnostic"):
|
||||
scope = {
|
||||
"scratch": str(scratch),
|
||||
"read_only_paths": [str(repo), str(evidence)],
|
||||
"writable_paths": [str(scratch), *([str(candidate)] if role == "optimizer" else [])],
|
||||
}
|
||||
backend = workflow_common.WorkflowSandbox(scope, memory_limit_mib=0)
|
||||
assert (await backend.read_file(str(target))) in (b"old\n", b"new\n")
|
||||
assert (await backend.stat(str(target))).size_bytes > 0
|
||||
assert "module.py" in await backend.list_dir(str(candidate))
|
||||
await backend.write_file(str(scratch / "analysis.py"), b"print('analysis')\n")
|
||||
assert (await backend.exec_shell([sys.executable, "-I", "-B", str(scratch / "analysis.py")])).ok()
|
||||
for path in (repo / ".git/config", evidence / "result.json", secret):
|
||||
with pytest.raises(OSError):
|
||||
await backend.write_file(str(path), b"forbidden")
|
||||
with pytest.raises(OSError):
|
||||
await backend.read_file(str(secret))
|
||||
if role == "optimizer":
|
||||
await backend.write_file(str(target), b"new\n")
|
||||
# Replaced roots after managed Git operations must not retain stale inodes.
|
||||
replacement = candidate / "replacement"
|
||||
replacement.write_text("new\n")
|
||||
replacement.replace(target)
|
||||
assert await backend.read_file(str(target)) == b"new\n"
|
||||
else:
|
||||
with pytest.raises(OSError):
|
||||
await backend.write_file(str(target), b"forbidden")
|
||||
assert secret.read_text() == "outside" and (evidence / "result.json").read_text() == "evidence"
|
||||
# A single-file candidate root must survive a managed checkout replacing
|
||||
# its inode without accidentally granting writes to its containing directory.
|
||||
file_scope = {
|
||||
"scratch": str(scratch),
|
||||
"read_only_paths": [str(repo)],
|
||||
"writable_paths": [str(scratch), str(target)],
|
||||
}
|
||||
backend = workflow_common.WorkflowSandbox(file_scope, memory_limit_mib=0)
|
||||
await backend.write_file(str(target), b"file root\n")
|
||||
replacement = candidate / "replacement"
|
||||
replacement.write_text("replaced root\n")
|
||||
replacement.replace(target)
|
||||
assert await backend.read_file(str(target)) == b"replaced root\n"
|
||||
with pytest.raises(OSError):
|
||||
await backend.write_file(str(candidate / "sibling.py"), b"forbidden")
|
||||
target.unlink()
|
||||
target.symlink_to(secret)
|
||||
with pytest.raises(ValueError, match="symlinks"):
|
||||
await backend.read_file(str(target))
|
||||
Loading…
Add table
Reference in a new issue