feat(home): migrate optimization workflows and evaluation budgets

This commit is contained in:
酱牛肉 2026-10-31 16:55:54 +08:00
parent 860f67b555
commit 9d80f4b14e
27 changed files with 3121 additions and 125 deletions

View file

@ -164,6 +164,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
advisory time between completed formal evaluations. It defaults to 150 minutes;
set a positive number to change it or null to disable timed reminders.
The top-level `max_validations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.
After preparation, run local one-shot optimization with the same configuration:
```bash
python benchmark/beam/run_home.py --optimize
# Use the same --config and --workspace-dir overrides as preparation, if supplied.
python benchmark/beam/run_home.py --optimize --instruction "Improve retrieval quality"
```
This starts a fresh application, applies both budgets and the sampling policy,
runs `home_optimize`, and closes the application. It does not repeat preparation.
Python callers can use `await optimize(config_path, workspace_dir=...)` from the
same module. The default `run()` and CLI behavior remain preparation only.
Python callers can use the same lifecycle:
```python

View file

@ -154,4 +154,21 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
准备完成后,用同一份配置进行本地一次性优化:
```bash
python benchmark/beam/run_home.py --optimize
```
若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
不加 `--optimize` 时仍只准备。
默认选择 BEAM **100K 全部 case**。

View file

@ -1,5 +1,10 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
# Preparation by default; --optimize applies these settings to the prepared workspace.
# Paths are relative to the ReMe repository root, not the cwd.
workspace_dir: benchmark/beam/workspaces/home-100K
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_validations: 10
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
validation_cycle_time_budget_minutes: 150
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
@ -18,8 +23,6 @@ preparation:
source_ref: HEAD # Committed source only; resolved and saved as a full SHA.
config: benchmark.yaml
plugins: [beam]
candidate:
paths: [plugins/beam/src]
debug:
enabled: true
sessions_per_case: 3

View file

@ -1,22 +1,145 @@
"""Prepare the beam Home workspace, dataset and candidate code; do not evaluate."""
"""Prepare the BEAM Home workspace, or optimize it with --optimize after preparation."""
from pathlib import Path
import argparse
import asyncio
import json
import sys
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
import yaml
# Support direct execution from any working directory as well as module imports.
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
# pylint: disable=wrong-import-position
from reme_home.preparation.spec import PreparationSpec # noqa: E402
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.sampling import SamplingPolicy # noqa: E402
from reme.application import Application # noqa: E402
# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
class RunHomeConfig(BaseModel):
"""Runner settings kept separate from Home's public preparation contract."""
model_config = ConfigDict(extra="forbid", strict=True)
workspace_dir: str = Field(min_length=1)
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_validations: int = Field(default=10, ge=1)
def project_path(value: str, project_root: Path) -> Path:
"""Use repository-relative paths consistently regardless of the caller's cwd."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Resolve source paths and validate explicit Home preparation parameters."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
config = RunHomeConfig.model_validate(raw)
config.preparation.setdefault("candidate", {"paths": ["plugins/beam/src"]})
spec = PreparationSpec.model_validate(config.preparation)
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
if spec.benchmark.evidence_source is not None:
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
return workspace, spec.model_dump(mode="json"), config
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
return workspace, spec
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
"max_validations": config.max_validations,
},
},
},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
if optimize_workspace:
arguments = {} if instruction is None else {"instruction": instruction}
response = await app.run_job("home_optimize", **arguments)
else:
response = await app.run_job("home_prepare", spec=spec)
if not response.success:
raise RuntimeError(response.answer)
return {"workspace_dir": str(workspace), **response.answer}
finally:
await app.close()
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
"""Prepare once without running evaluations or optimization."""
return await _run(config_path, workspace_dir=workspace_dir)
async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
"""Optimize an already prepared workspace using this config's actual model budgets."""
return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
def main(default_config: Path = DEFAULT_CONFIG) -> None:
"""Run preparation or explicit local one-shot optimization with the benchmark config."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
args = parser.parse_args()
if args.instruction is not None and not args.optimize:
parser.error("--instruction requires --optimize")
if args.optimize:
result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
print(json.dumps(result, ensure_ascii=False, indent=2))
return
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
print(
json.dumps(
{
"workspace_dir": result["workspace_dir"],
"case_count": result["dataset"]["case_count"],
"query_count": result["dataset"]["query_count"],
"source_commit": result["code"]["source_commit"],
"initial_commit": result["code"]["initial_commit"],
},
indent=2,
),
)
if __name__ == "__main__":
main(DEFAULT_CONFIG)
main()

View file

@ -1,93 +0,0 @@
"""Shared preparation-only entry point for benchmark Home experiments."""
import argparse
import asyncio
import json
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
import yaml
from reme_home.preparation.spec import PreparationSpec
from reme_home.judge import JUDGE_PLUGINS
from reme_home.sampling import SamplingPolicy
from reme.application import Application
PROJECT_ROOT = Path(__file__).resolve().parents[1]
class RunHomeConfig(BaseModel):
"""Runner settings kept separate from Home's public preparation contract."""
model_config = ConfigDict(extra="forbid", strict=True)
workspace_dir: str = Field(min_length=1)
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
def project_path(value: str, project_root: Path) -> Path:
"""Use repository-relative paths consistently regardless of the caller's cwd."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Resolve source paths and validate explicit Home preparation parameters."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
config = RunHomeConfig.model_validate(raw)
spec = PreparationSpec.model_validate(config.preparation)
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
if spec.benchmark.evidence_source is not None:
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
return workspace, spec.model_dump(mode="json"), config.sampling_policy
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
return workspace, spec
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
"""Start Home, prepare once, and close all resources without evaluating cases."""
workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
response = await app.run_job("home_prepare", spec=spec)
if not response.success:
raise RuntimeError(response.answer)
return {"workspace_dir": str(workspace), **response.answer}
finally:
await app.close()
def main(default_config: Path) -> None:
"""Run a benchmark's default preparation from the command line."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
args = parser.parse_args()
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
print(
json.dumps(
{
"workspace_dir": result["workspace_dir"],
"case_count": result["dataset"]["case_count"],
"query_count": result["dataset"]["query_count"],
"source_commit": result["code"]["source_commit"],
"initial_commit": result["code"]["initial_commit"],
},
indent=2,
),
)

View file

@ -137,6 +137,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
advisory time between completed formal evaluations. It defaults to 150 minutes;
set a positive number to change it or null to disable timed reminders.
The top-level `max_validations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.
After preparation, run local one-shot optimization with the same configuration:
```bash
python benchmark/longmemeval/run_home.py --optimize
# Use the same --config and --workspace-dir overrides as preparation, if supplied.
python benchmark/longmemeval/run_home.py --optimize --instruction "Improve retrieval quality"
```
This starts a fresh application, applies both budgets and the sampling policy,
runs `home_optimize`, and closes the application. It does not repeat preparation.
Python callers can use `await optimize(config_path, workspace_dir=...)` from the
same module. The default `run()` and CLI behavior remain preparation only.
Python callers can use the same lifecycle:
```python

View file

@ -125,6 +125,23 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
准备完成后,用同一份配置进行本地一次性优化:
```bash
python benchmark/longmemeval/run_home.py --optimize
```
若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
不加 `--optimize` 时仍只准备。
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
运行时无需读取分组文件,group_6–group_10 保持为留出集。

View file

@ -1,6 +1,9 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
# Preparation by default; --optimize applies these settings to the prepared workspace.
# Paths are relative to the ReMe repository root, not the cwd.
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_validations: 10
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5

View file

@ -1,22 +1,145 @@
"""Prepare the longmemeval Home workspace, dataset and candidate code; do not evaluate."""
"""Prepare the LongMemEval Home workspace, or optimize it with --optimize after preparation."""
from pathlib import Path
import argparse
import asyncio
import json
import sys
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
import yaml
# Support direct execution from any working directory as well as module imports.
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
# pylint: disable=wrong-import-position
from reme_home.preparation.spec import PreparationSpec # noqa: E402
from reme_home.judge import JUDGE_PLUGINS # noqa: E402
from reme_home.sampling import SamplingPolicy # noqa: E402
from reme.application import Application # noqa: E402
# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
class RunHomeConfig(BaseModel):
"""Runner settings kept separate from Home's public preparation contract."""
model_config = ConfigDict(extra="forbid", strict=True)
workspace_dir: str = Field(min_length=1)
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_validations: int = Field(default=10, ge=1)
def project_path(value: str, project_root: Path) -> Path:
"""Use repository-relative paths consistently regardless of the caller's cwd."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Resolve source paths and validate explicit Home preparation parameters."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
config = RunHomeConfig.model_validate(raw)
config.preparation.setdefault("candidate", {"paths": ["plugins/lme/src"]})
spec = PreparationSpec.model_validate(config.preparation)
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
if spec.benchmark.evidence_source is not None:
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
return workspace, spec.model_dump(mode="json"), config
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
return workspace, spec
async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
"max_validations": config.max_validations,
},
},
},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
if optimize_workspace:
arguments = {} if instruction is None else {"instruction": instruction}
response = await app.run_job("home_optimize", **arguments)
else:
response = await app.run_job("home_prepare", spec=spec)
if not response.success:
raise RuntimeError(response.answer)
return {"workspace_dir": str(workspace), **response.answer}
finally:
await app.close()
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
"""Prepare once without running evaluations or optimization."""
return await _run(config_path, workspace_dir=workspace_dir)
async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
"""Optimize an already prepared workspace using this config's actual model budgets."""
return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
def main(default_config: Path = DEFAULT_CONFIG) -> None:
"""Run preparation or explicit local one-shot optimization with the benchmark config."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
args = parser.parse_args()
if args.instruction is not None and not args.optimize:
parser.error("--instruction requires --optimize")
if args.optimize:
result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
print(json.dumps(result, ensure_ascii=False, indent=2))
return
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
print(
json.dumps(
{
"workspace_dir": result["workspace_dir"],
"case_count": result["dataset"]["case_count"],
"query_count": result["dataset"]["query_count"],
"source_commit": result["code"]["source_commit"],
"initial_commit": result["code"]["initial_commit"],
},
indent=2,
),
)
if __name__ == "__main__":
main(DEFAULT_CONFIG)
main()

View file

@ -838,3 +838,150 @@ reme home_case_study evaluation_path=evaluations/candidate/abcdef0/1 'case_ids=[
Calls with `_allowed_paths` restrictions fail closed, as other Home artifact
Jobs do. Structured inspection tools run in the host with their existing
containment checks; only Bash/Python subprocesses run inside the OS sandbox.
## Diagnostic and optimizer agents
`home_diagnose` and `home_optimize` provide the minimal Meta-ReMe workflow using
Home's existing Jobs. Enable Home with a benchmark judge and prepare the
workspace first. Start the configured service in one terminal:
```bash
reme start config=/absolute/path/to/home.yaml
```
Call from another terminal:
```bash
reme home_diagnose 'evaluation_paths=["evaluations/init/abcdef0/1"]' 'instruction=Explain the lowest-scoring cases'
reme home_optimize 'instruction=Improve evidence retrieval while reducing query tokens'
```
Diagnosis requires explicit, unique result directories, including failed runs
with saved evidence. Each call creates a fresh diagnostic agent, delegates deep
case investigation to `home_case_study`, and returns ranked, evidence-supported
causes without prescribing fixes. An optimizer invocation automatically injects
its available caller messages and compression summary; the model cannot supply
or override that trajectory. Standalone calls record an empty trajectory.
Reports and trajectories are saved under `home/logs/diagnostic-reports/` and
`home/logs/diagnostic-trajectories/` with their evidence paths and session IDs.
Optimization reuses a completed full initial baseline covering every prepared
training case and query, with no infrastructure errors. Otherwise it runs that
baseline first, requiring clean `init` to be checked out. It never switches away
from an existing candidate to repair a missing baseline. Baseline observations
feed sampling history. Candidate execution failures retain Home's existing
failure thresholds; low scores are valid evidence. Infrastructure failures stop
baseline startup and are distinguished from candidate defects during iteration.
The optimizer diagnoses, edits declared `candidate_paths`, commits through
managed Git Jobs, runs debug until execution passes, then evaluates using the
host's adaptive sampling policy. Formal evaluations always rebuild memory.
The agent cannot change execution settings, selections, sampling mode, or
construction reuse through its evaluation tool; only `debug` is model-selected.
Normal `home_evaluate` callers retain their existing interface and replay support.
Unchanged candidates and infrastructure retries follow the existing repetition
rules. There is no admission gate, architecture agent,
construction-equivalence reviewer, or query-only optimizer policy in this workflow.
Both agents share the case-study OS sandbox implementation. All file tools,
Bash and Python use it, including file metadata and Edit/Write checks. Evidence,
fixed runtime and Git metadata are read-only. Each call receives a private
scratch directory under `home/tmp/`; only the optimizer additionally receives
writes to declared candidate roots. Network is denied. Diagnostic writes are
limited to scratch. Host Jobs alone write reports, evaluation artifacts and Git
metadata. Sandbox startup is probed before model calls and never falls back to
unrestricted execution. Declared write roots must exist and remain real paths.
Only one optimizer can run per Home workspace; cancellation releases its lock
after nested tool cleanup. Separate diagnostic calls and case studies may run
concurrently with fresh state.
The final optimizer response must identify `branch_name`, full `commit_sha`,
`evaluation_path` and `report`. The host verifies a clean checkout at that exact
branch/commit and a matching completed, scored, end-to-end evaluation without
infrastructure failures. Sampled formal evidence qualifies; debug or query-only
evidence does not. Keeping the initial baseline is valid. Invalid final claims
fail while preserving changes and the raw report in
`home/logs/optimization-reports/`. The host verifies provenance rather than
ranking candidates by a single average score.
Dedicated `as_llm` and `agent_wrapper` components are named `home_optimizer` and
`home_diagnostic`; override them through ordinary application configuration.
Both default to `qwen3.8-max`, with context sizes 160,000 and 128,000, respectively.
The optimizer has 3,000 iterations and a 150-minute advisory validation-cycle timer;
configure `components.agent_wrapper.home_optimizer.validation_cycle_time_budget_minutes`
(null disables timed reminders). The previous `time_budget_minutes` setting is a
fallback when the cycle setting is absent. Diagnosis retains 80 iterations and
a 60-minute whole-invocation timer. Neither timer enforces a time limit.
The optimizer follows dev_meta b53048e4: before reasoning it checks 50%, 75%, 90%,
then 100% and each additional 10% of the cycle budget. Reminders include elapsed
time, budget percentage, completed context compressions and guidance to finish
editing, commit, pass the smoke test and begin validation. Temporary and durable
reminders alternate, beginning with temporary. The cycle includes nested diagnosis,
debug and evaluation time. Only a new counted formal evaluation returning
`success` or `completed` resets the clock, thresholds, alternation and compression
count; scores and candidate failures do not control reset. As in dev_meta, a
completed result can reset even when its case results contain infrastructure errors;
an incomplete failure, rejected call, cancellation, debug or pre-sandbox failure
cannot. The initial full baseline precedes the first optimizer cycle and is excluded
from its formal-validation numbering. A durable hint records the completed
validation number and commit prefix, expires that cycle's old timer reminders and
announces the next cycle with the remaining formal-validation count.
Persisted old reminders are retained as in dev_meta.
Hints use AgentScope's current reply message so accumulated token usage is retained.
Evaluation tools queue transitions in request state; the optimizer injects them
before the next reasoning call, after all tool results have been written. Thus a
transition never separates an API tool call from its matching result.
`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching
dev_meta. `get_validation_budget` reports consumed and remaining slots, the search
phase and next-candidate guidance. A non-init formal run consumes one slot when
its first sandbox initializes successfully, including runs that later fail or are
cancelled. Initial baselines, debug, rejected calls and pre-start failures are free.
Accounting reuses persistent started-run evidence and survives optimizer restarts;
removing the rebuildable sampling cache does not restore the budget. The evaluator
checks the limit while holding the formal-validation lock, before starting another
run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects
from existing evidence. The first `max_validations // 2` starts favor structural
exploration; later starts favor evidence-driven refinement. This guidance is advisory.
Ordinary `home_evaluate` calls remain unbounded unless the host supplies
`max_validations`; all recorded non-init formal starts in that workspace count.
Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage.
The optimizer follows dev_meta's compression recovery policy. Recoverable model
or summary-output failures leave context intact below the emergency ratio
`(trigger_ratio + 1) / 2`. At or above that ratio (90% for an 80% trigger), old
messages are atomically saved under `logs/context-offloads/`; recent context and
the existing summary are retained, with a reminder pointing to the readable JSON
file. This fallback makes no additional model call and counts as one completed
compression. Cancellation, invalid configuration and uncompressible system prompts
still propagate during normal operation. Diagnostic and case-study agents retain
their existing compression behavior. Bounded draft notebook tools are not included.
Host-only execution settings belong in the optimizer Step; model, iteration and
timer settings belong in its wrapper. For example:
```yaml
jobs:
home_optimize:
steps:
- backend: home_optimize_step
agent_wrapper: home_optimizer
evaluation_options:
concurrency: 4
timeout_seconds: 7200
max_construction_failure_rate: 0.03
# memory_limit_mib: 0
components:
agent_wrapper:
home_optimizer:
react_config:
max_iters: 3000
validation_cycle_time_budget_minutes: 150
max_validations: 10
```
`evaluation_options` also accepts `base_image`, `fail_fast` and
`max_artifact_bytes`. Omit `fail_fast` to retain Home's debug/scored defaults.
Restricted `_allowed_paths` calls fail closed. No real model or Docker benchmark
is required by the isolated workflow tests.

View file

@ -116,7 +116,8 @@ def _format_minutes(minutes: float) -> str:
"""Format a timer duration compactly."""
if minutes < 1:
return f"{max(0, round(minutes * 60))} seconds"
seconds = max(0, round(minutes * 60))
return f"{seconds} {'second' if seconds == 1 else 'seconds'}"
rounded = round(minutes, 1)
value = int(rounded) if rounded.is_integer() else rounded
unit = "minute" if value == 1 else "minutes"

View file

@ -0,0 +1,126 @@
"""Meta-ReMe compression recovery adapted to AgentScope's native context pipeline."""
import asyncio
import logging
from pathlib import Path
from uuid import uuid4
from agentscope.exception import ToolJSONDecodeError
from agentscope.message import TextBlock
import jsonschema
import openai
from pydantic import ValidationError
from ..evaluation.artifacts import contained, write_json
from .bounded import BoundedReActAgent
logger = logging.getLogger(__name__)
class RecoverableCompressionAgent(BoundedReActAgent):
"""Skip recoverable failures until emergency offload is needed.
Override the native implementation so compression middleware, successful-change
counters, finalization and cancellation remain owned by their existing layers.
"""
def __init__(self, *args, fallback_directory=None, **kwargs):
super().__init__(*args, **kwargs)
self.fallback_directory = Path(fallback_directory) if fallback_directory is not None else None
async def _compress_context_impl(self, context_config=None, instructions=None):
"""Keep the old trigger-derived emergency boundary without silent truncation."""
config = context_config or self.context_config
model_input = await self._prepare_model_input()
context_ratio = await self.model.count_tokens(**model_input) / self.model.context_size
if context_ratio < config.trigger_ratio:
return
# The SDK's default truncation would hide the failure and discard old
# evidence before Home can decide whether emergency offload is needed.
config = config.model_copy(update={"compression_fallback_to_truncation": False})
try:
await super()._compress_context_impl(context_config=config, instructions=instructions)
except Exception as exc:
if not self._is_recoverable_compression_error(exc):
raise
emergency_ratio = (config.trigger_ratio + 1.0) / 2.0
if context_ratio < emergency_ratio:
logger.warning(
"Context compression failed at %.1f%%, below the %.1f%% emergency threshold; skipping: %s: %s",
context_ratio * 100,
emergency_ratio * 100,
type(exc).__name__,
exc,
)
return
logger.warning(
"Context compression failed at %.1f%%; using local offload at the %.1f%% emergency threshold: %s: %s",
context_ratio * 100,
emergency_ratio * 100,
type(exc).__name__,
exc,
)
await self._fallback_compress_context(exc, context_ratio, config)
@staticmethod
def _is_recoverable_compression_error(exc):
"""Match dev_meta's model/output failures, excluding uncompressible prompts."""
if isinstance(exc, RuntimeError):
return not str(exc).startswith("The system prompt")
return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError))
async def _fallback_compress_context(self, cause, context_ratio, config):
"""Persist older messages before replacing state; never call a second model."""
if self.fallback_directory is None:
raise RuntimeError("Optimizer fallback_directory is required for local context compression") from cause
model_input = await self._prepare_model_input()
old_messages, recent_messages = await self._split_context_for_compression(
config.reserve_ratio * self.model.context_size,
model_input.get("tools", []),
)
if not old_messages:
raise RuntimeError(
"Local fallback compression cannot remove any context while preserving the configured reserve",
) from cause
async def apply_change():
contained(self.fallback_directory.parent, self.fallback_directory, directory=True)
path = contained(self.fallback_directory, f"context-{uuid4().hex}.json")
write_json(
path,
{
"session_id": self.state.session_id,
"context_ratio": context_ratio,
"trigger_ratio": config.trigger_ratio,
"emergency_ratio": (config.trigger_ratio + 1.0) / 2.0,
"failure": {"type": type(cause).__name__, "message": str(cause)},
"messages": [message.model_dump(mode="json", exclude_none=False) for message in old_messages],
},
)
notice = (
"<system-reminder>Model-based context compression failed near the context limit. "
f"Older context was offloaded without semantic summarization to '{path}'. "
"The recent context is preserved; use Read on that file if older experiments, decisions, "
"validation results, or tool outputs are needed.</system-reminder>"
)
await self._clear_unreserved_read_cache(recent_messages)
# Recent SDK versions also carry current-reply usage through compression.
usage = self._get_reply_usage()
summary = self.state.summary
self.state.summary = (
"\n".join(part for part in (summary, notice) if part)
if isinstance(summary, str)
else [*summary, TextBlock(text=notice)]
)
self.state.context = recent_messages
if usage is not None and self._get_reply_usage() is None:
self.state.append_context(self.name, [])
self.state.context[-1].usage = usage
logger.warning("Local compression offloaded %d message(s) to %s", len(old_messages), path)
task = asyncio.create_task(apply_change())
try:
await asyncio.shield(task)
except asyncio.CancelledError:
await task
raise

View file

@ -0,0 +1,159 @@
"""Diagnose saved Home evaluations with an isolated evidence-driven agent."""
import asyncio
import copy
import json
from agentscope.state import AgentState
from agentscope.tool import FunctionTool
from pydantic import Field, field_validator
from reme.steps.base_step import BaseStep, Ref
from ..evaluation.artifacts import contained, read_json, write_json
from ..evaluation.component import blocking
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace
from .workflow_common import required_job, save_record, workflow_reply
from .wrapper import BoundedAgentWrapper
class DiagnosticInput(SpecModel):
"""Require explicit evaluation evidence; caller state is never model-selected."""
evaluation_paths: list[str] = Field(min_length=1)
instruction: str = Field(default="Identify and rank the evidence-supported root causes.", min_length=1)
@field_validator("evaluation_paths")
@classmethod
def validate_paths(cls, values):
"""Reject empty and duplicate input references before creating an agent."""
if any(not value.strip() for value in values) or len(set(values)) != len(values):
raise ValueError("evaluation_paths must contain unique nonempty paths")
return values
def evaluation_paths(home, values):
"""Require explicit, unique local runs with actual evidence, including failed runs."""
if not values:
raise ValueError("evaluation_paths must identify at least one run")
paths = []
root = contained(home, "evaluations", directory=True)
for value in values:
path = contained(home, value, directory=True)
if not path.is_relative_to(root) or path == root:
raise ValueError("evaluation_paths must be inside home/evaluations")
read_json(contained(path, "manifest.json"))
evidence = [path / "summary.json", path / "failure.json"]
evidence += list(path.glob("cases/*/memory_construction/result.json"))
evidence += list(path.glob("cases/*/queries/result.json"))
if not any(contained(path, item).is_file() for item in evidence):
raise ValueError(f"Evaluation contains no results: {path}")
paths.append(str(path))
if len(paths) != len(set(paths)):
raise ValueError("evaluation_paths must be unique")
return paths
def trajectory(state):
"""Serialize only caller messages and compression summary, never credentials or permissions."""
if state is not None and not isinstance(state, AgentState):
raise TypeError("Injected caller state must be an AgentState")
if state is None:
return {"messages": [], "summary": None}
summary = state.summary
if isinstance(summary, list):
summary = [block.model_dump(mode="json") for block in summary]
return {
"session_id": state.session_id,
"messages": [message.model_dump(mode="json") for message in state.context],
"summary": summary,
}
def diagnostic_tool(wrapper):
"""Inject the optimizer's real trajectory into a fresh diagnostic invocation."""
job = required_job(wrapper, "home_diagnose")
async def diagnose(
evaluation_paths: list[str], # pylint: disable=redefined-outer-name
instruction: str = "Identify and rank the evidence-supported root causes.",
**kwargs,
):
if set(kwargs) - {"_agent_state"}:
raise ValueError("Caller trajectory is host-owned and cannot be supplied by the model")
response = await job(
instruction=instruction,
evaluation_paths=evaluation_paths,
caller_trajectory=trajectory(kwargs.get("_agent_state")),
)
return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
tool = FunctionTool(
diagnose,
name="home_diagnose",
description=job.description,
is_concurrency_safe=False,
is_state_injected=True,
)
tool.input_schema = copy.deepcopy(job.parameters)
return tool
class HomeDiagnoseStep(BaseStep):
"""Aggregate explicit evaluation evidence through a read-only diagnostic agent."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
async def execute(self):
"""Validate evidence, save the host trajectory and persist a provenance-bearing report."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home diagnosis requires access to its managed evidence")
if not self.workspace.is_started or not self.agent_wrapper.is_started:
raise RuntimeError("Start the Home workspace and diagnostic wrapper first")
spec = DiagnosticInput.model_validate(
{key: self.context[key] for key in DiagnosticInput.model_fields if key in self.context},
)
home = self.workspace.home_path
paths = await blocking(evaluation_paths, home, spec.evaluation_paths)
caller = self.context.get("caller_trajectory", trajectory(None))
if not isinstance(caller, dict):
raise ValueError("Caller trajectory must be a host-supplied object")
source = await blocking(save_record, home, "diagnostic-trajectories", caller)
payload = {
"status": "running",
"instruction": spec.instruction,
"evaluation_paths": paths,
"trajectory_path": source,
}
report_path = await blocking(save_record, home, "diagnostic-reports", payload)
self.context.response.metadata.update(diagnostic_report_path=report_path, trajectory_path=source)
try:
result = await workflow_reply(
self.agent_wrapper,
home,
"diagnostic",
f"Diagnostic request:\n{spec.instruction}\n\nComplete available caller trajectory: {source}\n"
f"Explicit evaluation paths: {json.dumps(paths, ensure_ascii=False)}",
job_names=("home_case_study",),
memory_limit_mib=self.kwargs.get("memory_limit_mib"),
)
if not isinstance(result.get("result"), str) or not result["result"].strip():
raise ValueError("Diagnostic agent returned no report")
payload.update(
status="completed",
diagnosis=result["result"],
session_id=result.get("session_id"),
usage=result.get("usage"),
)
self.context.response.answer = {**payload, "diagnostic_report_path": report_path}
self.context.response.metadata.update(
{key: result[key] for key in ("session_id", "usage") if key in result},
)
except BaseException as exc:
payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
raise
finally:
await blocking(write_json, contained(home, report_path), payload)
return self.context.response

View file

@ -0,0 +1,32 @@
system_prompt: |-
You are Home's read-only code diagnostician, migrated from Meta-ReMe.
Identify and rank evidence-supported mechanisms that caused evaluation failures or limited candidate quality.
The candidate repository is {code_repository}; guides are under {docs}; private scratch is {scratch}.
The caller provides explicit evaluation paths and its available complete trajectory, including any compression summary.
Produce one concise report organized as Priority scope, Failure clusters, Ranked shared root causes, Outliers,
and Missing evidence. State which low-scoring cases and question types you prioritized and what you deferred.
For each cause, name affected cases and queries, lifecycle stage, causal chain, supporting and contradicting
evidence, confidence, and the smallest missing observation. Separate fixed-runtime limitations.
Read architecture-guide.md and validation-artifact-guide.md when needed. Confirm each run's exact commit,
status, score contract and coverage. Compare runs only where contracts and case/query coverage permit it.
Prioritize the lowest-scoring cases and question types; you need not explain every failure.
Use home_case_study batches early as the default for deep case investigation, even for one case. Pass one
evaluation_path, case_ids and a shared focus; do not put case-specific instructions in the shared focus.
Use its reports to identify shared causes, then inspect only remaining cross-case or code-level evidence.
Direct investigation is appropriate for quick lookups, aggregate comparisons and specific evidence checks.
Classify infrastructure, candidate-execution, answer-quality and judging failures separately. Trace extraction,
persistence, overwrite, indexing, retrieval, ranking, context presentation or reasoning as evidence requires.
Treat partial_success construction as requiring repair. Do not replace persisted runtime failure statuses.
Original sessions and derived memory are both evidence; absence from summaries does not prove unavailability.
Prefer independent evidence_session_ids; inferred evidence is a lead, not proof. Explicitly state uncertainty.
Use home_list_validation_results and home_inspect_validation to locate evidence. Pin revision to the evaluated
full commit when using home_list_code_tree, home_search_code and home_read_code_file. The current checkout may
differ. Read artifacts and runtime docs with Read, Grep, Glob, AutoBash or python_execute, using absolute paths.
Keep stderr visible; denied reads are not proof of missing content.
The OS sandbox allows evidence reads and writes only to private scratch. You may create disposable scripts
and analysis files there. Do not modify candidate code, datasets, evaluations, reports or Git metadata.
Do not run ReMe, evaluations or network operations. Do not implement or prescribe fixes; report supported causes.

View file

@ -0,0 +1,513 @@
"""Optimize Home candidates and verify their baseline and final evaluation evidence."""
import asyncio
import json
import math
import time
from pathlib import Path
from typing import Callable
from agentscope.message import HintBlock
from agentscope.state import AgentState
from agentscope.tool import Edit, FunctionTool
from pydantic import Field
from reme.steps.base_step import BaseStep, Ref
from ..code.repository import validate_repository, require_clean, validate_tree, status
from ..evaluation.artifacts import contained, read_json, write_json
from ..evaluation.component import blocking
from ..evaluation.inputs import EvaluationSpec, load_inputs
from ..evaluation.history import read_state
from ..evaluation.budget import validate_max_validations, validation_budget_status
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from .diagnostic import diagnostic_tool, evaluation_paths
from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity
from .compression import RecoverableCompressionAgent
from .workflow_common import required_job, save_record, workflow_reply
from .wrapper import CaseStudyAgentWrapper
MUTATION_JOBS = (
"home_create_candidate_branch",
"home_switch_candidate_branch",
"home_merge_candidate_branch",
"home_commit_candidate",
)
PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions"
class ValidationCycleTimer:
"""Emit escalating reminders as a scored-validation cycle ages."""
def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None:
if (
not isinstance(time_budget_minutes, (int, float))
or isinstance(time_budget_minutes, bool)
or not math.isfinite(time_budget_minutes)
or time_budget_minutes <= 0
):
raise ValueError("time_budget_minutes must be a positive number")
self.time_budget_minutes = float(time_budget_minutes)
self._clock = clock
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
def reset(self) -> None:
"""Start a fresh cycle after a scored validation finishes."""
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
def record_context_compression(self) -> None:
"""Record one completed compression since the last scored validation."""
self.context_compression_count += 1
def next_reminder(self) -> TimeBudgetReminder | None:
"""Return one reminder for the latest newly reached deadline."""
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
reached_ratio: float | None = None
if elapsed_ratio >= 1.0:
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
else:
for threshold in (0.9, 0.75, 0.5):
if elapsed_ratio >= threshold:
reached_ratio = threshold
break
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
return None
self._last_reminded_ratio = reached_ratio
persistent = self._next_reminder_is_persistent
self._next_reminder_is_persistent = not persistent
elapsed = _format_minutes(elapsed_minutes)
percentage = round(reached_ratio * 100)
timing = (
f"The optimizer agent run has used {elapsed} since the last scored validation "
f"({percentage}% of its {_format_minutes(self.time_budget_minutes)} soft time budget). "
)
compression = (
f"It has completed {self.context_compression_count} context compressions since the last "
"scored validation. "
)
if elapsed_ratio >= 1.0:
return TimeBudgetReminder(
content=(
f"<system-reminder>{timing}{compression}"
"Prioritize completing the code changes, committing them, passing the smoke test, and starting "
"validation as soon as possible.</system-reminder>"
),
persistent=persistent,
)
remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes)
return TimeBudgetReminder(
content=(
f"<system-reminder>{timing}{compression}"
f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
"</system-reminder>"
),
persistent=persistent,
)
class OptimizerAgent(RecoverableCompressionAgent):
"""Add Meta-ReMe's per-validation reminders to Home's bounded agent."""
def __init__(self, *args, validation_cycle_timer=None, **kwargs):
super().__init__(*args, **kwargs)
self.validation_cycle_timer = validation_cycle_timer
async def _reasoning(self, *args, **kwargs):
"""Deliver hints after tool results without splitting the SDK's reply message."""
transitions = self.state.middle_context.pop(PENDING_CYCLE_TRANSITIONS, [])
for content in transitions:
self.state.append_context(
self.name,
[HintBlock(source="meta_reme_validation_cycle_transition", hint=content)],
)
timer = self.validation_cycle_timer
reminder = timer.next_reminder() if timer is not None else None
message = None
if reminder is not None:
message = HintBlock(source="meta_reme_validation_timer", hint=reminder.content)
self.state.append_context(self.name, [message])
try:
async for item in super()._reasoning(*args, **kwargs):
yield item
finally:
if reminder is not None and not reminder.persistent:
_remove_hint_by_identity(self.state.context, message)
async def compress_context(self, *args, **kwargs):
"""Count completed context replacements in the current validation cycle."""
summary_before, context_before = self.state.summary, self.state.context
await super().compress_context(*args, **kwargs)
if self.validation_cycle_timer is not None and (
self.state.summary is not summary_before or self.state.context is not context_before
):
self.validation_cycle_timer.record_context_compression()
class OptimizerAgentWrapper(CaseStudyAgentWrapper):
"""Keep each optimizer's tools and cycle timer bound to one reply invocation."""
def create_cycle_timer(self, options=None):
"""Use the legacy time setting as a fallback; explicit null disables reminders."""
options = self._merged_kwargs(options or {})
budget = options.get("validation_cycle_time_budget_minutes", options.get("time_budget_minutes"))
return None if budget is None else ValidationCycleTimer(budget)
def validation_limit(self):
"""Keep the formal count limit host-owned and independent of elapsed time."""
return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10))
def _create_agent(self, *, options, **kwargs):
timer = (
options["_validation_cycle_timer"]
if "_validation_cycle_timer" in options
else self.create_cycle_timer(options)
)
return OptimizerAgent(
validation_cycle_timer=timer,
fallback_directory=options.get("_context_fallback_directory", self.session_path / "context-offloads"),
**kwargs,
)
def validation_cycle_transition(manifest, validation_number, remaining):
"""Expire old reminders and announce the next cycle with its remaining budget."""
commit = manifest["commit_sha"][:7]
return HintBlock(
source="meta_reme_validation_cycle_transition",
hint=(
f"<system-reminder>Formal validation {validation_number} completed successfully for commit {commit}. "
f"The previous cycle timer has been reset, and all timer reminders for cycle {validation_number} "
f"are now expired. Remaining formal validations: {remaining}. Start cycle {validation_number + 1}."
"</system-reminder>"
),
)
def completed_validation(home, result, started_before):
"""Match dev_meta's new-count plus completed-status boundary, independent of scores."""
if not isinstance(result, dict) or result.get("status") not in {"success", "completed"}:
return None
run = contained(home, result["result_dir"], directory=True)
key = run.relative_to(contained(home, "evaluations", directory=True)).as_posix()
started = read_state(home)["started_validations"]
# The initial full baseline precedes optimizer creation in dev_meta and
# never consumes its validation budget. Home records it in sampling history.
if key in started_before or key not in started or key.startswith("init/"):
return None
manifest = read_json(contained(run, "manifest.json"))
if (
manifest.get("status") != "completed"
or manifest.get("mode") != "scored"
or manifest.get("dataset_kind") != "training-set"
or not manifest.get("sandbox_started_at")
):
return None
return manifest, sum(not name.startswith("init/") for name in started)
class OptimizationInput(SpecModel):
"""The host owns runtime settings; the caller supplies only the optimization focus."""
instruction: str = Field(default="Improve the candidate using the completed baseline evaluation.", min_length=1)
class OptimizationResult(SpecModel):
"""An agent claim that must be checked against Git and saved formal evidence."""
branch_name: str = Field(min_length=1)
commit_sha: str = Field(pattern="^[0-9a-f]{40}$")
evaluation_path: str = Field(min_length=1)
report: str = Field(min_length=1)
def parse_result(text):
"""Accept a JSON object, optionally enclosed in one Markdown JSON fence."""
text = text.strip()
if text.startswith("```") and text.endswith("```"):
text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
return OptimizationResult.model_validate(json.loads(text))
EVALUATION_OPTIONS = {
"concurrency",
"timeout_seconds",
"base_image",
"fail_fast",
"max_construction_failure_rate",
"max_artifact_bytes",
}
def evaluation_options(raw):
"""Validate host-only execution settings without exposing selection or replay."""
if not isinstance(raw, dict) or set(raw) - EVALUATION_OPTIONS:
raise ValueError("evaluation_options may contain only Home execution settings")
return EvaluationSpec.model_validate(raw).model_dump(exclude_unset=True)
def verify_evaluation(home, path, commit, *, baseline=False):
"""Require completed end-to-end evidence with no infrastructure failures."""
run = Path(evaluation_paths(home, [str(path)])[0])
manifest = read_json(contained(run, "manifest.json"))
summary = read_json(contained(run, "summary.json"))
required = {
"commit_sha": commit,
"status": "completed",
"success": True,
"mode": "scored",
"dataset_kind": "training-set",
"execution_mode": "full_validation",
}
if any(manifest.get(key) != value for key, value in required.items()):
raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
if (
not manifest.get("sandbox_started_at")
or not summary.get("case_count")
or summary.get("scoring_enabled") is not True
or any(summary.get(key, {}).get("infra_error") != 0 for key in ("case_status_counts", "query_status_counts"))
):
raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
_record, dataset, cases = load_inputs(home, EvaluationSpec())
if manifest.get("fingerprints", {}).get("dataset") != dataset.normalized_fingerprint:
raise ValueError("Evaluation dataset no longer matches the prepared dataset")
selected = manifest.get("case_ids", [])
queries = manifest.get("query_ids", {})
expected = {case.case_id: {q.query_id for q in case.queries} for case in cases}
if (
not selected
or len(set(selected)) != len(selected)
or set(selected) - expected.keys()
or set(queries) != set(selected)
or any(not ids or len(set(ids)) != len(ids) or set(ids) - expected[key] for key, ids in queries.items())
):
raise ValueError("Evaluation has an invalid case/query selection")
if baseline and (set(selected) != set(expected) or any(set(queries[key]) != ids for key, ids in expected.items())):
raise ValueError("Initial baseline must cover every prepared training case and query")
rows = summary.get("cases", {})
if (
set(rows) != set(selected)
or summary["case_count"] != len(selected)
or summary.get("expected_query_count") != sum(len(ids) for ids in queries.values())
):
raise ValueError("Evaluation summary has incomplete case coverage")
# Failed construction can legitimately omit answers; the recorded candidate
# failure is evidence, not an infrastructure failure or a missing result.
for key, row in rows.items():
if set(row.get("queries", {})) != set(queries[key]) and row.get("construction", {}).get("status") != "failure":
raise ValueError(f"Evaluation summary has incomplete query coverage: {key}")
return {"evaluation_path": str(run), "manifest": manifest, "summary": summary}
def baseline_state(home):
"""Inspect a clean repository and locate a usable initial full evaluation."""
_validate_directory(home, home / "metadata")
with _initialization_lock(home / "metadata/workspace.lock"):
repo, record = validate_repository(home)
require_clean(repo)
validate_tree(repo, record, "HEAD")
initial = record["code"]["initial_commit"]
current = status(repo, record)
for manifest in sorted((home / "evaluations/init").glob("*/*/manifest.json"), reverse=True):
try:
evidence = verify_evaluation(home, manifest.parent, initial, baseline=True)
return current, evidence
except (ValueError, OSError, KeyError, TypeError):
continue
if current["current_branch"] != "init" or current["commit_sha"] != initial:
raise ValueError("A complete initial baseline is missing; check out clean init before home_optimize")
return current, None
def verify_final(home, result):
"""Check the actual checkout and evidence; never repair or discard agent edits."""
_validate_directory(home, home / "metadata")
with _initialization_lock(home / "metadata/workspace.lock"):
repo, record = validate_repository(home)
require_clean(repo)
validate_tree(repo, record, "HEAD")
current = status(repo, record)
if current["current_branch"] != result.branch_name or current["commit_sha"] != result.commit_sha:
raise ValueError("Final branch/commit does not match the checked-out HEAD")
evidence = verify_evaluation(home, result.evaluation_path, result.commit_sha)
if evidence["manifest"]["branch_name"] != result.branch_name:
raise ValueError("Final evaluation belongs to a different branch")
return evidence["evaluation_path"]
def validation_budget_tool(home, max_validations):
"""Expose dev_meta's read-only count, phase and next-candidate guidance."""
validate_max_validations(max_validations)
async def get_validation_budget():
"""Return started formal validations, remaining count and optimization guidance."""
return await blocking(validation_budget_status, home, max_validations)
return FunctionTool(
get_validation_budget,
name="get_validation_budget",
description=(
"Report how many formal optimizer validations have started at least one sandbox, how many remain, "
"and whether the next candidate should emphasize structural code changes or evidence-driven refinement. "
"Debug-case smoke validations, the initial baseline and failures before sandbox startup are excluded."
),
is_read_only=True,
)
def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
"""Only debug mode is model-selected; all formal runs rebuild construction."""
job = required_job(wrapper, "home_evaluate")
validate_max_validations(max_validations)
async def evaluate(debug: bool = False, **kwargs):
"""Run debug after committing; after debug passes, run a sampled formal evaluation.
Debug checks execution, not answer quality. Inspect infrastructure failures
separately; retries keep the existing commit and selection recovery rules.
"""
if not isinstance(debug, bool):
raise ValueError("debug must be boolean")
if set(kwargs) - {"_agent_state"}:
raise ValueError("Optimizer state is host-owned and cannot be supplied by the model")
state = kwargs.get("_agent_state")
if not isinstance(state, AgentState):
raise TypeError("Cycle reminders require the host-injected AgentState")
started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set()
response = await job(
**options,
**({"max_validations": max_validations} if not debug else {}),
debug=debug,
sampling=not debug,
record_history=True,
case_ids=None,
query_ids=None,
reuse_construction_from=None,
)
if not debug:
budget = await blocking(validation_budget_status, home, max_validations)
if isinstance(response.answer, dict):
response.answer = {**response.answer, "validation_budget": budget}
completed = await blocking(completed_validation, home, response.answer, started_before)
if completed is not None:
manifest, validation_number = completed
if timer is not None:
timer.reset()
transition = validation_cycle_transition(manifest, validation_number, budget["remaining_validations"])
state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint)
return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
tool = FunctionTool(evaluate, name="home_evaluate", is_concurrency_safe=False, is_state_injected=True)
tool.input_schema = {
"type": "object",
"properties": {"debug": {"type": "boolean", "default": False}},
"additionalProperties": False,
}
return tool
class HomeOptimizeStep(BaseStep):
"""Require a full initial baseline, run the coding agent, and verify its final claim."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
agent_wrapper: OptimizerAgentWrapper = Ref(OptimizerAgentWrapper, "agent_wrapper")
async def execute(self):
"""One optimizer per workspace; never hold the evaluation/Git lock across model calls."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home optimization requires access to its managed workspace")
if not self.workspace.is_started or not self.agent_wrapper.is_started:
raise RuntimeError("Start the Home workspace and optimizer wrapper first")
spec = OptimizationInput.model_validate(
{key: self.context[key] for key in OptimizationInput.model_fields if key in self.context},
)
options = evaluation_options(self.kwargs.get("evaluation_options", {}))
self.agent_wrapper.validation_limit()
home = self.workspace.home_path
_validate_directory(home, self.workspace.state_path)
with _initialization_lock(self.workspace.state_path / "optimization.lock"):
_current, baseline = await blocking(baseline_state, home)
if baseline is None:
response = await required_job(self.agent_wrapper, "home_evaluate")(
**options,
debug=False,
sampling=False,
record_history=True,
case_ids=None,
query_ids=None,
reuse_construction_from=None,
)
if not response.success:
raise RuntimeError(f"Initial baseline failed: {response.answer}")
current, _ = await blocking(baseline_state, home)
baseline = await blocking(
verify_evaluation,
home,
response.answer["result_dir"],
current["commit_sha"],
baseline=True,
)
return await self._optimize(home, spec, baseline, options)
async def _optimize(self, home, spec, baseline, options):
payload = {"status": "running", "instruction": spec.instruction, "baseline_path": baseline["evaluation_path"]}
report_path = await blocking(save_record, home, "optimization-reports", payload)
self.context.response.metadata["optimization_report_path"] = report_path
try:
timer = self.agent_wrapper.create_cycle_timer()
max_validations = self.agent_wrapper.validation_limit()
budget = await blocking(validation_budget_status, home, max_validations)
reply = await workflow_reply(
self.agent_wrapper,
home,
"optimizer",
f"Begin optimization.\n{spec.instruction}\n\n"
f"Completed full initial baseline: {baseline['evaluation_path']}\n"
f"Formal validation budget: {json.dumps(budget)}\n"
"Return exactly one JSON object with branch_name, commit_sha (full SHA), evaluation_path and report. "
"Leave that evaluated branch and commit checked out with no uncommitted changes.",
tool_factory=lambda backend: [
Edit(backend=backend),
diagnostic_tool(self.agent_wrapper),
validation_budget_tool(home, max_validations),
evaluation_tool(
self.agent_wrapper,
options,
home=home,
timer=timer,
max_validations=max_validations,
),
],
job_names=MUTATION_JOBS,
agent_options={
"_validation_cycle_timer": timer,
"_context_fallback_directory": contained(home, "logs/context-offloads", directory=True),
},
memory_limit_mib=self.kwargs.get("memory_limit_mib"),
)
payload.update(
agent_report=reply.get("result"),
session_id=reply.get("session_id"),
usage=reply.get("usage"),
)
result = parse_result(reply["result"])
evidence_path = await blocking(verify_final, home, result)
payload.update(result.model_dump(), evaluation_path=evidence_path, status="completed")
self.context.response.answer = {**payload, "optimization_report_path": report_path}
self.context.response.metadata.update({key: reply[key] for key in ("session_id", "usage") if key in reply})
except BaseException as exc:
payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
raise
finally:
await blocking(write_json, contained(home, report_path), payload)
return self.context.response

View file

@ -0,0 +1,60 @@
system_prompt: |-
You are Home's primary optimization agent, migrated from Meta-ReMe. Execute the optimization loop;
your output is the best formally evaluated candidate, with its branch and exact commit checked out.
ReMe is local-first and file-native. User-owned memory is the source of truth; indexes and caches are rebuildable.
Improve general memory mechanisms, robustness, recoverability and transparent behavior. Never introduce
benchmark-specific shortcuts, held-out answer leakage or case-specific rules. Fixed runtime is host-owned.
Candidate source is {code_repository}; guides are under {docs}; disposable scratch is {scratch}.
Optimize answer quality by supplying precise evidence at lower query-time token cost. Score improvement is
more important than query token reduction. Every proposed change should aim for both. A candidate branch
represents one coherent, falsifiable hypothesis. Candidate paths are the only editable source roots.
Repeat this loop:
1. Start from the completed full baseline supplied by the host or the current candidate's completed formal
evaluation. Inventory evaluations and branches, commits and coverage before editing.
2. Diagnose before editing. Strongly prefer home_diagnose with the actual evaluation_paths; it automatically
receives your available full trajectory. Prioritize low-scoring cases and question types, identify at least
two distinct supported causes when evidence permits, estimate impact, coverage and confidence, and rank
directions. Direct investigation is appropriate when evidence is already narrow and clear. Treat partial
construction as a defect. Distinguish infrastructure errors from candidate behavior; do not patch candidate
code to compensate for infrastructure outages. Defer unrelated, redundant and low-impact investigation.
3. Choose lineage before editing. Continue the current direction on its branch; fork a promising branch for
a distinct direction with home_create_candidate_branch. Never edit init. Implement the highest-impact
supported direction, or two or three compatible changes supporting one coherent claim. Inspect diffs before
merging. Create, switch, merge and commit exclusively through the managed Home Jobs; keep evaluated history.
Preserve a branch at each evaluated commit you may want to select finally: before advancing such a branch,
fork a new candidate. Managed switching selects existing branch heads, not arbitrary historical commits.
4. Commit before testing, then call home_evaluate(debug=true). Repair concrete execution failures, commit and
repeat until debug passes. Debug establishes execution only; it has no answer-quality score.
5. Call home_evaluate(debug=false) on clean committed code after debug passes. Formal evaluation uses host-owned
adaptive sampling and always rebuilds memory before answering. Inspect the resulting evidence before the
next iteration. Candidate failures and low scores require a code change; infrastructure failures may retry
the same commit using the host's frozen previous selection. Empty commits do not bypass repetition checks.
Call get_validation_budget before planning the next candidate. The host limits formal validations; one run
consumes one slot when its first sandbox starts successfully, even if it later fails or is cancelled. Debug,
initial baseline and pre-sandbox failures do not consume slots. The budget persists across optimizer invocations.
During structural_exploration prefer substantial mechanisms or coherent architecture changes; during
evidence_driven_refinement prefer evidence-driven parameters, prompts or scoped changes. When exhausted,
select the best existing evaluated branch without another formal validation. There is no candidate admission
gate; do not call architecture or construction-equivalence tools. Use the configured iteration cap and advisory time
reminders to finish a useful search; time reminders do not terminate execution. The cycle clock includes diagnosis,
editing, debug and formal evaluation. Only a newly counted, completed formal evaluation resets it and its context
compression count; debug, rejected calls and incomplete evaluations do not. A cycle transition expires earlier
timer reminders and announces the next cycle. You may stop once the evidence
supports a final selection. Do not keep evaluating without a concrete improvement hypothesis.
Compare scores on common cases and queries, weigh coverage, difficulty, regressions and lineage, and do not
rank sampled candidates by aggregate score alone. Prefer quality, stability, query cost and simplicity.
Do not primarily improve scores by increasing top-k, iteration limits, token budgets or output length.
Investigate better extraction, persistence, routing, retrieval, fusion, deduplication, reranking and context selection.
Read the runtime guides when needed. Tools can inspect memory checkpoints, evidence and commit-pinned code.
Use revision explicitly when examining code associated with an evaluation. Shell and Python run within the same
OS file sandbox as Read/Edit/Write; only declared candidate roots and scratch are writable. Network is denied.
Never write Git metadata, runtime, datasets, evaluation artifacts or reports. Run ReMe only through home_evaluate.
Finish with a clean checkout of the best evaluated branch and full commit. Choosing the baseline is valid if
improvements are unsupported. Return exactly one JSON object with branch_name, commit_sha, evaluation_path,
and report. report must explain the chosen evidence, comparisons, regressions, costs and limitations. The host
will reject a mismatched checkout or missing, incomplete, debug, query-only or infrastructure-failed evidence.

View file

@ -0,0 +1,230 @@
"""Shared sandbox, tools and invocation helpers for Home workflow agents."""
import json
import sys
from pathlib import Path
from uuid import uuid4
from agentscope.tool import BackendBase, DirEntry, Glob, Grep, Read, Toolkit, Write
import yaml
from ..auto_bash import AutoBash
from ..code.repository import validate_repository
from ..evaluation.artifacts import contained, write_json
from ..evaluation.component import blocking
from .case_study import python_tool
from .sandbox import create_case_study_sandbox
INSPECTION_JOBS = (
"home_list_validation_results",
"home_inspect_validation",
"home_trace_query_evidence",
"home_inspect_memory_history",
"home_compare_memory",
"home_code_status",
"home_list_candidate_branches",
"home_code_diff",
"home_list_code_tree",
"home_read_code_file",
"home_search_code",
)
def required_job(wrapper, name):
"""Do not silently bypass a disabled or unstarted Home Job."""
job = wrapper.app_context.jobs[name]
if not job.enabled or not job.is_started:
raise RuntimeError(f"Workflow requires a started Job: {name}")
return job
def create_toolkit(wrapper, backend, scope, *, extra_tools=(), job_names=()):
"""All builtin tools share the OS sandbox; privileged mutations use managed Jobs."""
tools = [
Read(backend=backend),
Glob(backend=backend),
Grep(backend=backend),
Write(backend=backend),
AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=False),
python_tool(backend, scope["scratch"]),
]
tools[-1].description = (
tools[-1]
.description.replace(
"No host path is writable.",
"Only the host-granted scratch and candidate paths are writable.",
)
.replace("case directory", "scratch directory")
)
tools[-2].description = (
"Run shell commands inside the OS file sandbox; network is denied. Use absolute allowed paths. "
"Only private scratch and, for the optimizer, declared candidate paths are writable. "
"Git metadata is read-only: commit, switch, create and merge only through Home Jobs. "
"Run ReMe evaluations only through home_evaluate, never through shell or Python. "
"Keep stderr visible and use set -o pipefail for pipelines. timeout is in milliseconds."
)
tools.extend(extra_tools)
tools.extend(
wrapper._make_tool(required_job(wrapper, name)) # pylint: disable=protected-access
for name in (*INSPECTION_JOBS, *job_names)
)
for tool in tools:
tool.is_concurrency_safe = False
return Toolkit(tools=tools)
async def workflow_reply(
wrapper,
home,
role,
task,
*,
tool_factory=None,
job_names=(),
agent_options=None,
memory_limit_mib=None,
):
"""Use fresh state, the dedicated configured model and sandboxed tools for every call."""
scope = await blocking(prepare_scope, home, role)
backend = WorkflowSandbox(scope, memory_limit_mib)
probe = await backend.exec_shell(
[sys.executable, "-I", "-B", "-c", "print('home workflow sandbox')"],
timeout=10,
)
if not probe.ok():
raise RuntimeError("Home workflow sandbox unavailable: " + probe.stderr.decode(errors="replace"))
prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8"))
return await wrapper.reply(
task,
system_prompt=prompt["system_prompt"].format(**scope),
toolkit=create_toolkit(
wrapper,
backend,
scope,
extra_tools=tool_factory(backend) if tool_factory is not None else (),
job_names=job_names,
),
builtin_tools=[],
job_tools=[],
skills=[],
resume=None,
session_id=None,
fork_session=False,
output_schema=None,
permission_mode="bypass",
**(agent_options or {}),
)
def prepare_scope(home, role):
"""Grant evidence reads and private scratch writes, plus declared optimizer roots."""
repo, record = validate_repository(home)
scratch = contained(home, f"tmp/{role}-{uuid4().hex}", directory=True)
scratch.mkdir()
roots = [
contained(home, name, directory=True)
for name in ("code", "datasets", "docs", "evaluations", "logs", "sampling")
]
roots.append(contained(home, "metadata/preparation.json"))
writable = [scratch]
if role == "optimizer":
for value in record["code"]["candidate_paths"]:
path = contained(repo, value)
if ".git" in path.relative_to(repo).parts or path == repo:
raise ValueError("Optimizer cannot write Git metadata or the repository root")
writable.append(path)
return {
"home": str(home),
"code_repository": str(repo),
"scratch": str(scratch),
"docs": str(home / "docs"),
"read_only_paths": [str(path) for path in roots],
"writable_paths": [str(path) for path in writable],
}
def save_record(home, category, payload):
"""Write a uniquely named host-owned provenance record, outside agent write grants."""
path = contained(home, f"logs/{category}/{uuid4().hex}.json")
write_json(path, payload)
return str(path)
_INSPECT = """\
import json, pathlib, sys
path = pathlib.Path(sys.argv[2])
def entry(p):
s = p.stat()
return dict(name=p.name, is_dir=p.is_dir(), size_bytes=None if p.is_dir() else s.st_size, mtime=s.st_mtime)
if sys.argv[1] == 'stat':
print(json.dumps(entry(path)))
else:
result = []
for p in path.iterdir():
try:
result.append(entry(p))
except FileNotFoundError:
pass
print(json.dumps(result))
"""
class WorkflowSandbox(BackendBase):
"""Route every builtin I/O primitive through Home's OS sandbox, including metadata.
Rebuild the backend before each operation: managed branch switches can
replace an allowed file's inode. Core validation still checks paths and
identities for each subprocess; no operation falls back to local I/O.
"""
def __init__(self, scope, memory_limit_mib=None):
self.scope = scope
self.memory_limit_mib = memory_limit_mib
def _backend(self):
return create_case_study_sandbox(
cwd=self.scope["scratch"],
read_only_paths=self.scope["read_only_paths"],
writable_paths=self.scope["writable_paths"],
memory_limit_mib=self.memory_limit_mib,
)
async def getcwd(self):
"""Use private scratch as the default, never the host process directory."""
return self.scope["scratch"]
async def exec_shell(self, command, *, cwd=None, timeout=None):
"""Keep command execution, environment and process cleanup inside isolation."""
return await self._backend().exec_shell(command, cwd=cwd, timeout=timeout)
async def read_file(self, path):
"""Use core sandboxed file reads, not LocalBackend's host reads."""
return await self._backend().backend.read_file(path)
async def write_file(self, path, data):
"""Use core sandboxed writes, including parent creation and stdin transport."""
await self._backend().backend.write_file(path, data)
async def _inspect(self, operation, path):
result = await self.exec_shell([sys.executable, "-I", "-B", "-c", _INSPECT, operation, path], timeout=10)
if not result.ok():
raise OSError(result.stderr.decode(errors="replace"))
return json.loads(result.stdout)
async def stat(self, path):
"""Use portable sandboxed metadata on Linux and macOS."""
return DirEntry(**await self._inspect("stat", path))
async def scandir(self, path):
"""Avoid GNU find assumptions and host filesystem metadata access."""
return [DirEntry(**item) for item in await self._inspect("scandir", path)]
async def stat_mtime(self, path):
"""Preserve the builtin Edit/Write optimistic modification checks."""
return (await self.stat(path)).mtime
async def list_dir(self, path, *, recursive=False):
"""Use the portable metadata implementation for immediate children."""
if recursive:
return await super().list_dir(path, recursive=True)
return [entry.name for entry in await self.scandir(path)]

View file

@ -0,0 +1,43 @@
"""Formal optimizer budgets derived from Home's durable sandbox-start evidence."""
from .history import read_state
def validate_max_validations(value):
"""Accept only a positive host-supplied integer, matching dev_meta."""
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
raise ValueError("max_validations must be a positive integer")
return value
def validation_budget_status(home, max_validations):
"""Report started formal runs, independently of completion, score or sampler success."""
validate_max_validations(max_validations)
used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"])
remaining = max(0, max_validations - used)
gated = max_validations // 2
if not remaining:
phase = "exhausted"
guidance = (
"The formal validation budget is exhausted; select among existing evidence without another validation."
)
elif used < gated:
phase = "structural_exploration"
guidance = (
"Prefer a substantial code-level mechanism or coherent plugin architecture change. "
"Do not spend an early validation on prompt-only or single-parameter tuning."
)
else:
phase = "evidence_driven_refinement"
guidance = (
"Prefer evidence-driven parameter tuning, fine-grained prompt refinement, or a small scoped code change. "
"Avoid a new broad architecture unless existing validation evidence clearly requires it."
)
return {
"used_validations": used,
"max_validations": max_validations,
"remaining_validations": remaining,
"gated_validation_count": gated,
"optimization_phase": phase,
"next_validation_guidance": guidance,
}

View file

@ -29,6 +29,7 @@ from ..judge import HomeJudge
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from ..sampling import SamplingPolicy, create_sampling_plan
from .history import HomeEvaluationHistory
from .budget import validation_budget_status
from .repetition import previous_candidate_result
@ -85,7 +86,25 @@ class HomeEvaluation(BaseComponent):
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
with lock:
return await self._evaluate(spec)
budgeted = spec.max_validations is not None and not spec.debug
if budgeted:
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations)
if budget["remaining_validations"] < 1:
return {
"status": "rejected",
"error_code": "VALIDATION_BUDGET_EXHAUSTED",
"error": "Optimizer validation budget is exhausted; no sandbox was started.",
"validation_budget": budget,
"result_dir": None,
}
result = await self._evaluate(spec)
if budgeted:
result["validation_budget"] = await blocking(
validation_budget_status,
self.workspace.home_path,
spec.max_validations,
)
return result
def _prepare(self, spec, environment):
home = self.workspace.home_path

View file

@ -29,6 +29,7 @@ class EvaluationSpec(SpecModel):
debug: bool = False
sampling: bool = False
record_history: bool = True
max_validations: int | None = Field(default=None, ge=1)
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
@ -67,6 +68,8 @@ class EvaluationSpec(SpecModel):
@model_validator(mode="after")
def resolve_mode(self):
"""Debug stops on failure by default and never reuses scored evidence."""
if self.max_validations is not None and not self.debug and not self.record_history:
raise ValueError("Budgeted formal evaluations require record_history=true")
if self.debug and self.reuse_construction_from:
raise ValueError("Construction reuse requires training-set")
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):

View file

@ -39,6 +39,9 @@ class HomeEvaluateStep(BaseStep):
self.context.response.metadata["result_dir"] = str(error.path)
return self.context.response
self.context.response.answer = result
if result.get("status") == "rejected":
self.context.response.success = False
return self.context.response
# Candidate failures are evaluation observations; infrastructure failures
# must remain visible to ordinary Job callers checking Response.success.
self.context.response.success = result["case_status_counts"]["infra_error"] == 0

View file

@ -1,4 +1,8 @@
backends:
home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper
home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep
home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep
home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep
@ -20,6 +24,30 @@ backends:
application_defaults:
components:
as_llm:
home_optimizer:
backend: ${LLM_BACKEND:-openai}
model: qwen3.8-max
stream: false
context_size: 160000
max_retries: 3
credential:
api_key: ${LLM_API_KEY:-}
base_url: ${LLM_BASE_URL:-}
parameters:
max_tokens: 65536
thinking_enable: false
home_diagnostic:
backend: ${LLM_BACKEND:-openai}
model: qwen3.8-max
stream: false
context_size: 128000
max_retries: 3
credential:
api_key: ${LLM_API_KEY:-}
base_url: ${LLM_BASE_URL:-}
parameters:
max_tokens: 65536
thinking_enable: false
home_case_study:
backend: ${LLM_BACKEND:-openai}
model: qwen3.7-max
@ -33,6 +61,22 @@ application_defaults:
max_tokens: 65536
thinking_enable: false
agent_wrapper:
home_optimizer:
backend: home_optimizer_agent
max_validations: 10
as_llm: home_optimizer
validation_cycle_time_budget_minutes: 150
context_config:
trigger_ratio: 0.85
reserve_ratio: 0.15
react_config:
max_iters: 3000
home_diagnostic:
backend: home_workflow_agent
as_llm: home_diagnostic
time_budget_minutes: 60
react_config:
max_iters: 80
home_case_study:
backend: home_case_study_agent
as_llm: home_case_study
@ -64,6 +108,41 @@ application_defaults:
default:
backend: home_workspace
jobs:
home_diagnose:
backend: base
description: "Diagnose explicit evaluation runs with a fresh read-only agent; aggregate case studies and return evidence-supported root causes without prescribing fixes"
parameters:
type: object
additionalProperties: false
required: [evaluation_paths]
properties:
evaluation_paths:
type: array
minItems: 1
uniqueItems: true
items: {type: string, minLength: 1}
description: "Explicit evaluation result directories, absolute or relative to home/"
instruction:
type: string
minLength: 1
description: "Diagnostic question or suspected regression"
steps:
- backend: home_diagnose_step
agent_wrapper: home_diagnostic
home_optimize:
backend: base
description: "Check or run the full initial baseline, then diagnose, edit, commit, debug and evaluate candidates; verify the final clean checkout against saved formal evidence"
parameters:
type: object
additionalProperties: false
properties:
instruction:
type: string
minLength: 1
description: "Optimization focus; runtime, selection and sandbox settings are host-owned"
steps:
- backend: home_optimize_step
agent_wrapper: home_optimizer
home_case_study:
backend: base
description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order"
@ -227,6 +306,10 @@ application_defaults:
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
max_artifact_bytes: {type: integer, minimum: 1}
max_validations:
type: [integer, "null"]
minimum: 1
description: "Optional host formal-evaluation budget; optimizer injects its configured limit"
steps:
- backend: home_evaluate_step
home_prepare:

View file

@ -1,6 +1,8 @@
"""Benchmark Home defaults and lifecycle wiring, without models or dataset writes."""
import hashlib
import json
import sys
from importlib import import_module
from pathlib import Path
from unittest.mock import AsyncMock
@ -10,7 +12,8 @@ import yaml
from reme.schema import Response
from benchmark import home_runner
from benchmark.beam import run_home as beam_runner
from benchmark.longmemeval import run_home as lme_runner
ROOT = Path(__file__).resolve().parents[3]
@ -21,7 +24,7 @@ def test_preserved_lme_split_and_evidence():
assert hashlib.sha256((directory / "home_evidence_sessions.jsonl").read_bytes()).hexdigest() == (
"35655c51210a96b595910923576fa97b93c81abd82e3cfcda801180007c23a7d"
)
workspace, spec = home_runner.load_config(directory / "config_home.yaml")
workspace, spec = lme_runner.load_config(directory / "config_home.yaml")
selected = spec["benchmark"]["case_ids"]
assert len(selected) == len(set(selected)) == 250
# Hash the original first-five-group sequence to preserve both IDs and order.
@ -36,7 +39,7 @@ def test_preserved_lme_split_and_evidence():
def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
"""Paths and selection do not depend on where Python was launched."""
monkeypatch.chdir(tmp_path)
workspace, spec = home_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
workspace, spec = beam_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
assert spec["benchmark"]["variant"] == "100K"
assert spec["benchmark"]["all_cases"] is True
assert spec["benchmark"]["case_ids"] is None
@ -47,6 +50,18 @@ def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
assert workspace == ROOT / "benchmark/beam/workspaces/home-100K"
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
def test_runner_preserves_explicit_candidate_paths(tmp_path, benchmark):
"""Runner defaults preserve custom scopes from existing configurations."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config["preparation"]["candidate"] = {"paths": ["reme/steps"]}
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
_, spec = module.load_config(path)
assert spec["candidate"]["paths"] == ["reme/steps"]
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, benchmark):
@ -60,7 +75,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
observed.update(kwargs)
return application
monkeypatch.setattr(home_runner, "Application", factory)
monkeypatch.setattr(module, "Application", factory)
result = await module.run(workspace_dir=str(tmp_path / "experiment"))
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
assert observed["service"] == {"backend": "cli"}
@ -72,6 +87,10 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
"ewma_alpha": 0.3,
"uniform_exploration": 0.2,
}
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"validation_cycle_time_budget_minutes": 150,
"max_validations": 10,
}
application.start.assert_awaited_once()
application.close.assert_awaited_once()
assert application.run_job.await_count == 1
@ -80,9 +99,11 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
@pytest.mark.asyncio
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch, benchmark):
"""Partial policies merge typed defaults and never leak into preparation inputs."""
config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
@ -94,8 +115,8 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
observed.update(kwargs)
return application
monkeypatch.setattr(home_runner, "Application", factory)
await home_runner.prepare(path)
monkeypatch.setattr(module, "Application", factory)
await module.prepare(path)
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
@ -103,22 +124,193 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
config["sampling_policy"] = invalid
path.write_text(yaml.safe_dump(config))
with pytest.raises(ValueError):
await home_runner.prepare(path)
await module.prepare(path)
assert application.start.await_count == 1
assert application.run_job.await_count == 1
assert application.run_job.call_args.args == ("home_prepare",)
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("budget", ["omitted", 45.5, None])
async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, benchmark, budget):
"""Both local runners apply the default, custom duration and explicit disable value."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
if budget == "omitted":
config.pop("validation_cycle_time_budget_minutes", None)
else:
config["validation_cycle_time_budget_minutes"] = budget
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(module, "Application", factory)
await module.run(path)
expected = 150 if budget == "omitted" else budget
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"validation_cycle_time_budget_minutes": expected,
"max_validations": 10,
}
assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"]
application.close.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("budget", [0, -1, True, "150", float("nan"), float("inf")])
async def test_runner_rejects_invalid_cycle_budget_before_start(tmp_path, monkeypatch, benchmark, budget):
"""Reject nonpositive, nonnumeric and nonfinite durations without creating an application."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config["validation_cycle_time_budget_minutes"] = budget
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
created = []
monkeypatch.setattr(module, "Application", lambda **kwargs: created.append(kwargs))
with pytest.raises(ValueError):
await module.run(path)
assert not created
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("failure", ["startup", "preparation"])
async def test_runner_propagates_failure_and_closes(monkeypatch, failure):
async def test_runner_propagates_failure_and_closes(monkeypatch, failure, benchmark):
"""A failed preparation is never printed or returned as success."""
application = AsyncMock()
if failure == "startup":
application.start.side_effect = RuntimeError("startup failed")
else:
application.run_job.return_value = Response(success=False, answer="already prepared")
monkeypatch.setattr(home_runner, "Application", lambda **kwargs: application)
module = import_module(f"benchmark.{benchmark}.run_home")
monkeypatch.setattr(module, "Application", lambda **kwargs: application)
with pytest.raises(RuntimeError):
await home_runner.prepare(ROOT / "benchmark/beam/config_home.yaml")
await module.prepare(module.DEFAULT_CONFIG)
application.close.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("limit", ["omitted", 3, 0, -1, True, "10", 1.5, None])
async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchmark, limit):
"""Validate host count budgets and pass them only to the optimizer wrapper."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
if limit == "omitted":
config.pop("max_validations", None)
else:
config["max_validations"] = limit
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(module, "Application", factory)
if limit in ("omitted", 3):
await module.prepare(path)
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == (
10 if limit == "omitted" else limit
)
assert "max_validations" not in application.run_job.call_args.kwargs["spec"]
else:
with pytest.raises(ValueError):
await module.prepare(path)
assert not observed
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("minutes", [30, None])
async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_path, monkeypatch, benchmark, minutes):
"""A separate optimization launch consumes the same count, timer and sampling settings."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes)
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={"status": "completed"})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(module, "Application", factory)
result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval")
assert result["status"] == "completed"
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
"max_validations": 3,
"validation_cycle_time_budget_minutes": minutes,
}
application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval")
application.close.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_optimizer_failure_closes_application(monkeypatch, benchmark):
"""Failed optimization retains the normal cleanup and error contract."""
module = import_module(f"benchmark.{benchmark}.run_home")
application = AsyncMock()
application.run_job.return_value = Response(success=False, answer="optimization failed")
monkeypatch.setattr(module, "Application", lambda **kwargs: application)
with pytest.raises(RuntimeError, match="optimization failed"):
await module.optimize()
application.run_job.assert_awaited_once_with("home_optimize")
application.close.assert_awaited_once()
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
def test_optimizer_cli_forwards_configuration_and_focus(tmp_path, monkeypatch, capsys, benchmark):
"""The explicit CLI mode reaches optimization and preserves the preparation default."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = tmp_path / "home.yaml"
optimize = AsyncMock(return_value={"status": "completed"})
prepare = AsyncMock()
monkeypatch.setattr(module, "optimize", optimize)
monkeypatch.setattr(module, "prepare", prepare)
monkeypatch.setattr(
sys,
"argv",
[
"run_home.py",
"--optimize",
"--config",
str(config),
"--workspace-dir",
str(tmp_path / "existing"),
"--instruction",
"Improve retrieval",
],
)
module.main()
optimize.assert_awaited_once_with(
config,
workspace_dir=str(tmp_path / "existing"),
instruction="Improve retrieval",
)
prepare.assert_not_called()
assert json.loads(capsys.readouterr().out) == {"status": "completed"}
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
def test_cli_rejects_optimization_focus_in_preparation_mode(monkeypatch, benchmark):
"""Do not silently discard an instruction when the caller forgot --optimize."""
module = import_module(f"benchmark.{benchmark}.run_home")
monkeypatch.setattr(sys, "argv", ["run_home.py", "--instruction", "Improve retrieval"])
with pytest.raises(SystemExit) as error:
module.main()
assert error.value.code == 2

View file

@ -0,0 +1,215 @@
"""Compression recovery parity with dev_meta, without model or filesystem services."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
# pylint: disable=redefined-outer-name,unused-argument
import asyncio
import json
from agentscope.agent import Agent, ContextConfig
from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage
from agentscope.state import AgentState
import jsonschema
import openai
import pytest
from reme_home.agent import compression, optimizer
class FailingCompressionModel:
def __init__(self, ratio=0.9, failure=None):
self.context_size = 90 / ratio
self.failure = failure or RuntimeError("Failed to generate structured output for model.")
self.calls = 0
async def count_tokens(self, messages, tools=None):
return sum(10 for message in messages if (message.get_text_content() or "").startswith("message-"))
async def generate_structured_output(self, **kwargs):
self.calls += 1
raise self.failure
def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing summary"):
return optimizer.OptimizerAgent(
name="optimizer",
system_prompt="system",
model=FailingCompressionModel(ratio, failure),
state=AgentState(
summary=summary,
context=[UserMsg(name="user", content=f"message-{index}") for index in range(9)],
),
context_config=ContextConfig(trigger_ratio=trigger, reserve_ratio=0.2),
fallback_directory=tmp_path / "offloads",
validation_cycle_timer=optimizer.ValidationCycleTimer(150),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("trigger", "ratio", "expected"),
[(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)],
)
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected):
agent = make_agent(tmp_path, ratio, trigger)
context_before, summary_before = agent.state.context, agent.state.summary
await agent.compress_context()
assert agent.validation_cycle_timer.context_compression_count == int(expected)
assert len(list(tmp_path.rglob("context-*.json"))) == int(expected)
assert agent.model.calls == int(ratio >= trigger)
assert agent.context_config.compression_fallback_to_truncation # The shared config was not mutated.
if not expected:
assert agent.state.context is context_before
assert agent.state.summary is summary_before
@pytest.mark.asyncio
@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]])
@pytest.mark.parametrize("reserve_ratio", [0.2, 0])
async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio):
agent = make_agent(tmp_path, summary=summary)
agent.context_config.reserve_ratio = reserve_ratio
agent.state.context[-1] = AssistantMsg(
id=agent.state.reply_id,
name=agent.name,
content="message-8",
usage=Usage(input_tokens=100, output_tokens=10),
)
original = [msg.model_dump(mode="json", exclude_none=False) for msg in agent.state.context]
await agent.compress_context()
(path,) = tmp_path.rglob("context-*.json")
data = json.loads(path.read_text())
assert data["messages"] == original[:7] if reserve_ratio else data["messages"] == original
if reserve_ratio:
assert [msg.get_text_content() for msg in agent.state.context] == ["message-7", "message-8"]
else:
assert len(agent.state.context) == 1
assert agent.state.context[0].content == [] # SDK usage carrier after full offload.
assert data["session_id"] == agent.state.session_id
assert data["context_ratio"] == 0.9
assert data["emergency_ratio"] == 0.9
assert data["failure"]["type"] == "RuntimeError"
text = agent.state.summary
if isinstance(text, list):
text = "\n".join(block.text for block in text)
assert "existing summary" in text
assert "without semantic summarization" in text
assert str(path) in text
assert agent._get_reply_usage().input_tokens == 100
assert not list(path.parent.glob(".result-*"))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure",
[RuntimeError("The system prompt exceeds threshold"), ValueError("bad configuration"), asyncio.CancelledError()],
)
async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failure):
agent = make_agent(tmp_path, failure=failure)
original = agent.state.context
with pytest.raises(type(failure)):
await agent.compress_context()
assert agent.state.context is original
assert not list(tmp_path.rglob("*.json"))
assert agent.validation_cycle_timer.context_compression_count == 0
@pytest.mark.parametrize(
("failure", "expected"),
[
(RuntimeError("provider failed"), True),
(RuntimeError("The system prompt exceeds threshold"), False),
(jsonschema.ValidationError("invalid summary"), True),
(openai.OpenAIError("provider unavailable"), True),
(ValueError("programming error"), False),
(OSError("disk full"), False),
],
)
def test_recoverable_error_classifier(failure, expected):
assert compression.RecoverableCompressionAgent._is_recoverable_compression_error(failure) is expected
@pytest.mark.asyncio
async def test_override_config_and_successful_compression_use_native_pipeline(tmp_path, monkeypatch):
agent = make_agent(tmp_path, ratio=0.85)
config = ContextConfig(trigger_ratio=0.65, reserve_ratio=0.1)
calls = []
async def compress(self, context_config=None, instructions=None):
calls.append((context_config, instructions))
self.state.summary = "semantic summary"
self.state.context = []
monkeypatch.setattr(Agent, "_compress_context_impl", compress)
await agent.compress_context(context_config=config)
assert calls[0][0].trigger_ratio == 0.65
assert calls[0][0].reserve_ratio == 0.1
assert not calls[0][0].compression_fallback_to_truncation
assert config.compression_fallback_to_truncation
assert agent.state.summary == "semantic summary"
assert agent.validation_cycle_timer.context_compression_count == 1
assert not list(tmp_path.rglob("*.json"))
@pytest.mark.asyncio
async def test_offload_write_failure_keeps_context_and_does_not_count(tmp_path, monkeypatch):
agent = make_agent(tmp_path)
original = agent.state.context
def fail(*args, **kwargs):
raise OSError("disk full")
monkeypatch.setattr(compression, "write_json", fail)
with pytest.raises(OSError, match="disk full"):
await agent.compress_context()
assert agent.state.context is original
assert agent.state.summary == "existing summary"
assert agent.validation_cycle_timer.context_compression_count == 0
@pytest.mark.asyncio
async def test_cancellation_during_apply_finishes_consistent_offload_then_propagates(tmp_path, monkeypatch):
agent = make_agent(tmp_path)
started, release = asyncio.Event(), asyncio.Event()
async def clear_cache(messages):
started.set()
await release.wait()
monkeypatch.setattr(agent, "_clear_unreserved_read_cache", clear_cache)
task = asyncio.create_task(agent.compress_context())
await started.wait()
task.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert len(list(tmp_path.rglob("context-*.json"))) == 1
assert "offloaded" in agent.state.summary
assert len(agent.state.context) == 2
@pytest.mark.asyncio
async def test_missing_directory_or_unsplittable_context_fails_explicitly(tmp_path, monkeypatch):
agent = make_agent(tmp_path)
agent.fallback_directory = None
with pytest.raises(RuntimeError, match="fallback_directory"):
await agent.compress_context()
agent.fallback_directory = tmp_path / "offloads"
async def no_old_messages(*args):
return [], agent.state.context
monkeypatch.setattr(agent, "_split_context_for_compression", no_old_messages)
with pytest.raises(RuntimeError, match="cannot remove any context"):
await agent.compress_context()
assert not list(tmp_path.rglob("*.json"))
def test_wrapper_uses_host_offload_directory(tmp_path):
wrapper = optimizer.OptimizerAgentWrapper(name="optimizer")
agent = wrapper._create_agent(
options={"_context_fallback_directory": tmp_path / "logs/context-offloads"},
name="optimizer",
system_prompt="system",
model=FailingCompressionModel(),
)
assert agent.fallback_directory == tmp_path / "logs/context-offloads"

View file

@ -0,0 +1,297 @@
"""Meta-ReMe cycle timing parity with fake clocks and isolated evaluation evidence."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
# pylint: disable=redefined-outer-name,unused-argument
import asyncio
import copy
from types import SimpleNamespace
import pytest
from agentscope.agent import ModelConfig, ReActConfig
from agentscope.message import HintBlock, TextBlock, UserMsg
from agentscope.model import ChatResponse, ChatUsage
from agentscope.state import AgentState
from reme_home.agent import optimizer
from reme_home.agent.bounded import BoundedReActAgent
from reme_home.evaluation.artifacts import write_json
from test_bounded_agent import RecordingModel
def make_agent(timer, model=None):
return optimizer.OptimizerAgent(
name="optimizer",
system_prompt="Improve general memory mechanisms.",
model=model or RecordingModel(),
validation_cycle_timer=timer,
model_config=ModelConfig(max_retries=0),
react_config=ReActConfig(max_iters=100),
)
def cycle_hints(messages, source="meta_reme_validation_timer"):
return [
block
for message in messages
for block in message.content
if isinstance(block, HintBlock) and block.source == source
]
@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
def test_cycle_rejects_invalid_budget(budget):
with pytest.raises(ValueError, match="positive"):
optimizer.ValidationCycleTimer(budget)
@pytest.mark.asyncio
async def test_cycle_injects_dev_meta_reminders_and_resets():
now = [0.0]
timer = optimizer.ValidationCycleTimer(100, clock=lambda: now[0])
class SnapshotModel(RecordingModel):
async def __call__(self, **kwargs):
return await super().__call__(**copy.deepcopy(kwargs))
agent = make_agent(timer, SnapshotModel())
timer.record_context_compression()
timer.record_context_compression()
delivered = []
for minute in (49, 50, 74, 75, 90, 100, 109, 110, 120):
now[0] = minute * 60
_ = [item async for item in agent._reasoning()]
reminders = [block.hint for block in cycle_hints(agent.model.calls[-1]["messages"])]
if minute in (50, 75, 90, 100, 110, 120):
delivered.append(reminders[-1])
assert timer.next_reminder() is None
for percentage, text in zip((50, 75, 90, 100, 110, 120), delivered):
assert f"({percentage}% of its 100 minutes soft time budget)" in text
assert "2 context compressions since the last scored validation" in text
assert "start validation within 50 minutes" in delivered[0]
assert "start validation within 10 minutes" in delivered[2]
assert "starting validation as soon as possible" in delivered[3]
persistent = cycle_hints(agent.state.context)
assert len(persistent) == 3
for percentage, message in zip((75, 100, 120), persistent):
assert f"({percentage}%" in message.hint
timer.reset()
assert timer.context_compression_count == 0
now[0] += 60
assert timer.next_reminder() is None
# Reset keeps historical persistent messages and restarts alternation.
assert cycle_hints(agent.state.context) == persistent
now[0] = 170 * 60
assert timer.next_reminder().persistent is False
def test_cycle_skips_old_thresholds_and_formats_seconds():
now = [0.0]
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
now[0] = 59
reminder = timer.next_reminder()
assert "(90%" in reminder.content
assert "within 1 second." in reminder.content
now[0] = 101
assert "(160%" in timer.next_reminder().content
assert timer.next_reminder() is None
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [RuntimeError("model unavailable"), asyncio.CancelledError()])
async def test_cycle_ephemeral_cleanup_on_error_and_cancellation(monkeypatch, failure):
now = [0.0]
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
agent = make_agent(timer)
async def fail(*args, **kwargs):
raise failure
yield # pylint: disable=unreachable
monkeypatch.setattr(BoundedReActAgent, "_reasoning", fail)
now[0] = 30
with pytest.raises(type(failure)):
_ = [item async for item in agent._reasoning()]
assert not cycle_hints(agent.state.context)
now[0] = 45
with pytest.raises(type(failure)):
_ = [item async for item in agent._reasoning()]
assert len(cycle_hints(agent.state.context)) == 1 # The second reminder is persistent.
@pytest.mark.asyncio
async def test_cycle_counts_only_completed_context_replacements(monkeypatch):
timer = optimizer.ValidationCycleTimer(1)
agent = make_agent(timer)
async def unchanged(self):
pass
monkeypatch.setattr(BoundedReActAgent, "compress_context", unchanged)
await agent.compress_context()
assert timer.context_compression_count == 0
async def compressed(self):
self.state.summary = "Retained previous evidence."
self.state.context = []
monkeypatch.setattr(BoundedReActAgent, "compress_context", compressed)
await agent.compress_context()
assert timer.context_compression_count == 1
async def failed(self):
raise RuntimeError("compression unavailable")
monkeypatch.setattr(BoundedReActAgent, "compress_context", failed)
with pytest.raises(RuntimeError, match="unavailable"):
await agent.compress_context()
assert timer.context_compression_count == 1
def write_history(home, started):
write_json(
home / "sampling/state.json",
{
"schema_version": 2,
"round": len(started),
"successful_round": 0,
"started_validations": {key: index for index, key in enumerate(started, 1)},
"processed_validations": [],
"cases": {},
"queries": {},
},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("timed", [False, True])
@pytest.mark.parametrize(
"outcome,reset",
[
("completed", True),
("success", True),
("candidate_failure", True),
("completed_infra_error", True),
("debug", False),
("rejected", False),
("failed", False),
("partial", False),
("not_started", False),
("preexisting", False),
("cancelled", False),
],
)
async def test_evaluation_reset_boundary_matches_dev_meta(tmp_path, outcome, reset, timed):
home = tmp_path
previous = ["init/initial/1", "candidate/previous/1"]
key = "candidate/current/1"
if outcome == "preexisting":
previous.append(key)
write_history(home, previous)
now = [0.0]
timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
timer.record_context_compression()
now[0] = 45
timer.next_reminder() # First reminder is ephemeral.
state = AgentState()
old = UserMsg(name="meta_reme_validation_timer", content="Previous persistent reminder.")
state.context.append(old)
class EvaluationJob:
enabled = is_started = True
async def __call__(self, **kwargs):
assert kwargs["debug"] is (outcome == "debug")
# Tool execution time remains part of the previous cycle until completion.
now[0] = 90
if outcome == "cancelled":
raise asyncio.CancelledError()
if outcome not in {"debug", "rejected", "not_started", "preexisting"}:
write_history(home, [*previous, key])
manifest = {
"commit_sha": "1b2cdcc" + "0" * 33,
"status": "failed" if outcome in {"failed", "partial"} else "completed",
"mode": "debug" if outcome == "debug" else "scored",
"dataset_kind": "training-set",
"sandbox_started_at": None if outcome == "not_started" else "2026-10-31T00:00:00Z",
}
run = home / "evaluations" / key
write_json(run / "manifest.json", manifest)
status = outcome if outcome in {"success", "rejected", "failed", "partial"} else "completed"
return SimpleNamespace(
success=outcome not in {"completed_infra_error", "rejected", "failed", "partial"},
answer={"status": status, "result_dir": str(run), "mean_query_score": 0.0},
metadata={},
)
wrapper = SimpleNamespace(app_context=SimpleNamespace(jobs={"home_evaluate": EvaluationJob()}))
tool = optimizer.evaluation_tool(wrapper, {}, home=home, timer=timer if timed else None)
assert set(tool.input_schema["properties"]) == {"debug"}
if outcome == "cancelled":
with pytest.raises(asyncio.CancelledError):
await tool._func(debug=False, _agent_state=state)
else:
await tool._func(debug=outcome == "debug", _agent_state=state)
assert len(state.context) == 1 # The tool cannot insert anything before its result is written.
pending = state.middle_context.get(optimizer.PENDING_CYCLE_TRANSITIONS, [])
assert len(pending) == int(reset)
agent = make_agent(None)
agent.state = state
_ = [item async for item in agent._reasoning()]
transitions = cycle_hints(state.context, "meta_reme_validation_cycle_transition")
assert len(transitions) == int(reset)
assert state.context[0] is old
if reset:
assert transitions[0].hint == (
"<system-reminder>Formal validation 2 completed successfully for commit 1b2cdcc. "
"The previous cycle timer has been reset, and all timer reminders for cycle 2 are now expired. "
"Remaining formal validations: 8. Start cycle 3.</system-reminder>"
)
if reset and timed:
assert timer.context_compression_count == 0
assert timer.next_reminder() is None
now[0] += 30
assert timer.next_reminder().persistent is False
else:
assert timer.context_compression_count == 1
assert "(150%" in timer.next_reminder().content
def test_wrapper_owns_fresh_cycle_timers_and_supports_disabling():
wrapper = optimizer.OptimizerAgentWrapper(name="optimizer", validation_cycle_time_budget_minutes=150)
first, second = wrapper.create_cycle_timer(), wrapper.create_cycle_timer()
assert first is not second
assert first.time_budget_minutes == 150
first.record_context_compression()
assert second.context_compression_count == 0
assert wrapper.create_cycle_timer({"validation_cycle_time_budget_minutes": None}) is None
legacy = optimizer.OptimizerAgentWrapper(name="optimizer", time_budget_minutes=75)
assert legacy.create_cycle_timer().time_budget_minutes == 75
assert legacy.create_cycle_timer({"validation_cycle_time_budget_minutes": 30}).time_budget_minutes == 30
@pytest.mark.asyncio
async def test_temporary_and_persistent_reminders_preserve_total_reply_usage():
now = [0.0]
class AdvancingModel(RecordingModel):
async def __call__(self, **kwargs):
result = await super().__call__(**kwargs)
now[0] = 30 if len(self.calls) == 1 else 45
return result
model = AdvancingModel(
final=ChatResponse(
content=[TextBlock(text="final report")],
is_last=True,
usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
),
)
agent = make_agent(optimizer.ValidationCycleTimer(1, clock=lambda: now[0]), model)
agent.react_config.max_iters = 2
reply = await agent.reply(UserMsg(name="user", content="Investigate"))
assert len(model.calls) == 3
assert reply.usage.input_tokens == 6
assert reply.usage.output_tokens == 9
retained = cycle_hints(agent.state.context)
assert len(retained) == 1
assert "(75%" in retained[0].hint

View file

@ -0,0 +1,142 @@
"""Formal budget accounting through real evaluation Jobs and fake Docker boundaries."""
# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument,protected-access
import asyncio
import pytest
from reme_home.agent.optimizer import OptimizerAgentWrapper, validation_budget_tool
from reme_home.evaluation import component as evaluation
from reme_home.evaluation.artifacts import write_json
from reme_home.evaluation.budget import validation_budget_status
from reme_home.evaluation.inputs import EvaluationSpec
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
commit_candidate,
home_application,
preparation_inputs,
prepared_home,
)
def seed_history(home, keys):
write_json(
home / "sampling/state.json",
{
"schema_version": 2,
"round": len(keys),
"successful_round": 0,
"cases": {},
"queries": {},
"started_validations": {key: index for index, key in enumerate(keys, 1)},
"processed_validations": [],
},
)
@pytest.mark.parametrize(
"used,phase,remaining",
[
(0, "structural_exploration", 10),
(4, "structural_exploration", 6),
(5, "evidence_driven_refinement", 5),
(9, "evidence_driven_refinement", 1),
(10, "exhausted", 0),
(11, "exhausted", 0),
],
)
def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining):
seed_history(tmp_path, ["init/first/1", *(f"candidate/hash/{index}" for index in range(used))])
result = validation_budget_status(tmp_path, 10)
assert result["used_validations"] == used
assert result["remaining_validations"] == remaining
assert result["optimization_phase"] == phase
assert result["gated_validation_count"] == 5
assert result["next_validation_guidance"]
@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None])
def test_invalid_optimizer_budgets_rejected(invalid):
wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid)
with pytest.raises(ValueError, match="positive integer"):
wrapper.validation_limit()
if invalid is not None:
with pytest.raises(ValueError):
EvaluationSpec(max_validations=invalid)
def test_budgeted_evaluation_cannot_disable_started_history():
with pytest.raises(ValueError, match="record_history"):
EvaluationSpec(max_validations=10, record_history=False)
assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug
assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10
@pytest.mark.asyncio
async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path):
seed_history(tmp_path, ["init/base/1", "candidate/hash/1"])
before = (tmp_path / "sampling/state.json").read_bytes()
tool = validation_budget_tool(tmp_path, 3)
assert tool.name == "get_validation_budget"
result = await tool._func()
assert result["remaining_validations"] == 2
assert result["optimization_phase"] == "evidence_driven_refinement"
assert (tmp_path / "sampling/state.json").read_bytes() == before
@pytest.mark.asyncio
async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success
assert validation_budget_status(prepared, 1)["used_validations"] == 0
commit_candidate(prepared)
result = await app.run_job("home_evaluate", max_validations=1)
assert result.success, result.answer
assert result.answer["validation_budget"]["remaining_validations"] == 0
count = len(FakeFactory.instances)
# A fresh status lookup and cache rebuild both retain the charge.
(prepared / "sampling/state.json").unlink()
assert validation_budget_status(prepared, 1)["used_validations"] == 1
commit_candidate(prepared)
rejected = await app.run_job("home_evaluate", max_validations=1)
assert not rejected.success
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
assert rejected.answer["result_dir"] is None
assert len(FakeFactory.instances) == count
debug = await app.run_job("home_evaluate", debug=True, max_validations=1)
assert debug.success, debug.answer
assert "validation_budget" not in debug.answer
assert validation_budget_status(prepared, 1)["used_validations"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["candidate_failure", "infra_failure", "cancelled", "prestart_failure"])
async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, monkeypatch, outcome):
commit_candidate(prepared)
if outcome == "candidate_failure":
monkeypatch.setattr(FakeFactory, "fail_build", "1")
elif outcome == "infra_failure":
monkeypatch.setattr(FakeFactory, "bad_export", True)
elif outcome == "cancelled":
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
else:
class PrestartFailureFactory(FakeFactory):
async def create_cases(self, ids, **kwargs):
raise RuntimeError("sandbox initialization failed")
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory)
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1))
if outcome == "cancelled":
while not FakeFactory.instances:
await asyncio.sleep(0)
await FakeFactory.instances[-1].entered.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
else:
await task
expected = int(outcome != "prestart_failure")
assert validation_budget_status(prepared, 1)["used_validations"] == expected
(prepared / "sampling/state.json").unlink(missing_ok=True)
assert validation_budget_status(prepared, 1)["used_validations"] == expected

View file

@ -0,0 +1,496 @@
"""Minimal Home agents exercise real Jobs, Git and AS tools with fake model/Docker boundaries."""
# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
# pylint: disable=unused-argument,protected-access
import asyncio
import json
import sys
from importlib import import_module
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from agentscope.formatter import OpenAIChatFormatter
from agentscope.message import HintBlock, TextBlock, ToolCallBlock
from agentscope.model import ChatResponse, ChatUsage
from agentscope.tool import LocalBackend
from jsonschema import Draft202012Validator
from reme_home.agent import optimizer as optimizer_module, diagnostic as diagnostic_module, workflow_common, case_study
from reme_home.evaluation.artifacts import read_json, write_json
from reme_home.preparation.code import git
from reme_home.workspace import _initialization_lock
from test_bounded_agent import RecordingModel
from test_case_study import RecordingSandbox
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
class ScriptedModel(RecordingModel):
def __init__(self, calls=(), final="Supported diagnostic report."):
super().__init__()
self.script = list(calls)
self.report = final
async def __call__(self, **kwargs):
index = len(self.calls)
self.calls.append(kwargs)
if index < len(self.script):
name, arguments = self.script[index]
arguments = arguments() if callable(arguments) else arguments
content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))]
else:
final = self.report() if callable(self.report) else self.report
content = [TextBlock(text=final)]
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
@pytest.fixture
def sandbox(monkeypatch):
policies = []
def create(**kwargs):
# Mock only subprocess isolation. Writes are restricted by the real
# sandbox in the dedicated OS tests below, not by this local fake.
policies.append(kwargs)
local = LocalBackend()
return SimpleNamespace(backend=local, exec_shell=local.exec_shell)
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", create)
monkeypatch.setattr(case_study, "create_case_study_sandbox", RecordingSandbox)
return policies
def set_model(app, name, model):
wrapper = app.context.components["agent_wrapper"][name]
wrapper.as_llm = SimpleNamespace(model=model)
return wrapper
async def baseline(app):
result = await app.run_job("home_evaluate")
assert result.success, result.answer
return result.answer["result_dir"]
def final_report(home, branch="init", path=None):
repo = home / "code/repo"
if path is None:
path = sorted((home / "evaluations" / branch).glob("*/*/summary.json"))[-1].parent
return json.dumps(
{
"branch_name": branch,
"commit_sha": git(repo, "rev-parse", "HEAD"),
"evaluation_path": str(path),
"report": "Selected using matching formal evidence; sampled coverage limits confidence.",
},
)
@pytest.mark.asyncio
async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepared, sandbox, monkeypatch):
home = prepared
now, timers = [0.0], []
timer_class = optimizer_module.ValidationCycleTimer
def create_timer(budget):
timer = timer_class(budget, clock=lambda: now[0])
timers.append(timer)
return timer
class TimedModel(ScriptedModel):
async def __call__(self, **kwargs):
formatted = await OpenAIChatFormatter().format(kwargs["messages"])
pending = set()
for message in formatted:
if pending:
assert message["role"] == "tool", formatted
pending.remove(message["tool_call_id"])
else:
assert message["role"] != "tool", formatted
pending.update(call["id"] for call in message.get("tool_calls", []))
assert not pending
response = await super().__call__(**kwargs)
now[0] += 60 * 60
return response
monkeypatch.setattr(optimizer_module, "ValidationCycleTimer", create_timer)
target = home / "code/repo/plugins/beam/src/reme_beam/__init__.py"
baseline_path = await baseline(app)
diagnostic = ScriptedModel(
[("home_case_study", {"evaluation_path": baseline_path, "case_ids": ["1"], "focus": "Trace wrong answers"})],
)
set_model(app, "home_diagnostic", diagnostic)
set_model(app, "home_case_study", ScriptedModel())
optimizer = TimedModel(
[
("get_validation_budget", {}),
("home_diagnose", {"evaluation_paths": [baseline_path]}),
("home_create_candidate_branch", {"branch_name": "mechanism"}),
("Read", {"file_path": str(target)}),
("Edit", {"file_path": str(target), "old_string": "value = 1", "new_string": "value = 2"}),
("home_commit_candidate", {"message": "Change mechanism"}),
("home_evaluate", {"debug": True}),
("home_evaluate", {"debug": False}),
],
final=lambda: final_report(home, "mechanism"),
)
set_model(app, "home_optimizer", optimizer)
response = await app.run_job("home_optimize", instruction="Improve general retrieval")
assert response.success, response.answer
assert target.read_text() == "value = 2\n"
assert response.answer["branch_name"] == "mechanism"
assert response.answer["status"] == "completed"
transitions = [
block.hint
for message in optimizer.calls[-1]["messages"]
for block in message.content
if isinstance(block, HintBlock) and block.source == "meta_reme_validation_cycle_transition"
]
assert len(transitions) == 1
assert "Formal validation 1 completed successfully" in transitions[0]
assert "Remaining formal validations: 9. Start cycle 2." in transitions[0]
assert len(timers) == 1
assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call.
assert any(
isinstance(block, HintBlock) and block.source == "meta_reme_validation_timer"
for message in optimizer.calls[-1]["messages"]
for block in message.content
)
assert response.metadata["usage"]["input_tokens"] == 2 * len(optimizer.calls)
assert response.metadata["usage"]["output_tokens"] == 3 * len(optimizer.calls)
assert "sampled coverage" in response.answer["report"]
assert read_json(Path(response.metadata["optimization_report_path"]))["status"] == "completed"
assert len(diagnostic.calls) == 2
source = next((home / "logs/diagnostic-trajectories").glob("*.json"))
trace = read_json(source)
assert trace["session_id"] and any("Improve general retrieval" in str(msg) for msg in trace["messages"])
assert "permission_context" not in trace
schemas = {item["function"]["name"]: item["function"]["parameters"] for item in optimizer.calls[0]["tools"]}
assert set(schemas["home_evaluate"]["properties"]) == {"debug"}
assert "caller_trajectory" not in schemas["home_diagnose"]["properties"]
assert "get_validation_budget" in schemas
assert not any("equivalence" in name for name in schemas)
for manifest_path in (home / "evaluations").glob("*/*/*/manifest.json"):
manifest = read_json(manifest_path)
assert manifest["execution_mode"] == "full_validation"
assert any(read_json(path)["mode"] == "debug" for path in (home / "evaluations").glob("*/*/*/manifest.json"))
diagnostic_schemas = {item["function"]["name"] for item in diagnostic.calls[0]["tools"]}
assert "home_case_study" in diagnostic_schemas
assert not set(optimizer_module.MUTATION_JOBS) & diagnostic_schemas
assert "Edit" not in diagnostic_schemas and "home_evaluate" not in diagnostic_schemas
assert "Write" in diagnostic_schemas # Private scratch only.
assert any(str(target.parent.parent) in policy["writable_paths"] for policy in sandbox)
assert all(str(home / "code/repo") not in policy["writable_paths"] for policy in sandbox)
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
app,
prepared,
inputs,
sandbox,
monkeypatch,
tmp_path,
benchmark,
):
baseline_path = await baseline(app)
await app.close()
module = import_module(f"benchmark.{benchmark}.run_home")
config = tmp_path / "home.yaml"
config.write_text(
yaml.safe_dump(
{
"workspace_dir": str(prepared.parent),
"preparation": inputs[0],
"max_validations": 3,
"validation_cycle_time_budget_minutes": 30,
},
),
)
application_class = type(app)
agents = []
model = ScriptedModel([("get_validation_budget", {})], final=lambda: final_report(prepared, path=baseline_path))
def factory(**kwargs):
application = application_class(**kwargs, log_to_console=False)
original_start = application.start
async def start():
await original_start()
wrapper = set_model(application, "home_optimizer", model)
original_create = wrapper._create_agent
def create(**options):
agent = original_create(**options)
agents.append(agent)
return agent
monkeypatch.setattr(wrapper, "_create_agent", create)
monkeypatch.setattr(application, "start", start)
return application
monkeypatch.setattr(module, "Application", factory)
result = await module.optimize(config, instruction="Keep the best existing candidate")
assert result["status"] == "completed"
assert result["branch_name"] == "init"
assert len(agents) == 1
assert agents[0].validation_cycle_timer.time_budget_minutes == 30
task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"])
assert '"max_validations": 3' in task
assert '"remaining_validations": 3' in task
assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.
@pytest.mark.asyncio
async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox):
model = ScriptedModel(final=lambda: final_report(prepared))
set_model(app, "home_optimizer", model)
response = await app.run_job("home_optimize")
assert response.success, response.answer
assert len(FakeFactory.instances) == 1
assert response.answer["branch_name"] == "init"
run = response.answer["baseline_path"]
manifest = read_json(Path(run) / "manifest.json")
assert set(manifest["case_ids"]) == {"1", "2"} and not manifest["options"]["sampling"]
assert manifest["options"]["record_history"]
set_model(app, "home_optimizer", ScriptedModel(final=lambda: final_report(prepared)))
second = await app.run_job("home_optimize")
assert second.success, second.answer
assert len(FakeFactory.instances) == 1
assert second.answer["session_id"] != response.answer["session_id"]
@pytest.mark.asyncio
async def test_standalone_diagnosis_fresh_state_and_persisted_provenance(app, prepared, sandbox):
path = await baseline(app)
model = ScriptedModel()
set_model(app, "home_diagnostic", model)
first = await app.run_job("home_diagnose", evaluation_paths=[path])
second = await app.run_job("home_diagnose", evaluation_paths=[path], instruction="Compare failure clusters")
assert first.success and second.success
assert first.answer["session_id"] != second.answer["session_id"]
assert first.answer["trajectory_path"] != second.answer["trajectory_path"]
assert first.answer["evaluation_paths"] == [path]
record = read_json(Path(first.answer["diagnostic_report_path"]))
assert record["diagnosis"] == "Supported diagnostic report."
assert read_json(Path(first.answer["trajectory_path"])) == {"messages": [], "summary": None}
assert all(
len(policy["writable_paths"]) == 1 and "/tmp/diagnostic-" in policy["writable_paths"][0] for policy in sandbox
)
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["infra", "partial", "debug", "replay", "wrong_commit"])
async def test_bad_baseline_rejected_before_optimizer(app, prepared, sandbox, change):
path = Path(await baseline(app))
manifest = read_json(path / "manifest.json")
summary = read_json(path / "summary.json")
if change == "infra":
summary["case_status_counts"]["infra_error"] = 1
elif change == "partial":
manifest["case_ids"] = ["1"]
manifest["query_ids"] = {"1": manifest["query_ids"]["1"]}
elif change == "debug":
manifest["mode"] = "debug"
elif change == "replay":
manifest["execution_mode"] = "query_only_replay"
else:
manifest["commit_sha"] = "0" * 40
write_json(path / "manifest.json", manifest)
write_json(path / "summary.json", summary)
# Missing valid baseline must not silently switch away from a candidate.
response = await app.run_job("home_create_candidate_branch", branch_name="existing")
assert response.success
model = ScriptedModel()
set_model(app, "home_optimizer", model)
response = await app.run_job("home_optimize")
assert not response.success and "baseline is missing" in str(response.answer)
assert not model.calls and not sandbox
assert git(prepared / "code/repo", "branch", "--show-current") == "existing"
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["dirty", "branch", "commit", "path", "infra", "debug", "replay", "invalid_json"])
async def test_final_claim_rejected_without_discarding_changes(app, prepared, sandbox, change):
path = Path(await baseline(app))
target = prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py"
def finish():
claim = json.loads(final_report(prepared, path=path))
if change == "dirty":
target.write_text("preserve unfinished changes\n")
elif change == "branch":
claim["branch_name"] = "invented"
elif change == "commit":
claim["commit_sha"] = "0" * 40
elif change == "path":
claim["evaluation_path"] = str(prepared / "tmp")
elif change == "infra":
summary = read_json(path / "summary.json")
summary["query_status_counts"]["infra_error"] = 1
write_json(path / "summary.json", summary)
elif change in {"debug", "replay"}:
manifest = read_json(path / "manifest.json")
manifest["mode" if change == "debug" else "execution_mode"] = (
"debug" if change == "debug" else "query_only_replay"
)
write_json(path / "manifest.json", manifest)
else:
return "not a structured final claim"
return json.dumps(claim)
set_model(app, "home_optimizer", ScriptedModel(final=finish))
response = await app.run_job("home_optimize")
assert not response.success
record = read_json(Path(response.metadata["optimization_report_path"]))
assert record["status"] == "failed" and record["error"]
if change == "dirty":
assert target.read_text() == "preserve unfinished changes\n"
@pytest.mark.asyncio
async def test_infrastructure_baseline_failure_never_calls_optimizer(app, prepared, sandbox, monkeypatch):
async def broken(*args, **kwargs):
raise RuntimeError("Docker unavailable")
monkeypatch.setattr(FakeFactory, "create_cases", broken)
model = ScriptedModel()
set_model(app, "home_optimizer", model)
response = await app.run_job("home_optimize")
assert not response.success and not model.calls
assert "baseline failed" in str(response.answer)
@pytest.mark.asyncio
async def test_invalid_scope_schema_and_injected_trajectory(app, prepared, sandbox):
path = await baseline(app)
wrapper = set_model(app, "home_optimizer", ScriptedModel())
tool = diagnostic_module.diagnostic_tool(wrapper)
with pytest.raises(ValueError, match="host-owned"):
await tool._func(evaluation_paths=[path], caller_trajectory={"spoof": True})
for values in ([str(prepared / "tmp")], [path, path], [], ["../escape"]):
result = await app.run_job("home_diagnose", evaluation_paths=values)
assert not result.success
result = await app.run_job("home_diagnose", evaluation_paths=[path], _allowed_paths=[path])
assert not result.success and not sandbox
for name in ("home_optimize", "home_diagnose"):
Draft202012Validator.check_schema(app.context.jobs[name].parameters)
assert "caller_trajectory" not in app.context.jobs["home_diagnose"].parameters["properties"]
with pytest.raises(ValueError, match="execution settings"):
optimizer_module.evaluation_options({"reuse_construction_from": path})
@pytest.mark.asyncio
async def test_single_optimizer_lock_and_cancellation_cleanup(app, prepared, sandbox, monkeypatch):
await baseline(app)
entered, release = asyncio.Event(), asyncio.Event()
async def blocked(*args, **kwargs):
entered.set()
await release.wait()
monkeypatch.setattr(optimizer_module, "workflow_reply", blocked)
running = asyncio.create_task(app.run_job("home_optimize"))
await asyncio.wait_for(entered.wait(), 5)
second = await app.run_job("home_optimize")
assert not second.success and "already in progress" in str(second.answer)
running.cancel()
with pytest.raises(asyncio.CancelledError):
await running
reports = list((prepared / "logs/optimization-reports").glob("*.json"))
assert len(reports) == 1 and read_json(reports[0])["status"] == "cancelled"
with _initialization_lock(prepared / "metadata/optimization.lock"):
pass
@pytest.mark.asyncio
async def test_metadata_link_rejected_before_opening_any_lock(app, prepared, tmp_path):
outside = tmp_path / "outside"
outside.mkdir()
state = prepared / "metadata"
saved = prepared / "saved-metadata"
state.rename(saved)
state.symlink_to(outside, target_is_directory=True)
try:
result = await app.run_job("home_optimize")
assert not result.success and "real directory" in str(result.answer)
with pytest.raises(ValueError, match="real directory"):
optimizer_module.baseline_state(prepared)
with pytest.raises(ValueError, match="real directory"):
optimizer_module.verify_final(prepared, None)
assert not list(outside.iterdir())
finally:
state.unlink()
saved.rename(state)
@pytest.mark.asyncio
async def test_workflow_sandbox_enforces_read_write_and_git_boundaries(tmp_path):
home = tmp_path / "home"
repo = home / "repo"
candidate = repo / "src"
scratch = home / "scratch"
evidence = home / "evidence"
for directory in (candidate, repo / ".git", scratch, evidence):
directory.mkdir(parents=True, exist_ok=True)
target = candidate / "module.py"
target.write_text("old\n")
secret = tmp_path / "outside.txt"
secret.write_text("outside")
(evidence / "result.json").write_text("evidence")
for role in ("optimizer", "diagnostic"):
scope = {
"scratch": str(scratch),
"read_only_paths": [str(repo), str(evidence)],
"writable_paths": [str(scratch), *([str(candidate)] if role == "optimizer" else [])],
}
backend = workflow_common.WorkflowSandbox(scope, memory_limit_mib=0)
assert (await backend.read_file(str(target))) in (b"old\n", b"new\n")
assert (await backend.stat(str(target))).size_bytes > 0
assert "module.py" in await backend.list_dir(str(candidate))
await backend.write_file(str(scratch / "analysis.py"), b"print('analysis')\n")
assert (await backend.exec_shell([sys.executable, "-I", "-B", str(scratch / "analysis.py")])).ok()
for path in (repo / ".git/config", evidence / "result.json", secret):
with pytest.raises(OSError):
await backend.write_file(str(path), b"forbidden")
with pytest.raises(OSError):
await backend.read_file(str(secret))
if role == "optimizer":
await backend.write_file(str(target), b"new\n")
# Replaced roots after managed Git operations must not retain stale inodes.
replacement = candidate / "replacement"
replacement.write_text("new\n")
replacement.replace(target)
assert await backend.read_file(str(target)) == b"new\n"
else:
with pytest.raises(OSError):
await backend.write_file(str(target), b"forbidden")
assert secret.read_text() == "outside" and (evidence / "result.json").read_text() == "evidence"
# A single-file candidate root must survive a managed checkout replacing
# its inode without accidentally granting writes to its containing directory.
file_scope = {
"scratch": str(scratch),
"read_only_paths": [str(repo)],
"writable_paths": [str(scratch), str(target)],
}
backend = workflow_common.WorkflowSandbox(file_scope, memory_limit_mib=0)
await backend.write_file(str(target), b"file root\n")
replacement = candidate / "replacement"
replacement.write_text("replaced root\n")
replacement.replace(target)
assert await backend.read_file(str(target)) == b"replaced root\n"
with pytest.raises(OSError):
await backend.write_file(str(candidate / "sibling.py"), b"forbidden")
target.unlink()
target.symlink_to(secret)
with pytest.raises(ValueError, match="symlinks"):
await backend.read_file(str(target))