This commit is contained in:
xyf2020 2026-10-29 18:26:05 +00:00 • committed by GitHub
commit 086cf27bb5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
90 changed files with 13506 additions and 41 deletions

3
.gitignore vendored
View file

@ -59,6 +59,9 @@ docs/_build/
site/
evaluation/
# Home's evaluation package is source code, not benchmark output.
!plugins/home/src/reme_home/evaluation/
!plugins/home/src/reme_home/evaluation/*.py
# The pi-Bench suite ships its own trace-history render config, which must
# stay in git even though it lives under an evaluation/ directory.
!benchmark/pibench/config/bench/evaluation/

View file

@ -135,3 +135,39 @@ Memory Construction average token consumption (default agent, full build over 35
| Agent | input tok/case | output tok/case | total tok/case |
|---|---|---|---|
| default | 31,943,817 | 1,417,061 | 33,360,878 |
## Prepare a Home experiment
`run_home.py` prepares all BEAM **100K** cases, the workspace, and committed candidate
code. It exits after preparation; it does not evaluate cases or start optimization.
Run from the repository root after obtaining the dataset described above:
```bash
reme plugins install plugins/home
python benchmark/beam/run_home.py
# Optional separate workspace (a successful preparation cannot be repeated):
python benchmark/beam/run_home.py --workspace-dir /path/to/new-workspace
```
[config_home.yaml](config_home.yaml) controls the preparation. Its paths are
repository-relative regardless of the current working directory; `--config`
accepts an alternative runner YAML file. The default workspace is
`benchmark/beam/workspaces/home-100K`. The evaluated runtime loads `beam`;
the host application loads `home`. `source_ref: HEAD` exports committed source
only, excluding uncommitted modifications. Preparation failure rolls back its
outputs; after success, another call fails instead of resetting the experiment.
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
`uniform_exploration`. The runner validates these and passes them to the host
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
Python callers can use the same lifecycle:
```python
from benchmark.beam.run_home import run
result = await run(workspace_dir="/path/to/new-workspace")
```

View file

@ -130,3 +130,28 @@ Memory Construction 平均 token 消耗(default agent,35 cases 全量构建
| Agent | input tok/case | output tok/case | total tok/case |
|---|---|---|---|
| default | 31,943,817 | 1,417,061 | 33,360,878 |
## 准备 Home 实验
安装 Home 插件后,在仓库根目录执行:
```bash
reme plugins install plugins/home
python benchmark/beam/run_home.py
```
该入口只创建工作区、准备数据和代码,不运行评测或优化。
参数见 [config_home.yaml](config_home.yaml),路径相对仓库根目录;
可通过 `--config` 更换配置,通过 `--workspace-dir` 指定新工作区。
默认工作区为 `benchmark/beam/workspaces/home-100K`。
代码从 `source_ref: HEAD` 对应的 Git 提交导出,不包含未提交修改。
准备失败会回滚;成功后禁止再次准备同一工作区。
Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
默认选择 BEAM **100K 全部 case**。

View file

@ -0,0 +1,26 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
workspace_dir: benchmark/beam/workspaces/home-100K
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
preparation:
benchmark:
name: beam
source: benchmark/beam/dataset/BEAM
variant: 100K
all_cases: true
runtime:
source_repo: .
source_ref: HEAD # Committed source only; resolved and saved as a full SHA.
config: benchmark.yaml
plugins: [beam]
candidate:
paths: [plugins/beam/src]
debug:
enabled: true
sessions_per_case: 3
messages_per_session: 6

View file

@ -0,0 +1,22 @@
"""Prepare the beam Home workspace, dataset and candidate code; do not evaluate."""
from pathlib import Path
import sys
# Support direct execution from any working directory as well as module imports.
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
if __name__ == "__main__":
main(DEFAULT_CONFIG)

93
benchmark/home_runner.py Normal file
View file

@ -0,0 +1,93 @@
"""Shared preparation-only entry point for benchmark Home experiments."""
import argparse
import asyncio
import json
from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field
import yaml
from reme_home.preparation.spec import PreparationSpec
from reme_home.judge import JUDGE_PLUGINS
from reme_home.sampling import SamplingPolicy
from reme.application import Application
PROJECT_ROOT = Path(__file__).resolve().parents[1]
class RunHomeConfig(BaseModel):
"""Runner settings kept separate from Home's public preparation contract."""
model_config = ConfigDict(extra="forbid", strict=True)
workspace_dir: str = Field(min_length=1)
preparation: dict
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
def project_path(value: str, project_root: Path) -> Path:
"""Use repository-relative paths consistently regardless of the caller's cwd."""
if value.startswith("~"):
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
return (project_root / value).resolve()
def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Resolve source paths and validate explicit Home preparation parameters."""
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
config = RunHomeConfig.model_validate(raw)
spec = PreparationSpec.model_validate(config.preparation)
spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
if spec.benchmark.evidence_source is not None:
spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
return workspace, spec.model_dump(mode="json"), config.sampling_policy
def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
"""Return resolved preparation inputs, preserving the preparation-only caller contract."""
workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
return workspace, spec
async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
"""Start Home, prepare once, and close all resources without evaluating cases."""
workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
)
try:
await app.start()
response = await app.run_job("home_prepare", spec=spec)
if not response.success:
raise RuntimeError(response.answer)
return {"workspace_dir": str(workspace), **response.answer}
finally:
await app.close()
def main(default_config: Path) -> None:
"""Run a benchmark's default preparation from the command line."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
args = parser.parse_args()
result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
print(
json.dumps(
{
"workspace_dir": result["workspace_dir"],
"case_count": result["dataset"]["case_count"],
"query_count": result["dataset"]["query_count"],
"source_commit": result["code"]["source_commit"],
"initial_commit": result["code"]["initial_commit"],
},
indent=2,
),
)

View file

@ -108,3 +108,48 @@ agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only (reusing prebuilt
| single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 |
| temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 |
| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** |
## Prepare a Home experiment
`run_home.py` prepares the fixed **group_1–group_5** training split (250 cases), the workspace, and committed candidate
code. It exits after preparation; it does not evaluate cases or start optimization.
Run from the repository root after obtaining the dataset described above:
```bash
reme plugins install plugins/home
python benchmark/longmemeval/run_home.py
# Optional separate workspace (a successful preparation cannot be repeated):
python benchmark/longmemeval/run_home.py --workspace-dir /path/to/new-workspace
```
[config_home.yaml](config_home.yaml) controls the preparation. Its paths are
repository-relative regardless of the current working directory; `--config`
accepts an alternative runner YAML file. The default workspace is
`benchmark/longmemeval/workspaces/home-groups-1-5`. The evaluated runtime loads `lme`;
the host application loads `home`. `source_ref: HEAD` exports committed source
only, excluding uncommitted modifications. Preparation failure rolls back its
outputs; after success, another call fails instead of resetting the experiment.
The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`,
`max_queries` (null means unlimited), `temperature`, `ewma_alpha` and
`uniform_exploration`. The runner validates these and passes them to the host
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
Python callers can use the same lifecycle:
```python
from benchmark.longmemeval.run_home import run
result = await run(workspace_dir="/path/to/new-workspace")
```
The 250 `benchmark.case_ids` in [config_home.yaml](config_home.yaml) are copied
in order from groups 1–5 of `dataset/groups.json` at `dev_meta` commit
`b53048e44833937df80a52fccceea78d054790e8`. The original splitter used seed 42
and type-stratified allocation into ten 50-case groups; groups 6–10 remain held
out. The runner reads the IDs directly from YAML without loading group files.
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl) is a byte-for-byte
copy of that revision's `dataset/longmemeval_evidence_sessions.jsonl`;
reviewed evidence-session annotations are preserved during normalization.

View file

@ -101,3 +101,32 @@ agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only(复用预构建
| single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 |
| temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 |
| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** |
## 准备 Home 实验
安装 Home 插件后,在仓库根目录执行:
```bash
reme plugins install plugins/home
python benchmark/longmemeval/run_home.py
```
该入口只创建工作区、准备数据和代码,不运行评测或优化。
参数见 [config_home.yaml](config_home.yaml),路径相对仓库根目录;
可通过 `--config` 更换配置,通过 `--workspace-dir` 指定新工作区。
默认工作区为 `benchmark/longmemeval/workspaces/home-groups-1-5`。
代码从 `source_ref: HEAD` 对应的 Git 提交导出,不包含未提交修改。
准备失败会回滚;成功后禁止再次准备同一工作区。
Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=...)`。
顶层 `sampling_policy` 配置采样默认值:`max_cases`、`max_queries`(null 表示不限)、
`temperature`、`ewma_alpha` 和 `uniform_exploration`。入口校验后将其传给宿主
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
运行时无需读取分组文件,group_6–group_10 保持为留出集。
独立审阅的证据标注保存在
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl),来源见英文说明。

View file

@ -0,0 +1,283 @@
# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
preparation:
benchmark:
name: longmemeval
source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json
evidence_source: benchmark/longmemeval/home_evidence_sessions.jsonl
# dev_meta b53048e44833937df80a52fccceea78d054790e8: dataset/groups.json, groups 1-5.
case_ids:
# group_1
- "gpt4_1e4a8aec"
- "e3038f8c"
- "gpt4_70e84552_abs"
- "edced276_abs"
- "c6853660"
- "gpt4_372c3eed"
- "gpt4_ec93e27f"
- "c8c3f81d"
- "dccbc061"
- "a06e4cfe"
- "a82c026e"
- "fca762bc"
- "603deb26"
- "bb7c3b45"
- "4baee567"
- "af082822"
- "1a1907b4"
- "gpt4_468eb064"
- "ef9cf60a"
- "gpt4_a2d1d1f6"
- "8fb83627"
- "1c0ddc50"
- "2ebe6c92"
- "e25c3b8d"
- "6aeb4375"
- "bbf86515"
- "8e91e7d9"
- "f0e564bc"
- "eaca4986"
- "gpt4_4fc4f797"
- "gpt4_5501fe77"
- "7a8d0b71"
- "982b5123"
- "f4f1d8a4_abs"
- "e831120c"
- "0977f2af"
- "51c32626"
- "gpt4_ab202e7f"
- "eace081b"
- "d01c6aa8"
- "8aef76bc"
- "d682f1a2"
- "gpt4_468eb063"
- "c5e8278d"
- "ccb36322"
- "affe2881"
- "75f70248"
- "6e984302"
- "ed4ddc30"
- "a11281a2"
# group_2
- "cc6d1ec1"
- "6613b389"
- "c4f10528"
- "1faac195"
- "3d86fd0a"
- "9d25d4e0"
- "b759caee"
- "gpt4_b4a80587"
- "b3c15d39"
- "370a8ff4"
- "bf659f65"
- "5a7937c8"
- "57f827a0"
- "545bd2b5"
- "a96c20ee"
- "caf03d32"
- "dad224aa"
- "50635ada"
- "21436231"
- "031748ae"
- "41275add"
- "51a45a95"
- "gpt4_4929293a"
- "3e321797"
- "0e5e2d1a"
- "b01defab"
- "gpt4_4ef30696"
- "91b15a6e"
- "d6233ab6"
- "c18a7dc8"
- "f0853d11"
- "031748ae_abs"
- "60d45044"
- "bcbe585f"
- "41698283"
- "0ddfec37"
- "46a3abf7"
- "dd2973ad"
- "129d1232"
- "60159905"
- "2b8f3739"
- "c8090214_abs"
- "8464fc84"
- "gpt4_93f6379c"
- "gpt4_2c50253f"
- "gpt4_18c2b244"
- "bc8a6e93_abs"
- "09ba9854"
- "2133c1b5_abs"
- "gpt4_4edbafa2"
# group_3
- "7e00a6cb"
- "0e4e4c46"
- "gpt4_0b2f1d21"
- "55241a1f"
- "77eafa52"
- "dc439ea3"
- "86f00804"
- "830ce83f"
- "cc5ded98"
- "2311e44b_abs"
- "gpt4_d12ceb0e"
- "71a3fd6b"
- "07741c45"
- "e6041065"
- "gpt4_68e94288"
- "32260d93"
- "db467c8c"
- "4adc0475"
- "6cb6f249"
- "c9f37c46"
- "4388e9dd"
- "9ea5eabc"
- "gpt4_fe651585_abs"
- "9a707b81"
- "e9327a54"
- "eac54adc"
- "gpt4_7fce9456"
- "f9e8c073"
- "ba358f49_abs"
- "561fabcd"
- "42ec0761"
- "gpt4_4929293b"
- "c19f7a0b"
- "0bc8ad93"
- "e47becba"
- "15745da0"
- "2e6d26dc"
- "efc3f7c2"
- "37f165cf"
- "gpt4_f420262d"
- "66f24dbb"
- "gpt4_194be4b3"
- "gpt4_c27434e8_abs"
- "f4f1d8a4"
- "a89d7624"
- "19b5f2b3"
- "gpt4_8279ba02"
- "gpt4_74aed68e"
- "1da05512"
- "gpt4_f420262c"
# group_4
- "gpt4_fa19884d"
- "6f9b354f"
- "a3045048"
- "81507db6"
- "61f8c8f8"
- "a346bb18"
- "87f22b4a"
- "gpt4_2ba83207"
- "01493427"
- "08e075c7"
- "09d032c9"
- "6a27ffc2"
- "edced276"
- "8550ddae"
- "e5ba910e_abs"
- "3f1e9474"
- "8a2466db"
- "gpt4_2f8be40d"
- "2698e78f"
- "gpt4_1a1dc16d"
- "6b168ec8"
- "d905b33f"
- "60472f9c"
- "6e984301"
- "gpt4_1e4a8aeb"
- "gpt4_98f46fc6"
- "e66b632c"
- "gpt4_1d4ab0c9"
- "gpt4_0a05b494"
- "72e3ee87"
- "16c90bf4"
- "5809eb10"
- "e8a79c70"
- "gpt4_213fd887"
- "0ddfec37_abs"
- "gpt4_cd90e484"
- "06f04340"
- "eeda8a6d"
- "9a707b82"
- "29f2956b"
- "09ba9854_abs"
- "gpt4_5dcc0aab"
- "gpt4_7bc6cf22"
- "5d3d2817"
- "4c36ccef"
- "e3fc4d6e"
- "aae3761f"
- "d596882b"
- "ce6d2d27"
- "ad7109d1"
# group_5
- "76d63226"
- "18dcd5a5"
- "e48988bc"
- "099778bb"
- "2c63a862"
- "gpt4_7abb270c"
- "f523d9fe"
- "06878be2"
- "6456829e_abs"
- "faba32e5"
- "gpt4_15e38248"
- "gpt4_8e165409"
- "35a27287"
- "gpt4_59c863d7"
- "gpt4_59149c77"
- "8ebdbe50"
- "f685340e"
- "21d02d0d"
- "4fd1909e"
- "3249768e"
- "gpt4_f49edff3"
- "1d4e3b97"
- "1192316e"
- "e61a7584"
- "0100672e"
- "8cf51dda"
- "95bcc1c8"
- "71017277"
- "a3838d2b"
- "c4ea545c"
- "2318644b"
- "18bc8abd"
- "gpt4_61e13b3c"
- "gpt4_e05b82a6"
- "b46e15ed"
- "3a704032"
- "eac54add"
- "9ee3ecd6"
- "80ec1f4f"
- "gpt4_1916e0ea"
- "b6019101"
- "45dc21b6"
- "gpt4_c27434e8"
- "bc149d6b"
- "6222b6eb"
- "06db6396"
- "29f2956b_abs"
- "gpt4_5438fa52"
- "cf22b7bf"
- "853b0a1d"
runtime:
source_repo: .
source_ref: HEAD # Committed source only; resolved and saved as a full SHA.
config: benchmark.yaml
plugins: [lme]
candidate:
paths: [plugins/lme/src]
debug:
enabled: true
sessions_per_case: 3
messages_per_session: 7

View file

@ -0,0 +1,500 @@
{"case_id":"e47becba","evidence_sessions":["answer_280352e9"],"is_certain":true}
{"case_id":"118b2229","evidence_sessions":["answer_40a90d51"],"is_certain":true}
{"case_id":"51a45a95","evidence_sessions":["answer_d61669c7"],"is_certain":true}
{"case_id":"58bf7951","evidence_sessions":["answer_355c48bb"],"is_certain":true}
{"case_id":"1e043500","evidence_sessions":["answer_3e012175"],"is_certain":true}
{"case_id":"c5e8278d","evidence_sessions":["answer_f6168136"],"is_certain":true}
{"case_id":"6ade9755","evidence_sessions":["answer_9398da02"],"is_certain":true}
{"case_id":"6f9b354f","evidence_sessions":["answer_feb5200f"],"is_certain":true}
{"case_id":"58ef2f1c","evidence_sessions":["answer_59547700"],"is_certain":true}
{"case_id":"f8c5f88b","evidence_sessions":["answer_c3567066"],"is_certain":true}
{"case_id":"5d3d2817","evidence_sessions":["answer_235eb6fb"],"is_certain":true}
{"case_id":"7527f7e2","evidence_sessions":["answer_7cb94507"],"is_certain":true}
{"case_id":"c960da58","evidence_sessions":["answer_e05e4612"],"is_certain":true}
{"case_id":"3b6f954b","evidence_sessions":["answer_94030872"],"is_certain":true}
{"case_id":"726462e0","evidence_sessions":["answer_f38f679b"],"is_certain":true}
{"case_id":"94f70d80","evidence_sessions":["answer_c63c0458"],"is_certain":true}
{"case_id":"66f24dbb","evidence_sessions":["answer_fea2e4d3"],"is_certain":true}
{"case_id":"ad7109d1","evidence_sessions":["answer_679840f8"],"is_certain":true}
{"case_id":"af8d2e46","evidence_sessions":["answer_82a04f59"],"is_certain":true}
{"case_id":"dccbc061","evidence_sessions":["answer_8f276838"],"is_certain":true}
{"case_id":"c8c3f81d","evidence_sessions":["answer_761acef8"],"is_certain":true}
{"case_id":"8ebdbe50","evidence_sessions":["answer_8ad8a34f"],"is_certain":true}
{"case_id":"6b168ec8","evidence_sessions":["answer_e623ae87"],"is_certain":true}
{"case_id":"75499fd8","evidence_sessions":["answer_723bf11f"],"is_certain":true}
{"case_id":"21436231","evidence_sessions":["answer_1e6d4567"],"is_certain":true}
{"case_id":"95bcc1c8","evidence_sessions":["answer_cb742a61"],"is_certain":true}
{"case_id":"0862e8bf","evidence_sessions":["answer_c6fd8ebd"],"is_certain":true}
{"case_id":"853b0a1d","evidence_sessions":["answer_69811d4a"],"is_certain":true}
{"case_id":"a06e4cfe","evidence_sessions":["answer_6fe9fb49"],"is_certain":true}
{"case_id":"37d43f65","evidence_sessions":["answer_55161935"],"is_certain":true}
{"case_id":"b86304ba","evidence_sessions":["answer_645b0329"],"is_certain":true}
{"case_id":"d52b4f67","evidence_sessions":["answer_0df6aa4b"],"is_certain":true}
{"case_id":"25e5aa4f","evidence_sessions":["answer_986de8c3"],"is_certain":true}
{"case_id":"caf9ead2","evidence_sessions":["answer_0714183a"],"is_certain":true}
{"case_id":"8550ddae","evidence_sessions":["answer_c8354ae9"],"is_certain":true}
{"case_id":"60d45044","evidence_sessions":["answer_9cddca88"],"is_certain":true}
{"case_id":"3f1e9474","evidence_sessions":["answer_57fc1954"],"is_certain":true}
{"case_id":"86b68151","evidence_sessions":["answer_dc11c1eb"],"is_certain":true}
{"case_id":"577d4d32","evidence_sessions":["answer_0dd4d99a"],"is_certain":true}
{"case_id":"ec81a493","evidence_sessions":["answer_ed1982fc"],"is_certain":true}
{"case_id":"15745da0","evidence_sessions":["answer_586de428"],"is_certain":true}
{"case_id":"e01b8e2f","evidence_sessions":["answer_5ca6cd28"],"is_certain":true}
{"case_id":"bc8a6e93","evidence_sessions":["answer_e6143162"],"is_certain":true}
{"case_id":"ccb36322","evidence_sessions":["answer_f1fbb330"],"is_certain":true}
{"case_id":"001be529","evidence_sessions":["answer_530960c1"],"is_certain":true}
{"case_id":"b320f3f8","evidence_sessions":["answer_5cc9b056"],"is_certain":true}
{"case_id":"19b5f2b3","evidence_sessions":["answer_5ff494b9"],"is_certain":true}
{"case_id":"4fd1909e","evidence_sessions":["answer_2952aee4"],"is_certain":true}
{"case_id":"545bd2b5","evidence_sessions":["answer_47ffab4c"],"is_certain":true}
{"case_id":"8a137a7f","evidence_sessions":["answer_15d63a22"],"is_certain":true}
{"case_id":"76d63226","evidence_sessions":["answer_bbdc7b0a"],"is_certain":true}
{"case_id":"86f00804","evidence_sessions":["answer_96b8c9ee"],"is_certain":true}
{"case_id":"8e9d538c","evidence_sessions":["answer_7bdcbd23"],"is_certain":true}
{"case_id":"311778f1","evidence_sessions":["answer_e40b054e"],"is_certain":true}
{"case_id":"c19f7a0b","evidence_sessions":["answer_f442ccbe"],"is_certain":true}
{"case_id":"4100d0a0","evidence_sessions":["answer_83c13ff9"],"is_certain":true}
{"case_id":"29f2956b","evidence_sessions":["answer_7cc5362f"],"is_certain":true}
{"case_id":"1faac195","evidence_sessions":["answer_d01949bf"],"is_certain":true}
{"case_id":"faba32e5","evidence_sessions":["answer_39b12014"],"is_certain":true}
{"case_id":"f4f1d8a4","evidence_sessions":["answer_f5b33470"],"is_certain":true}
{"case_id":"c14c00dd","evidence_sessions":["answer_304511ce"],"is_certain":true}
{"case_id":"36580ce8","evidence_sessions":["answer_93e1bd22"],"is_certain":true}
{"case_id":"3d86fd0a","evidence_sessions":["answer_19c24c11"],"is_certain":true}
{"case_id":"a82c026e","evidence_sessions":["answer_787e6a6d"],"is_certain":true}
{"case_id":"0862e8bf_abs","evidence_sessions":["answer_c6fd8ebd_abs"],"is_certain":true}
{"case_id":"15745da0_abs","evidence_sessions":["answer_586de428_abs"],"is_certain":true}
{"case_id":"bc8a6e93_abs","evidence_sessions":["answer_e6143162_abs"],"is_certain":true}
{"case_id":"19b5f2b3_abs","evidence_sessions":["answer_5ff494b9_abs"],"is_certain":true}
{"case_id":"29f2956b_abs","evidence_sessions":["answer_7cc5362f_abs"],"is_certain":true}
{"case_id":"f4f1d8a4_abs","evidence_sessions":["answer_f5b33470_abs"],"is_certain":true}
{"case_id":"0a995998","evidence_sessions":["answer_afa9873b_2","answer_afa9873b_3"],"is_certain":true}
{"case_id":"6d550036","evidence_sessions":["answer_ec904b3c_1","answer_ec904b3c_4","answer_ec904b3c_3","answer_ec904b3c_2"],"is_certain":false}
{"case_id":"gpt4_59c863d7","evidence_sessions":["answer_593bdffd_4","answer_593bdffd_1","answer_593bdffd_3","answer_593bdffd_2"],"is_certain":true}
{"case_id":"b5ef892d","evidence_sessions":["answer_a8b4290f_3","answer_a8b4290f_1","answer_a8b4290f_2"],"is_certain":true}
{"case_id":"e831120c","evidence_sessions":["answer_86c505e7_1","answer_86c505e7_2"],"is_certain":true}
{"case_id":"3a704032","evidence_sessions":["answer_c2204106_2","answer_c2204106_3","answer_c2204106_1"],"is_certain":false}
{"case_id":"gpt4_d84a3211","evidence_sessions":["answer_2880eb6c_2","answer_2880eb6c_1"],"is_certain":true}
{"case_id":"aae3761f","evidence_sessions":["answer_526354c8_1","answer_526354c8_3","answer_526354c8_2"],"is_certain":true}
{"case_id":"gpt4_f2262a51","evidence_sessions":["answer_55a6940c_3","answer_55a6940c_1","answer_55a6940c_2"],"is_certain":true}
{"case_id":"dd2973ad","evidence_sessions":["answer_f9de4602_2","answer_f9de4602_1"],"is_certain":true}
{"case_id":"c4a1ceb8","evidence_sessions":["answer_56d02cab_3","answer_56d02cab_4","answer_56d02cab_2","answer_56d02cab_1"],"is_certain":false}
{"case_id":"gpt4_a56e767c","evidence_sessions":["answer_cf9e3940_2","answer_cf9e3940_1","answer_cf9e3940_3"],"is_certain":true}
{"case_id":"6cb6f249","evidence_sessions":["answer_a4204937_2","answer_a4204937_1"],"is_certain":true}
{"case_id":"46a3abf7","evidence_sessions":["answer_c65042d7_1","answer_c65042d7_2","answer_c65042d7_3"],"is_certain":true}
{"case_id":"36b9f61e","evidence_sessions":["answer_ef74281f_2","answer_ef74281f_1","answer_ef74281f_3"],"is_certain":true}
{"case_id":"28dc39ac","evidence_sessions":["answer_8d015d9d_3","answer_8d015d9d_5","answer_8d015d9d_2","answer_8d015d9d_4","answer_8d015d9d_1"],"is_certain":true}
{"case_id":"gpt4_2f8be40d","evidence_sessions":["answer_e7b0637e_2","answer_e7b0637e_1","answer_e7b0637e_3"],"is_certain":true}
{"case_id":"2e6d26dc","evidence_sessions":["answer_fa526fc0_4","answer_fa526fc0_3","answer_fa526fc0_1","answer_fa526fc0_2"],"is_certain":true}
{"case_id":"gpt4_15e38248","evidence_sessions":["answer_8858d9dc_3","answer_8858d9dc_1","answer_8858d9dc_4","answer_8858d9dc_2"],"is_certain":true}
{"case_id":"88432d0a","evidence_sessions":["answer_733e443a_3","answer_733e443a_4","answer_733e443a_2","answer_733e443a_1"],"is_certain":false}
{"case_id":"80ec1f4f","evidence_sessions":["answer_990c8992_2","answer_990c8992_1","answer_990c8992_3"],"is_certain":true}
{"case_id":"d23cf73b","evidence_sessions":["answer_5a0d28f8_4","answer_5a0d28f8_2","answer_5a0d28f8_3","answer_5a0d28f8_1"],"is_certain":true}
{"case_id":"gpt4_7fce9456","evidence_sessions":["answer_a679a86a_5","answer_a679a86a_4","answer_a679a86a_2","answer_a679a86a_3","answer_a679a86a_1"],"is_certain":true}
{"case_id":"d682f1a2","evidence_sessions":["answer_c008e5df_1","answer_c008e5df_2","answer_c008e5df_3"],"is_certain":true}
{"case_id":"7024f17c","evidence_sessions":["answer_a21f3697_1","answer_a21f3697_2"],"is_certain":true}
{"case_id":"gpt4_5501fe77","evidence_sessions":["answer_203bf3fa_1","answer_203bf3fa_3","answer_203bf3fa_2"],"is_certain":false}
{"case_id":"gpt4_2ba83207","evidence_sessions":["answer_6a3b5c13_3","answer_6a3b5c13_1","answer_6a3b5c13_2","answer_6a3b5c13_4"],"is_certain":true}
{"case_id":"2318644b","evidence_sessions":["answer_eaa8e3ef_1","answer_eaa8e3ef_2"],"is_certain":true}
{"case_id":"2ce6a0f2","evidence_sessions":["answer_901a6763_2","answer_901a6763_4","answer_901a6763_1","answer_901a6763_3"],"is_certain":true}
{"case_id":"gpt4_d12ceb0e","evidence_sessions":["answer_2504635e_3","answer_2504635e_2","answer_2504635e_1"],"is_certain":true}
{"case_id":"00ca467f","evidence_sessions":["answer_39900a0a_3","answer_39900a0a_2","answer_39900a0a_1"],"is_certain":true}
{"case_id":"b3c15d39","evidence_sessions":["answer_05d808e6_1","answer_05d808e6_2"],"is_certain":true}
{"case_id":"gpt4_31ff4165","evidence_sessions":["answer_02b63d04_1","answer_02b63d04_5","answer_02b63d04_2","answer_02b63d04_3","answer_02b63d04_4"],"is_certain":true}
{"case_id":"eeda8a6d","evidence_sessions":["answer_3e5fea0e_1","answer_3e5fea0e_2"],"is_certain":true}
{"case_id":"2788b940","evidence_sessions":["answer_8f6b938d_1","answer_8f6b938d_3","answer_8f6b938d_2"],"is_certain":true}
{"case_id":"60bf93ed","evidence_sessions":["answer_e0956e0a_1","answer_e0956e0a_2"],"is_certain":true}
{"case_id":"9d25d4e0","evidence_sessions":["answer_fcff2dc4_2","answer_fcff2dc4_1","answer_fcff2dc4_3"],"is_certain":true}
{"case_id":"129d1232","evidence_sessions":["answer_1de862d6_1","answer_1de862d6_3","answer_1de862d6_2"],"is_certain":true}
{"case_id":"60472f9c","evidence_sessions":["answer_e7fe8c8b_1","answer_e7fe8c8b_2","answer_e7fe8c8b_3"],"is_certain":true}
{"case_id":"gpt4_194be4b3","evidence_sessions":["answer_3826dc55_1","answer_3826dc55_3","answer_3826dc55_5","answer_3826dc55_2","answer_3826dc55_4"],"is_certain":true}
{"case_id":"a9f6b44c","evidence_sessions":["answer_cc021f81_2","answer_cc021f81_3","answer_cc021f81_1"],"is_certain":true}
{"case_id":"d851d5ba","evidence_sessions":["answer_5cdf9bd2_2","answer_5cdf9bd2_1","answer_5cdf9bd2_3","answer_5cdf9bd2_4"],"is_certain":false}
{"case_id":"5a7937c8","evidence_sessions":["answer_4cef8a3c_3","answer_4cef8a3c_1","answer_4cef8a3c_2"],"is_certain":true}
{"case_id":"gpt4_ab202e7f","evidence_sessions":["answer_728deb4d_5","answer_728deb4d_2","answer_728deb4d_3","answer_728deb4d_1","answer_728deb4d_4"],"is_certain":true}
{"case_id":"gpt4_e05b82a6","evidence_sessions":["answer_6350aa4f_1","answer_6350aa4f_2","answer_6350aa4f_3","answer_6350aa4f_4"],"is_certain":true}
{"case_id":"gpt4_731e37d7","evidence_sessions":["answer_826d51da_3","answer_826d51da_4","answer_826d51da_2","answer_826d51da_1"],"is_certain":true}
{"case_id":"edced276","evidence_sessions":["answer_60e8941a_2","answer_60e8941a_1"],"is_certain":true}
{"case_id":"10d9b85a","evidence_sessions":["answer_e0585cb5_2","answer_e0585cb5_1"],"is_certain":true}
{"case_id":"e3038f8c","evidence_sessions":["answer_b6018747_2","answer_b6018747_4","answer_b6018747_1","answer_b6018747_3"],"is_certain":true}
{"case_id":"2b8f3739","evidence_sessions":["answer_23759615_2","answer_23759615_3","answer_23759615_1"],"is_certain":true}
{"case_id":"1a8a66a6","evidence_sessions":["answer_2bd23659_3","answer_2bd23659_2"],"is_certain":true}
{"case_id":"c2ac3c61","evidence_sessions":["answer_923c0221_1","answer_923c0221_2"],"is_certain":true}
{"case_id":"bf659f65","evidence_sessions":["answer_7726e7e9_1","answer_7726e7e9_3","answer_7726e7e9_2"],"is_certain":false}
{"case_id":"gpt4_372c3eed","evidence_sessions":["answer_35c5419d_3","answer_35c5419d_2","answer_35c5419d_1"],"is_certain":false}
{"case_id":"gpt4_2f91af09","evidence_sessions":["answer_669318cf_2","answer_669318cf_1","answer_669318cf_3"],"is_certain":true}
{"case_id":"81507db6","evidence_sessions":["answer_da3c1266_4","answer_da3c1266_1","answer_da3c1266_5","answer_da3c1266_2"],"is_certain":true}
{"case_id":"88432d0a_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"80ec1f4f_abs","evidence_sessions":["answer_990c8992_abs_2","answer_990c8992_abs_3","answer_990c8992_abs_1"],"is_certain":true}
{"case_id":"eeda8a6d_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"60bf93ed_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"edced276_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_372c3eed_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"8a2466db","evidence_sessions":["answer_edb03329"],"is_certain":true}
{"case_id":"06878be2","evidence_sessions":["answer_555dfb94"],"is_certain":true}
{"case_id":"75832dbd","evidence_sessions":["answer_d87a6ef8"],"is_certain":true}
{"case_id":"0edc2aef","evidence_sessions":["answer_d586e9cd"],"is_certain":true}
{"case_id":"35a27287","evidence_sessions":["answer_9b182436"],"is_certain":true}
{"case_id":"32260d93","evidence_sessions":["answer_0250ae1c"],"is_certain":true}
{"case_id":"195a1a1b","evidence_sessions":["answer_6dc4305e"],"is_certain":true}
{"case_id":"afdc33df","evidence_sessions":["answer_8549e5e0"],"is_certain":true}
{"case_id":"caf03d32","evidence_sessions":["answer_2fc6aabb"],"is_certain":true}
{"case_id":"54026fce","evidence_sessions":["answer_f7b22c66"],"is_certain":true}
{"case_id":"06f04340","evidence_sessions":["answer_92d5f7cd"],"is_certain":true}
{"case_id":"6b7dfb22","evidence_sessions":["answer_f6502d0f"],"is_certain":true}
{"case_id":"1a1907b4","evidence_sessions":["answer_719502eb"],"is_certain":true}
{"case_id":"09d032c9","evidence_sessions":["answer_b10dce5e"],"is_certain":true}
{"case_id":"38146c39","evidence_sessions":["answer_772472c8"],"is_certain":true}
{"case_id":"d24813b1","evidence_sessions":["answer_7c0ade93"],"is_certain":true}
{"case_id":"57f827a0","evidence_sessions":["answer_1bde8d3b"],"is_certain":true}
{"case_id":"95228167","evidence_sessions":["answer_5e613445"],"is_certain":true}
{"case_id":"505af2f5","evidence_sessions":["answer_f3164f2c"],"is_certain":true}
{"case_id":"75f70248","evidence_sessions":["answer_8ee04a2e"],"is_certain":true}
{"case_id":"d6233ab6","evidence_sessions":["answer_b0fac439"],"is_certain":true}
{"case_id":"1da05512","evidence_sessions":["answer_4d3be2ab"],"is_certain":true}
{"case_id":"fca70973","evidence_sessions":["answer_a1e169b1"],"is_certain":true}
{"case_id":"b6025781","evidence_sessions":["answer_8414cc57"],"is_certain":true}
{"case_id":"a89d7624","evidence_sessions":["answer_8f15ac24"],"is_certain":true}
{"case_id":"b0479f84","evidence_sessions":["answer_30f63ddb"],"is_certain":true}
{"case_id":"1d4e3b97","evidence_sessions":["answer_e6b6353d"],"is_certain":true}
{"case_id":"07b6f563","evidence_sessions":["answer_d03098f9"],"is_certain":true}
{"case_id":"1c0ddc50","evidence_sessions":["answer_8da8c7e0"],"is_certain":true}
{"case_id":"0a34ad58","evidence_sessions":["answer_cebb7159"],"is_certain":true}
{"case_id":"d3ab962e","evidence_sessions":["answer_5237bb0b_2","answer_5237bb0b_1"],"is_certain":true}
{"case_id":"2311e44b","evidence_sessions":["answer_bf633415_2","answer_bf633415_1"],"is_certain":true}
{"case_id":"cc06de0d","evidence_sessions":["answer_4fb01417_2","answer_4fb01417_1"],"is_certain":true}
{"case_id":"a11281a2","evidence_sessions":["answer_c69ee1f9_2","answer_c69ee1f9_1"],"is_certain":true}
{"case_id":"4f54b7c9","evidence_sessions":["answer_50940cb7_2","answer_50940cb7_1"],"is_certain":true}
{"case_id":"85fa3a3f","evidence_sessions":["answer_dcb18b9b_2","answer_dcb18b9b_1"],"is_certain":true}
{"case_id":"9aaed6a3","evidence_sessions":[],"is_certain":true}
{"case_id":"1f2b8d4f","evidence_sessions":["answer_d8454588_1","answer_d8454588_2"],"is_certain":true}
{"case_id":"e6041065","evidence_sessions":["answer_4eb6d671_2","answer_4eb6d671_1"],"is_certain":true}
{"case_id":"51c32626","evidence_sessions":["answer_58820c75_1","answer_58820c75_2"],"is_certain":false}
{"case_id":"d905b33f","evidence_sessions":["answer_85a77c48_2","answer_85a77c48_1"],"is_certain":true}
{"case_id":"7405e8b1","evidence_sessions":["answer_80323f3f_1","answer_80323f3f_2"],"is_certain":true}
{"case_id":"f35224e0","evidence_sessions":["answer_e9bb9500_2","answer_e9bb9500_1"],"is_certain":true}
{"case_id":"6456829e","evidence_sessions":["answer_743f03a1_2","answer_743f03a1_1"],"is_certain":true}
{"case_id":"a4996e51","evidence_sessions":["answer_feb5f98a_1","answer_feb5f98a_2"],"is_certain":true}
{"case_id":"3c1045c8","evidence_sessions":["answer_c8cc60d6_2","answer_c8cc60d6_1"],"is_certain":true}
{"case_id":"60036106","evidence_sessions":["answer_e552e1f9_1","answer_e552e1f9_2"],"is_certain":true}
{"case_id":"681a1674","evidence_sessions":["answer_3be95d43_1","answer_3be95d43_2"],"is_certain":true}
{"case_id":"e25c3b8d","evidence_sessions":["answer_6702277b_1","answer_6702277b_2"],"is_certain":true}
{"case_id":"4adc0475","evidence_sessions":["answer_6efce493_1","answer_6efce493_2"],"is_certain":true}
{"case_id":"4bc144e2","evidence_sessions":["answer_9ef115d4_1","answer_9ef115d4_2"],"is_certain":true}
{"case_id":"ef66a6e5","evidence_sessions":["answer_f7fd1029_2","answer_f7fd1029_1"],"is_certain":true}
{"case_id":"5025383b","evidence_sessions":["answer_688f9a3f_1","answer_688f9a3f_2"],"is_certain":true}
{"case_id":"a1cc6108","evidence_sessions":["answer_17dc2f5b_2","answer_17dc2f5b_1"],"is_certain":true}
{"case_id":"9ee3ecd6","evidence_sessions":["answer_66c23110_1"],"is_certain":true}
{"case_id":"3fdac837","evidence_sessions":["answer_419d21d5_2","answer_419d21d5_1"],"is_certain":true}
{"case_id":"91b15a6e","evidence_sessions":["answer_5404a208_1","answer_5404a208_2"],"is_certain":true}
{"case_id":"27016adc","evidence_sessions":["answer_a37bdf22_2","answer_a37bdf22_1"],"is_certain":true}
{"case_id":"720133ac","evidence_sessions":["answer_c9dfeaea_1","answer_c9dfeaea_2"],"is_certain":true}
{"case_id":"77eafa52","evidence_sessions":["answer_33c251f0_2","answer_33c251f0_1"],"is_certain":true}
{"case_id":"8979f9ec","evidence_sessions":["answer_35e36341_1","answer_35e36341_2"],"is_certain":true}
{"case_id":"0100672e","evidence_sessions":["answer_35c9798c_2","answer_35c9798c_1"],"is_certain":true}
{"case_id":"a96c20ee","evidence_sessions":["answer_ef84b994_1","answer_ef84b994_2"],"is_certain":true}
{"case_id":"92a0aa75","evidence_sessions":["answer_6cb8f792_1","answer_6cb8f792_2"],"is_certain":true}
{"case_id":"3fe836c9","evidence_sessions":["answer_1bb63ea5_2","answer_1bb63ea5_1"],"is_certain":true}
{"case_id":"1c549ce4","evidence_sessions":["answer_4cb841a8_1","answer_4cb841a8_2"],"is_certain":true}
{"case_id":"6c49646a","evidence_sessions":["answer_cc1ced21_2","answer_cc1ced21_1"],"is_certain":true}
{"case_id":"1192316e","evidence_sessions":["answer_e184e4c3_2","answer_e184e4c3_1"],"is_certain":true}
{"case_id":"0ea62687","evidence_sessions":["answer_dc5e537d_1","answer_dc5e537d_2"],"is_certain":true}
{"case_id":"67e0d0f2","evidence_sessions":["answer_3a5010af_2","answer_3a5010af_1"],"is_certain":true}
{"case_id":"bb7c3b45","evidence_sessions":["answer_de64539a_1","answer_de64539a_2"],"is_certain":true}
{"case_id":"ba358f49","evidence_sessions":["answer_cbd08e3c_1","answer_cbd08e3c_2"],"is_certain":true}
{"case_id":"61f8c8f8","evidence_sessions":["answer_872e8da2_2","answer_872e8da2_1"],"is_certain":true}
{"case_id":"60159905","evidence_sessions":["answer_75eca223_2","answer_75eca223_1"],"is_certain":true}
{"case_id":"ef9cf60a","evidence_sessions":["answer_87e3a1cb_1","answer_87e3a1cb_2"],"is_certain":true}
{"case_id":"73d42213","evidence_sessions":["answer_1881e7db_1","answer_1881e7db_2"],"is_certain":true}
{"case_id":"bc149d6b","evidence_sessions":["answer_92147866_1","answer_92147866_2"],"is_certain":true}
{"case_id":"099778bb","evidence_sessions":["answer_80d6d664_2","answer_80d6d664_1"],"is_certain":true}
{"case_id":"09ba9854","evidence_sessions":["answer_96c743d0_2","answer_96c743d0_1"],"is_certain":true}
{"case_id":"d6062bb9","evidence_sessions":["answer_23f3a657_2","answer_23f3a657_1"],"is_certain":true}
{"case_id":"157a136e","evidence_sessions":["answer_8de18468_2","answer_8de18468_1"],"is_certain":true}
{"case_id":"c18a7dc8","evidence_sessions":["answer_2e2085fa_2","answer_2e2085fa_1"],"is_certain":true}
{"case_id":"a3332713","evidence_sessions":["answer_16ece55f_2","answer_16ece55f_1"],"is_certain":true}
{"case_id":"55241a1f","evidence_sessions":["answer_fa08bf49_1","answer_fa08bf49_2"],"is_certain":true}
{"case_id":"a08a253f","evidence_sessions":["answer_47152166_2","answer_47152166_1"],"is_certain":true}
{"case_id":"f0e564bc","evidence_sessions":["answer_cfcf5340_1","answer_cfcf5340_2"],"is_certain":true}
{"case_id":"078150f1","evidence_sessions":["answer_254d8b09_1","answer_254d8b09_2"],"is_certain":true}
{"case_id":"8cf4d046","evidence_sessions":["answer_e2278b24_1","answer_e2278b24_2"],"is_certain":true}
{"case_id":"a346bb18","evidence_sessions":["answer_4934b2d7_2","answer_4934b2d7_1"],"is_certain":true}
{"case_id":"37f165cf","evidence_sessions":["answer_6b9b2b1e_2","answer_6b9b2b1e_1"],"is_certain":false}
{"case_id":"8e91e7d9","evidence_sessions":["answer_477ae455_1","answer_477ae455_2"],"is_certain":true}
{"case_id":"87f22b4a","evidence_sessions":["answer_f56e6152_2","answer_f56e6152_1"],"is_certain":true}
{"case_id":"e56a43b9","evidence_sessions":["answer_430d0a87_1","answer_430d0a87_2"],"is_certain":true}
{"case_id":"efc3f7c2","evidence_sessions":["answer_fc81d1a2_1","answer_fc81d1a2_2"],"is_certain":true}
{"case_id":"21d02d0d","evidence_sessions":["answer_2c637141_2","answer_2c637141_1"],"is_certain":true}
{"case_id":"2311e44b_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"6456829e_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"e5ba910e_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"a96c20ee_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"ba358f49_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"09ba9854_abs","evidence_sessions":["answer_96c743d0_abs_1","answer_96c743d0_abs_2"],"is_certain":true}
{"case_id":"gpt4_59149c77","evidence_sessions":["answer_d00ba6d0_1","answer_d00ba6d0_2"],"is_certain":true}
{"case_id":"gpt4_f49edff3","evidence_sessions":["answer_3e9fce53_1","answer_3e9fce53_2","answer_3e9fce53_3"],"is_certain":true}
{"case_id":"71017276","evidence_sessions":["answer_0b4a8adc_1"],"is_certain":true}
{"case_id":"b46e15ed","evidence_sessions":["answer_4bfcc250_4","answer_4bfcc250_3","answer_4bfcc250_2","answer_4bfcc250_1"],"is_certain":true}
{"case_id":"gpt4_fa19884c","evidence_sessions":["answer_ff201786_1","answer_ff201786_2"],"is_certain":true}
{"case_id":"0bc8ad92","evidence_sessions":["answer_f4ea84fb_3","answer_f4ea84fb_2","answer_f4ea84fb_1"],"is_certain":true}
{"case_id":"af082822","evidence_sessions":["answer_b51b6115_1"],"is_certain":true}
{"case_id":"gpt4_4929293a","evidence_sessions":["answer_add9b012_2","answer_add9b012_1"],"is_certain":true}
{"case_id":"gpt4_b5700ca9","evidence_sessions":["answer_a17423e7_1"],"is_certain":true}
{"case_id":"9a707b81","evidence_sessions":["answer_dba89487_2","answer_dba89487_1"],"is_certain":false}
{"case_id":"gpt4_1d4ab0c9","evidence_sessions":["answer_febde667_1","answer_febde667_2"],"is_certain":true}
{"case_id":"gpt4_e072b769","evidence_sessions":["answer_c19bd2bf_1"],"is_certain":true}
{"case_id":"0db4c65d","evidence_sessions":["answer_b9e32ff8_1","answer_b9e32ff8_2"],"is_certain":true}
{"case_id":"gpt4_1d80365e","evidence_sessions":["answer_661b711f_1","answer_661b711f_2"],"is_certain":true}
{"case_id":"gpt4_7f6b06db","evidence_sessions":["answer_5d8c99d3_1","answer_5d8c99d3_2","answer_5d8c99d3_3"],"is_certain":true}
{"case_id":"gpt4_6dc9b45b","evidence_sessions":["answer_c4df007f_1"],"is_certain":true}
{"case_id":"gpt4_8279ba02","evidence_sessions":["answer_56521e65_1"],"is_certain":true}
{"case_id":"gpt4_18c2b244","evidence_sessions":["answer_c862f65a_2","answer_c862f65a_3","answer_c862f65a_1"],"is_certain":true}
{"case_id":"gpt4_a1b77f9c","evidence_sessions":["answer_e9ad5914_1","answer_e9ad5914_2","answer_e9ad5914_3","answer_e9ad5914_4","answer_e9ad5914_5","answer_e9ad5914_6"],"is_certain":false}
{"case_id":"gpt4_1916e0ea","evidence_sessions":["answer_447052a5_2","answer_447052a5_1"],"is_certain":true}
{"case_id":"gpt4_7a0daae1","evidence_sessions":["answer_4d5490f1_1","answer_4d5490f1_2"],"is_certain":true}
{"case_id":"gpt4_468eb063","evidence_sessions":["answer_9b09d95b_1"],"is_certain":true}
{"case_id":"gpt4_7abb270c","evidence_sessions":["answer_7093d898_1","answer_7093d898_2","answer_7093d898_3","answer_7093d898_4","answer_7093d898_5","answer_7093d898_6"],"is_certain":false}
{"case_id":"gpt4_1e4a8aeb","evidence_sessions":["answer_16bd5ea5_1","answer_16bd5ea5_2"],"is_certain":true}
{"case_id":"gpt4_4fc4f797","evidence_sessions":["answer_be07688f_1","answer_be07688f_2"],"is_certain":true}
{"case_id":"4dfccbf7","evidence_sessions":["answer_4bebc782_1","answer_4bebc782_2"],"is_certain":true}
{"case_id":"gpt4_61e13b3c","evidence_sessions":["answer_e831a29f_1","answer_e831a29f_2"],"is_certain":true}
{"case_id":"gpt4_45189cb4","evidence_sessions":["answer_e6c20e52_3","answer_e6c20e52_2","answer_e6c20e52_1"],"is_certain":true}
{"case_id":"2ebe6c90","evidence_sessions":["answer_c9d35c00_1","answer_c9d35c00_2"],"is_certain":true}
{"case_id":"gpt4_e061b84f","evidence_sessions":["answer_8c64ce25_2","answer_8c64ce25_1","answer_8c64ce25_3"],"is_certain":true}
{"case_id":"370a8ff4","evidence_sessions":["answer_61d1be50_1","answer_61d1be50_2"],"is_certain":true}
{"case_id":"gpt4_d6585ce8","evidence_sessions":["answer_f999b05b_1","answer_f999b05b_4","answer_f999b05b_2","answer_f999b05b_5","answer_f999b05b_3"],"is_certain":true}
{"case_id":"gpt4_4ef30696","evidence_sessions":["answer_f964cea3_1","answer_f964cea3_2"],"is_certain":true}
{"case_id":"gpt4_ec93e27f","evidence_sessions":["answer_e3aa84c4_2","answer_e3aa84c4_1"],"is_certain":true}
{"case_id":"6e984301","evidence_sessions":["answer_88841f26_1","answer_88841f26_2"],"is_certain":true}
{"case_id":"8077ef71","evidence_sessions":["answer_0dd54b7c_1"],"is_certain":true}
{"case_id":"gpt4_f420262c","evidence_sessions":["answer_d8a1af6b_1","answer_d8a1af6b_2","answer_d8a1af6b_3","answer_d8a1af6b_4","answer_d8a1af6b_5"],"is_certain":true}
{"case_id":"gpt4_8e165409","evidence_sessions":["answer_d97db962_1","answer_d97db962_2"],"is_certain":true}
{"case_id":"gpt4_74aed68e","evidence_sessions":["answer_aed8cf17_1","answer_aed8cf17_2"],"is_certain":true}
{"case_id":"bcbe585f","evidence_sessions":["answer_43ba3965_1"],"is_certain":true}
{"case_id":"gpt4_21adecb5","evidence_sessions":["answer_1e2369c9_1","answer_1e2369c9_2"],"is_certain":true}
{"case_id":"5e1b23de","evidence_sessions":["answer_c18d480b_1"],"is_certain":true}
{"case_id":"gpt4_98f46fc6","evidence_sessions":["answer_5850de18_2","answer_5850de18_1"],"is_certain":true}
{"case_id":"gpt4_af6db32f","evidence_sessions":["answer_184c8f56_1"],"is_certain":true}
{"case_id":"eac54adc","evidence_sessions":["answer_0d4d0347_1","answer_0d4d0347_2"],"is_certain":true}
{"case_id":"gpt4_7ddcf75f","evidence_sessions":["answer_ad8a76cb_1"],"is_certain":true}
{"case_id":"gpt4_a2d1d1f6","evidence_sessions":["answer_f6d6e33f_1"],"is_certain":true}
{"case_id":"gpt4_85da3956","evidence_sessions":["answer_581ab834_1"],"is_certain":true}
{"case_id":"gpt4_b0863698","evidence_sessions":["answer_550bb2d1_1"],"is_certain":true}
{"case_id":"gpt4_68e94287","evidence_sessions":["answer_9793daa3_2","answer_9793daa3_1"],"is_certain":true}
{"case_id":"gpt4_e414231e","evidence_sessions":["answer_e28c1f0d_1","answer_e28c1f0d_2"],"is_certain":true}
{"case_id":"gpt4_7ca326fa","evidence_sessions":["answer_cf021b36_1","answer_cf021b36_2","answer_cf021b36_3"],"is_certain":true}
{"case_id":"gpt4_7bc6cf22","evidence_sessions":["answer_e22d6aef_1"],"is_certain":true}
{"case_id":"2ebe6c92","evidence_sessions":["answer_c9d35c00_1","answer_c9d35c00_2"],"is_certain":true}
{"case_id":"gpt4_e061b84g","evidence_sessions":["answer_8c64ce26_2","answer_8c64ce26_1","answer_8c64ce26_3"],"is_certain":true}
{"case_id":"71017277","evidence_sessions":["answer_0b4a8adc_1"],"is_certain":true}
{"case_id":"b46e15ee","evidence_sessions":["answer_4bfcc251_4","answer_4bfcc251_3","answer_4bfcc251_2","answer_4bfcc251_1"],"is_certain":true}
{"case_id":"gpt4_d6585ce9","evidence_sessions":["answer_f999b05c_1","answer_f999b05c_4","answer_f999b05c_2","answer_f999b05c_5","answer_f999b05c_3"],"is_certain":true}
{"case_id":"gpt4_1e4a8aec","evidence_sessions":["answer_16bd5ea6_1","answer_16bd5ea6_2"],"is_certain":true}
{"case_id":"gpt4_f420262d","evidence_sessions":["answer_d8a1af6c_1","answer_d8a1af6c_2","answer_d8a1af6c_5","answer_d8a1af6c_4","answer_d8a1af6c_3"],"is_certain":false}
{"case_id":"gpt4_59149c78","evidence_sessions":["answer_d00ba6d1_2"],"is_certain":true}
{"case_id":"gpt4_e414231f","evidence_sessions":["answer_e28c1f0e_1","answer_e28c1f0e_2"],"is_certain":true}
{"case_id":"gpt4_4929293b","evidence_sessions":["answer_add9b013_2","answer_add9b013_1"],"is_certain":true}
{"case_id":"gpt4_468eb064","evidence_sessions":["answer_9b09d95b_1"],"is_certain":true}
{"case_id":"gpt4_fa19884d","evidence_sessions":["answer_ff201787_1","answer_ff201787_2"],"is_certain":true}
{"case_id":"9a707b82","evidence_sessions":["answer_dba89488_2","answer_dba89488_1"],"is_certain":true}
{"case_id":"eac54add","evidence_sessions":["answer_0d4d0348_1","answer_0d4d0348_2"],"is_certain":true}
{"case_id":"4dfccbf8","evidence_sessions":["answer_4bebc783_1","answer_4bebc783_2"],"is_certain":true}
{"case_id":"0bc8ad93","evidence_sessions":["answer_f4ea84fc_3","answer_f4ea84fc_2","answer_f4ea84fc_1"],"is_certain":false}
{"case_id":"6e984302","evidence_sessions":["answer_88841f27_2"],"is_certain":true}
{"case_id":"gpt4_8279ba03","evidence_sessions":["answer_56521e66_1"],"is_certain":true}
{"case_id":"gpt4_b5700ca0","evidence_sessions":["answer_a17423e8_1"],"is_certain":true}
{"case_id":"gpt4_68e94288","evidence_sessions":["answer_9793daa4_2","answer_9793daa4_1"],"is_certain":true}
{"case_id":"gpt4_2655b836","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_2487a7cb","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_76048e76","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_2312f94c","evidence_sessions":[],"is_certain":true}
{"case_id":"0bb5a684","evidence_sessions":[],"is_certain":true}
{"case_id":"08f4fc43","evidence_sessions":["answer_6ea1541e_2","answer_6ea1541e_1"],"is_certain":true}
{"case_id":"2c63a862","evidence_sessions":["answer_d39b7977_1","answer_d39b7977_2"],"is_certain":true}
{"case_id":"gpt4_385a5000","evidence_sessions":[],"is_certain":true}
{"case_id":"2a1811e2","evidence_sessions":[],"is_certain":true}
{"case_id":"bbf86515","evidence_sessions":["answer_b3763b6b_1","answer_b3763b6b_2"],"is_certain":true}
{"case_id":"gpt4_5dcc0aab","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_0b2f1d21","evidence_sessions":[],"is_certain":true}
{"case_id":"f0853d11","evidence_sessions":["answer_932ea3bf_2","answer_932ea3bf_1"],"is_certain":true}
{"case_id":"gpt4_6ed717ea","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_70e84552","evidence_sessions":[],"is_certain":true}
{"case_id":"a3838d2b","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_93159ced","evidence_sessions":["answer_e5131a1b_2","answer_e5131a1b_1"],"is_certain":true}
{"case_id":"gpt4_2d58bcd6","evidence_sessions":["answer_3e11e0ae_1","answer_3e11e0ae_2"],"is_certain":true}
{"case_id":"gpt4_65aabe59","evidence_sessions":[],"is_certain":true}
{"case_id":"982b5123","evidence_sessions":[],"is_certain":true}
{"case_id":"b9cfe692","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_4edbafa2","evidence_sessions":[],"is_certain":true}
{"case_id":"c8090214","evidence_sessions":["answer_70dc7d08_2","answer_70dc7d08_1"],"is_certain":true}
{"case_id":"gpt4_483dd43c","evidence_sessions":[],"is_certain":true}
{"case_id":"e4e14d04","evidence_sessions":[],"is_certain":true}
{"case_id":"c9f37c46","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_2c50253f","evidence_sessions":["answer_9af4e346_1","answer_9af4e346_2"],"is_certain":true}
{"case_id":"dcfa8644","evidence_sessions":["answer_5e3eeb12_2","answer_5e3eeb12_1"],"is_certain":true}
{"case_id":"gpt4_b4a80587","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_9a159967","evidence_sessions":["answer_8a42fedf_2"],"is_certain":true}
{"case_id":"cc6d1ec1","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_8c8961ae","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_d9af6064","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_7de946e7","evidence_sessions":["answer_6a78e959_2","answer_6a78e959_1"],"is_certain":true}
{"case_id":"d01c6aa8","evidence_sessions":[],"is_certain":true}
{"case_id":"993da5e2","evidence_sessions":[],"is_certain":true}
{"case_id":"a3045048","evidence_sessions":["answer_016f6bd4_1"],"is_certain":true}
{"case_id":"gpt4_d31cdae3","evidence_sessions":["answer_8464304d_1","answer_8464304d_2"],"is_certain":true}
{"case_id":"gpt4_cd90e484","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_88806d6e","evidence_sessions":["answer_e60a93ff_1","answer_e60a93ff_2"],"is_certain":true}
{"case_id":"gpt4_4cd9eba1","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_93f6379c","evidence_sessions":["answer_544fe66c_2","answer_544fe66c_3"],"is_certain":true}
{"case_id":"b29f3365","evidence_sessions":["answer_436d4309_2","answer_436d4309_1"],"is_certain":true}
{"case_id":"gpt4_2f56ae70","evidence_sessions":[],"is_certain":true}
{"case_id":"6613b389","evidence_sessions":["answer_aaf71ce2_2","answer_aaf71ce2_1"],"is_certain":true}
{"case_id":"gpt4_78cf46a3","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_0a05b494","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_1a1dc16d","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_2f584639","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_213fd887","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_5438fa52","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_c27434e8","evidence_sessions":["answer_d8e33f5c_1","answer_d8e33f5c_2"],"is_certain":true}
{"case_id":"gpt4_fe651585","evidence_sessions":[],"is_certain":true}
{"case_id":"8c18457d","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_70e84552_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_93159ced_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"982b5123_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"c8090214_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_c27434e8_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"gpt4_fe651585_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"6a1eabeb","evidence_sessions":["answer_a25d4a91_1","answer_a25d4a91_2"],"is_certain":true}
{"case_id":"6aeb4375","evidence_sessions":["answer_3f9693b7_1","answer_3f9693b7_2"],"is_certain":true}
{"case_id":"830ce83f","evidence_sessions":["answer_0b1a0942_2"],"is_certain":true}
{"case_id":"852ce960","evidence_sessions":["answer_3a6f1e82_2"],"is_certain":true}
{"case_id":"945e3d21","evidence_sessions":["answer_6a4f8626_1","answer_6a4f8626_2"],"is_certain":true}
{"case_id":"d7c942c3","evidence_sessions":["answer_eecb10d9_1","answer_eecb10d9_2"],"is_certain":true}
{"case_id":"71315a70","evidence_sessions":["answer_c44b9df4_1","answer_c44b9df4_2"],"is_certain":true}
{"case_id":"89941a93","evidence_sessions":["answer_e1403127_1","answer_e1403127_2"],"is_certain":true}
{"case_id":"ce6d2d27","evidence_sessions":["answer_73540165_2"],"is_certain":true}
{"case_id":"9ea5eabc","evidence_sessions":["answer_02e66dec_1","answer_02e66dec_2"],"is_certain":true}
{"case_id":"07741c44","evidence_sessions":["answer_7e9ad7b4_1","answer_7e9ad7b4_2"],"is_certain":true}
{"case_id":"a1eacc2a","evidence_sessions":["answer_0eb23770_1","answer_0eb23770_2"],"is_certain":true}
{"case_id":"184da446","evidence_sessions":["answer_e2f4f947_1","answer_e2f4f947_2"],"is_certain":true}
{"case_id":"031748ae","evidence_sessions":["answer_8748f791_1","answer_8748f791_2"],"is_certain":true}
{"case_id":"4d6b87c8","evidence_sessions":["answer_766ab8da_1","answer_766ab8da_2"],"is_certain":true}
{"case_id":"0f05491a","evidence_sessions":["answer_d6d2eba8_2"],"is_certain":true}
{"case_id":"08e075c7","evidence_sessions":["answer_cdbe2250_1","answer_cdbe2250_2"],"is_certain":true}
{"case_id":"f9e8c073","evidence_sessions":["answer_b191df5b_1","answer_b191df5b_2"],"is_certain":true}
{"case_id":"41698283","evidence_sessions":["answer_c7ddc051_1","answer_c7ddc051_2"],"is_certain":true}
{"case_id":"2698e78f","evidence_sessions":["answer_9282283d_1","answer_9282283d_2"],"is_certain":true}
{"case_id":"b6019101","evidence_sessions":["answer_67074b4b_1","answer_67074b4b_2"],"is_certain":true}
{"case_id":"45dc21b6","evidence_sessions":["answer_07664d43_1","answer_07664d43_2"],"is_certain":true}
{"case_id":"5a4f22c0","evidence_sessions":["answer_b0f3dfff_1","answer_b0f3dfff_2"],"is_certain":true}
{"case_id":"6071bd76","evidence_sessions":["answer_4dac77cb_1","answer_4dac77cb_2"],"is_certain":true}
{"case_id":"e493bb7c","evidence_sessions":["answer_1a374afa_1","answer_1a374afa_2"],"is_certain":true}
{"case_id":"618f13b2","evidence_sessions":["answer_caf5b52e_1","answer_caf5b52e_2"],"is_certain":true}
{"case_id":"72e3ee87","evidence_sessions":["answer_d7de9a6a_1","answer_d7de9a6a_2"],"is_certain":true}
{"case_id":"c4ea545c","evidence_sessions":["answer_d3bf812b_1","answer_d3bf812b_2"],"is_certain":true}
{"case_id":"01493427","evidence_sessions":["answer_a7b44747_1","answer_a7b44747_2"],"is_certain":true}
{"case_id":"6a27ffc2","evidence_sessions":["answer_77f32504_1","answer_77f32504_2"],"is_certain":true}
{"case_id":"2133c1b5","evidence_sessions":["answer_52382508_1"],"is_certain":true}
{"case_id":"18bc8abd","evidence_sessions":["answer_fff743f5_1","answer_fff743f5_2"],"is_certain":true}
{"case_id":"db467c8c","evidence_sessions":["answer_611b6e83_1","answer_611b6e83_2"],"is_certain":true}
{"case_id":"7a87bd0c","evidence_sessions":["answer_d08a934d_1","answer_d08a934d_2"],"is_certain":true}
{"case_id":"e61a7584","evidence_sessions":["answer_f25c32f5_1","answer_f25c32f5_2"],"is_certain":true}
{"case_id":"1cea1afa","evidence_sessions":["answer_79c395a9_1","answer_79c395a9_2"],"is_certain":true}
{"case_id":"ed4ddc30","evidence_sessions":["answer_babbaccb_1","answer_babbaccb_2"],"is_certain":true}
{"case_id":"8fb83627","evidence_sessions":["answer_966cecbb_1","answer_966cecbb_2"],"is_certain":true}
{"case_id":"b01defab","evidence_sessions":["answer_8c0712af_1","answer_8c0712af_2"],"is_certain":true}
{"case_id":"22d2cb42","evidence_sessions":["answer_bcce0b73_1","answer_bcce0b73_2"],"is_certain":true}
{"case_id":"0e4e4c46","evidence_sessions":["answer_f2f998c7_1","answer_f2f998c7_2"],"is_certain":true}
{"case_id":"4b24c848","evidence_sessions":["answer_2cec623b_1","answer_2cec623b_2"],"is_certain":true}
{"case_id":"7e974930","evidence_sessions":["answer_c9f5693c_1","answer_c9f5693c_2"],"is_certain":true}
{"case_id":"603deb26","evidence_sessions":["answer_8afdebac_1","answer_8afdebac_2"],"is_certain":true}
{"case_id":"59524333","evidence_sessions":["answer_b28f2c7a_2"],"is_certain":true}
{"case_id":"5831f84d","evidence_sessions":["answer_8d63a897_1","answer_8d63a897_2"],"is_certain":true}
{"case_id":"eace081b","evidence_sessions":["answer_8a791264_2"],"is_certain":true}
{"case_id":"affe2881","evidence_sessions":["answer_90de9b4d_1","answer_90de9b4d_2"],"is_certain":true}
{"case_id":"50635ada","evidence_sessions":["answer_dcd74827_1","answer_dcd74827_2"],"is_certain":true}
{"case_id":"e66b632c","evidence_sessions":["answer_ac0140ce_1","answer_ac0140ce_2"],"is_certain":true}
{"case_id":"0ddfec37","evidence_sessions":["answer_a22b654d_1","answer_a22b654d_2"],"is_certain":true}
{"case_id":"f685340e","evidence_sessions":["answer_25df025b_1","answer_25df025b_2"],"is_certain":true}
{"case_id":"cc5ded98","evidence_sessions":["answer_a5b68517_1","answer_a5b68517_2"],"is_certain":true}
{"case_id":"dfde3500","evidence_sessions":["answer_35d6c0be_1","answer_35d6c0be_2"],"is_certain":true}
{"case_id":"69fee5aa","evidence_sessions":["answer_d6028d6e_1","answer_d6028d6e_2"],"is_certain":true}
{"case_id":"7401057b","evidence_sessions":["answer_94650bfa_1","answer_94650bfa_2"],"is_certain":true}
{"case_id":"cf22b7bf","evidence_sessions":["answer_ae3a122b_1","answer_ae3a122b_2"],"is_certain":true}
{"case_id":"a2f3aa27","evidence_sessions":["answer_5126c02d_1","answer_5126c02d_2"],"is_certain":true}
{"case_id":"c7dc5443","evidence_sessions":["answer_0cdbca92_1","answer_0cdbca92_2"],"is_certain":true}
{"case_id":"06db6396","evidence_sessions":["answer_da72b1b4_1","answer_da72b1b4_2"],"is_certain":true}
{"case_id":"3ba21379","evidence_sessions":["answer_cd345582_1","answer_cd345582_2"],"is_certain":true}
{"case_id":"9bbe84a2","evidence_sessions":["answer_c6a0c6c2_1","answer_c6a0c6c2_2"],"is_certain":true}
{"case_id":"10e09553","evidence_sessions":["answer_67be2c38_1","answer_67be2c38_2"],"is_certain":true}
{"case_id":"dad224aa","evidence_sessions":["answer_4a97ae40_2"],"is_certain":true}
{"case_id":"ba61f0b9","evidence_sessions":["answer_f377cda7_2"],"is_certain":true}
{"case_id":"42ec0761","evidence_sessions":["answer_e3892371_1","answer_e3892371_2"],"is_certain":true}
{"case_id":"5c40ec5b","evidence_sessions":["answer_1cb52d0a_1","answer_1cb52d0a_2"],"is_certain":true}
{"case_id":"c6853660","evidence_sessions":["answer_626e93c4_1","answer_626e93c4_2"],"is_certain":true}
{"case_id":"26bdc477","evidence_sessions":["answer_f762ad8d_1","answer_f762ad8d_2"],"is_certain":true}
{"case_id":"0977f2af","evidence_sessions":["answer_3bf5b73b_1","answer_3bf5b73b_2"],"is_certain":true}
{"case_id":"6aeb4375_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"031748ae_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"2698e78f_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"2133c1b5_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"0ddfec37_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"f685340e_abs","evidence_sessions":[],"is_certain":true}
{"case_id":"89941a94","evidence_sessions":["answer_e1403127_1","answer_e1403127_2"],"is_certain":true}
{"case_id":"07741c45","evidence_sessions":["answer_7e9ad7b4_1","answer_7e9ad7b4_2"],"is_certain":true}
{"case_id":"7161e7e2","evidence_sessions":["answer_sharegpt_5Lzox6N_0"],"is_certain":true}
{"case_id":"c4f10528","evidence_sessions":["answer_ultrachat_234453"],"is_certain":true}
{"case_id":"89527b6b","evidence_sessions":["answer_sharegpt_YkWn1Ne_0"],"is_certain":true}
{"case_id":"e9327a54","evidence_sessions":["answer_ultrachat_480665"],"is_certain":true}
{"case_id":"4c36ccef","evidence_sessions":["answer_ultrachat_448704"],"is_certain":true}
{"case_id":"6ae235be","evidence_sessions":["answer_sharegpt_IUWQYGQ_0"],"is_certain":true}
{"case_id":"7e00a6cb","evidence_sessions":["answer_ultrachat_370515"],"is_certain":true}
{"case_id":"1903aded","evidence_sessions":["answer_sharegpt_hA7AkP3_0"],"is_certain":true}
{"case_id":"ceb54acb","evidence_sessions":["answer_sharegpt_cGdjmYo_0"],"is_certain":true}
{"case_id":"f523d9fe","evidence_sessions":["answer_sharegpt_m2xJfjo_0"],"is_certain":true}
{"case_id":"0e5e2d1a","evidence_sessions":["answer_ultrachat_113156"],"is_certain":true}
{"case_id":"fea54f57","evidence_sessions":["answer_ultrachat_187684"],"is_certain":true}
{"case_id":"cc539528","evidence_sessions":["answer_ultrachat_374124"],"is_certain":true}
{"case_id":"dc439ea3","evidence_sessions":["answer_ultrachat_459954"],"is_certain":true}
{"case_id":"18dcd5a5","evidence_sessions":["answer_sharegpt_hn3IS1q_0"],"is_certain":true}
{"case_id":"488d3006","evidence_sessions":["answer_ultrachat_275993"],"is_certain":true}
{"case_id":"58470ed2","evidence_sessions":["answer_sharegpt_U4oCSfU_7"],"is_certain":true}
{"case_id":"8cf51dda","evidence_sessions":["answer_sharegpt_HFMn2ZX_0"],"is_certain":true}
{"case_id":"1d4da289","evidence_sessions":["answer_ultrachat_348449"],"is_certain":true}
{"case_id":"8464fc84","evidence_sessions":["answer_ultrachat_467053"],"is_certain":true}
{"case_id":"8aef76bc","evidence_sessions":["answer_ultrachat_563222"],"is_certain":true}
{"case_id":"71a3fd6b","evidence_sessions":["answer_ultrachat_417348"],"is_certain":true}
{"case_id":"2bf43736","evidence_sessions":["answer_sharegpt_2kpncbX_13"],"is_certain":true}
{"case_id":"70b3e69b","evidence_sessions":["answer_ultrachat_334948"],"is_certain":true}
{"case_id":"8752c811","evidence_sessions":["answer_sharegpt_6pWK9yx_0"],"is_certain":true}
{"case_id":"3249768e","evidence_sessions":["answer_sharegpt_CaxTGYP_0"],"is_certain":true}
{"case_id":"1b9b7252","evidence_sessions":["answer_ultrachat_115151"],"is_certain":true}
{"case_id":"1568498a","evidence_sessions":["answer_sharegpt_d6JJiqH_76"],"is_certain":true}
{"case_id":"6222b6eb","evidence_sessions":["answer_sharegpt_H9PiM5G_0"],"is_certain":true}
{"case_id":"e8a79c70","evidence_sessions":["answer_ultrachat_13075"],"is_certain":true}
{"case_id":"d596882b","evidence_sessions":["answer_ultrachat_252214"],"is_certain":true}
{"case_id":"e3fc4d6e","evidence_sessions":["answer_sharegpt_5m7gg5F_0"],"is_certain":true}
{"case_id":"51b23612","evidence_sessions":["answer_ultrachat_427265"],"is_certain":true}
{"case_id":"3e321797","evidence_sessions":["answer_ultrachat_94624"],"is_certain":true}
{"case_id":"e982271f","evidence_sessions":["answer_ultrachat_195444"],"is_certain":true}
{"case_id":"352ab8bd","evidence_sessions":["answer_sharegpt_NoDZzot_7"],"is_certain":true}
{"case_id":"fca762bc","evidence_sessions":["answer_ultrachat_39395"],"is_certain":true}
{"case_id":"7a8d0b71","evidence_sessions":["answer_sharegpt_i0tMT9q_9"],"is_certain":true}
{"case_id":"a40e080f","evidence_sessions":["answer_ultrachat_269020"],"is_certain":true}
{"case_id":"8b9d4367","evidence_sessions":["answer_ultrachat_289157"],"is_certain":true}
{"case_id":"5809eb10","evidence_sessions":["answer_sharegpt_4aJsGCH_0"],"is_certain":true}
{"case_id":"41275add","evidence_sessions":["answer_sharegpt_81riySf_0"],"is_certain":true}
{"case_id":"4388e9dd","evidence_sessions":["answer_sharegpt_qTi81nS_0"],"is_certain":true}
{"case_id":"4baee567","evidence_sessions":["answer_sharegpt_i9adwQn_0"],"is_certain":true}
{"case_id":"561fabcd","evidence_sessions":["answer_sharegpt_hChsWOp_97"],"is_certain":true}
{"case_id":"b759caee","evidence_sessions":["answer_sharegpt_2BSXlAr_0"],"is_certain":true}
{"case_id":"ac031881","evidence_sessions":["answer_sharegpt_GYqnAhC_190"],"is_certain":true}
{"case_id":"28bcfaac","evidence_sessions":["answer_ultrachat_446979"],"is_certain":true}
{"case_id":"16c90bf4","evidence_sessions":["answer_ultrachat_294807"],"is_certain":true}
{"case_id":"c8f1aeed","evidence_sessions":["answer_ultrachat_519486"],"is_certain":true}
{"case_id":"eaca4986","evidence_sessions":["answer_sharegpt_SS141vi_0"],"is_certain":true}
{"case_id":"c7cf7dfd","evidence_sessions":["answer_ultrachat_456407"],"is_certain":true}
{"case_id":"e48988bc","evidence_sessions":["answer_ultrachat_174360"],"is_certain":true}
{"case_id":"1de5cff2","evidence_sessions":["answer_ultrachat_440262"],"is_certain":true}
{"case_id":"65240037","evidence_sessions":["answer_ultrachat_403752"],"is_certain":true}
{"case_id":"778164c6","evidence_sessions":["answer_ultrachat_399000"],"is_certain":true}

View file

@ -0,0 +1,22 @@
"""Prepare the longmemeval Home workspace, dataset and candidate code; do not evaluate."""
from pathlib import Path
import sys
# Support direct execution from any working directory as well as module imports.
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
if str(_PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(_PROJECT_ROOT))
from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
if __name__ == "__main__":
main(DEFAULT_CONFIG)

View file

@ -1,5 +1,5 @@
# Build from repository root:
# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.9 .
# docker build -f deploy/benchmark_docker/Dockerfile.base -t reme-benchmark-base:agentscope-2.0.10 .
FROM python:3.11-slim
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
@ -18,7 +18,7 @@ RUN mkdir reme \
&& printf '__version__ = "0.0.0"\n' > reme/__init__.py \
&& python -m pip install '.[core]' \
&& python -m pip uninstall -y reme-ai \
&& python -c 'import agentscope; assert agentscope.__version__ == "2.0.9"' \
&& python -c 'import agentscope; assert agentscope.__version__ == "2.0.10"' \
&& rm -rf /tmp/reme-dependencies
WORKDIR /workspace

View file

@ -1,5 +1,5 @@
# Build from repository root with this Dockerfile-specific ignore file.
ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.9
ARG BASE_IMAGE=reme-benchmark-base:agentscope-2.0.10
FROM ${BASE_IMAGE}
COPY . /opt/reme

View file

@ -2,14 +2,18 @@
This package runs ReMe benchmark jobs directly inside isolated AgentScope
Docker workspaces. It does not start the ReMe HTTP service. Use AgentScope
**2.0.9** on the host and in the image.
**2.0.10** on the host and in the image.
Rebuild the base image when upgrading from the previous AgentScope 2.0.9 image;
existing images must also use 2.0.10 to pass candidate validation. New runs use
the `reme-benchmark-base:agentscope-2.0.10` tag by default, including Home evaluations.
From the repository root:
```bash
pip install -e '.[benchmark-docker]'
docker build -f deploy/benchmark_docker/Dockerfile.base \
-t reme-benchmark-base:agentscope-2.0.9 .
-t reme-benchmark-base:agentscope-2.0.10 .
```
`SourceCandidate` uploads an immutable source snapshot and installs the root
@ -57,6 +61,18 @@ case's workspace **as it stood immediately after construction**, or provide
`workspace_source` pointing at a host workspace directory or a saved archive.
A reused workspace may be queried without additional build jobs.
`export_workspace_to` persists a portable snapshot beyond the pool run.
`run(..., on_case_finished=callback)` awaits an async `(plan, result)` callback
after each case releases its worker, allowing incremental host-side persistence
and judging. Final worker-close errors can still be attached afterward; reconcile
returned results before publishing a completed summary. Callback errors abort the
batch and trigger worker cleanup. `fail_fast=True` stops the batch on an execution
failure. An optional `accept_build` predicate allows trusted callers to define
partial-build acceptance; the default requires successful construction.
`CasePlan.export_diagnostics_to` requests a best-effort full archive on execution
failure before resetting a still-accessible worker. Interrupted containers may
only retain a cached build log; failed diagnostic export never replaces the
original error. `CaseResult.failure_stage` and `exception` identify execution
failures separately from ordinary structured build/query results.
To build memory once and query it in later runs using the same factory:
```python
@ -137,6 +153,6 @@ explicitly enabled:
REME_DOCKER_INTEGRATION=1 pytest tests/integration/test_benchmark_docker.py -v
```
The host must have AgentScope 2.0.9 and a working Docker daemon. To use an
The host must have AgentScope 2.0.10 and a working Docker daemon. To use an
already built base image, set `REME_BENCHMARK_TEST_IMAGE` to its tag; the tests
do not remove a caller-supplied image.

View file

@ -146,7 +146,7 @@ class SourceCandidate:
"""
snapshot: SourceSnapshot
base_image: str = "reme-benchmark-base:agentscope-2.0.9"
base_image: str = "reme-benchmark-base:agentscope-2.0.10"
install_projects: tuple[str, ...] = (".",)
def __post_init__(self) -> None:

View file

@ -1,4 +1,4 @@
"""Minimal AgentScope 2.0.9 DockerWorkspace for direct ReMe jobs.
"""Minimal AgentScope 2.0.10 DockerWorkspace for direct ReMe jobs.
The stock workspace builds and starts an MCP gateway. The benchmark harness
uses only DockerBackend's exec/read/write primitives, so building that gateway

View file

@ -239,13 +239,13 @@ def _archive_workspace(source: str | Path) -> bytes:
def _default_workspace_builder(workspace_id: str, image: str, env: dict[str, str]) -> Workspace:
"""Create the AgentScope 2.0.9 Docker backend lazily."""
"""Create the AgentScope 2.0.10 Docker backend lazily."""
import agentscope
from .direct_docker_workspace import DirectDockerWorkspace
if agentscope.__version__ != "2.0.9":
raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.9, got {agentscope.__version__}")
if agentscope.__version__ != "2.0.10":
raise RuntimeError(f"benchmark Docker host requires agentscope 2.0.10, got {agentscope.__version__}")
return DirectDockerWorkspace(
workspace_id=workspace_id,
base_image=image,
@ -272,8 +272,11 @@ class DockerReMeSandboxFactory:
plugins: tuple[str, ...] | list[str] = (),
command_timeout: float = 1800.0,
workspace_builder: WorkspaceBuilder | None = None,
on_sandbox_started: Callable[[], None] | None = None,
) -> None:
self.candidate = candidate
self._on_sandbox_started = on_sandbox_started
self._sandbox_started_reported = False
self.env = dict(env or {})
self.config = config
self.plugins = tuple(plugins)
@ -309,6 +312,9 @@ class DockerReMeSandboxFactory:
)
try:
await case.initialize()
if self._on_sandbox_started is not None and not self._sandbox_started_reported:
self._on_sandbox_started()
self._sandbox_started_reported = True
except BaseException as exc:
await cleanup_on_error([(f"workspace {container_id!r}", workspace.close())], exc)
raise
@ -489,9 +495,9 @@ class DockerReMeSandbox:
)
result = await self._exec_checked("validate_candidate", [python, "-c", script])
validation = json.loads(_decode(result.stdout))
if validation.get("agentscope") != "2.0.9":
if validation.get("agentscope") != "2.0.10":
raise SandboxCommandError(
f"candidate environment requires agentscope 2.0.9, got {validation.get('agentscope')!r}",
f"candidate environment requires agentscope 2.0.10, got {validation.get('agentscope')!r}",
)
candidate_root = PurePosixPath(CANDIDATE_ROOT)

View file

@ -9,6 +9,7 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
from collections.abc import Awaitable, Callable
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
@ -30,6 +31,7 @@ class CasePlan:
export_workspace_to: str | Path | None = None
export_build_log_to: str | Path | None = None
export_evaluation_to: str | Path | None = None
export_diagnostics_to: str | Path | None = None
@dataclass(slots=True)
@ -42,6 +44,8 @@ class CaseResult:
workspace_snapshot: Path | None = None
build_log: str | None = None
error: str | None = None
failure_stage: str | None = None
exception: dict[str, Any] | None = None
@property
def success(self) -> bool:
@ -79,7 +83,14 @@ class BenchmarkDockerPool:
self.factory = factory
self.concurrency = concurrency
async def run(self, plans: list[CasePlan]) -> list[CaseResult]:
async def run(
self,
plans: list[CasePlan],
*,
on_case_finished: Callable[[CasePlan, CaseResult], Awaitable[None]] | None = None,
accept_build: Callable[[dict[str, Any]], bool] | None = None,
fail_fast: bool = False,
) -> list[CaseResult]:
"""Return results in input order, closing every container on exit."""
if not plans:
return []
@ -105,8 +116,7 @@ class BenchmarkDockerPool:
# a container. The graph check above rules out deadlocks.
tasks: dict[str, asyncio.Task[CaseResult]] = {}
async def execute(plan: CasePlan) -> CaseResult:
result = CaseResult(plan.case_id)
async def execute(plan: CasePlan, result: CaseResult) -> CaseResult:
source = plan.workspace_source
if plan.reuse_workspace_from is not None:
parent = await tasks[plan.reuse_workspace_from]
@ -121,43 +131,83 @@ class BenchmarkDockerPool:
return CaseResult(plan.case_id, error=slots.exhaustion_error)
worker_results.setdefault(id(worker), []).append(result)
operation = "prepare"
try:
await worker.reset_case(plan.case_id)
if source is not None:
operation = "restore"
await worker.upload_workspace(source)
if plan.build_jobs:
operation = "construction"
try:
result.build = await worker.run_build(list(plan.build_jobs))
except Exception as exc:
result.error = format_error(exc)
result.failure_stage = operation
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
await self._retain_failed_build_log(worker, plan, result)
return result
if not result.build.get("success"):
accepted = accept_build(result.build) if accept_build else result.build.get("success")
if not accepted:
result.error = "build failed"
result.failure_stage = operation
await self._retain_failed_build_log(worker, plan, result)
return result
if plan.export_build_log_to is not None:
operation = "artifacts"
await worker.export_build_log(plan.export_build_log_to)
# Capture before queries, since answering may change the
# workspace. Dependents receive construction state.
if plan.export_workspace_to is not None or plan.case_id in reused_case_ids:
operation = "artifacts"
destination = plan.export_workspace_to or snapshot_root / f"{plan.case_id}.tar.gz"
result.workspace_snapshot = await worker.export_workspace(destination)
if plan.queries:
operation = "answer"
result.queries = await worker.run_queries(list(plan.queries))
if not result.queries.get("success"):
result.error = "query failed"
result.failure_stage = operation
if plan.export_evaluation_to is not None:
operation = "artifacts"
await worker.export_evaluation(plan.export_evaluation_to)
return result
except Exception as exc: # Keep other cases running and release this worker.
result.error = format_error(exc)
result.failure_stage = operation
result.exception = {"type": type(exc).__name__, "module": type(exc).__module__}
return result
finally:
if result.error and plan.export_diagnostics_to is not None and worker.reusable:
try:
await worker.export_full(plan.export_diagnostics_to)
except Exception as exc:
result.error += f"; diagnostic export failed: {format_error(exc)}"
await self._release_worker(worker, result, slots)
async def run_plan(plan: CasePlan) -> CaseResult:
result = CaseResult(plan.case_id)
try:
result = await execute(plan, result)
except asyncio.CancelledError:
result.error = result.error or "interrupted"
raise
finally:
if on_case_finished is not None:
await on_case_finished(plan, result)
failed = not result.success
if accept_build is not None and result.build is not None:
failed = (
result.error is not None
or not accept_build(result.build)
or (result.queries is not None and not result.queries.get("success"))
)
if fail_fast and failed:
raise RuntimeError(f"case {plan.case_id!r} failed: {result.error or 'execution failed'}")
return result
for plan in plans:
tasks[plan.case_id] = asyncio.create_task(execute(plan))
tasks[plan.case_id] = asyncio.create_task(run_plan(plan))
try:
results = await asyncio.gather(*(tasks[plan.case_id] for plan in plans))
finally:

View file

@ -40,12 +40,30 @@ def _exception_details(exc: BaseException) -> dict[str, Any]:
"message": str(exc),
}
response = getattr(exc, "response", None)
status_code = getattr(response, "status_code", None)
status_code = getattr(exc, "status_code", None)
if not isinstance(status_code, int):
status_code = getattr(response, "status_code", None)
if isinstance(status_code, int):
details["status_code"] = status_code
error_number = getattr(exc, "errno", None)
if isinstance(error_number, int):
details["errno"] = error_number
causes = []
seen = {id(exc)}
current = exc
while True:
current = current.__cause__ or (None if current.__suppress_context__ else current.__context__)
if current is None or id(current) in seen:
break
seen.add(id(current))
cause = {"type": type(current).__name__, "module": type(current).__module__}
status = getattr(current, "status_code", None)
if not isinstance(status, int):
status = getattr(getattr(current, "response", None), "status_code", None)
if isinstance(status, int) and not isinstance(status, bool):
cause["status_code"] = status
causes.append(cause)
details["causes"] = causes
return details
@ -158,6 +176,9 @@ async def _run_job_on_app(app: Any, job: str, arguments: dict[str, Any]) -> dict
"metadata": response.metadata,
"error": None if response.success else str(response.answer),
}
exception = response.metadata.get("reme.job_error")
if not response.success and isinstance(exception, dict):
result["exception"] = exception
result["token_usage"] = token_usage
result["job_call_counts"] = {name: count for name, count in job_call_counts.items() if count}
return result

View file

@ -1,16 +1,78 @@
# Home plugin
Provides the `auto_bash` job through an independently enabled ReMe plugin.
Adds a self-evolution workspace, the one-time `home_prepare` job, `home_evaluate`, and
`auto_bash` through an independently enabled ReMe plugin.
Source code is grouped by workflow under `src/reme_home/preparation/` and
`src/reme_home/evaluation/`. Preparation contains its component and Step, config,
code export, transaction recovery, and benchmark data adapters. Evaluation contains
its component, Step, input validation, Docker request adapters, and artifacts with
failure classification. Shared workspace and host judge components live at the
package root.
Install it into the same Python environment as ReMe:
```bash
reme plugins install plugins/home
reme plugins install plugins/beam-judge # Use plugins/lme-judge for LongMemEval.
```
Enable it in an application config with `plugins: [home]`. The plugin then
contributes the `auto_bash` job, which can be run with
`reme start config=<your-config.yaml> job=auto_bash command=pwd`.
Enable it with the matching judge, e.g. `plugins: [home, beam-judge]`, then start the
service and call its jobs from another terminal:
```bash
reme start config=<your-config.yaml>
# In another terminal:
reme auto_bash command=pwd
```
At application startup, Home automatically adds these directories inside the
configured ReMe `workspace_dir/home/`:
```text
code/repo/
code/runtime/
datasets/training-set/cases/
datasets/debug-cases/cases/
docs/
evaluations/
sampling/plans/
tmp/
logs/
metadata/
```
ReMe's existing memory, resource, and session directories remain in place.
On first initialization, each managed directory under `home/` must be absent or
completely empty (an empty child directory still counts as content). File
conflicts, symbolic links in managed paths, and overlap with configured ReMe
directories cause startup to fail without overwriting their contents.
Home atomically publishes `home/metadata/workspace.json` after creating
the layout. It records layout schema version 1, a workspace UUID, and an aware
creation timestamp. Subsequent starts validate this marker, preserve its identity
and existing contents, and recreate missing directories. Recreating directories
does not recover missing data. Invalid or unsupported markers fail explicitly.
A process lock at `home/metadata/workspace.lock` prevents concurrent
initialization; the lock file remains on disk, but process exit releases the
lock. A competing initializer can retry after the first finishes.
Failed initialization removes only newly created empty layout directories and
temporary manifest files. Existing directories and files are preserved. After
an abrupt process termination before the marker is published, inspect any
nonempty Home directories before retrying; they are not automatically adopted.
Older flat Home workspaces are rejected without moving or deleting files; use a new
workspace for this layout. Relative input paths and `spec_path` still resolve against
the outer ReMe workspace, not `home/`.
Initialization creates the layout only: it does not install datasets, copy
candidate code or documentation, initialize Git, or generate experiment/domain
configuration, sampling state, or evaluation results.
For explicitly one-shot CLI execution, use:
```bash
reme start config=<your-config.yaml> job=auto_bash command=pwd
```
The job defaults to read-only command validation and a 60-second timeout. Set
`is_read_only=false` to disable command validation. Supplying `read_only_paths`
@ -23,3 +85,709 @@ nonzero value is supplied.
Standard output and standard error each have a 1 MiB (1,048,576-byte) limit on
all backends. Exceeding either limit terminates the command and returns a
failure with bounded partial output and a diagnostic in standard error.
## Prepare an experiment once
`home_prepare` runs explicitly after startup. It converts BEAM or LongMemEval
inputs, exports a committed ReMe runtime and selected plugins, and initializes a
candidate Git repository on branch `init`. It does not install dependencies, run
source code, build Docker images, call models, or evaluate the baseline.
Copy [examples/prepare.yaml](examples/prepare.yaml) into your workspace as
`prepare.yaml` and edit the source paths, revision and case IDs. For LongMemEval,
set `benchmark.name: longmemeval`, point `source` at its JSON file, remove
`variant`, and choose `runtime.plugins: [lme]` and `candidate.paths:
[plugins/lme/src]`. Optional `benchmark.evidence_source` accepts reviewed JSONL
session evidence. Choose nonempty, unique string `case_ids`, or explicitly set
`all_cases: true` without `case_ids`. Relative input paths are workspace-relative;
explicit absolute dataset and repository paths are read-only source inputs.
`spec_path` must resolve to a file inside the workspace. Home-relative paths and
relative traversal are rejected.
Start a service configured with `plugins: [home]` and your `workspace_dir`, then
call it from another terminal using the same service selection as other jobs:
```bash
reme start config=<your-config.yaml>
# In another terminal:
reme home_prepare spec_path=prepare.yaml
```
For explicitly one-shot local execution:
```bash
reme start config=<your-config.yaml> job=home_prepare spec_path=prepare.yaml
```
Python uses the same Job and normal Application lifecycle; see
[examples/prepare.py](examples/prepare.py). Pass exactly one of `spec_path` or
`spec` (a configuration dictionary):
```python
from reme.application import Application
app = Application(workspace_dir="/experiments/run-1", plugins=["home", "beam-judge"], service={"backend": "cli"})
try:
await app.start()
response = await app.run_job("home_prepare", spec=preparation_config)
if not response.success:
raise RuntimeError(response.answer)
finally:
await app.close()
```
All prepared outputs live under `workspace_dir/home/` (paths below are relative to `home/`):
- `domain_spec.yaml`: resolved configuration, including actual selected case IDs
and the full source commit SHA.
- `datasets/training-set/`: normalized Case/Session/Query JSON and a
manifest with case IDs and content fingerprints.
- `datasets/debug-cases/`: deterministic small smoke cases with source case IDs
in their metadata, or an empty `cases/` directory when debug derivation is disabled.
- `code/runtime/`: fixed committed ReMe sources, packaging files and configurations,
without plugin projects. Uncommitted source edits are excluded.
- `code/repo/`: an independent Git repository containing complete selected plugin
projects (including packaging and configuration) and any selected ReMe core paths,
retaining repository-relative locations. `candidate.paths` defines the editable
scope, not the complete file inventory. The initial commit preserves the baseline.
Future runners assemble an isolated evaluation tree from runtime plus repo,
excluding `.git`; `install_projects` paths are relative to that tree. Install
plugins from the candidate, and never edit the fixed runtime.
- `docs/architecture-guide.md` and `docs/validation-artifact-guide.md`: guides
shipped with the installed Home plugin, covering candidate development and
investigation of evaluation artifacts.
- `metadata/preparation.json`: completion record, config/data/code
fingerprints, initial candidate commit, plugin and installation plan.
Before preparation, code and docs directories must be empty, dataset directories may
contain only their empty `cases/` skeleton, and `domain_spec.yaml` must be absent.
Candidate paths must be present in the selected source, cannot overlap, and may
select ReMe core or plugin files. Runtime plugins are selected explicitly; the host plugin list is not copied. All selected source files must be regular Git blobs; symlinks and
submodules are rejected. Source references are explicit Git revisions; this
version does not support uncommitted working-tree snapshots or remote fetching.
Preparation holds the same process lock as layout initialization. Inputs are
staged beneath `home/metadata/preparation-transaction/`, flushed, and
published with a durable rollback journal. Reported preparation failures restore
pre-existing files and directories (filesystem timestamps are not part of
the contract); the corrected request can be retried. Interrupted transactions
are recovered at startup or before another preparation. A completed preparation
rejects **every** later call, including identical parameters. There is no reset or
force-overwrite option: use a new workspace for another experiment.
## Evaluate prepared candidates
`home_evaluate` runs committed candidate code through
[the shared Docker pool](../../deploy/benchmark_docker/README.md), then scores
answers through the host Application's `answer_judge` Job. Enable `beam-judge`
for BEAM or `lme-judge` for LongMemEval. Home requires exactly one supported judge
plugin; enabling additional benchmark judge plugins is an error. On a prepared
workspace, startup checks that the judge matches the recorded benchmark. On a
new workspace, `home_prepare` checks the match before preparing data or code.
A disabled or replaced `answer_judge` Job is rejected.
Install ReMe's `benchmark-docker` extra and build the documented base image first.
The host and containers need the configured LLM/embedding environment; the host
judge additionally uses `JUDGE_MODEL_NAME`. Install the corresponding judge plugin
on the host. Its code and prompts come from the host installation, never the
candidate repo; runs record a fingerprint of the judge code and configuration. Credentials are never written into
the result manifest. No initial evaluation is triggered by `home_prepare`.
Start a service with Home enabled, then call from another terminal:
```bash
reme start config=<your-config.yaml>
# In another terminal: all prepared training cases, all their queries
reme home_evaluate
```
Programs inject selections directly, without putting case IDs in the Job YAML:
```python
# app is already started with plugins=["home", "beam-judge"].
result = await app.run_job("home_evaluate", case_ids=["1", "2"], concurrency=4)
# Recover a prior pre-query memory snapshot, skip construction, answer and score again.
replayed = await app.run_job(
"home_evaluate",
case_ids=["1"],
reuse_construction_from="evaluations/init/<commit-prefix>/<run-id>",
)
```
Omitted/null `case_ids` selects the entire installed dataset, including when a
reuse source is supplied. Empty, duplicate and unknown selections fail before
Docker starts. Optional `query_ids` maps case IDs to nonempty query-ID lists;
omitted entries run all queries for that case. Evaluation always uses the current
HEAD of `home/code/repo` and records its commit in the results. Run the baseline
before modifying the prepared candidate. Uncommitted modifications, staged changes
and untracked files reject evaluation before Docker starts, as do changed fixed
runtimes/datasets.
The source snapshot combines fixed ReMe with complete plugins from the selected
candidate commit; candidate core roots replace their baseline roots, including
deleted files. Source snapshots exclude `.git` and credentials.
`reuse_construction_from` must identify a completed training-set run inside this
Home's `evaluations` directory, either absolutely or relative to `home/`. There
is no `eval_only` toggle. Every selected case must have a successful construction
and a compatible, checksum-verified tar; any missing/invalid case rejects the
entire request without silently skipping or rebuilding it. Query-only replay
uses current candidate code over the donor's memory, not donor answers/scores.
The caller must establish construction equivalence when choosing reuse; this
Job does not invoke a code-equivalence review agent. Runs record both commits.
Results use `home/evaluations/<branch>/<commit-prefix>/<ascending-run-id>/`:
```text
manifest.json
summary.json
cases/<case-id>/
memory_construction/
result.json
build.log
reme_workspace.tar.gz
reme_workspace/
queries/
result.json
<query-id>/answer.log
```
Structured scores in query results and summaries use at most four decimal places.
Raw judge text and reference answers are preserved.
The memory tar is exported **before** queries, retains session Git checkpoints,
and is independent of the query execution state. Replayed runs keep an
independent donor tar and inherited build log without charging construction
tokens again. Answers are persisted before host judging. Per-query results keep
reference/actual answers, scores, compact judge output, reported answer token usage
and failures. `queries/result.json` is the sole authoritative answer record;
reconstructable prompts and repeated payloads are removed using the reviewed
`dev_meta` rules. Construction results similarly compact redundant links, Job
counts, token records and cumulative daily-index snapshots. Summary
scores use the expected-query denominator; infrastructure/judge/artifact errors
make the comparable mean unavailable while retaining the observed-score mean.
The Job returns `success=false` for infrastructure/judge/artifact failures, while
preserving its structured answer and result directory; candidate failures remain
ordinary evaluation observations.
Base jobs preserve structured exception types, HTTP status codes and causes in
`Response.metadata["reme.job_error"]`; the Docker worker carries them into failure
artifacts. Network/provider failures anywhere in the cause chain invalidate the
comparison. Failed jobs without structured error details (including older
candidates) are conservatively treated as infrastructure errors, without parsing
their error text. Successful jobs returning empty answers still count as candidate
failures. The construction failure-rate tolerance applies only to candidate
errors: infrastructure or unclassified failures always reject construction, even
when another failed job was a candidate error.
The manifest retains run identity, selection, mode and execution status. The
summary retains only outcome counts, expected/scored query counts, score means,
low-score cases, question-type statistics and compact case/query mappings.
Candidate answer token summaries average only total tokens per model; detailed
construction costs remain in construction results. Temporary post-query archives
are discarded after extracting logs; separate execution, checksum and raw-answer
JSON copies are not saved. See the
[artifact guide](src/reme_home/runtime_docs/validation-artifact-guide.md) for the
exact fields, rebuild helper and migration from the previous verbose format.
Before comparing runs, verify completed scored execution, infrastructure outcomes,
candidate code and complete case/query coverage.
Run/case/query `failure.json` files preserve diagnostics. Interrupted/fail-fast
runs publish `partial_summary.json`, not a completed summary. New executions
always allocate a new run; they never overwrite previous results. For scored runs, default
`fail_fast=false` keeps independent cases running. The shared pool executes each
case's build followed by queries, rather than Meta-ReMe's global construction
barrier/query-stealing schedule. `max_construction_failure_rate` defaults to the
old strict-below-3% rule; other pool callers retain strict build-success behavior.
`debug=true` selects prepared debug cases and runs smoke inputs without a judge.
It defaults to `fail_fast=true`; scored runs (`debug=false`, the default) use
`fail_fast=false`. An explicitly supplied `fail_fast` overrides either default.
Debug requires successful construction and nonempty, error-free answers, and
returns `success=false` for candidate failures as well as infrastructure failures.
It cannot reuse construction. Case/query selections refer to the selected mode's
dataset; omitted selections run all its cases/queries.
Debug results retain execution counts and artifacts, but all mean scores are
null. `scored_query_count` and `query_status_counts.unscored` are zero, while
`expected_query_count` still records the expected execution count. Debug/scored
mode is recorded in the manifest.
The former `dataset_kind` input is rejected; use `debug` instead. Debug results
must not be used to assess answer correctness or candidate quality. No real
Docker/model evaluation is required by Home's isolated unit tests.
Only Home's writers cooperate with this lock. Keep the preparation target
locations exclusively owned by this task while preparing and recovering; other
processes must not write there. Recovery requires a writable, functioning
filesystem; if rollback fails, its journal remains for the next attempt and the
Job reports the error. Calls with `_allowed_paths` restrictions are rejected
because preparation owns multiple workspace output locations.
## Bounded agent foundation
`reme_home.agent.BoundedAgentWrapper` inherits ReMe's `AsAgentWrapper` and is
registered as the `home_bounded` agent-wrapper backend when Home is enabled.
It reuses ReMe's Job tools, hidden `injected_job_kwargs`, `tool_context_id`,
built-in tools, permissions, model configuration, skills, token accounting and
persisted sessions. Home configures a wrapper for the single-case investigation
Job below; no investigation or optimization runs automatically. Configure other
agents explicitly, alongside the required model and Home plugins:
```yaml
components:
agent_wrapper:
investigator:
backend: home_bounded
as_llm: default
react_config:
max_iters: 40
time_budget_minutes: 30 # Optional advisory budget; null disables it.
job_tools: [home_evaluate]
injected_job_kwargs:
case_ids: ["1"] # Hidden from the model and enforced by the host.
```
Call `reply()` or `reply_stream()` through the normal ReMe wrapper interface;
call-time options override component defaults. Each built agent owns its own
timer and request state. The internal `BoundedReActAgent` still subclasses the AS
runtime to implement loop hooks; callers use `BoundedAgentWrapper` for ReMe wiring.
The lower-level `BoundedReActAgent` and `RunTimeBudgetTimer` exports remain available.
Each reply invocation starts and stops its advisory timer, including on errors and
cancellation. Iteration reminders occur at 50%, 75%, 90% and 95%; coincident or
skipped thresholds produce one reminder. Time reminders occur at 60%, 75%, 90%,
100%, then each additional 10%, alternating temporary and retained messages. Time
budgets do not cancel model calls or tools. Completed context replacements are
counted in time reminders. Reminders use native AS hint blocks, preserving the
reply's cumulative token usage across reasoning and tool calls. Use separate
agent instances for concurrent work.
The AgentScope loop owns the iteration cap and its single finalization call.
For text reports, Home removes all tools from that call, retains the original
task, and trims oldest messages if needed to fit the context budget. The final
report is still attempted if its preceding context compression fails; ordinary
compression errors outside finalization and cancellation continue to propagate.
The final response is buffered until complete, including for streaming models; unsolicited
tool calls are discarded. If finalization fails or returns no text, the reply
explicitly identifies any retained prior model output as incomplete. Cancellation
and provider interruption keep AgentScope's interruption behavior. Native
structured-output requests retain the SDK's schema and grace-iteration behavior.
Events and final-message token usage retain AgentScope's existing semantics.
## Candidate versions and code inspection
Home registers the `home.code` Component and ten ordinary Step-backed Jobs. They
operate only on the prepared `home/code/repo`, sharing the preparation/evaluation
process lock. Enable Home and prepare the workspace before calling them.
| Job | Inputs and behavior |
| --- | --- |
| `home_code_status` | Current branch, full commit SHA, exact changed paths and pending merge state. |
| `home_list_candidate_branches` | Local branches and candidates, excluding protected `init` from the candidate count. |
| `home_create_candidate_branch` | `branch_name`: fork current HEAD and switch to a new candidate. |
| `home_switch_candidate_branch` | `branch_name`: switch to an existing local branch, including `init`. |
| `home_merge_candidate_branch` | `branch_name`: merge another candidate without committing; conflicts are aborted. |
| `home_commit_candidate` | `message`: stage and commit all visible changes within prepared `candidate_paths`. |
| `home_code_diff` | Omit inputs for tracked changes against HEAD; supply `base_revision` and `revision` to compare commits. With only `base_revision`, compare the working tree against that commit. |
| `home_list_code_tree` | Optional `revision`, literal `path`, and `max_results` (default 500, maximum 2000). |
| `home_read_code_file` | Required `path`; optional `revision`, one-based `start_line` and inclusive `end_line`. |
| `home_search_code` | Required Python regex `pattern`; optional `revision`, literal `path`, and `max_results` (maximum 30). |
Start the configured service in one terminal, then use a second terminal:
```bash
reme start config=/absolute/path/to/home.yaml
```
```bash
reme home_code_status
reme home_create_candidate_branch branch_name=retrieval-routing
# Edit files under the prepared candidate_paths, then inspect and commit.
reme home_code_diff
reme home_commit_candidate message='Improve evidence routing'
reme home_evaluate debug=true
reme home_evaluate
```
Branch names must be single path-safe names without slashes. Creation, switching
and merging require a clean checkout, including untracked files and pending
merges. `init` must continue to point to the preparation's initial commit; commits
and merges into it are rejected. Detached candidates, symlinks, submodules and
changes outside `candidate_paths` are rejected by the managed mutation tools.
Merges require an explicit later commit. No remote operations, branch deletion,
amend, reset or rebase are exposed. Empty ordinary commits fail through Git.
These guarantees apply to cooperating Home Jobs; do not grant an optimizer
unrestricted shell writes or direct access to `.git` if it must obey this boundary.
Cancellation waits for already-started Git work to finish before returning.
Code-navigation Jobs resolve `revision` (default `HEAD`) to a full SHA once per
call, read raw committed objects and never execute candidate code. Dirty working
files and Git archive attributes do not affect these reads. Pin the same SHA
across a diagnostic session through ReMe's hidden injection:
```yaml
job_tools: [home_list_code_tree, home_read_code_file, home_search_code]
injected_job_kwargs:
revision: "<full-commit-sha>"
```
The wrapper hides injected fields from the model and rejects attempts to supply
them. For comparisons, configure two agent invocations or two named Job aliases
with separately pinned revisions. `HomeCodeStep` takes a fixed `operation` in
Step configuration; repository location and operation are never model-selected.
The Python entry point is `await home_code.run(operation, **arguments)`.
Reads are limited to 300 lines and 100,000 content characters, with explicit
truncation flags. Search accepts Python multiline regex, merges adjacent match
contexts, and displays at most five lines per result block and 100,000 content
characters. Files above 5 MB and searches above 100 MB are rejected; regex
matching runs in a separate process with a ten-second timeout. Use narrower
paths or line ranges to continue inspecting. These tool-operation timeouts are
independent of the bounded agent's advisory time budget.
Evaluation also checks the standalone repository identity and protected baseline
before taking its existing clean-HEAD snapshot. Formal evaluation deduplication,
automatic ancestor/donor selection and optimization scheduling remain separate
policies; these Jobs do not enable an optimizer or change evaluation selection.
## Evaluation sampling and history
`home_evaluate` is the single evaluation Job. Set `sampling=true` for adaptive
case/query selection, or leave it false for a full baseline or an explicit
host-provided `case_ids`/`query_ids` selection. `sampling=true` conflicts with
explicit selections in scored mode. `debug=true` ignores the sampling flag,
uses debug cases and never reads, writes or registers evaluation history.
The former `home_validate` Job is removed; use `home_evaluate sampling=true`.
Selection and optimization-history recording are independent. `record_history`
defaults to `true` for compatibility: a first full optimization baseline contributes
observations even with `sampling=false`. Set `record_history=false` to score code
without feeding optimization history. This still saves the manifest, scores and
result artifacts, but never registers a history entry, increments either counter
or updates sampling observations. Rebuilding history preserves that exclusion.
Debug always skips history regardless of either flag.
Evaluation parameters have distinct names and lifetimes:
| Parameter | Where it belongs | Meaning |
| --- | --- | --- |
| `sampling` | Job input, default `false` | Enable adaptive selection for this invocation. |
| `record_history` | Job input, default `true` | Record started rounds and successful results for optimization. |
| `sampling_policy` | Component defaults or host Job input | Case/query limits and weighting settings. |
Home registers these Component defaults automatically; override the policy in
application configuration when needed:
```yaml
components:
home.evaluation:
default:
backend: home_evaluation
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
```
The Component policy is immutable. Each invocation validates its own policy,
merging a partial `sampling_policy` over Component defaults and persisting the
effective values with that run. Explicit null `max_queries` removes the query
budget. The previous Component configuration key `sampling` is now
`sampling_policy`; passing a policy object as the Job's boolean `sampling` fails.
Normal ReMe Job defaults also work: set `sampling: true` and optionally
`sampling_policy` directly under `jobs.home_evaluate`. Call-time values take
precedence over same-named Job defaults; a call-time policy object replaces the
Job-default policy object, then merges with Component defaults.
With the configured service running, use a second terminal:
```bash
reme home_evaluate sampling=false # Full baseline.
reme home_evaluate sampling=false record_history=false # Standalone full evaluation.
reme home_evaluate sampling=true # Adaptive formal evaluation.
reme home_evaluate debug=true sampling=true # Debug only; no history feedback.
```
Host Python callers can override the policy for one invocation without changing
the next invocation's defaults:
```python
result = await app.run_job(
"home_evaluate",
sampling=True,
sampling_policy={"max_cases": 3, "max_queries": 12},
)
# Score specified prepared cases without affecting the optimizer's history.
standalone = await app.run_job(
"home_evaluate",
sampling=False,
record_history=False,
case_ids=["case-a", "case-b"], # Omit to run the full prepared dataset.
)
```
`sampling_policy` and explicit case/query IDs stay out of the default agent tool
schema. An optimizer that must obey a fixed budget should receive hidden,
enforced values through the normal wrapper injection:
```yaml
job_tools: [home_evaluate]
injected_job_kwargs:
sampling: true
record_history: true
sampling_policy:
max_cases: 3
max_queries: 12
```
The wrapper hides injected parameters and rejects model attempts to provide
those keys. Use ordinary call arguments when the host should decide the mode
per invocation; use injection when that decision must remain fixed for an agent.
Case/query selection retains dev_meta b53048e4's EWMA loss, staleness, observation
uncertainty, un-rechecked error priority, normalized adaptive/uniform mixture,
weighted drawing without replacement and fair-share query budget allocation.
Policy defaults are unchanged. Positive integer case/query limits, positive
finite temperature, alpha in `(0, 1]` and uniform exploration in `[0, 1]` are
validated. Seeds, draw weights, eligibility and exact selections are saved.
An explicit construction donor uses the existing dev_meta eligibility rules and
checksum checks. A later full rebuild considers the full training set again.
### Successful evaluation history
`reme_home.evaluation.history.HomeEvaluationHistory` is the dedicated
`home.evaluation_history` Component, automatically bound by `HomeEvaluation`.
It owns successful-result registration, lookup, deduplication, cache rebuilding
and the observations used for the next draw. Steps remain invocation adapters;
they do not own cross-evaluation state.
A successful history entry requires `record_history=true` and a started, completed, scored training-set
run with a full summary and no infrastructure errors, following the existing
`Response.success` contract. Low or zero quality scores remain valid observations.
Classified candidate failures retain the existing response semantics; only their
actual numeric scores train EWMA. A run with any infrastructure error is entirely
excluded, including scores from its healthy cases. Debug, cancelled, fail-fast
and pre-start failures never enter successful history. A full summary is the durable
completion boundary: cancelling the caller during subsequent history bookkeeping
does not relabel that already-completed evaluation. Failed registration discards
only derived caches so they can rebuild from the final source run status.
```python
history = app.context.components["home.evaluation_history"]["default"]
results = history.list_results() # Typed EvaluationRecord objects, chronological.
result = history.get_result(results[-1].run_key)
history.rebuild() # Rebuild sampling observations from saved artifacts.
```
The Component owns the workspace lock for these synchronous filesystem methods;
async host code can run them in a worker thread. Successful results are discovered
from each run's manifest and summary; no `history.json` index is written or read.
Legacy index files are ignored and left untouched. Each returned record points to
its run via `run_key`; questions, answers and memory files stay in that result directory.
The separate startup ledger advances only after the first successfully initialized
sandbox, including when a later worker or execution fails. Extra/replacement
workers do not count twice. `home/sampling/state.json` distinguishes started
`round` from `successful_round`. Only successful history advances the probability
clock or changes EWMA; failed runs cannot age cases or alter the next draw's
weights. Started plans live under
`home/sampling/plans/<branch>/<commit-prefix>/<run-id>.json`. Their `round` is the
startup ledger round; `selection_round` records the successful-history snapshot
used for an adaptive draw.
For `record_history=false`, full or explicit evaluation does not access the history
manager. Adaptive selection can still read existing observations, while leaving
them unchanged; its draw is saved locally as `manifest.selection_plan`, without
a startup-ledger round or shared sampling plan. `manifest.options.record_history`
persists the requested setting, and the response's `record_history` indicates
whether the run participates in history (always false for debug). Legacy manifests
without the setting retain the previous history behavior.
Formal runs remain serialized by `home/metadata/validation.lock`; debug bypasses
that lock. Missing caches reconstruct from started plans and successful scored
artifacts. Older sampling caches are rebuilt under the success-only rule.
`history.rebuild()` rewrites only the catalog and sampling cache. Preserve source
manifests, plans and result artifacts when rebuilding; candidate code, datasets
and memory files are never modified.
## Evaluation result inspection
Two read-only Jobs navigate saved evaluations independently of successful
optimization history. They include debug, failed, cancelled, running and standalone
(`record_history=false`) runs. Reads never register results, update sampling,
rebuild summaries or write artifacts.
`home_list_validation_results` accepts optional `branch_name` and `commit_sha`
filters plus `offset=0`, `limit=20` (1–100). Without filters it lists all runs,
newest first, with a stable path tie-breaker. It returns `row_count`, `rows` and
`next_offset`, compact identity/options, saved summary counts and scores, failure
previews and artifact paths. A malformed run is reported as a `read_error` row
without stopping the unfiltered catalog.
`commit_sha` accepts a full SHA or hexadecimal prefix. A prefix must match exactly
one of **all commits reachable from the selected local branch**, including commits
that have never been evaluated. Omitting `branch_name` with a commit filter uses
the current candidate branch. Missing or ambiguous matches fail explicitly; the
response includes the resolved branch and full SHA. A branch-only filter can also
browse historical runs from a deleted branch.
`home_inspect_validation` requires `validation_path` (absolute or relative to
`home/`) and accepts optional `case_id`, `query_id`, `offset=0` and `limit=20`:
| Selection | Returned rows |
| --- | --- |
| Run only | Case summaries, missing scores first, then low scores. |
| Run and case | Query summaries in manifest order, plus independently paged failed construction Jobs. |
| Run, case and query | One query's question, answer, rubric, score and judge/error previews. |
`query_id` requires `case_id`; both must be selected by the run manifest. Cases or
queries without published evidence are explicitly marked `missing`, with null
scores. The tool uses a saved full summary, or an existing partial summary when
the full summary is absent. Missing summaries are reported without reconstruction.
Errors are previewed to 600 characters, and selected query fields to 2,000 each;
truncated fields carry a marker. `rows_source`, JSON `pointer` values, and log and
memory paths identify the original evidence. Failed construction Jobs have their
own `row_count`, `rows`, `next_offset` under `construction.failed_jobs`; their
pagination is independent of the query rows, using the supplied offset/limit.
With the configured service running, use a second terminal:
```bash
reme home_list_validation_results branch_name=candidate commit_sha=abc123 limit=10
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1
reme home_inspect_validation validation_path=evaluations/candidate/abc1234/1 case_id=case-a query_id=query-a
```
Host callers use the same Jobs through `app.run_job()`. For a case-scoped agent,
inject `validation_path` and `case_id` with `injected_job_kwargs`; the wrapper hides
and enforces these values. Both Jobs reject path escapes and symlinks, and fail
closed for restricted `_allowed_paths` contexts, like the other Home Jobs.
### Memory checkpoint inspection
Read-only Jobs investigate a case's exported memory Git repository. Locate
an evaluation and case with `home_list_validation_results` and
`home_inspect_validation` first. These Jobs never change artifacts, Git refs, the
index, evaluation history or sampling state.
- `home_trace_query_evidence(validation_path, case_id, query_id,
evidence_session_ids=null)` joins a selected query to its source sessions and
memory checkpoints without calling a model. Dataset evidence annotations take
precedence over optional inferred session IDs, including an empty annotation.
Returns `query` (question, expected/actual answer, score, error and
`result_missing`), `evidence_source` (`independent_annotation`,
`diagnostician_inference`, or `unavailable`), `evidence_is_certain`,
`memory_status`, `sessions` (ID, checkpoint and changed files), and `paths` to
the normalized case, memory repository, query results and query artifacts.
A null checkpoint means no matching `session: <id>` commit was found; missing
results keep scores null. Answer/question previews are limited to 2,000
characters and errors to 600. At most 50 sessions and 50 changed files per
session are returned, with omitted counts when needed. Session text, diffs,
later history and full answer/judge results remain at the returned paths or
in the inspection/comparison Jobs below. Original transcripts and derived
memories can both be searchable; these coordinates alone do not prove a cause.
- `home_inspect_memory_history(validation_path, case_id, path=null, offset=0,
limit=20)` returns `status`, `workspace_exists`, `has_git_history`, `head`,
`repository`, and newest-first checkpoint `rows` with `row_count` and
`next_offset`. Limits are 1–100. Each row includes the full commit SHA, parents,
timestamp, subject (up to 2,000 characters) and a `session_id` when the subject
follows `session: <id>`. `path` follows a literal workspace-relative file or
directory. Missing exports, absent Git history and empty repositories have
distinct statuses; corrupt repositories fail explicitly.
- `home_compare_memory(validation_path, case_id, base_ref=null,
target_ref="HEAD", target_validation_path=null, path=null)` compares two memory
commits. Within one evaluation, `base_ref` is required. For another evaluation,
supply `target_validation_path`; both sides then default to `HEAD`. The case
must be selected in both manifests. Returns resolved commits, candidate commit
identities, manifest fingerprints, added/removed/modified paths, content-tree
hashes, a unified `diff`, and `truncated`.
References accept HEAD-relative expressions and hexadecimal SHA prefixes unique
among commits reachable from that memory repository's HEAD. Resolved expressions
must also be HEAD-reachable. This is independent of candidate-code branch history.
Comparisons read committed regular files from Git objects, ignoring dirty or
untracked checkout contents. File mode changes alone are not content differences.
Binary changes are identified without printing their bytes. Each snapshot is
limited to 2,000 files, 20 MB total and 5 MB per file; diff output is limited to
100,000 characters. Narrow `path` if a limit is reached. Tree hashes describe the
selected path scope. Linked Git storage, alternates, shallow/partial histories and
symbolic links in Git metadata are rejected.
After starting the service with the Home configuration, call from another terminal:
```bash
reme home_inspect_memory_history validation_path=evaluations/candidate/abcdef0/1 case_id=case1
reme home_trace_query_evidence validation_path=evaluations/candidate/abcdef0/1 case_id=case1 query_id=query1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 case_id=case1 base_ref=HEAD~1
reme home_compare_memory validation_path=evaluations/candidate/abcdef0/1 target_validation_path=evaluations/candidate/1234567/1 case_id=case1
```
`validation_path` is absolute or relative to `home/`; for example the relative
form is `evaluations/candidate/abcdef0/1`. For agent tools, host `injected_job_kwargs` can pin
`validation_path` and `case_id`; the wrapper hides injected fields and rejects
model overrides. Like the other Home inspection Jobs, these Jobs reject
`_allowed_paths` restrictions rather than bypassing them. Use these tools directly
or through the single-case investigator below.
## Single-case investigation
`home_case_study(validation_path, case_id, focus=...)` runs one fresh diagnostic
agent. It follows dev_meta's case-study workflow: trace judging and answer
failures backward through retrieval, memory construction, source sessions and
the evaluated implementation. The text report cites evidence paths/lines or
JSON fields and distinguishes supported causes, hypotheses and missing evidence;
it does not prescribe fixes. There is no batch fan-out or parallel case-study
dispatcher. Each invocation has its own agent state and timer.
The `home_case_study` component uses `home_case_study_agent`, a thin subclass of
the bounded wrapper, defaulting to
the host's existing `as_llm: judge` model, 60 iterations and a 30-minute advisory
budget, matching dev_meta's default budgets. Configure a separate host model by
overriding `components.agent_wrapper.home_case_study.as_llm`. ReMe owns model
startup, token accounting and saved session transcripts. The response answer is
the report; metadata includes the validation, case, evaluated commit, session ID
and `usage`. No case scratch directory or intermediate analysis files are created.
The toolkit includes validation inspection, compact query-evidence tracing,
memory history/comparison, and the three committed-code navigation Jobs.
Validation/case selectors and the evaluated code revision are hidden and enforced
by the host; memory comparison stays within the selected evaluation. General
Read/Grep/Glob, Write/Edit, validation and code-mutation tools are not exposed.
`AutoBash` and `python_execute` share a required OS sandbox. Commands have no
static whitelist (`is_read_only=false`); filesystem permissions enforce access.
Read-only roots are the selected case artifact directory, normalized source-case
JSON, validation manifest, all of `home/code/`, and `home/docs/`. No host writable
roots are granted. System/interpreter runtime paths and disposable backend
temporary storage remain available. Direct code reads can see current worktree
changes; use the revision-pinned Jobs to establish the evaluated behavior.
`python_execute` accepts either `code` or an existing `file`, optional file-only
`args`, and a `timeout` in seconds (default 60, maximum 600). It uses `python -I -B`
and returns returncode/stdout/stderr/timed_out. Both executors reuse bounded output,
timeouts, cancellation cleanup and network isolation from the platform backend.
Linux requires Bubblewrap and, with the default 4096-MiB limit, a writable
delegated cgroup v2. macOS requires Seatbelt (`sandbox-exec`); memory limits are
not enforced there. The Job probes sandbox startup before calling the model and
never falls back to an unrestricted executor. The command whitelist parser is
not used by this agent. Sandbox policy, output-report token guidance and executor
memory policy are host Step configuration, not model arguments:
```yaml
jobs:
home_case_study:
steps:
- backend: home_case_study_step
agent_wrapper: home_case_study
output_token_limit: 2500
# memory_limit_mib: 0 # Explicitly disable Linux cgroup memory limiting if needed.
```
With the configured service running, call from another terminal:
```bash
reme home_case_study validation_path=evaluations/candidate/abcdef0/1 case_id=case1 'focus=Trace low-scoring queries'
```
Calls with `_allowed_paths` restrictions fail closed, as other Home artifact
Jobs do. Structured inspection tools run in the host with their existing
containment checks; only Bash/Python subprocesses run inside the OS sandbox.

View file

@ -0,0 +1,28 @@
"""Start a local Home application and prepare once: python prepare.py WORKSPACE."""
import asyncio
from pathlib import Path
import sys
from reme.application import Application
async def main(workspace: str):
"""Read WORKSPACE/prepare.yaml; a dictionary can instead be passed as spec=."""
app = Application(
workspace_dir=str(Path(workspace).resolve()),
plugins=["home"],
service={"backend": "cli"},
)
try:
await app.start()
response = await app.run_job("home_prepare", spec_path="prepare.yaml")
if not response.success:
raise RuntimeError(response.answer)
print(response.answer)
finally:
await app.close()
if __name__ == "__main__":
asyncio.run(main(sys.argv[1]))

View file

@ -0,0 +1,17 @@
benchmark:
name: beam
variant: 100K
source: /data/BEAM
case_ids: ["1", "3", "8"]
# For an explicit full selection, remove case_ids and set all_cases: true.
runtime:
source_repo: /projects/ReMe
source_ref: HEAD # Resolved once to a full commit SHA; uncommitted edits are excluded.
config: benchmark.yaml
plugins: [beam]
candidate:
paths: [plugins/beam/src]
debug:
enabled: true
sessions_per_case: 3
messages_per_session: 6

View file

@ -1,7 +1,7 @@
[project]
name = "reme-home"
version = "0.1.0"
description = "Local command execution jobs for ReMe."
description = "Self-evolution workspace preparation and local command execution jobs for ReMe."
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
@ -17,7 +17,7 @@ where = ["src"]
include = ["reme_home*"]
[tool.setuptools.package-data]
reme_home = ["plugin.yaml"]
reme_home = ["plugin.yaml", "runtime_docs/*.md", "agent/*.yaml"]
[tool.pytest.ini_options]
asyncio_default_fixture_loop_scope = "function"

View file

@ -1,4 +1,4 @@
"""Home plugin: local command execution jobs for ReMe."""
"""Home plugin: self-evolution workspace and local command execution jobs."""
from .auto_bash import AutoBash, AutoBashStep

View file

@ -0,0 +1,6 @@
"""Agent foundations for Home's investigation and optimization workflows."""
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
from .wrapper import BoundedAgentWrapper
__all__ = ["BoundedAgentWrapper", "BoundedReActAgent", "RunTimeBudgetTimer"]

View file

@ -0,0 +1,308 @@
"""AgentScope agent that turns an exhausted ReAct budget into a final answer."""
from __future__ import annotations
import inspect
import logging
import math
import time
from dataclasses import dataclass
from typing import Any, AsyncGenerator, Callable
from agentscope.agent import Agent
from agentscope.message import HintBlock, Msg, TextBlock, UserMsg
from agentscope.model import ChatResponse, FinishedReason
from agentscope.tool import ToolChoice
logger = logging.getLogger(__name__)
_REMINDER_RATIOS = (0.50, 0.75, 0.90, 0.95)
@dataclass(frozen=True)
class TimeBudgetReminder:
"""One reminder plus whether it should survive the current reasoning call."""
content: str
persistent: bool
class RunTimeBudgetTimer:
"""Track one agent run and emit advisory reminders without stopping it."""
def __init__(
self,
time_budget_minutes: float,
*,
agent_label: str,
clock: Callable[[], float] = time.monotonic,
) -> None:
if (
not isinstance(time_budget_minutes, (int, float))
or isinstance(time_budget_minutes, bool)
or not math.isfinite(time_budget_minutes)
or time_budget_minutes <= 0
):
raise ValueError("time_budget_minutes must be a positive number")
if not isinstance(agent_label, str) or not agent_label.strip():
raise ValueError("agent_label must be a non-empty string")
self.time_budget_minutes = float(time_budget_minutes)
self.agent_label = agent_label.strip()
self._clock = clock
self._started_at: float | None = None
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
@property
def active(self) -> bool:
"""Return whether a run is currently being timed."""
return self._started_at is not None
def start(self) -> None:
"""Start a fresh run timer."""
self._started_at = self._clock()
self._last_reminded_ratio = 0.0
self._next_reminder_is_persistent = False
self.context_compression_count = 0
def stop(self) -> None:
"""Stop timing the current run."""
self._started_at = None
def record_context_compression(self) -> None:
"""Record one completed context compression in the current run."""
self.context_compression_count += 1
def next_reminder(self) -> TimeBudgetReminder | None:
"""Return the latest newly reached soft-budget reminder."""
if self._started_at is None:
return None
elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
elapsed_ratio = elapsed_minutes / self.time_budget_minutes
reached_ratio: float | None = None
if elapsed_ratio >= 1.0:
reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
else:
for threshold in (0.90, 0.75, 0.60):
if elapsed_ratio >= threshold:
reached_ratio = threshold
break
if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
return None
self._last_reminded_ratio = reached_ratio
persistent = self._next_reminder_is_persistent
self._next_reminder_is_persistent = not persistent
elapsed = _format_minutes(elapsed_minutes)
percentage = round(reached_ratio * 100)
return TimeBudgetReminder(
content=(
f"<system-reminder>The {self.agent_label} run has used {elapsed} ({percentage}% of its "
f"{_format_minutes(self.time_budget_minutes)} soft time budget). "
f"The run has completed {self.context_compression_count} context compressions. "
"This reminder does not terminate the run. Prioritize the highest-value remaining investigation and "
"produce the required final report as soon as the evidence is sufficient.</system-reminder>"
),
persistent=persistent,
)
def _format_minutes(minutes: float) -> str:
"""Format a timer duration compactly."""
if minutes < 1:
return f"{max(0, round(minutes * 60))} seconds"
rounded = round(minutes, 1)
value = int(rounded) if rounded.is_integer() else rounded
unit = "minute" if value == 1 else "minutes"
return f"{value} {unit}"
def _remove_hint_by_identity(messages: list[Msg], target: HintBlock) -> None:
"""Remove only the temporary hint, retaining the reply message and its usage."""
for message in messages:
for index, block in enumerate(message.content):
if block is target:
del message.content[index]
return
class BoundedReActAgent(Agent):
"""Add advisory budgets and safe finalization to AgentScope's bounded loop.
AgentScope owns iteration accounting, tool execution, events and cancellation.
Its one final reasoning call after ``max_iters`` receives no tools. Native
structured-output requests retain AgentScope's schema and grace-iteration rules.
Instances, like AgentScope agents, must not serve concurrent replies.
"""
def __init__(self, *args: Any, run_time_budget_timer: RunTimeBudgetTimer | None = None, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.run_time_budget_timer = run_time_budget_timer
self._sent_iteration_reminders: set[int] = set()
self._original_task_prompt = ""
async def _reply_impl(self, inputs: Any = None, **kwargs: Any) -> AsyncGenerator[Any, None]:
"""Own the advisory timer for this invocation, including error cleanup."""
self._sent_iteration_reminders.clear()
if isinstance(inputs, Msg):
self._original_task_prompt = inputs.get_text_content() or ""
elif isinstance(inputs, list):
self._original_task_prompt = "\n\n".join(message.get_text_content() or "" for message in inputs)
elif inputs is None:
# ReMe observes inputs before reply(); retain that task for finalization.
self._original_task_prompt = next(
(
message.get_text_content() or ""
for message in reversed(self.state.context)
if message.role == "user"
),
"",
)
timer = self.run_time_budget_timer
if timer is not None:
timer.start()
try:
async for item in super()._reply_impl(inputs=inputs, **kwargs):
yield item
finally:
if timer is not None:
timer.stop()
async def compress_context(self, *args: Any, **kwargs: Any) -> None:
"""Count successful compressions; allow final reports after compression failure."""
summary_before, context_before = self.state.summary, self.state.context
try:
await super().compress_context(*args, **kwargs)
except Exception as exc:
if not self._is_finalizing():
raise
logger.warning(
"Compression before the final report failed for Home agent %s; attempting finalization: %s",
self.name,
type(exc).__name__,
)
return
if self.run_time_budget_timer is not None and (
self.state.summary is not summary_before or self.state.context is not context_before
):
self.run_time_budget_timer.record_context_compression()
async def _reasoning(self, *args: Any, **kwargs: Any) -> AsyncGenerator[Any, None]:
"""Deliver each iteration threshold once and alternate time reminders."""
timer = self.run_time_budget_timer
reminder = timer.next_reminder() if timer is not None else None
reminder_hint = None
if reminder is not None:
reminder_hint = HintBlock(source="home_run_time_guard", hint=reminder.content)
self.state.append_context(self.name, [reminder_hint])
completed = self.state.cur_iter
thresholds = {math.ceil(self.react_config.max_iters * ratio) for ratio in _REMINDER_RATIOS}
reached = thresholds - self._sent_iteration_reminders
reached = {threshold for threshold in reached if threshold <= completed}
if reached:
remaining = max(0, self.react_config.max_iters - completed)
self.state.append_context(
self.name,
[
HintBlock(
source="home_iteration_guard",
hint=(
f"<system-reminder>You have completed {completed} reasoning-acting iterations; "
f"{remaining} iteration(s) remain.</system-reminder>"
),
),
],
)
self._sent_iteration_reminders.update(reached)
try:
async for item in super()._reasoning(*args, **kwargs):
yield item
finally:
if reminder is not None and not reminder.persistent and reminder_hint is not None:
_remove_hint_by_identity(self.state.context, reminder_hint)
def _is_finalizing(self) -> bool:
"""Only the iteration cap triggers finalization; elapsed time is advisory."""
return self.state.cur_iter >= self.react_config.max_iters and self.state.reply_context.structured_schema is None
async def _call_model(self, messages: list[Msg], tools: list[dict], tool_choice: ToolChoice | None = None):
"""Use the native finalization slot; never start a second synthesis call."""
if not self._is_finalizing():
return await super()._call_model(messages=messages, tools=tools, tool_choice=tool_choice)
last_text = next(
(
message.get_text_content()
for message in reversed(self.state.context)
if message.role == "assistant" and message.name == self.name and message.get_text_content()
),
"(no previous displayable model response)",
)
messages = list(messages)
messages.append(
UserMsg(
name="home_iteration_guard",
content=(
"The iteration budget is exhausted. Return the best final report from collected evidence. "
"Do not call tools or propose more investigation. Distinguish findings from uncertainty.\n"
f"Original task: {self._original_task_prompt or '(see the conversation above)'}"
),
),
)
response_usage = None
try:
limit = int(self.model.context_size * self.context_config.trigger_ratio)
while await self.model.count_tokens(messages=messages, tools=[]) >= limit:
if len(messages) <= 2:
raise ValueError("Final instructions exceed the model context budget")
del messages[1]
response = await super()._call_model(messages=messages, tools=[], tool_choice=ToolChoice(mode="none"))
response = await self._consume_final_response(response)
response_usage = response.usage
if response.finished_reason == FinishedReason.INTERRUPTED:
return response
content = [block for block in response.content if isinstance(block, TextBlock) and block.text.strip()]
if not content:
raise ValueError("Final model response contains no displayable answer")
# Ignore unsolicited tool calls, even when accompanied by valid text.
response.content = content
return response
except Exception as exc:
logger.exception("Final report failed for Home agent %s", self.name)
return ChatResponse(
content=[
TextBlock(
text=(
f"Reached the maximum of {self.react_config.max_iters} reasoning-acting iterations. "
f"Final report generation failed ({type(exc).__name__}); "
f"the following is incomplete prior model output:\n\n{last_text}"
),
),
],
is_last=True,
usage=response_usage,
)
@staticmethod
async def _consume_final_response(response: ChatResponse | AsyncGenerator[ChatResponse, None]) -> ChatResponse:
"""Require a completed response for streaming and non-streaming models."""
if isinstance(response, ChatResponse):
return response
if not inspect.isasyncgen(response):
raise TypeError(f"Unexpected final response: {type(response)}")
completed = None
try:
async for chunk in response:
if chunk.is_last:
completed = chunk
finally:
await response.aclose()
if completed is None:
raise ValueError("Final response stream ended without a completed response")
return completed

View file

@ -0,0 +1,209 @@
"""One case-local investigation using ReMe Jobs and sandboxed command tools."""
import sys
from pathlib import Path
from agentscope.tool import FunctionTool, Toolkit
from pydantic import Field, field_validator
import yaml
from reme.steps.base_step import BaseStep, Ref
from reme.utils.file_sandbox import create_file_sandbox_backend
from ..auto_bash import AutoBash
from ..code.repository import resolve_revision, validate_repository
from ..evaluation.artifacts import contained
from ..evaluation.component import blocking
from ..evaluation.evidence_step import _source_case
from ..evaluation.inputs import entity_id
from ..evaluation.memory_inspection_step import _repository
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace
from .wrapper import BoundedAgentWrapper
PROMPT = Path(__file__).with_suffix(".yaml")
CASE_TOOLS = (
"home_inspect_validation",
"home_trace_query_evidence",
"home_inspect_memory_history",
"home_compare_memory",
)
CODE_TOOLS = ("home_list_code_tree", "home_search_code", "home_read_code_file")
class CaseStudyInput(SpecModel):
"""Select one existing validation case; runtime and permissions are host-owned."""
validation_path: str = Field(min_length=1)
case_id: str = Field(min_length=1)
focus: str = Field(
default="Investigate the failed or low-scoring queries and their earliest supported causes.",
min_length=1,
)
@field_validator("case_id")
@classmethod
def validate_case(cls, value):
"""Keep the case inside its manifest-selected artifact directory."""
return entity_id(value)
def prepare_scope(home, spec):
"""Resolve evidence and the validation's immutable candidate revision before model calls."""
run, manifest, _memory = _repository(home, spec.validation_path, spec.case_id)
case_path = contained(run, f"cases/{spec.case_id}", directory=True)
if not case_path.is_dir():
raise ValueError("Case artifacts do not exist")
source_path, _case = _source_case(home, manifest, spec.case_id)
repo, _record = validate_repository(home)
commit = manifest.get("commit_sha")
if not isinstance(commit, str) or len(commit) != 40 or any(char not in "0123456789abcdef" for char in commit):
raise ValueError("Evaluation manifest must record a full candidate commit_sha")
revision = resolve_revision(repo, commit)
roots = [
case_path,
source_path,
contained(run, "manifest.json"),
contained(home, "code", directory=True),
contained(home, "docs", directory=True),
]
return {
"validation_path": str(run),
"case_id": spec.case_id,
"case_path": str(case_path),
"candidate_commit": revision,
"code_repository": str(repo),
"source_case": str(source_path),
"docs": str(home / "docs"),
"read_only_paths": [str(path) for path in roots],
}
def python_tool(backend, cwd):
"""Run source or an existing script with the same read-only sandbox as AutoBash."""
async def python_execute(
code: str | None = None,
timeout: float = 60,
*,
file: str | None = None,
args: list[str] | None = None,
) -> dict:
"""Execute code or file, exclusively; print results to stdout. No host path is writable.
Args:
code: Python source, mutually exclusive with file.
timeout: Execution timeout in seconds, greater than zero and at most 600.
file: Existing script path, absolute or relative to the case directory; sandbox read rules apply.
args: String arguments for file mode only.
"""
if (code is not None and not isinstance(code, str)) or (file is not None and not isinstance(file, str)):
raise ValueError("code and file must be strings or null")
has_code, has_file = bool(code and code.strip()), bool(file and file.strip())
if has_code == has_file:
raise ValueError("Provide exactly one of code or file")
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0 < timeout <= 600:
raise ValueError("timeout must be greater than zero and at most 600 seconds")
if args is not None and (
not has_file or not isinstance(args, list) or any(not isinstance(a, str) for a in args)
):
raise ValueError("args must be a list of strings in file mode only")
command = [sys.executable, "-I", "-B"]
command += ["-c", code] if has_code else [str(Path(cwd) / file), *(args or [])]
result = await backend.exec_shell(command, cwd=cwd, timeout=timeout)
return {
"returncode": result.exit_code,
"stdout": result.stdout.decode(errors="replace"),
"stderr": result.stderr.decode(errors="replace"),
"timed_out": result.exit_code == -1 and result.stderr == b"timed out",
}
return FunctionTool(python_execute, is_concurrency_safe=False)
def create_toolkit(wrapper, scope, backend):
"""Expose only read-only Jobs with hidden, enforced scope and two sandboxed executors."""
tools = []
for name in CASE_TOOLS + CODE_TOOLS:
job = wrapper.app_context.jobs[name]
if not job.enabled or not job.is_started:
raise RuntimeError(f"Case study requires a started Job: {name}")
injected = (
{"revision": scope["candidate_commit"]}
if name in CODE_TOOLS
else {"validation_path": scope["validation_path"], "case_id": scope["case_id"]}
)
if name == "home_compare_memory":
injected["target_validation_path"] = None
tools.append(wrapper._make_tool(job, injected_job_kwargs=injected)) # pylint: disable=protected-access
tools.extend(
[
AutoBash(cwd=scope["case_path"], backend=backend, is_read_only=False),
python_tool(backend, scope["case_path"]),
],
)
for tool in tools:
tool.is_concurrency_safe = False
return Toolkit(tools=tools)
class HomeCaseStudyStep(BaseStep):
"""Run one fresh agent through the application's bounded wrapper and model lifecycle."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
async def execute(self):
"""Prepare scope, require isolation, then return the case report and framework token usage."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home case study requires unrestricted access to its managed artifacts")
if not self.workspace.is_started or not self.agent_wrapper.is_started:
raise RuntimeError("Start the Home workspace and case-study agent wrapper first")
spec = CaseStudyInput.model_validate(
{key: self.context[key] for key in CaseStudyInput.model_fields if key in self.context},
)
scope = await blocking(prepare_scope, self.workspace.home_path, spec)
backend = create_file_sandbox_backend(
cwd=scope["case_path"],
read_only_paths=scope["read_only_paths"],
writable_paths=[],
memory_limit_mib=self.kwargs.get("memory_limit_mib"),
)
# Fail before spending model tokens if the OS sandbox or its runtime cannot start.
probe = await backend.exec_shell(
[sys.executable, "-I", "-B", "-c", "pass"],
cwd=scope["case_path"],
timeout=10,
)
if not probe.ok():
raise RuntimeError("Case-study sandbox unavailable: " + probe.stderr.decode(errors="replace"))
toolkit = create_toolkit(self.agent_wrapper, scope, backend)
output_limit = self.kwargs.get("output_token_limit", 2500)
if isinstance(output_limit, bool) or not isinstance(output_limit, int) or output_limit < 1:
raise ValueError("output_token_limit must be a positive integer")
prompts = yaml.safe_load(PROMPT.read_text(encoding="utf-8"))
if not isinstance(prompts, dict) or any(
not isinstance(prompts.get(key), str) or not prompts[key].strip()
for key in ("system_prompt", "task_prompt")
):
raise ValueError("Case-study YAML requires nonempty system_prompt and task_prompt strings")
prompt = prompts["system_prompt"].format(**scope, output_token_limit=output_limit)
result = await self.agent_wrapper.reply(
prompts["task_prompt"].format(focus=spec.focus),
system_prompt=prompt,
toolkit=toolkit,
builtin_tools=[],
job_tools=[],
skills=[],
resume=None,
session_id=None,
fork_session=False,
output_schema=None,
permission_mode="bypass",
)
self.context.response.answer = result["result"]
self.context.response.metadata.update(
{key: scope[key] for key in ("validation_path", "case_id", "candidate_commit")},
)
self.context.response.metadata.update({key: result[key] for key in ("session_id", "usage") if key in result})
return self.context.response

View file

@ -0,0 +1,42 @@
system_prompt: |-
Investigate exactly one validation case. Trace observed failures backward from judging and answer generation through
retrieval/ranking, final memory, construction checkpoints, original sessions, and the responsible implementation.
At each layer explain what should have happened, what happened, and the supporting or contradicting evidence.
Continue to the earliest evidence-supported mechanism; "not found", "ranked too low" and "wrong answer" are symptoms.
Compare plausible explanations and state the smallest missing observation when no cause is established.
Assigned scope:
- Validation: {validation_path}
- Case: {case_id}; authoritative case artifacts: {case_path}
- Normalized source case: {source_case}
- Candidate repository: {code_repository}; evaluated commit: {candidate_commit}
- Read architecture-guide.md and validation-artifact-guide.md in {docs} when needed.
Investigate no other case. Do not modify code or evidence, run validation, execute ReMe, or prescribe fixes.
Treat construction partial_success as requiring repair even when answers succeed. Distinguish infrastructure,
candidate execution, answer-quality and judging failures without replacing their saved runtime statuses.
Use home_inspect_validation to select anomalous queries and locate result fields and logs. Start query evidence
tracing with home_trace_query_evidence. Respect evidence_source and evidence_is_certain: inferred or uncertain
session annotations are leads, not established facts. Original transcripts and derived memory may both be searchable;
absence from a summary does not prove that the answerer could not retrieve the source session.
Use home_inspect_memory_history and home_compare_memory for construction checkpoints and focused diffs.
Use home_list_code_tree, home_search_code and home_read_code_file to examine the evaluated commit; their revision
and case-tool scopes are fixed by the host. Follow returned paths for full source details and omitted evidence.
AutoBash and python_execute share an OS file sandbox. The case artifacts, source case, validation manifest,
home/docs and all of home/code are readable; no host paths are writable. System tools and interpreter libraries
are also available. No scratch directory or Write/Edit tool is provided. Commands and Python analysis are allowed
without a command whitelist, but network access and writes outside sandbox-internal temporary storage are denied.
Print analysis results instead of saving intermediate files. Python accepts source code or an existing script file.
The directly readable code worktree can differ from the evaluated commit; establish commit alignment before
attributing a failure to code. Prefer the pinned code tools when checking historical behavior.
Return a concise report, conclusion and strongest evidence first, within approximately {output_token_limit} tokens.
Name affected queries and lifecycle stages, explain the causal chain, include contrary evidence and confidence,
and separate established causes, hypotheses and unknowns. Cite every error and causal claim with an exact
path:line location or path#key.path for structured JSON. Keep citations tight. Make no unsupported cross-case claims.
task_prompt: |-
Pay special attention to:
{focus}

View file

@ -0,0 +1,42 @@
"""ReMe agent wrapper with Home's bounded investigation runtime."""
from typing import Any
from reme.components.agent_wrapper import AsAgentWrapper
from reme.components.as_llm import BaseAsLLM
from ..judge import HomeJudge
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
class BoundedAgentWrapper(AsAgentWrapper):
"""Reuse ReMe Job tools, injected arguments, permissions and persisted sessions.
``react_config.max_iters`` configures the iteration cap. Optional
``time_budget_minutes`` adds advisory reminders; each built SDK agent owns
its own timer so requests never share budget state.
"""
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
budget = options.get("time_budget_minutes")
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)
class CaseStudyAgentWrapper(BoundedAgentWrapper):
"""Validate Home's host configuration before requiring the investigation model.
A configured model and the host judge start before this wrapper. Optional
model binding preserves Home's missing-judge diagnostic; startup still
rejects an absent investigation model after the judge is validated.
"""
def __init__(self, as_llm: str = "judge", **kwargs):
super().__init__(as_llm="", **kwargs)
self.home_judge = self.bind("default", HomeJudge, optional=False)
self.as_llm = self.bind(as_llm, BaseAsLLM, optional=True)
async def _start(self):
await super()._start()
if self.as_llm is None:
raise ValueError("Case-study agent requires a configured as_llm component")

View file

@ -0,0 +1 @@
"""Home candidate version management and immutable code inspection."""

View file

@ -0,0 +1,72 @@
"""Application-owned Git work serialized with Home preparation and snapshots."""
import asyncio
from reme.components.base_component import BaseComponent
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from . import repository, snapshot
from .models import ARGUMENTS
OPERATIONS = {
"status": repository.status,
"list_branches": lambda repo, _record: repository.branches(repo),
"create_branch": repository.create_branch,
"switch_branch": repository.switch_branch,
"merge_branch": repository.merge_branch,
"commit": repository.commit,
"diff": repository.diff,
"list_code_tree": snapshot.list_code_tree,
"read_code_file": snapshot.read_code_file,
"search_code": snapshot.search_code,
}
class HomeCode(BaseComponent):
"""Manage only the prepared Home candidate; no remotes or history rewrites."""
component_type = "home.code"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
self._pending: set[asyncio.Task] = set()
async def _close(self):
if self._pending:
await asyncio.gather(*self._pending, return_exceptions=True)
async def run(self, operation: str, **kwargs) -> dict:
"""Validate before dispatch, and finish owned Git work before propagating cancellation."""
if not self.is_started:
raise RuntimeError("Start the Home application before managing candidate code")
if operation not in OPERATIONS:
raise ValueError(f"Unknown code operation: {operation}")
arguments = ARGUMENTS[operation].model_validate(kwargs).model_dump()
task = asyncio.create_task(asyncio.to_thread(self._run, operation, arguments))
self._pending.add(task)
cancelled = False
try:
while True:
try:
result = await asyncio.shield(task)
if cancelled:
raise asyncio.CancelledError
return result
except asyncio.CancelledError:
if task.done():
raise
cancelled = True
except Exception:
if cancelled:
raise asyncio.CancelledError from None
raise
finally:
self._pending.discard(task)
def _run(self, operation: str, arguments: dict) -> dict:
"""Hold the same cross-process lock used when evaluations snapshot HEAD."""
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
repo, record = repository.validate_repository(self.workspace.home_path)
return OPERATIONS[operation](repo, record, **arguments)

View file

@ -0,0 +1,68 @@
"""Strict input contracts for candidate version and code-navigation Jobs."""
from pydantic import Field
from ..preparation.spec import SpecModel
class NoArguments(SpecModel):
"""An operation with no public inputs."""
class BranchArguments(SpecModel):
"""One existing or new local candidate branch."""
branch_name: str = Field(min_length=1, max_length=100)
class CommitArguments(SpecModel):
"""Commit the complete candidate change with an explicit message."""
message: str = Field(min_length=1, max_length=10_000, pattern=r"\S")
class DiffArguments(SpecModel):
"""Omit revisions for tracked working changes against HEAD."""
base_revision: str | None = None
revision: str | None = None
class TreeArguments(SpecModel):
"""Resolve HEAD by default; hosts can pin revision with injected_job_kwargs."""
revision: str = Field(default="HEAD", min_length=1)
path: str | None = None
max_results: int = Field(default=500, ge=1, le=2000)
class ReadArguments(SpecModel):
"""A bounded line range from a committed regular text file."""
path: str = Field(min_length=1)
revision: str = Field(default="HEAD", min_length=1)
start_line: int = Field(default=1, ge=1)
end_line: int | None = Field(default=None, ge=1)
class SearchArguments(SpecModel):
"""Python regex search with multiline matching and bounded result blocks."""
pattern: str = Field(min_length=1, max_length=500)
revision: str = Field(default="HEAD", min_length=1)
path: str | None = None
max_results: int = Field(default=30, ge=1, le=30)
ARGUMENTS = {
"status": NoArguments,
"list_branches": NoArguments,
"create_branch": BranchArguments,
"switch_branch": BranchArguments,
"merge_branch": BranchArguments,
"commit": CommitArguments,
"diff": DiffArguments,
"list_code_tree": TreeArguments,
"read_code_file": ReadArguments,
"search_code": SearchArguments,
}

View file

@ -0,0 +1,84 @@
"""Stdlib-only, subprocess-isolated Python regex matching for committed text."""
from bisect import bisect_right
from itertools import accumulate
import json
import re
import sys
def search(files: dict[str, str], pattern: str, max_results: int) -> dict:
"""Merge adjacent contexts, retaining at most five lines per displayed block."""
compiled = re.compile(pattern, re.MULTILINE)
blocks = []
occurrences = 0
for path, text in files.items():
lines = text.splitlines()
starts = [0, *accumulate(map(len, text.splitlines(keepends=True)))]
for match in compiled.finditer(text):
if match.start() == match.end():
raise ValueError("pattern must consume at least one character")
occurrences += 1
if occurrences > 100_000:
raise ValueError("Too many regex matches; narrow pattern or path")
first = bisect_right(starts, match.start())
last = bisect_right(starts, match.end() - 1)
start, end = max(1, first - 1), min(len(lines), last + 1)
if (
blocks
and blocks[-1]["path"] == path
and start <= blocks[-1]["end"] + 1
and (max(end, blocks[-1]["end"]) - blocks[-1]["start"] < 5)
):
block = blocks[-1]
block["end"] = max(end, block["end"])
block["ranges"].append((first, last))
else:
blocks.append({"path": path, "start": start, "end": end, "ranges": [(first, last)]})
matches = []
characters = 0
for block in blocks[:max_results]:
lines = files[block["path"]].splitlines()
displayed_end = min(block["end"], block["start"] + 4)
content = "\n".join(
f"{'>' if any(first <= index <= last for first, last in block['ranges']) else ' '} "
f"{index} | {lines[index - 1]}"
for index in range(block["start"], displayed_end + 1)
)
remaining = 100_000 - characters
if remaining <= 0:
break
content = content[:remaining]
characters += len(content)
matches.append(
{
"path": block["path"],
"start_line": block["start"],
"end_line": block["end"],
"displayed_end_line": displayed_end,
"occurrence_count": len(block["ranges"]),
"content": content,
"content_truncated": displayed_end < block["end"] or characters >= 100_000,
},
)
return {
"occurrence_count": occurrences,
"merged_match_count": len(blocks),
"returned_match_count": len(matches),
"matches": matches,
"truncated": len(blocks) > len(matches) or any(match["content_truncated"] for match in matches),
}
def main():
"""Emit structured errors without candidate text or traceback noise."""
arguments = json.load(sys.stdin)
try:
result = search(**arguments)
except (ValueError, re.error) as exc:
result = {"error": str(exc)}
json.dump(result, sys.stdout)
if __name__ == "__main__":
main()

View file

@ -0,0 +1,213 @@
"""Local candidate Git operations shared by Home Jobs and evaluation snapshots."""
from pathlib import Path
from ..evaluation.artifacts import contained, read_json
from ..preparation.code import git
from ..preparation.spec import relative_path
INITIAL_BRANCH = "init"
MAX_OUTPUT = 100_000
def resolve_revision(repository: Path, revision: str) -> str:
"""Resolve a revision once, without accepting command-line options."""
if not isinstance(revision, str) or not revision.strip() or revision.startswith("-"):
raise ValueError("revision must be a nonempty Git revision without a leading '-'")
return git(repository, "rev-parse", "--verify", "--end-of-options", revision + "^{commit}")
def validate_repository(home: Path, record: dict | None = None) -> tuple[Path, dict]:
"""Require the prepared standalone repository and its unchanged baseline ref."""
record = record if record is not None else read_json(contained(home, "metadata/preparation.json"))
repository = contained(home, "code/repo", directory=True)
metadata = contained(repository, ".git", directory=True)
if not metadata.is_dir() or git(repository, "rev-parse", "--show-toplevel") != str(repository):
raise ValueError("Candidate must be a standalone Git repository")
if resolve_revision(repository, "refs/heads/init") != record["code"]["initial_commit"]:
raise ValueError("The protected init branch has changed since preparation")
git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], "HEAD")
return repository, record
def current_branch(repository: Path) -> str:
"""Reject detached HEAD so every candidate commit remains branch-reachable."""
branch = git(repository, "branch", "--show-current")
validate_branch(repository, branch)
return branch
def validate_branch(repository: Path, branch: str) -> str:
"""Branch names also serve as single evaluation-directory names."""
if not branch or branch.startswith("-") or any(char in branch for char in "/\\\0\n\r"):
raise ValueError("branch_name must be one path-safe local branch name")
if git(repository, "check-ref-format", "--branch", branch) != branch:
raise ValueError("Use a literal local branch name")
return branch
def require_clean(repository: Path) -> None:
"""Do not carry uncommitted or untracked files across branch operations."""
if git(repository, "status", "--porcelain=v2", "--untracked-files=all", "--ignore-submodules=none"):
raise ValueError("Candidate repository must be clean before changing branches")
if (repository / ".git/MERGE_HEAD").exists():
raise ValueError("Finish the pending merge first")
def branches(repository: Path) -> dict:
"""Describe local refs and the current immutable commit."""
local = sorted(git(repository, "for-each-ref", "--format=%(refname:short)", "refs/heads").splitlines())
candidates = [branch for branch in local if branch != INITIAL_BRANCH]
return {
"current_branch": current_branch(repository),
"commit_sha": resolve_revision(repository, "HEAD"),
"local_branches": local,
"candidate_branches": candidates,
"candidate_branch_count": len(candidates),
}
def status(repository: Path, _record: dict) -> dict:
"""Report exact paths, including names containing whitespace."""
result = branches(repository)
output = git(repository, "status", "--porcelain=v1", "-z", "--untracked-files=all", strip=False)
entries = iter(output.split("\0") if output else [])
changes = []
for entry in entries:
if not entry:
continue
change = {"status": entry[:2], "path": entry[3:]}
if "R" in entry[:2] or "C" in entry[:2]:
change["original_path"] = next(entries)
changes.append(change)
result.update(changes=changes, clean=not changes, merge_pending=(repository / ".git/MERGE_HEAD").exists())
return result
def create_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Fork from the current candidate or baseline and switch to the new branch."""
validate_branch(repository, branch_name)
if branch_name in branches(repository)["local_branches"]:
raise ValueError("Local branch already exists")
require_clean(repository)
validate_tree(repository, record, "HEAD")
git(repository, "switch", "--no-track", "--create", branch_name)
return {"status": "created", **branches(repository)}
def switch_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Switch only to an existing local ref, including the baseline."""
validate_branch(repository, branch_name)
if branch_name not in branches(repository)["local_branches"]:
raise ValueError("Unknown local branch")
require_clean(repository)
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
git(repository, "switch", "--no-guess", branch_name)
return {"status": "switched", **branches(repository)}
def validate_tree(repository: Path, record: dict, revision: str) -> None:
"""Only declared editable roots may differ from the prepared baseline."""
roots = [Path(relative_path(path)) for path in record["code"]["candidate_paths"]]
changed = git(
repository,
"diff",
"--no-ext-diff",
"--no-textconv",
"--name-only",
"--no-renames",
"-z",
record["code"]["initial_commit"],
revision,
"--",
strip=False,
)
for name in filter(None, changed.split("\0")):
path = Path(relative_path(name))
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
raise ValueError(f"Change outside candidate_paths: {name}")
for entry in filter(None, git(repository, "ls-tree", "-r", "-z", revision, strip=False).split("\0")):
header, name = entry.split("\t", 1)
mode, kind, _oid = header.split()
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in Path(relative_path(name)).parts:
raise ValueError(f"Candidate must contain regular files only: {name}")
def merge_branch(repository: Path, record: dict, *, branch_name: str) -> dict:
"""Stage compatible candidates; automatically abort conflicts."""
validate_branch(repository, branch_name)
state = branches(repository)
if state["current_branch"] == INITIAL_BRANCH:
raise ValueError("Cannot merge into init")
if branch_name in (INITIAL_BRANCH, state["current_branch"]):
raise ValueError("Select another candidate branch")
if branch_name not in state["candidate_branches"]:
raise ValueError("Unknown candidate branch")
require_clean(repository)
validate_tree(repository, record, "HEAD")
validate_tree(repository, record, resolve_revision(repository, "refs/heads/" + branch_name))
try:
git(
repository,
"-c",
"user.name=Home",
"-c",
"user.email=home@localhost",
"merge",
"--no-commit",
"--no-ff",
branch_name,
)
except Exception:
if (repository / ".git/MERGE_HEAD").exists():
git(repository, "merge", "--abort")
raise
merge_status = "merged_not_committed" if (repository / ".git/MERGE_HEAD").exists() else "already_up_to_date"
return {"status": merge_status, "source_branch": branch_name, **branches(repository)}
def commit(repository: Path, record: dict, *, message: str) -> dict:
"""Commit all candidate changes without rewriting existing history."""
if current_branch(repository) == INITIAL_BRANCH:
raise ValueError("Create a candidate branch before committing; init is protected")
# Validate existing staged changes before staging anything else, then the complete index.
validate_tree(repository, record, git(repository, "write-tree"))
changes = status(repository, record)["changes"]
roots = [Path(value) for value in record["code"]["candidate_paths"]]
for change in changes:
for name in (change["path"], change.get("original_path", change["path"])):
path = Path(relative_path(name))
if ".git" in path.parts or not any(path.is_relative_to(root) for root in roots):
raise ValueError(f"Change outside candidate_paths: {name}")
target = repository / path
if target.is_symlink():
raise ValueError(f"Candidate must contain regular files only: {name}")
git(repository, "add", "--all", "--", ".")
validate_tree(repository, record, git(repository, "write-tree"))
git(repository, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "--message", message)
return {"status": "committed", **branches(repository)}
def diff(repository: Path, _record: dict, *, base_revision: str | None = None, revision: str | None = None) -> dict:
"""Compare commits or all tracked changes against HEAD; never run external diff helpers."""
if revision is not None and base_revision is None:
raise ValueError("revision requires base_revision")
base = resolve_revision(repository, base_revision or "HEAD")
target = resolve_revision(repository, revision) if revision is not None else None
output = git(
repository,
"diff",
"--no-ext-diff",
"--no-textconv",
"--no-renames",
"--ignore-submodules=all",
base,
*([target] if target else []),
"--",
)
return {
"base_revision": base,
"revision": target,
"content": output[:MAX_OUTPUT],
"truncated": len(output) > MAX_OUTPUT,
}

View file

@ -0,0 +1,143 @@
"""Bounded navigation of immutable Git objects, independent of the checkout."""
import json
from pathlib import Path
import subprocess
import sys
from ..preparation.code import git, git_bytes
from ..preparation.spec import relative_path
from .repository import MAX_OUTPUT, resolve_revision
MAX_FILE_BYTES = 5_000_000
MAX_SEARCH_BYTES = 100_000_000
MAX_READ_LINES = 300
def code_path(path: str) -> str:
"""Accept literal repository-relative paths, never Git pathspec expressions."""
value = relative_path(path)
if ".git" in Path(value).parts or "\0" in value:
raise ValueError("Invalid code path")
return value
def tree(repository: Path, revision: str, path: str | None = None) -> list[tuple[str, str]]:
"""Read regular-file object IDs with NUL-delimited names and no archive filters."""
# ls-tree does not support pathspec magic. Filter exact names/prefixes ourselves.
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", revision, strip=False)
prefix = code_path(path) if path is not None else None
result = []
for entry in filter(None, entries.split("\0")):
header, name = entry.split("\t", 1)
if prefix is not None and name != prefix and not name.startswith(prefix + "/"):
continue
mode, kind, oid = header.split()
code_path(name)
if mode not in {"100644", "100755"} or kind != "blob":
raise ValueError(f"Code snapshot must contain regular files only: {name}")
result.append((name, oid))
return result
def blob(repository: Path, oid: str) -> bytes:
"""Check blob size before loading its bytes; never follow tracked symlinks."""
if int(git(repository, "cat-file", "-s", oid)) > MAX_FILE_BYTES:
raise ValueError("Code file exceeds 5 MB; narrow the path")
return git_bytes(repository, "cat-file", "blob", oid)
def list_code_tree(
repository: Path,
_record: dict,
*,
revision: str = "HEAD",
path: str | None = None,
max_results: int = 500,
) -> dict:
"""List tracked paths at a resolved commit."""
revision = resolve_revision(repository, revision)
entries = tree(repository, revision, path)
return {
"revision": revision,
"path": path,
"paths": [name for name, _ in entries[:max_results]],
"truncated": len(entries) > max_results,
}
def read_code_file(
repository: Path,
_record: dict,
*,
path: str,
revision: str = "HEAD",
start_line: int = 1,
end_line: int | None = None,
) -> dict:
"""Return at most 300 lines and 100,000 characters from one committed text file."""
path = code_path(path)
revision = resolve_revision(repository, revision)
entries = dict(tree(repository, revision, path))
if path not in entries:
raise ValueError(f"No tracked regular file at {path}")
data = blob(repository, entries[path])
if b"\0" in data:
raise ValueError("Code file is binary")
lines = data.decode("utf-8", errors="replace").splitlines()
requested_end = end_line if end_line is not None else start_line + MAX_READ_LINES - 1
if requested_end < start_line:
raise ValueError("end_line must be greater than or equal to start_line")
displayed_end = min(requested_end, start_line + MAX_READ_LINES - 1, len(lines))
content = "\n".join(lines[start_line - 1 : displayed_end])
if content:
content += "\n"
truncated = len(content) > MAX_OUTPUT or (bool(content) and (start_line > 1 or displayed_end < len(lines)))
content = content[:MAX_OUTPUT]
if truncated:
content += "\n<output truncated; request another line range to continue>"
return {
"revision": revision,
"path": path,
"start_line": start_line,
"end_line": displayed_end,
"total_lines": len(lines),
"truncated": truncated,
"content": content,
}
def search_code(
repository: Path,
_record: dict,
*,
pattern: str,
revision: str = "HEAD",
path: str | None = None,
max_results: int = 30,
) -> dict:
"""Search with Python multiline regex; isolate regex execution under a hard subprocess timeout."""
revision = resolve_revision(repository, revision)
files = {}
total = 0
for name, oid in tree(repository, revision, path):
size = int(git(repository, "cat-file", "-s", oid))
total += size
if total > MAX_SEARCH_BYTES:
raise ValueError("Code search exceeds 100 MB; narrow path")
data = blob(repository, oid)
if b"\0" not in data:
files[name] = data.decode("utf-8", errors="replace")
# Arbitrary Python regex can backtrack indefinitely. Run only the text matcher
# in a short-lived stdlib-only process; it never imports or executes candidate code.
result = subprocess.run(
[sys.executable, "-I", str(Path(__file__).with_name("regex_worker.py"))],
input=json.dumps({"files": files, "pattern": pattern, "max_results": max_results}).encode(),
capture_output=True,
check=True,
timeout=10,
)
payload = json.loads(result.stdout)
if "error" in payload:
raise ValueError(payload["error"])
return {"revision": revision, "pattern": pattern, "path": path, **payload}

View file

@ -0,0 +1,22 @@
"""Thin Job adapters; operations are fixed by Step configuration, never model input."""
from reme.steps.base_step import BaseStep, Ref
from .component import HomeCode
from .models import ARGUMENTS
class HomeCodeStep(BaseStep):
"""Expose one configured candidate operation using the normal ReMe Job lifecycle."""
code: HomeCode = Ref(HomeCode, "home.code")
async def execute(self):
"""Leave repository location and operation selection under host control."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home code Jobs require unrestricted access to their managed workspace")
operation = self.kwargs["operation"]
fields = ARGUMENTS[operation].model_fields
arguments = {name: self.context.get(name) for name in fields if name in self.context}
self.context.response.answer = await self.code.run(operation, **arguments)
return self.context.response

View file

@ -0,0 +1 @@
"""Home evaluation implementation."""

View file

@ -0,0 +1,70 @@
"""Translate normalized Home cases into existing benchmark Docker contracts."""
from datetime import datetime
from typing import Any
from deploy.benchmark_docker import EvaluationQuery, JobRequest
from ..preparation.data.models import CaseSpec, QuerySpec, SessionSpec
from .artifacts import construction_exception, failure_kind
def session_time(session: SessionSpec) -> str:
"""Match Meta-ReMe's timestamp ordering and date inputs."""
times = [message.get("created_at") for message in session.messages if message.get("created_at")]
if not times or not isinstance(times[0], str):
raise ValueError(f"Session {session.session_id!r} has no timestamp")
datetime.fromisoformat(times[0])
return times[0]
def lme_time(value: str) -> str:
"""Convert the source question date to ReMe's query-time format."""
return datetime.strptime(value, "%Y/%m/%d (%a) %H:%M").strftime("%Y-%m-%dT%H:%M:%S")
def build_jobs(dataset: str, case: CaseSpec) -> tuple[JobRequest, ...]:
"""Build in time order, checkpointing memory after each session's index refresh."""
sessions = sorted(case.sessions, key=session_time)
if dataset == "longmemeval":
dates = [query.metadata.get("question_date") for query in case.queries]
if any(dates):
if len(dates) != 1 or not dates[0]:
raise ValueError("LongMemEval requires one question_date per case")
cutoff = lme_time(str(dates[0]))
sessions = [session for session in sessions if session_time(session) <= cutoff]
jobs = []
for session in sessions:
jobs.extend(
[
JobRequest(
"auto_memory",
{
"session_id": session.session_id,
"messages": session.messages,
"date": session_time(session)[:10],
},
),
JobRequest("index_update", memory_checkpoint=f"session: {session.session_id}"),
],
)
if not jobs:
raise ValueError(f"Case {case.case_id!r} has no sessions to construct")
return tuple(jobs)
def evaluation_query(dataset: str, query: QuerySpec) -> EvaluationQuery:
"""Only answer inputs cross the sandbox boundary; metadata stays on the host."""
arguments: dict[str, Any] = {}
if dataset == "longmemeval" and query.metadata.get("question_date"):
arguments["query_time"] = lme_time(query.metadata["question_date"])
return EvaluationQuery(query.query_id, query.question, query.golden_answer, answer_arguments=arguments)
def accepts_build(build: dict, threshold: float) -> bool:
"""Preserve Meta-ReMe's strict-below-threshold partial-construction rule."""
jobs = build.get("jobs", [])
failed = sum(not isinstance(job.get("result"), dict) or not job["result"].get("success") for job in jobs)
if failed and failure_kind(construction_exception(build), "construction") == "infra_error":
return False
return failed / len(jobs) < threshold if failed else build.get("success") is True

View file

@ -0,0 +1,402 @@
"""Safe evaluation artifacts, failure classification, and result summaries."""
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import shutil
import tarfile
import tempfile
from uuid import uuid4
from ..preparation.data.models import CaseSpec, QuerySpec
from ..workspace import _validate_directory
from .compaction import round_scores
def contained(root: Path, value: str | Path, *, directory: bool = False) -> Path:
"""Reject traversal and symlinks instead of resolving through managed paths."""
path = Path(value)
if str(value).startswith("~") or ".." in path.parts:
raise ValueError(f"Invalid artifact path: {value}")
path = path if path.is_absolute() else root / path
if not path.is_relative_to(root):
raise ValueError(f"Artifact path escapes {root}: {value}")
_validate_directory(root, path if directory else path.parent)
if path.is_symlink():
raise ValueError(f"Symbolic link at artifact: {path}")
return path
def read_json(path: Path):
"""Read a regular persisted JSON artifact."""
if path.is_symlink() or not path.is_file():
raise ValueError(f"Missing regular artifact: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def write_json(path: Path, value, *, compact_leaf_containers: bool = False) -> None:
"""Atomically replace a rebuildable or progressively published result."""
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, name = tempfile.mkstemp(prefix=".result-", dir=path.parent)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
if compact_leaf_containers:
stream.write(_dumps_compact_leaf_containers(value))
else:
json.dump(value, stream, ensure_ascii=False, allow_nan=False, indent=2)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(name, path)
finally:
Path(name).unlink(missing_ok=True)
def _dumps_compact_leaf_containers(value) -> str:
"""Pretty-print JSON while keeping small scalar-only containers on one line."""
def is_atom(item) -> bool:
return isinstance(item, (str, int, float, bool, type(None)))
strings: set[str] = set()
def collect_strings(item) -> None:
if isinstance(item, str):
strings.add(item)
elif isinstance(item, dict):
for key, child in item.items():
if isinstance(key, str):
strings.add(key)
collect_strings(child)
elif isinstance(item, (list, tuple)):
for child in item:
collect_strings(child)
collect_strings(value)
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
while any(item.startswith(marker_prefix) for item in strings):
marker_prefix = f"\0meta-reme-singleton-{uuid4().hex}-"
replacements: dict[str, str] = {}
def replace_leaf_containers(item):
if isinstance(item, dict):
if len(item) == 1 and is_atom(next(iter(item.values()))):
marker = f"{marker_prefix}{len(replacements)}"
replacements[marker] = json.dumps(
item,
ensure_ascii=False,
sort_keys=True,
allow_nan=False,
separators=(",", ": "),
)
return marker
return {key: replace_leaf_containers(child) for key, child in item.items()}
if isinstance(item, (list, tuple)):
if 1 <= len(item) <= 2 and all(is_atom(child) for child in item):
marker = f"{marker_prefix}{len(replacements)}"
replacements[marker] = json.dumps(
item,
ensure_ascii=False,
sort_keys=True,
allow_nan=False,
separators=(",", ": "),
)
return marker
children = [replace_leaf_containers(child) for child in item]
return children if isinstance(item, list) else tuple(children)
return item
transformed = replace_leaf_containers(value)
content = json.dumps(transformed, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False)
for marker, replacement in replacements.items():
encoded_marker = json.dumps(marker, ensure_ascii=False)
content = content.replace(encoded_marker, replacement)
return content
def file_sha256(path: Path) -> str:
"""Hash potentially large snapshots without loading them into memory."""
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def inspect_archive(path: Path, max_bytes: int, destination: Path | None = None) -> None:
"""Validate every member before extracting regular files; never follow links."""
if path.is_symlink() or not path.is_file():
raise ValueError(f"Missing regular archive: {path}")
if path.stat().st_size > max_bytes:
raise ValueError(f"Archive exceeds byte limit: {path}")
with tarfile.open(path, "r:gz") as archive:
members = []
seen = set()
size = 0
for member in archive:
name = PurePosixPath(member.name)
if name.is_absolute() or ".." in name.parts or "\\" in member.name or name in seen:
raise ValueError(f"Unsafe archive path: {member.name}")
if not member.isdir() and not member.isfile():
raise ValueError(f"Unsupported archive member: {member.name}")
seen.add(name)
size += max(0, member.size)
if size > max_bytes or len(seen) > 100_000:
raise ValueError("Archive exceeds extraction limits")
members.append((member, name))
if not any(member.isfile() for member, _ in members):
raise ValueError("Archive contains no files")
if destination is None:
return
destination.mkdir(parents=True, exist_ok=False)
try:
for member, name in members:
target = destination.joinpath(*name.parts)
if member.isdir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
with archive.extractfile(member) as source, target.open("xb") as output:
shutil.copyfileobj(source, output)
target.chmod(0o755 if member.mode & 0o111 else 0o644)
except BaseException:
shutil.rmtree(destination)
raise
def average_tokens(queries: list[dict]) -> dict:
"""Average candidate answer total tokens over returned queries, like Meta-ReMe."""
totals = {}
for query in queries:
usage_by_model = (query.get("answer_result") or {}).get("token_usage", {})
if not isinstance(usage_by_model, dict):
continue
for model, usage in usage_by_model.items():
if not isinstance(usage, dict):
continue
total = usage.get("total_tokens")
if isinstance(total, int) and not isinstance(total, bool) and total != 0:
totals[model] = totals.get(model, 0) + total
return {model: round(total / len(queries), 3) for model, total in totals.items()} if queries else {}
def summarize(run: Path, cases: list, *, scoring_required: bool = True) -> dict:
"""Rebuild compact optimizer evidence with Meta-ReMe's expected-query denominator."""
rows, all_scores, query_results = {}, [], []
case_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
construction_counts = {"success": 0, "partial_success": 0, "failure": 0}
query_counts = {"completed": 0, "candidate_failure": 0, "infra_error": 0}
stages = {"construction": 0, "query": 0}
expected = sum(len(case.queries) for case in cases)
for case in cases:
folder = contained(run, f"cases/{case.case_id}", directory=True)
construction_path = contained(folder, "memory_construction/result.json")
construction = read_json(construction_path) if construction_path.is_file() else {}
query_path = contained(folder, "queries/result.json")
queries = read_json(query_path).get("queries", []) if query_path.is_file() else []
failure = read_json(folder / "failure.json") if (folder / "failure.json").is_file() else {}
if not construction and not queries and not failure:
continue
scores = [query["score"] for query in queries if query.get("score") is not None]
all_scores.extend(scores)
query_results.extend(queries)
failed = sum(query.get("error") is not None for query in queries)
has_infra = failure.get("failure_kind") == "infra_error" or any(q.get("infrastructure_error") for q in queries)
case_status = (
"infra_error"
if has_infra
else "candidate_failure" if failed or failure or not construction else "completed"
)
case_counts[case_status] += 1
construction_status = construction.get("status", "failure")
if construction_status == "reused":
construction_status = "success"
construction_counts[construction_status] += 1
jobs = [job for job in construction.get("jobs", []) if isinstance(job, dict)]
job_results = [job["result"] if isinstance(job.get("result"), dict) else {} for job in jobs]
construction_errors = sum(not result.get("success") for result in job_results)
stages["construction"] += construction_status == "failure" or bool(construction_errors)
stages["query"] += bool(failed)
compact_queries, query_errors = {}, {}
for query in queries:
query_status = (
"completed"
if query.get("error") is None
else "infra_error" if query.get("infrastructure_error") else "candidate_failure"
)
query_counts[query_status] += 1
compact_queries[query["query_id"]] = {
"question_type": query.get("question_type"),
"score": query.get("score"),
}
if query_status != "completed":
error = str(query["error"])
query_errors[query["query_id"]] = {
"failure_kind": query_status,
"stage": query.get("failure_stage"),
"error": error if len(error) <= 500 else error[:500] + "…",
}
rows[case.case_id] = {
"construction": {
"status": construction_status,
"success_job_count": len(jobs) - construction_errors,
"fail_job_count": construction_errors,
},
"mean_score": None if has_infra or not scoring_required else sum(scores) / len(case.queries),
"average_query_total_token_usage": average_tokens(queries),
"queries": compact_queries,
}
if failure.get("stage") is not None:
rows[case.case_id]["stage"] = failure["stage"]
if construction_errors:
rows[case.case_id]["construction_error_count"] = construction_errors
if failed:
rows[case.case_id]["query_error_count"] = failed
rows[case.case_id]["query_errors"] = query_errors
groups = {}
for query in query_results:
groups.setdefault(str(query.get("question_type") or "unknown"), []).append(query)
statistics = {}
for name, queries in sorted(groups.items()):
scores = [query["score"] for query in queries if query.get("score") is not None]
statistics[name] = {
"query_count": len(queries),
"average_score": sum(scores) / len(scores) if scores else None,
"average_query_total_token_usage": average_tokens(queries),
}
return round_scores(
{
"case_count": len(rows),
"scoring_enabled": scoring_required,
"case_status_counts": case_counts,
"construction_status_counts": construction_counts,
"failure_stage_case_counts": stages,
"expected_query_count": expected,
"scored_query_count": len(all_scores),
"query_status_counts": {
**query_counts,
"unscored": max(expected - len(all_scores), 0) if scoring_required else 0,
},
"mean_query_score": (
sum(all_scores) / expected if scoring_required and expected and not case_counts["infra_error"] else None
),
"observed_mean_query_score": sum(all_scores) / len(all_scores) if all_scores else None,
"low_score_cases": [
{key: row["mean_score"]}
for key, row in sorted(
rows.items(),
key=lambda item: (item[1]["mean_score"] if item[1]["mean_score"] is not None else 2, item[0]),
)
if row["mean_score"] is not None
and any(query.get("score") is not None and query["score"] < 1 for query in row["queries"].values())
],
"question_type_statistics": statistics,
"cases": rows,
},
)
def rebuild_summary(run: Path) -> dict:
"""Rebuild compact evidence from the manifest and authoritative phase results."""
manifest = read_json(run / "manifest.json")
cases = [
CaseSpec(
case_id=case_id,
sessions=[],
queries=[QuerySpec(query_id=query_id, question="", golden_answer=None) for query_id in query_ids],
)
for case_id, query_ids in manifest["query_ids"].items()
]
by_id = {case.case_id: case for case in cases}
if manifest.get("status") == "completed":
for case_id in manifest["case_ids"]:
folder = contained(run, f"cases/{case_id}", directory=True)
read_json(contained(folder, "memory_construction/result.json"))
query_path = contained(folder, "queries/result.json")
queries = read_json(query_path)["queries"] if query_path.is_file() else []
actual_ids = [query["query_id"] for query in queries]
if actual_ids != manifest["query_ids"][case_id]:
if queries or not (folder / "failure.json").is_file():
raise ValueError(f"Completed evaluation has incomplete query results: {case_id}")
summary = summarize(
run,
[by_id[case_id] for case_id in manifest["case_ids"]],
scoring_required=manifest.get("mode") != "debug",
)
name = "summary.json" if manifest.get("status") == "completed" else "partial_summary.json"
write_json(run / name, summary, compact_leaf_containers=True)
return summary
INFRA_TYPES = {
"APIConnectionError",
"APITimeoutError",
"ConnectError",
"ConnectionError",
"ConnectTimeout",
"NetworkError",
"PoolTimeout",
"RateLimitError",
"ReadError",
"ReadTimeout",
"RemoteProtocolError",
"ServiceUnavailableError",
"TimeoutError",
"TimeoutException",
"WriteError",
"WriteTimeout",
"SandboxCommandTimeoutError",
}
def failure_kind(exception, stage: str) -> str:
"""Only candidate execution errors count against candidate scores."""
if stage not in {"construction", "answer"}:
return "infra_error"
exception = exception if isinstance(exception, dict) else {}
if not isinstance(exception.get("type"), str) or not exception["type"]:
return "infra_error" # Legacy/unknown failures cannot support a fair comparison.
causes = exception.get("causes", [])
if isinstance(causes, list) and any(failure_kind(cause, stage) == "infra_error" for cause in causes):
return "infra_error"
status = exception.get("status_code")
if isinstance(status, int) and (status in {408, 429} or status >= 500):
return "infra_error"
name = str(exception.get("type", ""))
module = str(exception.get("module", ""))
if name in INFRA_TYPES or (
module.startswith(("httpx", "httpcore", "aiohttp", "requests", "urllib3"))
and any(word in name.lower() for word in ("connection", "network", "timeout", "protocol"))
):
return "infra_error"
return "candidate_failure"
def _failure_exception(exceptions: list, stage: str):
"""Infrastructure or unclassified failures take priority over code errors."""
return next(
(exception for exception in exceptions if failure_kind(exception, stage) == "infra_error"),
exceptions[0] if exceptions else None,
)
def construction_exception(build: dict):
"""Inspect every failed Job so an earlier code error cannot hide an outage."""
exceptions = []
for job in build.get("jobs", []):
result = job.get("result")
if not isinstance(result, dict):
exceptions.append(None)
elif not result.get("success"):
exceptions.append(result.get("exception"))
if build.get("exception"):
exceptions.append(build["exception"])
return _failure_exception(exceptions, "construction")
def answer_exception(queries: dict):
"""Select the representative error from failed answer results."""
return _failure_exception(
[row.get("exception") for row in queries.get("queries", []) if row.get("error")],
"answer",
)

View file

@ -0,0 +1,201 @@
"""Result compaction rules preserved from dev_meta b53048e4.
Keep diagnostic evidence while removing reconstructable prompts and duplicate data.
Compaction copies inputs so scoring and failure classification retain raw responses.
"""
from collections.abc import Mapping
from copy import deepcopy
from pathlib import PurePosixPath
from typing import Any
def round_scores(value: Any, *, score: bool = False) -> Any:
"""Copy structured scores with four decimal places, preserving text and other numbers."""
if isinstance(value, dict):
return {
key: round_scores(
child,
score=score
or isinstance(key, str)
and (
key in {"score", "precision", "recall", "f1", "tau_norm", "low_score_cases"}
or key.endswith("_score")
),
)
for key, child in value.items()
}
if isinstance(value, list):
return [round_scores(child, score=score) for child in value]
return round(value, 4) if score and isinstance(value, float) else value
def compact_query(result: dict) -> dict:
"""Copy and compact one authoritative query result."""
result = deepcopy(result)
_compact_query_result(result)
# Reference answers and provider raw text remain source evidence.
for key in ("score", "judge_result", "answer_result"):
if key in result:
result[key] = round_scores(result[key], score=key == "score")
return result
def compact_construction(build: dict) -> dict:
"""Copy and compact ordered construction jobs using the reviewed rules."""
persisted_build = deepcopy(build)
_compact_construction_source_conversations(persisted_build)
_compact_construction_index_snapshots(persisted_build)
for job in persisted_build.get("jobs", []):
if not isinstance(job, Mapping):
continue
if isinstance(job, dict):
job.pop("memory_checkpoint", None)
result = job.get("result")
if not isinstance(result, dict):
continue
_compact_outer_job_call_count(job.get("job"), result)
token_usage = result.get("token_usage")
if not isinstance(token_usage, dict):
continue
for backend, usage in list(token_usage.items()):
if isinstance(usage, Mapping) and all(
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
):
del token_usage[backend]
if not token_usage:
del result["token_usage"]
elif all(
isinstance(usage, Mapping)
and all(usage.get(field) == 0 for field in ("input_tokens", "output_tokens", "total_tokens"))
for usage in token_usage.values()
):
result["token_usage"] = 0
return persisted_build
def _compact_query_result(result: dict[str, Any]) -> None:
"""Remove reconstructable prompts and uninformative token records before persistence."""
answer_result = result.get("answer_result")
if isinstance(answer_result, dict):
if answer_result.get("answer") == result.get("answer"):
answer_result.pop("answer", None)
if "error" in result and answer_result.get("error") == result.get("error"):
answer_result.pop("error", None)
_compact_outer_job_call_count("agentic_answer", answer_result)
metadata = answer_result.get("metadata")
if isinstance(metadata, dict):
if metadata.get("query") == result.get("question"):
metadata.pop("query", None)
if metadata.get("response") == result.get("answer"):
metadata.pop("response", None)
judge_result = result.get("judge_result")
if isinstance(judge_result, dict):
judge_result.pop("token_usage", None)
if result.get("score") is not None:
judge_result.pop("answer", None)
_compact_outer_job_call_count("answer_judge", judge_result)
metadata = judge_result.get("metadata")
if isinstance(metadata, dict):
if metadata.get("raw_answer_judgement") is not None:
metadata.pop("answer_judgement", None)
for field in (
"agent_answer",
"golden_answer",
"llm_response",
"probing_question",
"query",
"question_type",
"rubric",
):
metadata.pop(field, None)
def compact(value: Any) -> None:
if isinstance(value, dict):
value.pop("sys_prompt", None)
token_usage = value.get("token_usage")
if isinstance(token_usage, dict):
for backend, usage in list(token_usage.items()):
if isinstance(usage, Mapping) and all(
usage.get(field) is None for field in ("input_tokens", "output_tokens", "total_tokens")
):
del token_usage[backend]
if not token_usage:
del value["token_usage"]
for child in value.values():
compact(child)
elif isinstance(value, list):
for child in value:
compact(child)
compact(result)
def _compact_outer_job_call_count(job_name: Any, result: dict[str, Any]) -> None:
"""Remove one call already represented by the enclosing result record."""
counts = result.get("job_call_counts")
if not isinstance(job_name, str) or not isinstance(counts, dict):
return
own_count = counts.get(job_name)
if isinstance(own_count, int) and not isinstance(own_count, bool) and own_count > 0:
remaining = own_count - 1
if remaining:
counts[job_name] = remaining
else:
del counts[job_name]
if not counts:
del result["job_call_counts"]
def _compact_construction_index_snapshots(value: Any) -> None:
"""Replace cumulative daily-index note snapshots with their counts."""
if isinstance(value, dict):
index = value.get("index")
if _is_daily_index_snapshot(index):
index["notes_count"] = len(index.pop("notes"))
for child in value.values():
_compact_construction_index_snapshots(child)
elif isinstance(value, list):
for child in value:
_compact_construction_index_snapshots(child)
def _is_daily_index_snapshot(value: Any) -> bool:
"""Recognize the stable ``refresh_day_index`` payload without assuming a configured directory name."""
if not isinstance(value, dict) or not isinstance(value.get("notes"), list):
return False
date = value.get("date")
path = value.get("path")
if not isinstance(date, str) or not isinstance(path, str):
return False
index_path = PurePosixPath(path)
return (
index_path.suffix == ".md"
and index_path.stem == date
and isinstance(value.get("created"), bool)
and isinstance(value.get("changed"), bool)
)
def _compact_construction_source_conversations(value: Any) -> None:
"""Drop a transcript link when its sibling session ID already identifies the same file."""
if isinstance(value, dict):
session_id = value.get("session_id")
source_conversation = value.get("source_conversation")
if isinstance(session_id, str) and isinstance(source_conversation, str):
source_path = source_conversation.strip()
if source_path.startswith("[[") and source_path[-2:] == "]]":
source_path = source_path[2:-2]
if PurePosixPath(source_path).name == f"{session_id}.jsonl":
del value["source_conversation"]
for child in value.values():
_compact_construction_source_conversations(child)
elif isinstance(value, list):
for child in value:
_compact_construction_source_conversations(child)

View file

@ -0,0 +1,488 @@
"""Home evaluation orchestration using the shared Docker pool and a host judge."""
import asyncio
import os
import shutil
import traceback
from contextlib import nullcontext
from deploy.benchmark_docker import BenchmarkDockerPool, CasePlan, DockerReMeSandboxFactory, SourceCandidate
from reme.components.base_component import BaseComponent
from ..preparation.data.models import fingerprint, utc_now
from .adapters import accepts_build, build_jobs, evaluation_query
from .artifacts import (
contained,
file_sha256,
inspect_archive,
read_json,
summarize,
write_json,
construction_exception,
answer_exception,
failure_kind,
)
from .inputs import EvaluationSpec, load_inputs, reuse_inputs, snapshot_code, replay_sampling_eligibility
from .compaction import compact_construction, compact_query
from ..judge import HomeJudge
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from ..sampling import SamplingPolicy, create_sampling_plan
from .history import HomeEvaluationHistory
async def blocking(function, *args, **kwargs):
"""Await owned filesystem work even when the calling job is cancelled."""
task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs))
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
await asyncio.gather(task, return_exceptions=True)
raise
class HomeEvaluation(BaseComponent):
"""Own evaluation tasks; the application owns the host judge Job lifecycle."""
component_type = "home.evaluation"
def __init__(self, sampling_policy: dict | None = None, **kwargs):
if "sampling" in kwargs:
raise ValueError("Use sampling_policy for Component defaults; sampling is a Job boolean")
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
self.judge = self.bind("default", HomeJudge, optional=False)
self.history = self.bind("default", HomeEvaluationHistory, optional=False)
self._pending: set[asyncio.Task] = set()
self.sampling_policy = SamplingPolicy.model_validate(sampling_policy if sampling_policy is not None else {})
async def _close(self):
if self._pending:
await asyncio.gather(*self._pending, return_exceptions=True)
async def evaluate(self, **kwargs) -> dict:
"""Run through the component lifecycle, including cancellation cleanup."""
if not self.is_started:
raise RuntimeError("Start the Home application before evaluation")
spec = EvaluationSpec.model_validate(kwargs)
override = spec.sampling_policy.model_dump(exclude_unset=True) if spec.sampling_policy is not None else {}
policy = SamplingPolicy.model_validate({**self.sampling_policy.model_dump(), **override})
spec = spec.model_copy(update={"sampling_policy": policy})
task = asyncio.create_task(self._serialized_evaluate(spec))
self._pending.add(task)
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
raise
finally:
self._pending.discard(task)
async def _serialized_evaluate(self, spec):
"""Match dev_meta's serialized formal validations; debug never owns sampling state."""
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
with lock:
return await self._evaluate(spec)
def _prepare(self, spec, environment):
home = self.workspace.home_path
sampling_state = self.history.sampling_state() if spec.sampling and not spec.debug else None
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
record, dataset, cases = load_inputs(home, spec)
plan = None
if spec.sampling and not spec.debug:
eligibility = replay_sampling_eligibility(home, spec, cases)
plan = create_sampling_plan(
cases,
sampling_state,
spec.sampling_policy,
eligible_query_ids=eligibility,
)
selected = {case.case_id: case for case in cases}
cases = []
for key in plan["case_ids"]:
case = selected[key]
queries = {query.query_id: query for query in case.queries}
cases.append(
case.model_copy(
update={
"queries": [queries[query_id] for query_id in plan["query_ids"][key]],
},
),
)
snapshot, commit, branch = snapshot_code(home, record)
fingerprints = {
"dataset": dataset.normalized_fingerprint,
"construction_config": fingerprint(
{
"runtime": record["code"]["runtime_fingerprint"],
"config": record["code"]["config"],
"plugins": record["code"]["plugins"],
"image": spec.base_image,
"models": {
k: v for k, v in environment.items() if "MODEL" in k and "KEY" not in k and "JUDGE" not in k
},
},
),
}
source, archives = reuse_inputs(home, spec, cases, fingerprints)
# Validate adapters before allocating results or starting containers.
jobs = {case.case_id: build_jobs(dataset.dataset, case) for case in cases} if source is None else {}
for case in cases:
for query in case.queries:
if dataset.dataset == "beam" and spec.dataset_kind == "training-set":
rubric = query.metadata.get("rubric", query.golden_answer)
if not isinstance(rubric, list) or not rubric:
raise ValueError(f"BEAM query {query.query_id} has no scoring rubric")
evaluation_query(dataset.dataset, query)
return record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, plan
def _sampling_started(self, run, manifest, plan, policy=None, *, record_history=True):
"""The factory invokes this only after a real sandbox successfully initializes."""
if not record_history:
manifest["sandbox_started_at"] = utc_now().isoformat()
if plan is not None:
# Keep draw evidence local without claiming a started-history round.
manifest["selection_plan"] = plan
write_json(run / "manifest.json", manifest)
return
self.history.record_started(run, policy if policy is not None else self.sampling_policy, plan)
manifest.update(read_json(run / "manifest.json"))
def _finish_sampling(self, run):
self.history.record(run)
def _allocate(self, branch, commit, manifest):
"""Allocate ascending IDs under a collision-safe commit prefix."""
home = self.workspace.home_path
with _initialization_lock(self.workspace.state_path / "workspace.lock"):
parent = contained(home, f"evaluations/{branch}", directory=True)
parent.mkdir(parents=True, exist_ok=True)
width = 7
while True:
folder = contained(parent, commit[:width], directory=True)
prior = list(folder.glob("*/manifest.json")) if folder.exists() else []
if all(read_json(path).get("commit_sha") == commit for path in prior):
break
width += 1
if width > len(commit):
raise ValueError("Invalid evaluation commit directory")
folder.mkdir(exist_ok=True)
numbers = [int(path.name) for path in folder.iterdir() if path.name.isdigit()]
run = folder / str(max(numbers, default=0) + 1)
run.mkdir()
manifest["validation_id"] = run.name
write_json(run / "manifest.json", manifest)
return run
async def _evaluate(self, spec):
environment = {**os.environ, **self.app_context.app_config.environment}
prepared = await blocking(self._prepare, spec, environment)
record, dataset, cases, snapshot, commit, branch, fingerprints, source, archives, jobs, sampling_plan = prepared
judge = None if spec.debug else self.judge
run = None
try:
manifest = {
"branch_name": branch,
"commit_sha": commit,
"case_ids": [case.case_id for case in cases],
"query_ids": {case.case_id: [q.query_id for q in case.queries] for case in cases},
"dataset": dataset.dataset,
"dataset_kind": spec.dataset_kind,
"mode": "debug" if judge is None else "scored",
"execution_mode": "query_only_replay" if source else "full_validation",
"fingerprints": fingerprints,
"options": spec.model_dump(
mode="json",
exclude={"case_ids", "query_ids", "reuse_construction_from", "debug", "sampling_policy"},
),
"judge": self.judge.identity(dataset.dataset),
"status": "running",
"started_at": utc_now().isoformat(),
}
if source is not None:
manifest["construction_source"] = source
run = self._allocate(branch, commit, manifest)
plans = []
for case in cases:
folder = run / "cases" / case.case_id
memory = folder / "memory_construction"
memory.mkdir(parents=True)
if source:
# Retain an independent pre-query snapshot and inherited log.
archive, expected_sha = archives[case.case_id]
await blocking(shutil.copyfile, archive, memory / "reme_workspace.tar.gz")
actual_sha = await blocking(file_sha256, memory / "reme_workspace.tar.gz")
if actual_sha != expected_sha:
raise ValueError(f"Construction source changed after preflight: {case.case_id}")
log = contained(archive.parent, "build.log")
if log.is_file():
await blocking(shutil.copyfile, log, memory / "build.log")
write_json(
memory / "result.json",
{
"success": True,
"status": "reused",
"source": source,
"workspace_sha256": actual_sha,
"jobs": [],
},
)
plans.append(
CasePlan(
case_id=case.case_id,
build_jobs=jobs.get(case.case_id, ()),
queries=tuple(evaluation_query(dataset.dataset, q) for q in case.queries),
workspace_source=memory / "reme_workspace.tar.gz" if source else None,
export_workspace_to=None if source else memory / "reme_workspace.tar.gz",
export_build_log_to=None if source else memory / "build.log",
export_evaluation_to=folder / "evaluation.tar.gz",
export_diagnostics_to=folder / "full.tar.gz",
),
)
factory = DockerReMeSandboxFactory(
SourceCandidate(
snapshot,
base_image=spec.base_image,
install_projects=tuple(record["code"]["install_projects"]),
),
config=record["code"]["config"],
plugins=tuple(record["code"]["plugins"]),
command_timeout=spec.timeout_seconds,
env=environment,
on_sandbox_started=(
None
if spec.debug
else lambda: self._sampling_started(
run,
manifest,
sampling_plan,
spec.sampling_policy,
record_history=spec.record_history,
)
),
)
semaphore = asyncio.Semaphore(spec.concurrency)
by_id = {case.case_id: case for case in cases}
async def completed(plan, result):
try:
await self._persist_case(run, by_id[plan.case_id], result, source, spec)
except Exception as error:
result.error = (
result.error + "; " if result.error else ""
) + f"artifact persistence failed: {error}"
write_json(
run / "cases" / plan.case_id / "failure.json",
{"stage": "artifacts", "error": result.error, "failure_kind": "infra_error"},
)
await self._score_case(
run,
by_id[plan.case_id],
dataset.dataset,
judge,
semaphore,
spec,
(result.queries or {}).get("queries", []),
)
results = await BenchmarkDockerPool(factory, concurrency=spec.concurrency).run(
plans,
on_case_finished=completed,
fail_fast=spec.fail_fast,
accept_build=lambda build: accepts_build(build, spec.max_construction_failure_rate),
)
# Pool teardown can attach late cleanup failures after callbacks.
for result in results:
if result.error:
failure = run / "cases" / result.case_id / "failure.json"
if not failure.exists():
write_json(failure, {"stage": "cleanup", "error": result.error, "failure_kind": "infra_error"})
else:
previous = read_json(failure)
if previous.get("error") != result.error:
previous.update(cleanup_error=result.error, failure_kind="infra_error")
write_json(failure, previous)
summary = summarize(run, cases, scoring_required=not spec.debug)
manifest.update(
status="completed",
success=summary["case_status_counts"]["infra_error"] == 0,
finished_at=utc_now().isoformat(),
)
write_json(run / "manifest.json", manifest)
write_json(run / "summary.json", summary, compact_leaf_containers=True)
if not spec.debug and spec.record_history:
await blocking(self._finish_sampling, run)
return {
"result_dir": str(run),
"commit_sha": commit,
"execution_mode": manifest["execution_mode"],
"record_history": spec.record_history and not spec.debug,
"status": "completed",
**{key: value for key, value in summary.items() if key != "cases"},
}
except BaseException as error:
# A full summary is the durable completion boundary. Cancellation
# during drained history bookkeeping cancels the caller, not the
# already completed evaluation or its successful history entry.
registration_failed = run is not None and manifest.get("status") == "completed"
if isinstance(error, asyncio.CancelledError) and registration_failed:
raise
if run is not None:
manifest.update(
status="cancelled" if isinstance(error, asyncio.CancelledError) else "failed",
success=False,
finished_at=utc_now().isoformat(),
)
write_json(run / "manifest.json", manifest)
write_json(
run / "failure.json",
{
"stage": "evaluation",
"error": f"{type(error).__name__}: {error}",
"traceback": traceback.format_exc(),
"failed_at": utc_now().isoformat(),
},
)
partial = summarize(
run,
cases,
scoring_required=not spec.debug,
)
write_json(run / "partial_summary.json", partial, compact_leaf_containers=True)
if registration_failed and not spec.debug and spec.record_history:
try:
await blocking(self.history.invalidate)
except Exception as cache_error:
error.add_note(f"History cache invalidation failed: {cache_error}")
if isinstance(error, asyncio.CancelledError) or run is None:
raise
raise RuntimeError(f"Evaluation failed; artifacts: {run}: {error}") from error
@staticmethod
async def _persist_case(run, case, result, source, spec):
folder = run / "cases" / case.case_id
memory = folder / "memory_construction"
# Freeze every returned answer in the sole authoritative query file
# before extraction or judging can fail. Unjudged answers stay present.
by_id = {row.get("query_id"): row for row in (result.queries or {}).get("queries", [])}
rows = [
compact_query(
{
**by_id[query.query_id],
"query_id": query.query_id,
"question": query.question,
"golden_answer": query.golden_answer,
"question_type": query.metadata.get("question_type"),
"score": None,
"judge_result": None,
},
)
for query in case.queries
if query.query_id in by_id
]
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
if result.error:
stage = result.failure_stage or "execution"
exception = result.exception
if exception is None:
exception = (
answer_exception(result.queries or {})
if stage == "answer"
else construction_exception(result.build or {})
)
write_json(
folder / "failure.json",
{
"stage": stage,
"error": result.error,
"exception": exception,
"failure_kind": failure_kind(exception, stage),
},
)
if source is None:
build = result.build or {"success": False, "error": result.error, "jobs": []}
accepted = accepts_build(build, spec.max_construction_failure_rate)
construction = {
**build,
"status": "success" if build.get("success") else "partial_success" if accepted else "failure",
}
archive = memory / "reme_workspace.tar.gz"
if archive.is_file():
construction["workspace_sha256"] = await blocking(file_sha256, archive)
write_json(memory / "result.json", compact_construction(construction), compact_leaf_containers=True)
archive = memory / "reme_workspace.tar.gz"
if archive.is_file():
await blocking(inspect_archive, archive, spec.max_artifact_bytes, memory / "reme_workspace")
evaluation = folder / "evaluation.tar.gz"
if evaluation.is_file():
extracted = folder / "exported"
await blocking(inspect_archive, evaluation, spec.max_artifact_bytes, extracted)
# Keep per-query logs beside host authoritative results. Container
# query scores are intentionally ignored; judging happens outside it.
for query in case.queries:
log = extracted / "queries" / query.query_id / "answer.log"
if log.is_file():
target = folder / "queries" / query.query_id / "answer.log"
target.parent.mkdir(parents=True, exist_ok=True)
await blocking(shutil.copyfile, log, target)
# Container results cannot overwrite host-frozen answers or scores.
await blocking(shutil.rmtree, extracted)
evaluation.unlink()
@staticmethod
async def _score_case(run, case, dataset, judge, semaphore, spec, answers):
folder = run / "cases" / case.case_id
by_id = {row.get("query_id"): row for row in answers}
rows = read_json(folder / "queries/result.json").get("queries", [])
positions = {row["query_id"]: index for index, row in enumerate(rows)}
case_failure = read_json(folder / "failure.json") if (folder / "failure.json").exists() else {}
for query in case.queries:
raw = by_id.get(query.query_id, {})
if not raw and case_failure:
continue # Construction failures leave queries unassessed.
row = {
**raw,
"query_id": query.query_id,
"question": query.question,
"golden_answer": query.golden_answer,
"question_type": query.metadata.get("question_type"),
"score": None,
"judge_result": None,
}
if not isinstance(row.get("answer"), str) or not row["answer"].strip() or row.get("error") is not None:
row["error"] = row.get("error") or "No successful answer (construction or execution failed)"
row["failure_stage"] = "answer"
row["failure_kind"] = (
failure_kind(row["exception"], "answer")
if row.get("exception")
else case_failure.get("failure_kind")
or ("candidate_failure" if raw and not raw.get("error") else failure_kind(None, "answer"))
)
row["infrastructure_error"] = row["failure_kind"] == "infra_error"
elif judge is not None:
try:
async with semaphore:
row["judge_result"], row["score"] = await judge.evaluate(dataset, query, row["answer"])
except Exception as error:
row.update(
error=f"{type(error).__name__}: {error}",
judge_result=getattr(error, "result", None),
failure_stage="judge",
failure_kind="infra_error",
infrastructure_error=True,
)
compact = compact_query(row)
if query.query_id in positions:
rows[positions[query.query_id]] = compact
else:
rows.append(compact)
# Publish each score; a later judge failure cannot erase frozen answers.
write_json(folder / "queries/result.json", {"queries": rows}, compact_leaf_containers=True)
if row.get("error") is not None:
write_json(folder / "queries" / query.query_id / "failure.json", compact, compact_leaf_containers=True)
if spec.fail_fast:
raise RuntimeError(f"Query {case.case_id}/{query.query_id} failed: {row['error']}")

View file

@ -0,0 +1,142 @@
"""Compact, read-only query evidence coordinates from saved evaluation artifacts."""
from typing import Annotated
from pydantic import Field, TypeAdapter, field_validator
from reme.steps.base_step import BaseStep, Ref
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace
from .artifacts import contained, read_json
from .inputs import entity_id
from .inspection_step import _execute, _object, _preview
from .memory_inspection_step import _head, _repository, git
SessionIds = list[Annotated[str, Field(min_length=1)]]
MAX_SESSIONS = 50
MAX_CHANGED_FILES = 50
class TraceQueryEvidenceInput(SpecModel):
"""Locate one selected query; inferred evidence is optional and never overrides annotations."""
validation_path: str = Field(min_length=1)
case_id: str = Field(min_length=1)
query_id: str = Field(min_length=1)
evidence_session_ids: SessionIds | None = None
@field_validator("case_id", "query_id")
@classmethod
def validate_entity(cls, value):
"""Keep artifact selectors within one directory segment."""
return entity_id(value)
def _source_case(home, manifest, case_id):
kind = manifest.get("dataset_kind")
if kind not in {"training-set", "debug-cases"}:
raise ValueError(f"Unsupported evaluation dataset_kind: {kind!r}")
root = contained(home, f"datasets/{kind}/cases", directory=True)
for candidate in sorted(root.glob("*.json")):
path = contained(root, candidate)
case = read_json(path)
if isinstance(case, dict) and case.get("case_id") == case_id:
return path, case
raise ValueError(f"Normalized case is missing: {case_id}")
def _session_checkpoints(repository, head, selected):
if head is None or not selected:
return {}
# Separate commit subjects and records without relying on line delimiters.
raw = git(repository, "log", "--no-show-signature", "--reverse", "--format=%H%x00%s", "-z", head, "--")
fields = raw.split("\0")
checkpoints = {}
for sha, subject in zip(fields[::2], fields[1::2]):
if subject.startswith("session: ") and subject[9:] in selected:
checkpoints[subject[9:]] = sha
return checkpoints
def trace_query_evidence(home, spec):
"""Join source annotations, committed checkpoint coordinates and query result previews."""
run, manifest, repository = _repository(home, spec.validation_path, spec.case_id)
if spec.query_id not in manifest.get("query_ids", {}).get(spec.case_id, []):
raise ValueError("query_id is not selected by this evaluation manifest")
source_path, case = _source_case(home, manifest, spec.case_id)
query = next((item for item in case.get("queries", []) if item.get("query_id") == spec.query_id), None)
if query is None:
raise ValueError(f"Normalized case has no query: {spec.query_id}")
metadata = query.get("metadata") or {}
annotated = metadata.get("evidence_session_ids")
selected = TypeAdapter(SessionIds).validate_python(
annotated if annotated is not None else spec.evidence_session_ids or [],
strict=True,
)
selected = list(dict.fromkeys(selected))
source = "independent_annotation" if annotated is not None else "diagnostician_inference"
if annotated is None and spec.evidence_session_ids is None:
source = "unavailable"
known = {session["session_id"] for session in case.get("sessions", [])}
if unknown := set(selected) - known:
raise ValueError(f"Evidence sessions are not present in case: {sorted(unknown)}")
status, head = _head(repository)
checkpoints = _session_checkpoints(repository, head, set(selected))
sessions = []
for session_id in selected[:MAX_SESSIONS]:
checkpoint = checkpoints.get(session_id)
files = (
git(repository, "diff-tree", "--root", "--no-commit-id", "--name-only", "-r", "-z", checkpoint, "--")
.rstrip("\0")
.split("\0")
if checkpoint
else []
)
files = [name for name in files if name]
row = {"id": session_id, "checkpoint": checkpoint, "changed_files": files[:MAX_CHANGED_FILES]}
if len(files) > MAX_CHANGED_FILES:
row["omitted_changed_file_count"] = len(files) - MAX_CHANGED_FILES
sessions.append(row)
results_path = contained(run, f"cases/{spec.case_id}/queries/result.json")
payload = _object(run, str(results_path.relative_to(run)))
result = next(
(item for item in (payload or {}).get("queries", []) if item.get("query_id") == spec.query_id),
None,
)
outcome = result or {}
response = {
"query": {
"question": _preview(outcome.get("question", query.get("question")), 2000),
"expected": _preview(outcome.get("golden_answer", query.get("golden_answer")), 2000),
"actual": _preview(outcome.get("answer"), 2000),
"score": outcome.get("score"),
"error": _preview(outcome.get("error")),
"result_missing": result is None,
},
"evidence_source": source,
"evidence_is_certain": metadata.get("evidence_is_certain"),
"memory_status": status,
"sessions": sessions,
"paths": {
"source_case": str(source_path),
"memory_repository": str(repository),
"query_result": str(results_path),
"query_artifacts": str(contained(run, f"cases/{spec.case_id}/queries/{spec.query_id}", directory=True)),
},
}
if len(selected) > MAX_SESSIONS:
response["omitted_session_count"] = len(selected) - MAX_SESSIONS
return response
class HomeTraceQueryEvidenceStep(BaseStep):
"""Expose compact evidence navigation through the normal Home Job lifecycle."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Read saved artifacts without model calls or workspace mutations."""
return await _execute(self, TraceQueryEvidenceInput, trace_query_evidence)

View file

@ -0,0 +1,325 @@
"""Successful evaluation history and rebuildable sampling evidence owned by Home."""
from pathlib import Path
from typing import Mapping
import math
from reme.components.base_component import BaseComponent
from ..preparation.data.models import utc_now
from ..preparation.spec import SpecModel
from ..sampling import SamplingPolicy, _empty_record, _observe, _query_key
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from .artifacts import contained, read_json, write_json
class EvaluationRecord(SpecModel):
"""A compact catalog entry backed by an immutable successful result directory."""
run_key: str
branch_name: str
commit_sha: str
validation_id: str
round: int
case_ids: list[str]
query_ids: dict[str, list[str]]
execution_mode: str
finished_at: str
case_count: int
candidate_failure_count: int
scored_query_count: int
mean_query_score: float | None
def successful_result(home: Path, run: Path) -> tuple[dict, dict] | None:
"""Match the non-debug Response.success contract, including valid low scores."""
manifest = read_json(contained(home, run / "manifest.json"))
if manifest.get("options", {}).get("record_history") is False:
return None
if (
manifest.get("mode") != "scored"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("status") != "completed"
or manifest.get("success") is False
or not manifest.get("sandbox_started_at")
):
return None
path = contained(home, run / "summary.json")
if not path.is_file():
return None
summary = read_json(path)
if summary.get("case_status_counts", {}).get("infra_error") != 0 or not summary.get("case_count"):
return None
return manifest, summary
def result_record(home: Path, run: Path) -> EvaluationRecord | None:
"""Exclude failed, cancelled, debug and incomplete directories from the catalog."""
result = successful_result(home, run)
if result is None:
return None
manifest, summary = result
return EvaluationRecord(
run_key=run.relative_to(home / "evaluations").as_posix(),
branch_name=manifest["branch_name"],
commit_sha=manifest["commit_sha"],
validation_id=manifest["validation_id"],
round=manifest["sampling"]["round"],
case_ids=manifest["case_ids"],
query_ids=manifest["query_ids"],
execution_mode=manifest["execution_mode"],
finished_at=manifest["finished_at"],
case_count=summary["case_count"],
candidate_failure_count=summary["case_status_counts"]["candidate_failure"],
scored_query_count=summary["scored_query_count"],
mean_query_score=summary["mean_query_score"],
)
def read_state(home: Path) -> dict:
"""Read a derived cache; started plans and scored artifacts retain its source evidence."""
path = contained(home, "sampling/state.json")
if not path.exists():
return rebuild_sampling_state(home, persist=False)
payload = read_json(path)
if not isinstance(payload, dict) or payload.get("schema_version") not in {1, 2}:
raise ValueError("Unsupported sampling state schema")
if not isinstance(payload.get("round"), int) or isinstance(payload["round"], bool) or payload["round"] < 0:
raise ValueError("Invalid sampling round")
for name in ("cases", "queries", "started_validations"):
if not isinstance(payload.get(name, {}), dict):
raise ValueError(f"Invalid sampling {name}")
payload.setdefault(name, {})
if not isinstance(payload.get("processed_validations"), list):
raise ValueError("Invalid processed validations")
if payload["schema_version"] == 1 or "successful_round" not in payload:
return rebuild_sampling_state(home, persist=False)
if (
not isinstance(payload["successful_round"], int)
or isinstance(payload["successful_round"], bool)
or (payload["successful_round"] < 0)
):
raise ValueError("Invalid successful-history round")
return payload
def start_validation(home: Path, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
"""Record once after the first successfully initialized sandbox, never before."""
manifest = read_json(contained(home, run / "manifest.json"))
if (
manifest.get("mode") == "debug"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("options", {}).get("record_history") is False
):
return {}
key = run.relative_to(home / "evaluations").as_posix()
state = read_state(home)
if key in state["started_validations"]:
return read_json(contained(home, manifest["sampling"]["plan_path"]))
plan = (
dict(plan)
if plan is not None
else {
"mode": "explicit",
"case_ids": manifest["case_ids"],
"query_ids": manifest["query_ids"],
"policy": policy.model_dump(),
}
)
# Preserve the draw's state snapshot separately from the committed start round.
if "round" in plan:
plan["selection_round"] = plan["round"]
plan.update(
round=state["round"] + 1,
validation_id=manifest["validation_id"],
validation_key=key,
sandbox_started_at=utc_now().isoformat(),
)
relative = f"sampling/plans/{key}.json"
write_json(contained(home, relative), plan)
manifest["sampling"] = {"round": plan["round"], "mode": plan["mode"], "plan_path": relative}
manifest["sandbox_started_at"] = plan["sandbox_started_at"]
write_json(run / "manifest.json", manifest)
state["round"] = plan["round"]
state["started_validations"][key] = plan["round"]
write_json(contained(home, "sampling/state.json"), state)
return plan
def update_sampling_state(home: Path, run: Path, *, _state: dict | None = None) -> None:
"""Update observations only from successful completed formal evaluations."""
result = successful_result(home, run)
if result is None:
return
manifest, summary = result
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
policy = SamplingPolicy.model_validate(plan["policy"])
state = read_state(home) if _state is None else _state
key = plan["validation_key"]
if key in state["processed_validations"]:
return
round_id = state["successful_round"] + 1
cases, queries = state["cases"], state["queries"]
summary_cases = summary.get("cases", {})
items = (
summary_cases.items()
if isinstance(summary_cases, Mapping)
else ((case.get("case_id"), case) for case in summary_cases)
)
for case_id, case_summary in items:
if not isinstance(case_id, str) or not isinstance(case_summary, Mapping):
continue
folder = contained(run, f"cases/{case_id}", directory=True)
query_path = contained(folder, "queries/result.json")
query_rows = read_json(query_path).get("queries", []) if query_path.is_file() else []
failure_path = contained(folder, "failure.json")
failure = read_json(failure_path) if failure_path.is_file() else {}
# Home summaries omit the redundant case status; reconstruct it using
# the same authoritative failure/query evidence as summarize().
status = case_summary.get("status") or (
"infra_error"
if failure.get("failure_kind") == "infra_error"
or any(query.get("infrastructure_error") for query in query_rows)
else "candidate_failure" if failure or any(query.get("error") for query in query_rows) else "completed"
)
case_record = cases.setdefault(case_id, _empty_record())
case_record["last_selected_round"] = max(case_record.get("last_selected_round") or 0, round_id)
if status == "infra_error":
case_record["infra_error_count"] += 1
continue
if status == "candidate_failure":
case_record["candidate_failure_count"] += 1
scores = []
for query in query_rows:
query_id = query.get("query_id")
if not isinstance(query_id, str):
continue
query_record = queries.setdefault(_query_key(case_id, query_id), _empty_record())
query_record["last_selected_round"] = max(query_record.get("last_selected_round") or 0, round_id)
if query.get("infrastructure_error"):
query_record["infra_error_count"] += 1
continue
if query.get("failure_kind") == "candidate_failure":
query_record["candidate_failure_count"] += 1
score = query.get("score")
if isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score):
scores.append(float(score))
_observe(query_record, 1.0 - score, round_id, policy.ewma_alpha)
if scores:
_observe(case_record, 1.0 - sum(scores) / len(scores), round_id, policy.ewma_alpha)
state["successful_round"] = round_id
state["processed_validations"].append(key)
if _state is None:
write_json(contained(home, "sampling/state.json"), state)
def rebuild_sampling_state(home: Path, *, persist: bool = True) -> dict:
"""Rebuild the disposable cache from immutable started plans and scored artifacts."""
state = {
"schema_version": 2,
"round": 0,
"successful_round": 0,
"cases": {},
"queries": {},
"started_validations": {},
"processed_validations": [],
}
runs = []
for path in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
manifest = read_json(contained(home, path))
if (
manifest.get("mode") == "debug"
or manifest.get("dataset_kind") != "training-set"
or manifest.get("options", {}).get("record_history") is False
or not (manifest.get("sandbox_started_at"))
):
continue
plan = read_json(contained(home, manifest["sampling"]["plan_path"]))
runs.append((plan["round"], path.parent, plan["validation_key"]))
for round_id, run, key in sorted(runs):
state["round"] = max(state["round"], round_id)
state["started_validations"][key] = round_id
update_sampling_state(home, run, _state=state)
if persist:
write_json(contained(home, "sampling/state.json"), state)
return state
class HomeEvaluationHistory(BaseComponent):
"""Own successful-result lookup, registration and sampling caches for one workspace."""
component_type = "home.evaluation_history"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
def _history_lock(self):
if not self.is_started:
raise RuntimeError("Start the Home application before reading evaluation history")
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
return _initialization_lock(self.workspace.state_path / "workspace.lock")
def _run_path(self, run: Path) -> Path:
home = self.workspace.home_path
run = contained(home, run, directory=True)
relative = run.relative_to(home / "evaluations")
if len(relative.parts) != 3:
raise ValueError("Expected one evaluation result directory")
return run
def _records(self) -> list[EvaluationRecord]:
"""Read source artifacts directly; legacy history.json files are ignored."""
home = self.workspace.home_path
records = []
for manifest in contained(home, "evaluations", directory=True).glob("*/*/*/manifest.json"):
record = result_record(home, contained(home, manifest.parent, directory=True))
if record is not None:
records.append(record)
return sorted(records, key=lambda record: (record.round, record.run_key))
def sampling_state(self) -> dict:
"""Return owned, reconstructed-on-demand observations for the next draw."""
with self._history_lock():
state = read_state(self.workspace.home_path)
return {"round": state["successful_round"], "cases": state["cases"], "queries": state["queries"]}
def record_started(self, run: Path, policy: SamplingPolicy, plan: dict | None = None) -> dict:
"""Record a started round, independently of whether it later enters successful history."""
with self._history_lock():
return start_validation(self.workspace.home_path, self._run_path(run), policy, plan)
def record(self, run: Path) -> EvaluationRecord | None:
"""Register a successful result once; unsuccessful runs never train the sampler."""
with self._history_lock():
home = self.workspace.home_path
run = self._run_path(run)
record = result_record(home, run)
if record is None:
return None
update_sampling_state(home, run)
return record
def list_results(self) -> list[EvaluationRecord]:
"""Return successful results in chronological order, including low-scoring candidates."""
with self._history_lock():
return self._records()
def get_result(self, run_key: str) -> EvaluationRecord:
"""Look up a successful result by its stable workspace-relative identity."""
for result in self.list_results():
if result.run_key == run_key:
return result
raise KeyError(f"No successful evaluation: {run_key}")
def rebuild(self) -> list[EvaluationRecord]:
"""Rebuild only derived indexes and sampling observations; never rewrite source evidence."""
with self._history_lock():
home = self.workspace.home_path
rebuild_sampling_state(home)
return self._records()
def invalidate(self) -> None:
"""Discard only derived caches after an unsuccessful registration; retain all source evidence."""
with self._history_lock():
contained(self.workspace.home_path, "sampling/state.json").unlink(missing_ok=True)

View file

@ -0,0 +1,290 @@
"""Validate prepared inputs and snapshot committed candidates before execution."""
from pathlib import Path
from typing import Literal
import shutil
import subprocess
import tempfile
import tarfile
from pydantic import Field, field_validator, model_validator
from deploy.benchmark_docker import SourceSnapshot
from ..preparation.code import git, tree_fingerprint
from ..preparation.data.models import CaseSpec, DatasetManifest, fingerprint
from .artifacts import contained, file_sha256, inspect_archive, read_json
from ..preparation.spec import SpecModel
from ..code.repository import validate_repository
from ..sampling import SamplingPolicy
class EvaluationSpec(SpecModel):
"""Selections are injected by callers; omitted cases mean the whole dataset."""
case_ids: list[str] | None = None
query_ids: dict[str, list[str]] | None = None
reuse_construction_from: str | None = Field(default=None, min_length=1)
debug: bool = False
sampling: bool = False
record_history: bool = True
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
base_image: str = Field(default="reme-benchmark-base:agentscope-2.0.10", min_length=1)
fail_fast: bool = False
max_construction_failure_rate: float = Field(default=0.03, ge=0, le=1, allow_inf_nan=False)
max_artifact_bytes: int = Field(default=2_000_000_000, ge=1)
@property
def dataset_kind(self) -> Literal["training-set", "debug-cases"]:
"""Resolve the internal dataset location from the requested mode."""
return "debug-cases" if self.debug else "training-set"
@field_validator("case_ids")
@classmethod
def validate_cases(cls, value):
"""An explicit selection must never silently expand or shrink."""
if value is not None and (
not value or any(not item.strip() for item in value) or len(set(value)) != len(value)
):
raise ValueError("case_ids must be nonempty, unique strings, or null for all cases")
return value
@field_validator("query_ids")
@classmethod
def validate_queries(cls, value):
"""Validate each explicit per-case query selection."""
if value is not None:
if not value:
raise ValueError("query_ids must not be empty")
for case_id, ids in value.items():
if not case_id.strip() or not ids or any(not item.strip() for item in ids) or len(set(ids)) != len(ids):
raise ValueError("query_ids must contain nonempty unique selections")
return value
@model_validator(mode="after")
def resolve_mode(self):
"""Debug stops on failure by default and never reuses scored evidence."""
if self.debug and self.reuse_construction_from:
raise ValueError("Construction reuse requires training-set")
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):
raise ValueError("sampling=true selects case/query IDs automatically; omit explicit selections")
if "fail_fast" not in self.model_fields_set:
self.fail_fast = self.debug
return self
def entity_id(value: str) -> str:
"""Keep case, query and branch IDs within a single artifact directory."""
if not value or value in {".", ".."} or "/" in value or "\\" in value or "\0" in value:
raise ValueError(f"Unsafe evaluation ID: {value!r}")
return value
def load_inputs(home: Path, spec: EvaluationSpec):
"""Validate selection and immutable dataset fingerprints before using Docker."""
record = read_json(contained(home, "metadata/preparation.json"))
directory = contained(home, f"datasets/{spec.dataset_kind}", directory=True)
manifest = DatasetManifest.model_validate(read_json(directory / "manifest.json"))
cases = {}
hashes = []
for path in sorted((directory / "cases").glob("*.json")):
case = CaseSpec.model_validate(read_json(path))
entity_id(case.case_id)
if case.case_id in cases or not case.queries:
raise ValueError("Dataset contains duplicate case IDs or a case without queries")
ids = [entity_id(query.query_id) for query in case.queries]
if len(ids) != len(set(ids)):
raise ValueError(f"Duplicate query IDs in case {case.case_id}")
hashes.append(fingerprint(case))
cases[case.case_id] = case
if list(cases) != manifest.case_ids or fingerprint(hashes) != manifest.normalized_fingerprint:
raise ValueError("Prepared dataset no longer matches its manifest")
if manifest.case_count != len(cases) or manifest.query_count != sum(len(c.queries) for c in cases.values()):
raise ValueError("Prepared dataset counts do not match its manifest")
if spec.dataset_kind == "training-set" and manifest != DatasetManifest.model_validate(record["dataset"]):
raise ValueError("Prepared training manifest has changed")
selected = spec.case_ids if spec.case_ids is not None else list(cases)
if not selected or set(selected) - cases.keys():
raise ValueError(f"Unknown or empty case selection: {sorted(set(selected) - cases.keys())}")
if spec.query_ids is not None and set(spec.query_ids) - set(selected):
raise ValueError("query_ids contains cases outside case_ids")
result = []
for case_id in selected:
case = cases[case_id]
if spec.query_ids is not None and case_id in spec.query_ids:
queries = {query.query_id: query for query in case.queries}
ids = spec.query_ids[case_id]
if set(ids) - queries.keys():
raise ValueError(f"Unknown query selection for {case_id}")
case = case.model_copy(update={"queries": [queries[key] for key in ids]})
result.append(case)
return record, manifest, result
def snapshot_code(home: Path, record: dict):
"""Combine fixed runtime with clean current HEAD; reject all uncommitted changes."""
repository, _ = validate_repository(home, record)
runtime = contained(home, "code/runtime", directory=True)
if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]:
raise ValueError("Fixed runtime has changed since preparation")
if git(repository, "status", "--porcelain=v2", "--untracked-files=all", "--ignore-submodules=none"):
raise ValueError("Commit candidate changes before evaluation")
commit = git(repository, "rev-parse", "--verify", "HEAD^{commit}")
branch = "init" if commit == record["code"]["initial_commit"] else git(repository, "branch", "--show-current")
entity_id(branch)
# NUL output validates exact paths without interpreting Git quoting rules.
entries = subprocess.check_output(
["git", "-C", str(repository), "ls-tree", "-r", "-z", commit],
timeout=120,
).split(b"\0")
parsed = []
for entry in entries:
if not entry:
continue
header, name = entry.split(b"\t", 1)
mode, kind, oid = header.decode().split()
relative = Path(name.decode())
if (
relative.is_absolute()
or ".." in relative.parts
or ".git" in relative.parts
or kind != "blob"
or mode not in {"100644", "100755"}
):
raise ValueError(f"Candidate must contain contained regular files: {relative}")
if not any(
relative.is_relative_to(path)
for path in [
Path("reme"),
Path("pyproject.toml"),
Path("README.md"),
Path("LICENSE"),
*[Path("plugins") / p for p in record["code"]["plugins"]],
]
):
raise ValueError(f"Candidate path is outside the prepared runtime projects: {relative}")
parsed.append((relative, mode, oid))
contents = subprocess.run(
["git", "-C", str(repository), "cat-file", "--batch"],
input="".join(oid + "\n" for _, _, oid in parsed).encode(),
capture_output=True,
check=True,
timeout=120,
).stdout
with tempfile.TemporaryDirectory(
prefix="evaluation-source-",
dir=contained(home, "tmp", directory=True),
) as temporary:
combined = Path(temporary) / "source"
shutil.copytree(runtime, combined)
# Core candidate roots replace their baseline roots, including deletions.
# Plugin projects exist only in repo and need no baseline overlay cleanup.
for value in record["code"]["candidate_paths"]:
if Path(value).parts[0] == "plugins":
continue
target = contained(combined, value)
if target.is_dir():
shutil.rmtree(target)
elif target.exists():
target.unlink()
offset = 0
for relative, mode, oid in parsed:
end = contents.index(b"\n", offset)
found, kind, size = contents[offset:end].decode().split()
if found != oid or kind != "blob":
raise ValueError("Invalid candidate Git object")
offset = end + 1
target = combined / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(contents[offset : offset + int(size)])
target.chmod(0o755 if mode == "100755" else 0o644)
offset += int(size) + 1
for project in record["code"]["install_projects"]:
if not (combined / project / "pyproject.toml").is_file():
raise ValueError(f"Candidate is missing install project {project}")
snapshot = SourceSnapshot.from_directory(combined)
return snapshot, commit, branch
def reuse_inputs(home: Path, spec: EvaluationSpec, cases: list, fingerprints: dict):
"""Preflight the entire requested selection; never silently skip or rebuild cases."""
if spec.reuse_construction_from is None:
return None, {}
evaluations = contained(home, "evaluations", directory=True)
value = spec.reuse_construction_from
source = contained(home, value, directory=True)
if not source.is_relative_to(evaluations) or source == evaluations:
raise ValueError("reuse_construction_from must be inside home/evaluations")
manifest = read_json(source / "manifest.json")
summary = read_json(source / "summary.json")
if manifest.get("dataset_kind") != "training-set" or manifest.get("status", summary.get("status")) != "completed":
raise ValueError("Reuse requires a completed training-set evaluation")
for key in ("dataset", "construction_config"):
if manifest.get("fingerprints", {}).get(key) != fingerprints[key]:
raise ValueError(f"Construction source has incompatible {key}")
archives, errors = {}, []
for case in cases:
try:
folder = contained(source, f"cases/{case.case_id}/memory_construction", directory=True)
result = read_json(folder / "result.json")
archive = contained(folder, "reme_workspace.tar.gz")
if result.get("success") is not True:
raise ValueError("construction was not successful")
if not result.get("workspace_sha256") or file_sha256(archive) != result["workspace_sha256"]:
raise ValueError("construction archive checksum mismatch")
inspect_archive(archive, spec.max_artifact_bytes)
archives[case.case_id] = (archive, result["workspace_sha256"])
except (ValueError, OSError, KeyError, tarfile.TarError) as error:
errors.append(f"{case.case_id}: {error}")
if errors:
raise ValueError("Invalid construction snapshots: " + "; ".join(errors))
return {"result_dir": str(source), "commit_sha": manifest["commit_sha"]}, archives
def replay_sampling_eligibility(home: Path, spec: EvaluationSpec, cases: list[CaseSpec]) -> dict | None:
"""Port dev_meta's replay eligibility without restricting future full rebuilds."""
if spec.reuse_construction_from is None:
return None
source = contained(home, spec.reuse_construction_from, directory=True)
evaluations = contained(home, "evaluations", directory=True)
if not source.is_relative_to(evaluations) or source == evaluations:
raise ValueError("reuse_construction_from must be inside home/evaluations")
manifest = read_json(contained(source, "manifest.json"))
summary = read_json(contained(source, "summary.json"))
if (
manifest.get("dataset_kind") != "training-set"
or manifest.get("mode") == "debug"
or (manifest.get("status", summary.get("status")) != "completed")
):
raise ValueError("Replay sampling requires a completed scored training-set run")
case_ids = manifest.get("case_ids")
if not isinstance(case_ids, list) or not case_ids or len(set(case_ids)) != len(case_ids):
raise ValueError("Replay source must select unique nonempty cases")
reusable = set()
for case_id in case_ids:
entity_id(case_id)
folder = contained(source, f"cases/{case_id}", directory=True)
result_path = contained(folder, "memory_construction/result.json")
archive = contained(folder, "memory_construction/reme_workspace.tar.gz")
if not result_path.is_file() or not archive.is_file() or read_json(result_path).get("success") is not True:
continue
reusable.add(case_id)
if (len(case_ids) - len(reusable)) / len(case_ids) > 0.05:
raise ValueError("Replay source construction failure ratio exceeds 5%")
for case_id in reusable:
payload = read_json(contained(source, f"cases/{case_id}/queries/result.json"))
queries = payload.get("queries")
if not queries or not all(
isinstance(query, dict)
and query.get("error") is None
and isinstance(query.get("answer"), str)
and query["answer"].strip()
for query in queries
):
raise ValueError(f"Replay source has no completed successful answers for {case_id}")
# A construction snapshot can answer new query subsets from the same case;
# dev_meta intentionally considers every installed query of a reusable case.
return {case.case_id: [query.query_id for query in case.queries] for case in cases if case.case_id in reusable}

View file

@ -0,0 +1,358 @@
"""Read-only evaluation navigation Jobs, independent of optimization history."""
import asyncio
import json
from pydantic import Field, field_validator, model_validator
from reme.steps.base_step import BaseStep, Ref
from ..code.repository import current_branch, resolve_revision, validate_branch, validate_repository
from ..preparation.code import git
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace, _initialization_lock
from .artifacts import contained, read_json
from .inputs import entity_id
TOTAL_FIELDS = (
"schema_version",
"scoring_policy",
"case_count",
"case_status_counts",
"construction_status_counts",
"failure_stage_case_counts",
"expected_query_count",
"scored_query_count",
"query_status_counts",
"mean_query_score",
"observed_mean_query_score",
)
class ListValidationResultsInput(SpecModel):
"""Filters affect catalog navigation, never which evaluations enter history."""
branch_name: str | None = None
commit_sha: str | None = Field(default=None, pattern=r"^[0-9a-fA-F]+$")
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
@field_validator("branch_name")
@classmethod
def validate_branch_name(cls, value):
"""Keep historical branch filters within one directory segment."""
return entity_id(value) if value is not None else None
class InspectValidationInput(SpecModel):
"""A query belongs to a case and every inspection belongs to one run."""
validation_path: str = Field(min_length=1)
case_id: str | None = None
query_id: str | None = None
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
@field_validator("case_id", "query_id")
@classmethod
def validate_entity(cls, value):
"""Reject path traversal before resolving any artifact."""
return entity_id(value) if value is not None else None
@model_validator(mode="after")
def validate_selection(self):
"""Require a case when selecting one query."""
if self.query_id is not None and self.case_id is None:
raise ValueError("query_id requires case_id")
return self
def _preview(value, limit=600):
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
return value if len(text) <= limit else text[:limit] + "… [truncated; read source field]"
def _pointer(value):
return str(value).replace("~", "~0").replace("/", "~1")
def _page(rows, spec):
return {
"row_count": len(rows),
"rows": rows[spec.offset : spec.offset + spec.limit],
"next_offset": spec.offset + spec.limit if spec.offset + spec.limit < len(rows) else None,
}
def _object(root, relative, *, required=False):
path = contained(root, relative)
if not path.exists() and not required:
return None
payload = read_json(path)
if not isinstance(payload, dict):
raise ValueError(f"Expected JSON object: {path}")
return payload
def _summary(run):
for name in ("summary.json", "partial_summary.json"):
value = _object(run, name)
if value is not None:
return value, str(run / name)
return None, None
def _run_path(home, value):
run = contained(home, value, directory=True)
relative = run.relative_to(home / "evaluations")
if len(relative.parts) != 3 or not run.is_dir():
raise ValueError("validation_path must identify one directory inside home/evaluations")
return run
def _commit_filter(home, spec):
if spec.commit_sha is None:
return spec.branch_name, None
# Resolve a local branch first, then match ALL of its reachable commits.
# Git's global abbreviation rules could otherwise reject a branch-unique prefix.
with _initialization_lock(contained(home, "metadata/workspace.lock")):
repo, _ = validate_repository(home)
branch = spec.branch_name if spec.branch_name is not None else current_branch(repo)
validate_branch(repo, branch)
tip = resolve_revision(repo, "refs/heads/" + branch)
prefix = spec.commit_sha.lower()
matches = [sha for sha in git(repo, "rev-list", tip, "--").splitlines() if sha.startswith(prefix)]
if len(matches) != 1:
raise ValueError(f"commit_sha prefix {prefix!r} has {len(matches)} matches on branch {branch!r}; require one")
return branch, matches[0]
def _catalog_row(run, manifest):
summary, summary_path = _summary(run)
options = manifest.get("options", {})
row = {
"validation_path": str(run),
"manifest_path": str(run / "manifest.json"),
**{
key: manifest.get(key)
for key in (
"branch_name",
"commit_sha",
"validation_id",
"status",
"success",
"started_at",
"finished_at",
"dataset",
"dataset_kind",
"mode",
"execution_mode",
"construction_source",
"fingerprints",
)
},
"sampling": options.get("sampling", False),
"record_history": options.get("record_history", True) and manifest.get("mode") != "debug",
"selected_case_count": len(manifest.get("case_ids", [])),
"selected_query_count": sum(len(ids) for ids in manifest.get("query_ids", {}).values()),
"summary_path": summary_path,
"summary_missing": summary is None,
"totals": {key: summary[key] for key in TOTAL_FIELDS if key in summary} if summary else {},
}
failure = _object(run, "failure.json")
row["failure"] = _preview(failure)
row["failure_path"] = str(run / "failure.json") if failure is not None else None
return row
def _list_results(home, spec):
branch, commit = _commit_filter(home, spec)
evaluations = contained(home, "evaluations", directory=True)
rows = []
for path in evaluations.glob("*/*/*/manifest.json"):
if branch is not None and path.parent.parent.parent.name != branch:
continue
row = {"validation_path": str(path.parent), "manifest_path": str(path)}
try:
run = _run_path(home, path.parent)
manifest = _object(run, "manifest.json", required=True)
row.update({key: manifest.get(key) for key in ("branch_name", "commit_sha", "started_at")})
if commit is not None and row["commit_sha"] != commit:
continue
row = _catalog_row(run, manifest)
except (ValueError, OSError, TypeError, KeyError, AttributeError) as error:
row["read_error"] = _preview(str(error))
if commit is None or row.get("commit_sha") == commit:
rows.append(row)
rows.sort(key=lambda row: (str(row.get("started_at") or ""), row["validation_path"]), reverse=True)
return {"branch_name": branch, "commit_sha": commit, **_page(rows, spec)}
def _case_rows(manifest, summary):
cases = (summary or {}).get("cases", {})
indexed = (
{key: (value, f"/cases/{_pointer(key)}") for key, value in cases.items()}
if isinstance(cases, dict)
else {value["case_id"]: (value, f"/cases/{index}") for index, value in enumerate(cases)}
)
rows = []
for case_id in manifest.get("case_ids", []):
case, pointer = indexed.get(case_id, ({}, None))
rows.append(
{
"case_id": case_id,
"mean_score": case.get("mean_score"),
"construction": case.get("construction"),
"construction_error_count": case.get("construction_error_count", 0),
"query_error_count": case.get("query_error_count", 0),
"query_tokens": case.get("average_query_total_token_usage"),
"missing": pointer is None,
"pointer": pointer,
},
)
rows.sort(
key=lambda row: (
row["mean_score"] is not None,
row["mean_score"] if row["mean_score"] is not None else -1,
-row["construction_error_count"],
row["case_id"],
),
)
return rows
def _case_inspection(run, manifest, spec):
folder = contained(run, f"cases/{spec.case_id}", directory=True)
build_path, query_path = "memory_construction/result.json", "queries/result.json"
build, payload = _object(folder, build_path), _object(folder, query_path)
failures = []
jobs = (build or {}).get("jobs", [])
for index, job in enumerate(jobs):
outcome = job.get("result") or {}
if outcome.get("success") is False:
failures.append(
{
"job": job.get("job"),
"error": _preview(outcome.get("error")),
"pointer": f"/jobs/{index}/result",
},
)
queries = (payload or {}).get("queries", [])
by_id = {query["query_id"]: (query, index) for index, query in enumerate(queries)}
selected = manifest["query_ids"][spec.case_id]
if spec.query_id is not None and spec.query_id not in selected:
raise ValueError("query_id is not selected by this evaluation manifest")
rows = []
for query_id in ([spec.query_id] if spec.query_id is not None else selected):
query, index = by_id.get(query_id, ({}, None))
row = {
"query_id": query_id,
"score": query.get("score"),
"question_type": query.get("question_type"),
"failure_kind": query.get("failure_kind"),
"error": _preview(query.get("error")),
"answer_success": (query.get("answer_result") or {}).get("success"),
"judge_success": (query.get("judge_result") or {}).get("success"),
"missing": index is None,
"pointer": f"/queries/{index}" if index is not None else None,
}
if spec.query_id is not None:
row["fields"] = {
key: _preview(query[key], 2000)
for key in (
"question",
"golden_answer",
"answer",
"score",
"question_type",
"answer_result",
"judge_result",
"error",
"exception",
"failure_kind",
"infrastructure_error",
)
if key in query
}
rows.append(row)
result = {
"case_id": spec.case_id,
"missing_files": [name for name, value in ((build_path, build), (query_path, payload)) if value is None],
"case_failure": _preview(_object(folder, "failure.json")),
"construction": {
"source": str(folder / build_path),
"success": (build or {}).get("success"),
"status": (build or {}).get("status"),
"job_count": len(jobs),
"failed_jobs": _page(failures, spec),
},
"build_log": str(contained(folder, "memory_construction/build.log")),
"memory_workspace": str(contained(folder, "memory_construction/reme_workspace", directory=True)),
"rows_source": str(folder / query_path),
**_page(rows, spec),
}
if spec.query_id is not None:
query_folder = contained(folder, f"queries/{spec.query_id}", directory=True)
result.update(
query_missing=rows[0]["missing"],
answer_log=str(contained(query_folder, "answer.log")),
query_failure=_preview(_object(query_folder, "failure.json")),
)
return result
def _inspect_result(home, spec):
run = _run_path(home, spec.validation_path)
manifest = _object(run, "manifest.json", required=True)
if spec.case_id is not None and spec.case_id not in manifest.get("case_ids", []):
raise ValueError("case_id is not selected by this evaluation manifest")
result = {"validation_path": str(run), "manifest_path": str(run / "manifest.json")}
if spec.case_id is not None:
return {**result, **_case_inspection(run, manifest, spec)}
summary, summary_path = _summary(run)
return {
**_catalog_row(run, manifest),
"rows_source": summary_path,
**_page(_case_rows(manifest, summary), spec),
}
async def _execute(step, model, function):
if step.context.get("_allowed_paths") is not None:
raise ValueError("Home inspection Jobs require unrestricted access to their managed evaluation artifacts")
spec = model.model_validate({name: step.context.get(name) for name in model.model_fields if name in step.context})
if not step.workspace.is_started:
raise RuntimeError("Start the Home workspace before inspecting evaluations")
task = asyncio.create_task(asyncio.to_thread(function, step.workspace.home_path, spec))
cancelled = False
while True:
try:
result = await asyncio.shield(task)
if cancelled:
raise asyncio.CancelledError
step.context.response.answer = result
return step.context.response
except asyncio.CancelledError:
if task.done():
raise
cancelled = True
class HomeListValidationResultsStep(BaseStep):
"""List all evaluation artifacts, including failures, debug and standalone runs."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Navigate runs without reading or rebuilding optimization history."""
return await _execute(self, ListValidationResultsInput, _list_results)
class HomeInspectValidationStep(BaseStep):
"""Inspect a run, a case or one query with bounded result previews."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Read only the requested result scope and return artifact coordinates."""
return await _execute(self, InspectValidationInput, _inspect_result)

View file

@ -0,0 +1,284 @@
"""Read-only navigation and comparison of exported memory Git checkpoints."""
import difflib
import hashlib
import re
import subprocess
from pydantic import Field, field_validator, model_validator
from reme.steps.base_step import BaseStep, Ref
from ..code.repository import MAX_OUTPUT, resolve_revision
from ..code.snapshot import blob, code_path, tree
from ..preparation.code import git as safe_git
from ..preparation.spec import SpecModel
from ..workspace import HomeWorkspace
from .inputs import entity_id
from .artifacts import contained
from .inspection_step import _execute, _object, _page, _run_path
MAX_TREE_FILES = 2000
MAX_TREE_BYTES = 20_000_000
def git(repository, *arguments):
"""Ignore replace refs while reusing the shared noninteractive Git executor."""
return safe_git(repository, "--no-replace-objects", *arguments)
class MemoryScopeInput(SpecModel):
"""An existing evaluation and one manifest-selected case."""
validation_path: str = Field(min_length=1)
case_id: str = Field(min_length=1)
path: str | None = None
@field_validator("case_id")
@classmethod
def validate_case(cls, value):
"""Accept a single case directory segment."""
return entity_id(value)
@field_validator("path")
@classmethod
def validate_path(cls, value):
"""Accept literal repository-relative paths only."""
return code_path(value) if value is not None else None
class MemoryHistoryInput(MemoryScopeInput):
"""Page through checkpoints, newest first."""
offset: int = Field(default=0, ge=0)
limit: int = Field(default=20, ge=1, le=100)
class CompareMemoryInput(MemoryScopeInput):
"""Compare two commits in one or two exported repositories."""
base_ref: str | None = Field(default=None, min_length=1)
target_ref: str = Field(default="HEAD", min_length=1)
target_validation_path: str | None = Field(default=None, min_length=1)
@model_validator(mode="after")
def validate_base(self):
"""Require an explicit earlier checkpoint for a same-repository comparison."""
if self.target_validation_path is None and self.base_ref is None:
raise ValueError("base_ref is required when target_validation_path is omitted")
return self
def _repository(home, validation_path, case_id):
run = _run_path(home, validation_path)
manifest = _object(run, "manifest.json", required=True)
if case_id not in manifest.get("case_ids", []):
raise ValueError("case_id is not selected by this evaluation manifest")
repository = contained(run, f"cases/{case_id}/memory_construction/reme_workspace", directory=True)
return run, manifest, repository
def _head(repository):
"""Reject indirect Git storage and distinguish absent/empty/corrupt repositories."""
if not repository.exists():
return "workspace_missing", None
metadata = contained(repository, ".git")
if not metadata.exists():
return "history_missing", None
if not metadata.is_dir():
raise ValueError("Memory history requires a standalone .git directory")
# Exported artifacts must not redirect reads to another repository or host path.
if any(path.is_symlink() for path in metadata.rglob("*")):
raise ValueError("Memory Git metadata must not contain symbolic links")
for name in ("commondir", "objects/info/alternates", "objects/info/http-alternates", "shallow"):
if (metadata / name).exists():
raise ValueError(f"Unsupported indirect or incomplete memory history: {name}")
if any(metadata.glob("objects/pack/*.promisor")):
raise ValueError("Partial memory repositories are unsupported")
if git(repository, "for-each-ref", "--format=%(refname)", "refs/replace"):
raise ValueError("Memory replace refs are unsupported")
configuration = git(repository, "config", "--local", "--list")
if any(line.lower().startswith(("include.", "includeif.")) for line in configuration.splitlines()):
raise ValueError("Memory Git configuration must not include external files")
if any(
"promisor=" in line.lower() or line.lower().startswith("extensions.partialclone=")
for line in configuration.splitlines()
):
raise ValueError("Partial memory repositories are unsupported")
if git(repository, "rev-parse", "--show-toplevel") != str(repository):
raise ValueError("Memory history must be a standalone repository")
if git(repository, "rev-parse", "--is-shallow-repository") != "false":
raise ValueError("Memory history must not be shallow")
try:
return "available", resolve_revision(repository, "HEAD")
except subprocess.CalledProcessError:
# Only an unborn symbolic HEAD is an empty repository, not a broken ref.
reference = git(repository, "symbolic-ref", "-q", "HEAD")
if git(repository, "for-each-ref", "--format=%(refname)", reference):
raise ValueError("Memory HEAD cannot be resolved") from None
if not (metadata / "objects").is_dir():
raise ValueError("Memory object database is missing") from None
return "history_empty", None
def _resolve(repository, reference, head):
if re.fullmatch(r"[0-9a-fA-F]+", reference or ""):
matches = [
sha for sha in git(repository, "rev-list", head, "--").splitlines() if sha.startswith(reference.lower())
]
if len(matches) != 1:
raise ValueError(f"Memory commit prefix has {len(matches)} matches reachable from HEAD; require one")
return matches[0]
revision = resolve_revision(repository, reference)
git(repository, "merge-base", "--is-ancestor", revision, head)
return revision
def inspect_history(home, spec):
"""Return saved Git history without refreshing the worktree or index."""
run, _manifest, repository = _repository(home, spec.validation_path, spec.case_id)
status, head = _head(repository)
result = {
"validation_path": str(run),
"case_id": spec.case_id,
"repository": str(repository),
"workspace_exists": repository.exists(),
"has_git_history": head is not None,
"status": status,
"head": head,
"path": spec.path,
}
if head is None:
return {**result, **_page([], spec)}
# Literal pathspecs prevent user-controlled wildcard/pathspec expansion.
path_args = ["--", ":(literal)" + spec.path] if spec.path is not None else ["--"]
count = int(git(repository, "rev-list", "--count", head, *path_args))
raw = git(
repository,
"log",
"--no-show-signature",
f"--skip={spec.offset}",
f"--max-count={spec.limit}",
"--format=%H%x00%P%x00%aI%x00%s",
head,
*path_args,
)
rows = []
for line in raw.splitlines():
sha, parents, timestamp, subject = line.split("\0", 3)
rows.append(
{
"commit_sha": sha,
"parents": parents.split(),
"timestamp": timestamp,
"subject": subject[:2000],
"subject_truncated": len(subject) > 2000,
"session_id": subject[9:2009] if subject.startswith("session: ") else None,
},
)
return {
**result,
"row_count": count,
"rows": rows,
"next_offset": spec.offset + len(rows) if spec.offset + len(rows) < count else None,
}
def _files(repository, revision, path):
entries = tree(repository, revision, path)
if len(entries) > MAX_TREE_FILES:
raise ValueError("Memory comparison exceeds 2000 files; narrow path")
result = {}
total = 0
for name, oid in entries:
total += int(git(repository, "cat-file", "-s", oid))
if total > MAX_TREE_BYTES:
raise ValueError("Memory comparison exceeds 20 MB per snapshot; narrow path")
result[name] = blob(repository, oid)
return result
def _hash(files):
digest = hashlib.sha256()
for name, data in sorted(files.items()):
digest.update(name.encode() + b"\0" + hashlib.sha256(data).digest())
return digest.hexdigest()
def compare_memory(home, spec):
"""Compare committed regular-file contents, independent of dirty checkout files."""
base_run, base_manifest, base_repo = _repository(home, spec.validation_path, spec.case_id)
target_run, target_manifest, target_repo = _repository(
home,
spec.target_validation_path or spec.validation_path,
spec.case_id,
)
base_status, base_head = _head(base_repo)
target_status, target_head = _head(target_repo)
if base_head is None or target_head is None:
raise ValueError(f"Comparison requires memory commits on both sides: {base_status}, {target_status}")
base_sha = _resolve(base_repo, spec.base_ref or "HEAD", base_head)
target_sha = _resolve(target_repo, spec.target_ref, target_head)
before = _files(base_repo, base_sha, spec.path)
after = _files(target_repo, target_sha, spec.path)
added = sorted(after.keys() - before.keys())
removed = sorted(before.keys() - after.keys())
modified = sorted(name for name in before.keys() & after.keys() if before[name] != after[name])
parts = []
length = 0
truncated = False
for name in sorted(added + removed + modified):
left, right = before.get(name, b""), after.get(name, b"")
try:
if b"\0" in left or b"\0" in right:
raise UnicodeDecodeError("utf-8", b"\0", 0, 1, "binary")
lines = difflib.unified_diff(
left.decode("utf-8").splitlines(keepends=True),
right.decode("utf-8").splitlines(keepends=True),
fromfile="base/" + name,
tofile="target/" + name,
)
except UnicodeDecodeError:
lines = iter([f"Binary files differ: {name}\n"])
for line in lines:
if not line.endswith("\n"):
line += "\n\\ No newline at end of file\n"
available = MAX_OUTPUT - length
parts.append(line[:available])
length += min(len(line), available)
if len(line) > available:
truncated = True
break
if truncated:
break
return {
"base_validation_path": str(base_run),
"target_validation_path": str(target_run),
"case_id": spec.case_id,
"path": spec.path,
"base_commit": base_sha,
"target_commit": target_sha,
"base_candidate_commit": base_manifest.get("commit_sha"),
"target_candidate_commit": target_manifest.get("commit_sha"),
"base_fingerprints": base_manifest.get("fingerprints"),
"target_fingerprints": target_manifest.get("fingerprints"),
"added": added,
"removed": removed,
"modified": modified,
"base_tree_sha256": _hash(before),
"target_tree_sha256": _hash(after),
"diff": "".join(parts),
"truncated": truncated,
}
class HomeMemoryInspectionStep(BaseStep):
"""Two read-only Jobs sharing one invocation-scoped Step implementation."""
workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
async def execute(self):
"""Select the operation from host configuration, never model input."""
operations = {"history": (MemoryHistoryInput, inspect_history), "compare": (CompareMemoryInput, compare_memory)}
model, function = operations[self.kwargs["operation"]]
return await _execute(self, model, function)

View file

@ -0,0 +1,36 @@
"""Expose immutable batch evaluation through ReMe's normal Job lifecycle."""
from reme.steps.base_step import BaseStep, Ref
from .component import HomeEvaluation
from .inputs import EvaluationSpec
class HomeEvaluateStep(BaseStep):
"""One evaluation Job with optional host-configured adaptive sampling."""
evaluation: HomeEvaluation = Ref(HomeEvaluation, "home.evaluation")
async def execute(self):
"""Evaluate via the Docker pool and persist host-side scores and evidence."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories")
arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context}
# Do not silently accept a removed toggle that conflicts with reuse paths.
if "eval_only" in self.context:
raise ValueError("Use reuse_construction_from instead of eval_only")
if "code_ref" in self.context:
raise ValueError("code_ref is unsupported; commit candidate changes and evaluate the current HEAD")
if "judge_concurrency" in self.context:
raise ValueError("Use concurrency to control both case and judge concurrency")
if "dataset_kind" in self.context:
raise ValueError("Use debug=true for debug cases; omit debug or use false for training cases")
result = await self.evaluation.evaluate(**arguments)
self.context.response.answer = result
# Candidate failures are evaluation observations; infrastructure failures
# must remain visible to ordinary Job callers checking Response.success.
self.context.response.success = result["case_status_counts"]["infra_error"] == 0
if arguments.get("debug"):
self.context.response.success &= result["case_status_counts"]["candidate_failure"] == 0
self.context.response.metadata["result_dir"] = result["result_dir"]
return self.context.response

View file

@ -0,0 +1,141 @@
"""Adapt host-installed benchmark judge Jobs without loading candidate code."""
import hashlib
import json
from importlib import import_module
import math
from pathlib import Path
from reme.components.base_component import BaseComponent
from reme.schema import ComponentConfig
from .preparation.data.models import fingerprint
from .evaluation.artifacts import contained, read_json
from .workspace import HomeWorkspace
JUDGE_PLUGINS = {"beam": "beam-judge", "longmemeval": "lme-judge"}
_BACKENDS = {"beam-judge": "beam_rubric_judge_step", "lme-judge": "lme_answer_judge_step"}
_MODULES = {"beam-judge": "judge_beam", "lme-judge": "judge_lme"}
class JudgeResultError(ValueError):
"""Preserve the host Job's response when scoring or result validation fails."""
def __init__(self, message, result):
super().__init__(message)
self.result = result
class HomeJudge(BaseComponent):
"""Validate the host judge selection and adapt its standard answer_judge Job."""
component_type = "home.judge"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
async def _start(self):
self.require_benchmark()
marker = contained(self.workspace.home_path, "metadata/preparation.json")
if marker.is_file():
self.require_benchmark(read_json(marker)["dataset"]["dataset"])
def require_benchmark(self, benchmark=None):
"""Reject missing, extra, mismatched or overridden benchmark judge Jobs."""
plugins = self.app_context.app_config.plugins
judges = [name for name in plugins if name.endswith("-judge")]
if len(judges) != 1 or judges[0] not in _BACKENDS:
raise ValueError("Home requires exactly one supported judge plugin: beam-judge or lme-judge")
plugin = judges[0]
if benchmark is not None and JUDGE_PLUGINS.get(benchmark) != plugin:
raise ValueError(f"Benchmark {benchmark!r} requires {JUDGE_PLUGINS.get(benchmark)!r}, enabled {plugin!r}")
job = self.app_context.jobs.get("answer_judge")
if job is None or not job.enabled:
raise ValueError("Home requires an enabled answer_judge Job")
steps = job.step_configs
if len(steps) != 1 or ComponentConfig.model_validate(steps[0]).backend != _BACKENDS[plugin]:
raise ValueError(f"Home requires answer_judge from {plugin}")
return plugin
def identity(self, benchmark):
"""Record host plugin code and effective non-secret scoring configuration."""
plugin = self.require_benchmark(benchmark)
root = Path(import_module(_MODULES[plugin]).__file__).parent
files = {
path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
for path in sorted(root.rglob("*"))
if path.is_file() and path.suffix in {".py", ".yaml", ".yml", ".json"}
}
config = self.app_context.app_config
step = ComponentConfig.model_validate(self.app_context.jobs["answer_judge"].step_configs[0])
wrapper_name = step.model_dump().get("agent_wrapper", "judge")
wrapper = config.components.get("agent_wrapper", {}).get(wrapper_name)
wrapper_config = wrapper.model_dump(mode="json") if wrapper else {}
model_name = wrapper_config.get("as_llm", "judge")
model = config.components.get("as_llm", {}).get(model_name)
model_config = model.model_dump(mode="json", exclude={"credential"}) if model else {}
return {
"plugin": plugin,
"model": model_config.get("model"),
"fingerprint": fingerprint(
{
"files": files,
"adapter": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
"step": step.model_dump(mode="json"),
"wrapper": wrapper_config,
"model": model_config,
},
),
}
async def evaluate(self, dataset, query, answer):
"""Call the host Job and reject invalid scores as infrastructure failures."""
self.require_benchmark(dataset)
job = self.app_context.jobs["answer_judge"]
if not job.is_started:
raise RuntimeError("Start answer_judge before evaluating")
if dataset == "beam":
arguments = {
"llm_response": answer,
"rubric": query.metadata.get("rubric", query.golden_answer),
"probing_question": query.question,
"question_type": query.metadata.get("question_type", ""),
}
else:
arguments = {
"query": query.question,
"agent_answer": answer,
"golden_answer": query.golden_answer,
"question_type": query.metadata.get("question_type", ""),
}
response = await job(**arguments)
result = json.loads(response.model_dump_json())
try:
score = self._score_response(dataset, response, arguments.get("rubric", []))
except (ValueError, TypeError) as error:
raise JudgeResultError(str(error), result) from error
return result, score
@staticmethod
def _score_response(dataset, response, rubric):
"""Keep benchmark-specific scores and reject failed/invalid Job output."""
if not response.success:
raise ValueError(f"answer_judge failed: {response.answer}")
if dataset == "beam":
scores = response.metadata.get("llm_judge_responses")
if not isinstance(scores, list) or len(scores) != len(rubric):
raise ValueError("BEAM judge returned incomplete rubric results")
for item in scores:
if not isinstance(item, dict) or str(item.get("reason", "")).startswith("Failed to parse:"):
raise ValueError("BEAM judge failed to parse rubric output")
if item.get("score") not in {0, 0.5, 1}:
raise ValueError("BEAM judge returned an invalid rubric score")
score = float(response.metadata.get("llm_judge_score", float("nan")))
if not math.isfinite(score) or not 0 <= score <= 1:
raise ValueError("BEAM judge returned an invalid aggregate score")
else:
if response.answer not in {"yes", "no"}:
raise ValueError("LME judge returned an invalid yes/no judgement")
score = 1.0 if response.answer == "yes" else 0.0
return score

View file

@ -1,8 +1,225 @@
backends:
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep
home_memory_inspection_step: reme_home.evaluation.memory_inspection_step:HomeMemoryInspectionStep
home_code: reme_home.code.component:HomeCode
home_code_step: reme_home.code.step:HomeCodeStep
home_bounded: reme_home.agent:BoundedAgentWrapper
home_preparation: reme_home.preparation.component:HomePreparation
home_prepare_step: reme_home.preparation.component:HomePrepareStep
home_workspace: reme_home.workspace:HomeWorkspace
home_judge: reme_home.judge:HomeJudge
home_evaluation: reme_home.evaluation.component:HomeEvaluation
home_evaluation_history: reme_home.evaluation.history:HomeEvaluationHistory
home_evaluate_step: reme_home.evaluation.step:HomeEvaluateStep
home_list_validation_results_step: reme_home.evaluation.inspection_step:HomeListValidationResultsStep
home_inspect_validation_step: reme_home.evaluation.inspection_step:HomeInspectValidationStep
auto_bash_step: reme_home.auto_bash:AutoBashStep
application_defaults:
components:
agent_wrapper:
home_case_study:
backend: home_case_study_agent
as_llm: judge
time_budget_minutes: 30
react_config:
max_iters: 60
home.code:
default:
backend: home_code
home.judge:
default:
backend: home_judge
home.evaluation_history:
default:
backend: home_evaluation_history
home.evaluation:
default:
backend: home_evaluation
sampling_policy:
max_cases: 5
max_queries: null
temperature: 1.0
ewma_alpha: 0.3
uniform_exploration: 0.2
home.preparation:
default:
backend: home_preparation
home.workspace:
default:
backend: home_workspace
jobs:
home_case_study:
backend: base
description: "Investigate one validation case with read-only sandboxed Bash/Python and commit-pinned evidence tools; return an evidence-supported diagnostic report"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
focus: {type: string, minLength: 1, description: "Questions or failure mechanisms to prioritize"}
steps:
- backend: home_case_study_step
agent_wrapper: home_case_study
output_token_limit: 2500
home_trace_query_evidence:
backend: base
description: "Locate a selected query's source evidence sessions, memory checkpoints and answer/score previews; use home_compare_memory and returned paths for details"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id, query_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
query_id: {type: string, minLength: 1}
evidence_session_ids:
type: [array, "null"]
items: {type: string, minLength: 1}
description: "Optional inferred session IDs, used only when the query has no dataset annotation"
steps:
- backend: home_trace_query_evidence_step
home_inspect_memory_history:
backend: base
description: "List a case's exported memory Git checkpoints, newest first; use home_inspect_validation to locate the case"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
path:
type: [string, "null"]
description: "Literal memory-workspace-relative file or directory to follow"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_memory_inspection_step
operation: history
home_compare_memory:
backend: base
description: "Compare a case's memory checkpoints or committed final memory across evaluations; narrow path when output is truncated"
parameters:
type: object
additionalProperties: false
required: [validation_path, case_id]
properties:
validation_path: {type: string, minLength: 1}
case_id: {type: string, minLength: 1}
base_ref:
type: [string, "null"]
description: "Required within one evaluation; defaults to HEAD for cross-evaluation comparisons"
target_ref:
type: string
default: HEAD
description: "Target memory commit or revision; hexadecimal prefixes must be unique among HEAD-reachable commits"
target_validation_path:
type: [string, "null"]
description: "Optional other evaluation containing the same case"
path:
type: [string, "null"]
description: "Literal memory-workspace-relative file or directory"
steps:
- backend: home_memory_inspection_step
operation: compare
home_list_validation_results:
backend: base
description: "List evaluation runs including debug, failed and standalone runs; inspect a returned validation_path with home_inspect_validation"
parameters:
type: object
additionalProperties: false
properties:
branch_name:
type: [string, "null"]
description: "Filter by branch; with commit_sha defaults to the current candidate branch"
commit_sha:
type: [string, "null"]
pattern: "^[0-9a-fA-F]+$"
description: "Full commit SHA or hexadecimal prefix matching exactly one reachable commit on the selected branch"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_list_validation_results_step
home_inspect_validation:
backend: base
description: "Read an evaluation's case summaries, a case's query summaries, or one query's answer/judge previews; follow next_offset and source paths for details"
parameters:
type: object
additionalProperties: false
required: [validation_path]
properties:
validation_path:
type: string
minLength: 1
description: "Result directory returned by home_evaluate or home_list_validation_results; absolute or relative to home/"
case_id:
type: [string, "null"]
description: "Selected case to inspect; omit to list cases"
query_id:
type: [string, "null"]
description: "Selected query to inspect in detail; requires case_id"
offset: {type: integer, minimum: 0, default: 0}
limit: {type: integer, minimum: 1, maximum: 100, default: 20}
steps:
- backend: home_inspect_validation_step
home_evaluate:
backend: base
description: "Evaluate committed code; optionally sample cases/queries or restore prior construction tar files"
parameters:
type: object
additionalProperties: false
properties:
sampling:
type: boolean
default: false
description: "Adaptively select case/query IDs using the host sampling_policy; debug ignores this flag"
record_history:
type: boolean
default: true
description: "Register successful scored results and started rounds for optimization; false saves results without history or sampling updates; debug always skips history"
# case_ids/query_ids are internal runtime inputs, supplied by the host
# through injected_job_kwargs or run_job(), never model-selected tools.
reuse_construction_from:
type: [string, "null"]
description: "Prior result directory inside home/evaluations; restores memory without rebuilding"
debug:
type: boolean
default: false
description: "Run prepared debug cases without judging; fail_fast defaults to true in this mode"
concurrency: {type: integer, minimum: 1}
timeout_seconds: {type: number, exclusiveMinimum: 0}
base_image: {type: string}
fail_fast:
type: boolean
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
max_artifact_bytes: {type: integer, minimum: 1}
steps:
- backend: home_evaluate_step
home_prepare:
backend: base
description: "Prepare experiment data and code once; failure rolls back, success forbids another call"
parameters:
type: object
properties:
spec_path:
type: string
description: "YAML preparation file inside the workspace"
spec:
type: object
description: "Preparation configuration, mutually exclusive with spec_path"
oneOf:
- required: [spec_path]
not: {required: [spec]}
- required: [spec]
not: {required: [spec_path]}
steps:
- backend: home_prepare_step
auto_bash:
backend: base
description: "Run a shell command in the workspace; read-only mode is enabled by default"
@ -38,3 +255,191 @@ application_defaults:
- command
steps:
- backend: auto_bash_step
home_code_status:
backend: base
description: Inspect candidate branch, commit and uncommitted files
parameters:
additionalProperties: false
properties: {}
type: object
steps:
- backend: home_code_step
operation: status
home_list_candidate_branches:
backend: base
description: List the protected init branch and local candidate branches
parameters:
additionalProperties: false
properties: {}
type: object
steps:
- backend: home_code_step
operation: list_branches
home_create_candidate_branch:
backend: base
description: Create and switch to a new candidate branch; requires clean code
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: create_branch
home_switch_candidate_branch:
backend: base
description: Switch to an existing local branch; requires clean code
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: switch_branch
home_merge_candidate_branch:
backend: base
description: Stage another candidate branch without committing; conflicts are aborted
parameters:
additionalProperties: false
properties:
branch_name:
maxLength: 100
minLength: 1
type: string
required:
- branch_name
type: object
steps:
- backend: home_code_step
operation: merge_branch
home_commit_candidate:
backend: base
description: Commit all changes within candidate_paths; init and existing history are protected
parameters:
additionalProperties: false
properties:
message:
maxLength: 10000
minLength: 1
pattern: \S
type: string
required:
- message
type: object
steps:
- backend: home_code_step
operation: commit
home_code_diff:
backend: base
description: Inspect tracked working changes against HEAD or compare immutable commits
parameters:
additionalProperties: false
properties:
base_revision:
anyOf:
- type: string
- type: 'null'
default: null
revision:
anyOf:
- type: string
- type: 'null'
default: null
type: object
steps:
- backend: home_code_step
operation: diff
home_list_code_tree:
backend: base
description: List tracked regular files at a committed revision
parameters:
additionalProperties: false
properties:
revision:
default: HEAD
minLength: 1
type: string
path:
anyOf:
- type: string
- type: 'null'
default: null
max_results:
default: 500
maximum: 2000
minimum: 1
type: integer
type: object
steps:
- backend: home_code_step
operation: list_code_tree
home_read_code_file:
backend: base
description: Read up to 300 lines from a committed code file
parameters:
additionalProperties: false
properties:
path:
minLength: 1
type: string
revision:
default: HEAD
minLength: 1
type: string
start_line:
default: 1
minimum: 1
type: integer
end_line:
anyOf:
- minimum: 1
type: integer
- type: 'null'
default: null
required:
- path
type: object
steps:
- backend: home_code_step
operation: read_code_file
home_search_code:
backend: base
description: Search committed code with a bounded Python multiline regular expression
parameters:
additionalProperties: false
properties:
pattern:
maxLength: 500
minLength: 1
type: string
revision:
default: HEAD
minLength: 1
type: string
path:
anyOf:
- type: string
- type: 'null'
default: null
max_results:
default: 30
maximum: 30
minimum: 1
type: integer
required:
- pattern
type: object
steps:
- backend: home_code_step
operation: search_code

View file

@ -0,0 +1 @@
"""Home preparation implementation."""

View file

@ -0,0 +1,166 @@
"""Export trusted source inputs without running their code or installing them."""
import hashlib
import os
from pathlib import Path
import shutil
import subprocess
import tomllib
from .spec import PreparationSpec, relative_path
def git_bytes(repository: Path, *arguments: str) -> bytes:
"""Run bounded, noninteractive Git commands without a shell or user hooks."""
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
environment.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0")
result = subprocess.run(
[
"git",
"-c",
"core.hooksPath=" + os.devnull,
"-c",
"commit.gpgSign=false",
"-c",
"core.fsmonitor=false",
"-C",
str(repository),
*arguments,
],
check=True,
capture_output=True,
env=environment,
timeout=120,
)
return result.stdout
def git(repository: Path, *arguments: str, strip: bool = True) -> str:
"""Decode Git output, optionally preserving porcelain whitespace and path bytes."""
output = git_bytes(repository, *arguments).decode("utf-8")
return output.strip() if strip else output
def tree_fingerprint(root: Path) -> str:
"""Hash relative names and file bytes, excluding candidate Git metadata."""
digest = hashlib.sha256()
for path in sorted(root.rglob("*")):
relative = path.relative_to(root)
if ".git" in relative.parts:
continue
if path.is_symlink():
raise ValueError(f"Symbolic link in prepared source: {relative}")
if path.is_file():
digest.update(relative.as_posix().encode() + b"\0")
digest.update(hashlib.sha256(path.read_bytes()).digest())
digest.update(b"\0")
digest.update(str(path.stat().st_mode & 0o111).encode() + b"\0")
return digest.hexdigest()
def _check_plugin(root: Path, name: str) -> None:
project = root / "plugins" / name
metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8"))
entries = metadata.get("project", {}).get("entry-points", {}).get("reme.plugins", {})
if name not in entries:
raise ValueError(f"Plugin project does not register {name!r}: {project}")
module = relative_path(entries[name].split(":", maxsplit=1)[0].replace(".", "/"))
if not any((project / prefix / module / "plugin.yaml").is_file() for prefix in ("src", "")):
raise ValueError(f"Plugin {name!r} is missing plugin.yaml")
def prepare_code(spec: PreparationSpec, source: Path, runtime: Path, candidate: Path) -> dict:
"""Export fixed ReMe sources and complete, independently versioned plugins.
Future evaluators must install plugins from the candidate, not the runtime.
Assemble an isolated evaluation tree from runtime plus candidate (excluding
.git), then resolve install_projects relative to that tree. candidate_paths
limits optimization scope; plugins are complete projects even when only src
is editable. The initial candidate commit supplies the original baseline.
"""
revision = git(source, "rev-parse", "--verify", "--end-of-options", spec.runtime.source_ref + "^{commit}")
# Read object contents directly, bypassing archive attributes and checkout filters.
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
raw_tree = subprocess.run(
["git", "-C", str(source), "ls-tree", "-r", "-z", "--full-tree", revision],
check=True,
capture_output=True,
timeout=120,
env=environment,
).stdout
roots = [
Path("reme"),
Path("pyproject.toml"),
Path("README.md"),
Path("LICENSE"),
*(Path("plugins") / name for name in spec.runtime.plugins),
]
entries = []
for entry in raw_tree.split(b"\0"):
if not entry:
continue
header, raw_name = entry.split(b"\t", maxsplit=1)
name = Path(relative_path(raw_name.decode("utf-8")))
if not any(name.is_relative_to(root) for root in roots):
continue
mode, kind, object_id = header.decode().split()
if mode not in {"100644", "100755"} or kind != "blob":
raise ValueError(f"Runtime source must contain regular files only: {name}")
entries.append((name, mode, object_id))
objects = subprocess.run(
["git", "-C", str(source), "cat-file", "--batch"],
input="".join(object_id + "\n" for _, _, object_id in entries).encode(),
check=True,
capture_output=True,
timeout=120,
env=environment,
).stdout
runtime.mkdir()
candidate.mkdir()
offset = 0
for name, mode, object_id in entries:
boundary = objects.index(b"\n", offset)
found, kind, size = objects[offset:boundary].decode().split()
if found != object_id or kind != "blob":
raise ValueError(f"Invalid Git object: {name}")
offset = boundary + 1
# Keep each selected plugin only in repo, including packaging/config files.
destination = (candidate if name.parts[0] == "plugins" else runtime) / name
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(objects[offset : offset + int(size)])
destination.chmod(0o755 if mode == "100755" else 0o644)
offset += int(size) + 1
if not (runtime / "reme/__init__.py").is_file() or not (runtime / "pyproject.toml").is_file():
raise ValueError("Source revision is not a ReMe project")
if not (runtime / "reme/config" / spec.runtime.config).is_file():
raise ValueError(f"Runtime configuration does not exist: {spec.runtime.config}")
for name in spec.runtime.plugins:
_check_plugin(candidate, name)
for name in spec.candidate.paths:
# Plugin candidates already include their full project. Core candidates
# still copy selected paths from the fixed ReMe runtime for later overlay.
if (candidate / name).exists():
continue
original = runtime / name
destination = candidate / name
if not original.exists():
raise ValueError(f"Candidate path is absent from the selected source: {name}")
destination.parent.mkdir(parents=True, exist_ok=True)
if original.is_dir():
shutil.copytree(original, destination)
else:
shutil.copy2(original, destination)
git(candidate, "init", "--initial-branch=init", "--template=")
git(candidate, "add", "--force", "--all")
git(candidate, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "-m", "Initial candidate")
return {
"source_commit": revision,
"initial_commit": git(candidate, "rev-parse", "HEAD"),
"runtime_fingerprint": tree_fingerprint(runtime),
"candidate_fingerprint": tree_fingerprint(candidate),
"candidate_paths": spec.candidate.paths,
"candidate_mount": ".",
"install_projects": [".", *[f"plugins/{name}" for name in spec.runtime.plugins]],
"config": spec.runtime.config,
"plugins": spec.runtime.plugins,
}

View file

@ -0,0 +1,200 @@
"""One-time experiment preparation, shared by Python and the Home Job."""
import asyncio
from pathlib import Path
from threading import Event
import shutil
from datetime import datetime, timezone
from importlib.resources import files
from uuid import uuid4
import yaml
from reme.components.base_component import BaseComponent
from reme.steps.base_step import BaseStep, Ref
from .code import prepare_code
from .data import prepare_dataset
from .data.basic import derive_debug_cases
from .data.models import fingerprint
from .spec import PreparationSpec
from ..judge import HomeJudge
from .transaction import (
TRANSACTION,
check_targets,
publish,
recover_preparation,
rollback,
write_json,
sync_directory,
)
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
def _source_path(root: Path, value: str) -> Path:
if value.startswith("~"):
raise ValueError("Home-relative paths are unsupported; use an absolute or workspace-relative path")
path = Path(value)
if not path.is_absolute() and ".." in path.parts:
raise ValueError("Relative source paths must stay inside the workspace")
resolved = (root / path).resolve(strict=True)
if not path.is_absolute() and not resolved.is_relative_to(root):
raise ValueError("Relative source paths must stay inside the workspace")
return resolved
class HomePreparation(BaseComponent):
"""Prepare once under Home's process lock; never reset an existing experiment."""
component_type = "home.preparation"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.workspace = self.bind("default", HomeWorkspace, optional=False)
self.judge = self.bind("default", HomeJudge, optional=False)
self._pending: set[asyncio.Task] = set()
async def _close(self) -> None:
if self._pending:
await asyncio.gather(*self._pending, return_exceptions=True)
async def prepare(self, *, spec: dict | None = None, spec_path: str | None = None) -> dict:
"""Keep blocking data/Git work off the event loop and await rollback on cancellation."""
stop = Event()
task = asyncio.create_task(asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop))
self._pending.add(task)
cancelled = False
try:
while True:
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
if task.cancelled():
raise
cancelled = True
stop.set()
except InterruptedError:
if cancelled:
raise asyncio.CancelledError from None
raise
finally:
self._pending.discard(task)
def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event) -> dict:
"""Prepare a started application's workspace from exactly one input form.
Local YAML paths must stay inside the workspace. Explicit dataset/repository
source paths may reference external read-only inputs. No source code is run.
"""
if not self.is_started:
raise RuntimeError("Start the Home application before preparation")
workspace = self.workspace_path.resolve()
root = self.workspace.home_path
state = self.workspace.state_path
_validate_directory(workspace, state)
with _initialization_lock(state / "workspace.lock"):
recover_preparation(root, state)
marker = state / "preparation.json"
if marker.exists() or marker.is_symlink():
raise RuntimeError("Home workspace is already prepared; use a new workspace")
if (spec is None) == (spec_path is None):
raise ValueError("Provide exactly one of spec or spec_path")
if spec_path is not None:
path = _source_path(workspace, spec_path)
if not path.is_relative_to(workspace) or not path.is_file():
raise ValueError("spec_path must be a file inside the workspace")
spec = yaml.safe_load(path.read_text(encoding="utf-8"))
config = PreparationSpec.model_validate(spec)
self.judge.require_benchmark(config.benchmark.name)
# Resolve explicitly supplied source inputs before creating transaction state.
source = _source_path(workspace, config.benchmark.source)
repository = _source_path(workspace, config.runtime.source_repo)
evidence = (
_source_path(workspace, config.benchmark.evidence_source) if config.benchmark.evidence_source else None
)
config.benchmark.source = str(source)
config.runtime.source_repo = str(repository)
config.benchmark.evidence_source = str(evidence) if evidence else None
existed = check_targets(root, state)
transaction = state / TRANSACTION
transaction.mkdir()
try:
sync_directory(state)
result = self._stage(config, transaction, source, repository, evidence)
def check_cancelled():
if stop.is_set():
raise InterruptedError("Home preparation cancelled")
check_cancelled()
# Catch conflicts introduced while staging before replacing any target.
if check_targets(root, state) != existed:
raise RuntimeError("Preparation targets changed during staging")
publish(root, state, existed, check_cancelled)
except BaseException:
rollback(root, state)
raise
# Completion is durable. Cleanup failure must not report a failed preparation.
try:
shutil.rmtree(transaction)
except OSError:
self.logger.warning("Preparation committed; transaction cleanup will retry at next startup")
return result
@staticmethod
def _stage(config, transaction, source, repository, evidence) -> dict:
dataset = prepare_dataset(
transaction / "output-2",
config.benchmark.name,
source,
train_case_ids=config.benchmark.case_ids,
variant=config.benchmark.variant,
evidence_source=evidence,
)
# Materialize the actual all-cases selection as part of the durable experiment contract.
config.benchmark.case_ids = dataset.case_ids
config.benchmark.all_cases = False
if config.debug.enabled:
derive_debug_cases(
transaction / "output-2",
transaction / "output-3",
sessions_per_case=config.debug.sessions_per_case,
messages_per_session=config.debug.messages_per_session,
)
else:
(transaction / "output-3/cases").mkdir(parents=True)
code = prepare_code(config, repository, transaction / "output-0", transaction / "output-1")
config.runtime.source_ref = code["source_commit"]
domain = config.model_dump(mode="json")
(transaction / "output-4").write_text(yaml.safe_dump(domain, sort_keys=False), encoding="utf-8")
docs = transaction / "output-5"
docs.mkdir()
resources = files("reme_home").joinpath("runtime_docs")
for name in ("architecture-guide.md", "validation-artifact-guide.md"):
(docs / name).write_bytes(resources.joinpath(name).read_bytes())
result = {
"preparation_id": uuid4().hex,
"created_at": datetime.now(timezone.utc).isoformat(),
"config_fingerprint": fingerprint(domain),
"dataset": dataset.model_dump(mode="json"),
"code": code,
}
write_json(transaction / "output-6", result)
return result
class HomePrepareStep(BaseStep):
"""Accept a workspace YAML file or a Python configuration dictionary."""
preparation: HomePreparation = Ref(HomePreparation, "home.preparation")
async def execute(self):
"""Prepare once and return the persisted completion information."""
if self.context.get("_allowed_paths") is not None:
raise ValueError("home_prepare requires unrestricted access to its workspace output directories")
result = await self.preparation.prepare(
spec=self.context.get("spec"),
spec_path=self.context.get("spec_path"),
)
self.context.response.answer = result
return self.context.response

View file

@ -0,0 +1,44 @@
"""Dataset-specific normalization for Meta-ReMe training data."""
from __future__ import annotations
from pathlib import Path
from typing import Iterable
from . import beam, lme
from .basic import write_normalized_dataset
from .models import DatasetManifest
def prepare_dataset(
directory: Path,
dataset: str,
source: Path,
train_case_ids: Iterable[str] | None = None,
variant: str | None = None,
evidence_source: Path | None = None,
) -> DatasetManifest:
"""Normalize the selected training cases into ``directory``."""
if dataset == "beam":
if evidence_source is not None:
raise ValueError("BEAM does not accept an evidence-session annotation source")
selected_variant = variant or "1M"
if selected_variant not in beam.VARIANTS:
raise ValueError(f"Unsupported BEAM variant: {selected_variant}")
selected = beam.selected_case_ids(source, selected_variant, train_case_ids)
cases = beam.iter_cases(source, selected_variant, selected)
return write_normalized_dataset(directory, dataset, selected_variant, cases)
if dataset == "longmemeval":
if variant is not None:
raise ValueError("LongMemEval does not accept a dataset variant")
return write_normalized_dataset(
directory,
dataset,
None,
lme.iter_cases(source, train_case_ids, evidence_source),
)
raise ValueError(f"Unsupported dataset: {dataset}")
__all__ = ["prepare_dataset"]

View file

@ -0,0 +1,146 @@
"""Shared helpers for writing normalized Meta-ReMe datasets."""
from __future__ import annotations
from pathlib import Path
import json
from datetime import datetime
import math
from typing import Iterable
from .models import CaseSpec, DatasetManifest, canonical_json_bytes, fingerprint
def select_case_ids(available: list[str], requested: Iterable[str] | None) -> list[str]:
"""Return requested IDs after checking duplicates and availability."""
requested_ids = list(requested or [])
if len(requested_ids) != len(set(requested_ids)):
raise ValueError("Training case IDs must not contain duplicates")
if not requested_ids:
return available
unknown = sorted(set(requested_ids) - set(available))
if unknown:
raise ValueError(f"Unknown training case IDs: {', '.join(unknown)}")
return requested_ids
def write_normalized_dataset(
directory: Path,
dataset: str,
variant: str | None,
cases: Iterable[CaseSpec],
) -> DatasetManifest:
"""Write normalized cases and their reproducibility manifest."""
cases_directory = directory / "cases"
cases_directory.mkdir(parents=True)
case_ids: list[str] = []
case_fingerprints: list[str] = []
query_count = 0
for index, case in enumerate(cases):
if case.case_id in case_ids:
raise ValueError(f"Duplicate normalized case ID: {case.case_id}")
(cases_directory / f"{index:06d}.json").write_bytes(canonical_json_bytes(case) + b"\n")
case_ids.append(case.case_id)
case_fingerprints.append(fingerprint(case))
query_count += len(case.queries)
if not case_ids:
raise ValueError("Dataset selection must contain at least one case")
normalized_fingerprint = fingerprint(case_fingerprints)
source_fingerprint = fingerprint(
{
"dataset": dataset,
"variant": variant,
"case_ids": case_ids,
"normalized_fingerprint": normalized_fingerprint,
},
)
manifest = DatasetManifest(
dataset=dataset,
source_fingerprint=source_fingerprint,
normalized_fingerprint=normalized_fingerprint,
case_count=len(case_ids),
query_count=query_count,
case_ids=case_ids,
variant=variant,
)
(directory / "manifest.json").write_bytes(canonical_json_bytes(manifest) + b"\n")
return manifest
def derive_debug_cases(
training_root: Path,
output: Path,
*,
sessions_per_case: int,
messages_per_session: int,
) -> DatasetManifest:
"""Build deterministic, unscored smoke cases from normalized training data."""
training_manifest = json.loads((training_root / "manifest.json").read_text(encoding="utf-8"))
source_paths = sorted((training_root / "cases").glob("*.json"))
case_count = _debug_case_count(len(source_paths))
case_indices = _evenly_spaced_indices(len(source_paths), case_count)
debug_cases: list[CaseSpec] = []
for source_case_index in case_indices:
case = CaseSpec.model_validate_json(source_paths[source_case_index].read_text(encoding="utf-8"))
eligible_indices = _eligible_session_indices(case)
selected_positions = _evenly_spaced_indices(len(eligible_indices), sessions_per_case)
session_indices = [eligible_indices[position] for position in selected_positions]
sessions = []
for index in session_indices:
source = case.sessions[index]
sessions.append(source.model_copy(update={"messages": source.messages[:messages_per_session]}))
debug_id = f"debug-{case.case_id}"
debug_cases.append(
case.model_copy(
update={
"case_id": debug_id,
"sessions": sessions,
# Debug cases only smoke-test execution, not answer correctness.
# Keep original query/evidence metadata intentionally; referenced
# evidence sessions or messages may be absent after truncation.
"queries": case.queries[:1],
"metadata": {
**case.metadata,
"debug_case": True,
"source_case_id": case.case_id,
},
},
),
)
return write_normalized_dataset(output, str(training_manifest["dataset"]), "debug", debug_cases)
def _debug_case_count(training_case_count: int) -> int:
"""Use ten percent of training cases, with a two-case floor when possible."""
return min(training_case_count, max(2, math.ceil(training_case_count * 0.1)))
def _evenly_spaced_indices(size: int, limit: int) -> list[int]:
"""Choose stable beginning/middle/end positions without duplicates."""
if size <= limit:
return list(range(size))
if limit == 1:
return [0]
return sorted({round(index * (size - 1) / (limit - 1)) for index in range(limit)})
def _eligible_session_indices(case: CaseSpec) -> list[int]:
"""Mirror LongMemEval's query-time cutoff so a debug case remains buildable."""
if case.metadata.get("dataset") != "longmemeval" or not case.queries:
return list(range(len(case.sessions)))
question_date = case.queries[0].metadata.get("question_date")
if not isinstance(question_date, str) or not question_date:
return list(range(len(case.sessions)))
query_time = datetime.strptime(question_date, "%Y/%m/%d (%a) %H:%M")
eligible = []
for index, session in enumerate(case.sessions):
timestamps = [message.get("created_at") for message in session.messages if message.get("created_at")]
if timestamps and datetime.fromisoformat(str(timestamps[0])) <= query_time:
eligible.append(index)
return eligible or list(range(len(case.sessions)))

View file

@ -0,0 +1,156 @@
"""Normalize BEAM cases into the shared Meta-ReMe contracts."""
from __future__ import annotations
from datetime import datetime
import json
from pathlib import Path
from typing import Any, Iterable, Iterator
from .basic import select_case_ids
from .models import CaseSpec, QuerySpec, SessionSpec
VARIANTS = ("100K", "500K", "1M", "10M")
def _parse_date(value: str) -> datetime:
for date_format in ("%B-%d-%Y", "%b-%d-%Y"):
try:
return datetime.strptime(value, date_format)
except ValueError:
continue
raise ValueError(f"Invalid BEAM time anchor: {value!r}")
def _sessions(raw_batches: list[dict[str, Any]], variant: str, case_id: str) -> list[SessionSpec]:
sessions: list[SessionSpec] = []
for batch in raw_batches:
fallback_anchor = batch.get("time_anchor") or "January-1-2024"
previous_time: datetime | None = None
first_time: datetime | None = None
messages: list[dict[str, Any]] = []
for turn in batch["turns"]:
anchor = next((message.get("time_anchor") for message in turn if message.get("time_anchor")), None)
current_time = _parse_date(anchor or fallback_anchor) if previous_time is None or anchor else previous_time
first_time = first_time or current_time
previous_time = current_time
for message in turn:
# BEAM raw messages carry integer ids while AgentScope Msg
# requires a string id, so normalize before ingestion.
if isinstance(message.get("id"), int):
message = {**message, "id": str(message["id"])}
messages.append(
{
**message,
"name": message["role"],
"created_at": current_time.strftime("%Y-%m-%dT%H:%M:%S"),
},
)
batch_number = str(batch["batch_number"])
sessions.append(
SessionSpec(
session_id=f"beam_{variant}_{case_id}_batch{batch_number}",
messages=messages,
metadata={"batch_number": batch["batch_number"], "date": first_time.strftime("%Y-%m-%d")},
),
)
return sessions
def _source_chat_ids(value: Any) -> list[int]:
"""Flatten BEAM's list- or group-shaped source-chat references."""
if isinstance(value, dict):
return [chat_id for nested in value.values() for chat_id in _source_chat_ids(nested)]
if isinstance(value, list):
return [chat_id for nested in value for chat_id in _source_chat_ids(nested)]
# One upstream 10M annotation contains the placeholder ``"--"``. It is not
# a chat reference, so retain only the integer IDs used by chat messages.
return [value] if isinstance(value, int) and not isinstance(value, bool) else []
def _queries(raw_questions: dict[str, list[dict[str, Any]]], sessions: list[SessionSpec]) -> list[QuerySpec]:
queries: list[QuerySpec] = []
session_ids_by_chat_id: dict[int, list[str]] = {}
for session in sessions:
for message in session.messages:
chat_id = message.get("id")
if isinstance(chat_id, str) and chat_id.isdigit():
chat_id = int(chat_id)
if isinstance(chat_id, int) and not isinstance(chat_id, bool):
session_ids = session_ids_by_chat_id.setdefault(chat_id, [])
if session.session_id not in session_ids:
session_ids.append(session.session_id)
for question_type in sorted(raw_questions):
for index, raw in enumerate(raw_questions[question_type], start=1):
golden_answer = raw.get("ideal_answer", raw.get("ideal_response", raw.get("answer")))
if golden_answer is None:
golden_answer = raw.get("rubric", [])
metadata = {
key: value
for key, value in raw.items()
if key not in {"question", "ideal_answer", "ideal_response", "answer"}
}
metadata["question_type"] = question_type
source_chat_ids = list(dict.fromkeys(_source_chat_ids(raw.get("source_chat_ids"))))
if source_chat_ids:
unknown_chat_ids = [chat_id for chat_id in source_chat_ids if chat_id not in session_ids_by_chat_id]
if unknown_chat_ids:
raise ValueError(
f"BEAM query {question_type}:{index} references unknown chat IDs: {unknown_chat_ids}",
)
metadata["evidence_session_ids"] = list(
dict.fromkeys(
session_id for chat_id in source_chat_ids for session_id in session_ids_by_chat_id[chat_id]
),
)
metadata["evidence_is_certain"] = all(
len(session_ids_by_chat_id[chat_id]) == 1 for chat_id in source_chat_ids
)
queries.append(
QuerySpec(
query_id=f"{question_type}:{index}",
question=raw["question"],
golden_answer=golden_answer,
metadata=metadata,
),
)
return queries
def selected_case_ids(source: Path, variant: str, requested: Iterable[str] | None) -> list[str]:
"""Select and validate case IDs for one BEAM variant."""
cases_root = source / "chats" / variant
if not cases_root.is_dir():
raise FileNotFoundError(f"BEAM variant does not exist: {cases_root}")
available = sorted(
(path.name for path in cases_root.iterdir() if path.is_dir()),
key=lambda value: (not value.isdigit(), int(value) if value.isdigit() else value),
)
return select_case_ids(available, requested)
def iter_cases(source: Path, variant: str, case_ids: Iterable[str]) -> Iterator[CaseSpec]:
"""Yield normalized BEAM cases one at a time."""
cases_root = source / "chats" / variant
for case_id in case_ids:
case_root = cases_root / case_id
raw_chat = json.loads((case_root / "chat.json").read_text(encoding="utf-8"))
raw_questions = json.loads(
(case_root / "probing_questions/probing_questions.json").read_text(encoding="utf-8"),
)
sessions = _sessions(raw_chat, variant, case_id)
yield CaseSpec(
case_id=case_id,
sessions=sessions,
queries=_queries(raw_questions, sessions),
metadata={"dataset": "beam", "variant": variant},
)
def load_cases(source: Path, variant: str, train_case_ids: Iterable[str] | None) -> list[CaseSpec]:
"""Load selected BEAM cases into the unified contracts."""
return list(iter_cases(source, variant, selected_case_ids(source, variant, train_case_ids)))

View file

@ -0,0 +1,123 @@
"""Normalize LongMemEval cases into the shared Meta-ReMe contracts."""
from __future__ import annotations
from datetime import datetime
import json
from pathlib import Path
from typing import Any, Iterable, Iterator
from .basic import select_case_ids
from .models import CaseSpec, QuerySpec, SessionSpec
def _load_evidence_sessions(source: Path | None) -> dict[str, dict[str, Any]]:
"""Load optional, independently reviewed evidence-session annotations."""
if source is None:
return {}
annotations: dict[str, dict[str, Any]] = {}
for line_number, line in enumerate(source.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
item = json.loads(line)
except json.JSONDecodeError as error:
raise ValueError(
f"Invalid JSONL evidence-session annotation on line {line_number}",
) from error
if not isinstance(item, dict):
raise ValueError(f"Evidence-session annotation on line {line_number} must be a JSON object")
case_id = item.get("case_id")
session_ids = item.get("evidence_sessions")
is_certain = item.get("is_certain")
if not isinstance(case_id, str) or not case_id:
raise ValueError("Each evidence-session annotation must have a non-empty case_id")
if case_id in annotations:
raise ValueError(f"Duplicate evidence-session annotation for case {case_id!r}")
if not isinstance(session_ids, list) or any(not isinstance(value, str) or not value for value in session_ids):
raise ValueError(f"Evidence sessions for case {case_id!r} must be a list of non-empty strings")
if not isinstance(is_certain, bool):
raise ValueError(f"Evidence certainty for case {case_id!r} must be a boolean")
annotations[case_id] = {
"evidence_session_ids": session_ids,
"evidence_is_certain": is_certain,
}
return annotations
def iter_cases(
source: Path,
train_case_ids: Iterable[str] | None,
evidence_source: Path | None = None,
) -> Iterator[CaseSpec]:
"""Yield selected LongMemEval question IDs as unified cases."""
raw_items = json.loads(source.read_text(encoding="utf-8"))
by_id = {str(item["question_id"]): item for item in raw_items}
if len(by_id) != len(raw_items):
raise ValueError("Duplicate LongMemEval question IDs")
evidence_by_id = _load_evidence_sessions(evidence_source)
for case_id in select_case_ids(list(by_id), train_case_ids):
item = by_id[case_id]
sessions = []
for date, session_id, messages in zip(
item["haystack_dates"],
item["haystack_session_ids"],
item["haystack_sessions"],
strict=True,
):
timestamp = datetime.strptime(date, "%Y/%m/%d (%a) %H:%M").strftime("%Y-%m-%dT%H:%M:%S")
normalized_messages = [
{**message, "name": message["role"], "created_at": timestamp} for message in messages
]
sessions.append(
SessionSpec(session_id=str(session_id), messages=normalized_messages, metadata={"date": date}),
)
query_metadata = {
key: value
for key, value in item.items()
if key
not in {
"question_id",
"question",
"answer",
"haystack_dates",
"haystack_session_ids",
"haystack_sessions",
}
}
if case_id in evidence_by_id:
annotation = evidence_by_id[case_id]
known_session_ids = {session.session_id for session in sessions}
unknown_session_ids = sorted(set(annotation["evidence_session_ids"]) - known_session_ids)
if unknown_session_ids:
raise ValueError(
f"Evidence annotation for case {case_id!r} references unknown sessions: "
f"{', '.join(unknown_session_ids)}",
)
query_metadata.update(annotation)
query_metadata["evidence_annotation_source"] = evidence_source.name
yield CaseSpec(
case_id=case_id,
sessions=sessions,
queries=[
QuerySpec(
query_id=case_id,
question=item["question"],
golden_answer=item["answer"],
metadata=query_metadata,
),
],
metadata={"dataset": "longmemeval"},
)
def load_cases(
source: Path,
train_case_ids: Iterable[str] | None,
evidence_source: Path | None = None,
) -> list[CaseSpec]:
"""Load selected LongMemEval cases into the unified contracts."""
return list(iter_cases(source, train_case_ids, evidence_source))

View file

@ -0,0 +1,76 @@
"""Normalized benchmark data persisted by Home preparation."""
from datetime import datetime, timezone
import hashlib
import json
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
SCHEMA_VERSION = 1
DatasetName = Literal["beam", "longmemeval"]
def utc_now() -> datetime:
"""Return an aware creation timestamp."""
return datetime.now(timezone.utc)
class ContractModel(BaseModel):
"""Reject unknown persisted fields."""
model_config = ConfigDict(extra="forbid")
def canonical_json_bytes(value: Any) -> bytes:
"""Encode normalized values deterministically."""
if isinstance(value, BaseModel):
value = value.model_dump(mode="json")
return json.dumps(value, allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
def fingerprint(value: Any) -> str:
"""Hash canonical content."""
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
class QuerySpec(ContractModel):
"""A question, reference answer and benchmark-specific scoring metadata."""
query_id: str = Field(min_length=1)
question: str
golden_answer: Any
metadata: dict[str, Any] = Field(default_factory=dict)
class SessionSpec(ContractModel):
"""An ordered conversation session with normalized message timestamps."""
session_id: str = Field(min_length=1)
messages: list[dict[str, Any]]
metadata: dict[str, Any] = Field(default_factory=dict)
class CaseSpec(ContractModel):
"""One selectable benchmark case with its sessions and questions."""
case_id: str = Field(min_length=1)
sessions: list[SessionSpec]
queries: list[QuerySpec]
metadata: dict[str, Any] = Field(default_factory=dict)
class DatasetManifest(ContractModel):
"""Selection identity, converter version and normalized content fingerprints."""
# Accept legacy files without emitting a version that has no reader dispatch.
schema_version: int = Field(default=SCHEMA_VERSION, ge=1, exclude=True)
dataset: DatasetName
converter_version: int = 1
variant: str | None = None
case_ids: list[str]
source_fingerprint: str = Field(min_length=1)
normalized_fingerprint: str = Field(min_length=1)
case_count: int = Field(ge=0)
query_count: int = Field(ge=0)
created_at: datetime = Field(default_factory=utc_now)

View file

@ -0,0 +1,116 @@
"""Validated inputs for a single Home experiment."""
from pathlib import Path
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class SpecModel(BaseModel):
"""Do not silently accept misspelled preparation options."""
model_config = ConfigDict(extra="forbid", strict=True)
def relative_path(value: str) -> str:
"""Require a portable, repository-relative path without traversal."""
if not value or value.startswith("~") or "\\" in value or ":" in value:
raise ValueError(f"Invalid relative path: {value!r}")
path = Path(value)
if path.is_absolute() or ".." in path.parts or path == Path("."):
raise ValueError(f"Invalid relative path: {value!r}")
return path.as_posix()
class BenchmarkSpec(SpecModel):
"""Benchmark adapter, explicit case selection, and source inputs."""
name: Literal["beam", "longmemeval"]
source: str = Field(min_length=1)
variant: str | None = None
case_ids: list[str] | None = None
all_cases: bool = False
evidence_source: str | None = None
@model_validator(mode="after")
def validate_selection(self):
"""Require either a nonempty unique selection or explicit all-cases."""
if self.all_cases == (self.case_ids is not None):
raise ValueError("Provide case_ids or all_cases=true, exclusively")
if self.case_ids is not None:
if not self.case_ids or any(not case_id.strip() for case_id in self.case_ids):
raise ValueError("case_ids must contain nonempty strings")
if len(set(self.case_ids)) != len(self.case_ids):
raise ValueError("case_ids must not contain duplicates")
if self.name == "beam":
if self.variant not in {"100K", "500K", "1M", "10M"}:
raise ValueError("BEAM requires variant: 100K, 500K, 1M, or 10M")
if self.evidence_source is not None:
raise ValueError("evidence_source is supported only for LongMemEval")
elif self.variant is not None:
raise ValueError("LongMemEval does not accept variant")
return self
class RuntimeSpec(SpecModel):
"""An immutable source revision; plugins belong to the evaluated ReMe."""
source_repo: str = Field(min_length=1)
source_ref: str = Field(min_length=1)
config: str = "benchmark.yaml"
plugins: list[str] = Field(default_factory=list)
@field_validator("config")
@classmethod
def validate_config(cls, value):
"""Select a single bundled application configuration."""
value = relative_path(value)
if Path(value).name != value or Path(value).suffix not in {".yaml", ".yml", ".json"}:
raise ValueError("runtime.config must name a YAML or JSON file in reme/config")
return value
@field_validator("plugins")
@classmethod
def validate_plugins(cls, values):
"""Select local repository plugin projects by name."""
if len(set(values)) != len(values):
raise ValueError("runtime.plugins must be unique")
for value in values:
if relative_path(value) != value or "/" in value:
raise ValueError("runtime.plugins must name plugin directories")
return values
class CandidateSpec(SpecModel):
"""Editable repository paths; selected plugins are exported as complete projects."""
paths: list[str] = Field(min_length=1)
@field_validator("paths")
@classmethod
def validate_paths(cls, values):
"""Reject traversal and overlapping candidate roots."""
paths = [relative_path(value) for value in values]
for index, value in enumerate(paths):
for other in paths[:index]:
if Path(value).is_relative_to(other) or Path(other).is_relative_to(value):
raise ValueError("candidate.paths must not overlap or contain duplicates")
return paths
class DebugSpec(SpecModel):
"""Deterministic smoke-data truncation, independent of scoring."""
enabled: bool = True
sessions_per_case: int = Field(default=3, ge=1)
messages_per_session: int = Field(default=6, ge=1)
class PreparationSpec(SpecModel):
"""Public Home preparation configuration, accepted as YAML or a dict."""
schema_version: Literal[1] = Field(default=1, exclude=True)
benchmark: BenchmarkSpec
runtime: RuntimeSpec
candidate: CandidateSpec
debug: DebugSpec = Field(default_factory=DebugSpec)

View file

@ -0,0 +1,165 @@
"""Recoverable publication under Home's lock.
Every root argument is HomeWorkspace.home_path (workspace/home), not the outer
ReMe workspace. Both outputs and recovery state stay inside this owned subtree.
"""
import json
import os
from pathlib import Path
import shutil
from typing import Literal
from pydantic import BaseModel, ConfigDict
from ..workspace import _validate_directory
OUTPUTS = ("code/runtime", "code/repo", "datasets/training-set", "datasets/debug-cases", "domain_spec.yaml", "docs")
TRANSACTION = "preparation-transaction"
class Journal(BaseModel):
"""Only fixed output locations may be restored or removed during recovery."""
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal[1, 2] = 2
existed: list[bool]
def sync_directory(path: Path) -> None:
"""Persist directory entries on POSIX."""
if os.name != "nt":
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def write_json(path: Path, value: dict) -> None:
"""Write a new durable transaction file."""
with path.open("x", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
sync_directory(path.parent)
def targets(root: Path, state: Path, *, schema_version: int = 2) -> list[Path]:
"""Keep completion publication last."""
# Version 1 journals predate docs publication; retain their original indices
# so an interrupted preparation can still be rolled back after an upgrade.
outputs = OUTPUTS[:-1] if schema_version == 1 else OUTPUTS
return [*[root / value for value in outputs], state / "preparation.json"]
def check_targets(root: Path, state: Path, *, schema_version: int = 2) -> list[bool]:
"""Only the original empty Home skeleton can be replaced."""
result = []
for path in targets(root, state, schema_version=schema_version):
_validate_directory(root, path.parent)
if path.is_symlink():
raise ValueError(f"Symbolic link at preparation output: {path}")
if path.exists():
if path in {root / "domain_spec.yaml", state / "preparation.json"}:
raise ValueError(f"Preparation output already exists: {path}")
if not path.is_dir():
raise ValueError(f"Preparation output already exists: {path}")
permitted = path.relative_to(root).as_posix() in {"datasets/training-set", "datasets/debug-cases"}
children = list(path.iterdir())
if permitted:
children = [
child
for child in children
if not (
child.name == "cases" and not child.is_symlink() and child.is_dir() and not any(child.iterdir())
)
]
if children:
raise ValueError(f"Preparation output is not empty: {path}")
result.append(path.exists())
return result
def _remove(path: Path) -> None:
if path.is_symlink() or path.is_file():
path.unlink()
elif path.exists():
shutil.rmtree(path)
def rollback(root: Path, state: Path) -> None:
"""Undo publication, restoring original skeleton directories by rename."""
transaction = state / TRANSACTION
journal_path = transaction / "journal.json"
if not journal_path.exists():
_remove(transaction)
return
if journal_path.is_symlink():
raise ValueError("Invalid preparation transaction journal")
journal = Journal.model_validate_json(journal_path.read_text(encoding="utf-8"))
locations = targets(root, state, schema_version=journal.schema_version)
if len(journal.existed) != len(locations):
raise ValueError("Invalid preparation transaction output count")
for index in reversed(range(len(locations))):
path = locations[index]
_validate_directory(root, path.parent)
backup = transaction / f"backup-{index}"
staged = transaction / f"output-{index}"
if backup.exists():
if backup.is_symlink() or not backup.is_dir():
raise ValueError("Invalid preparation backup")
_remove(path)
os.replace(backup, path)
elif not journal.existed[index] and not staged.exists():
_remove(path)
sync_directory(path.parent)
if check_targets(root, state, schema_version=journal.schema_version) != journal.existed:
raise RuntimeError("Preparation rollback could not restore the original output locations")
_remove(transaction)
sync_directory(state)
def recover_preparation(root: Path, state: Path) -> None:
"""Recover interrupted work before initialization or another preparation."""
transaction = state / TRANSACTION
if transaction.is_symlink() or (transaction.exists() and not transaction.is_dir()):
raise ValueError("Invalid Home preparation transaction directory")
if not transaction.exists():
return
marker = state / "preparation.json"
if marker.is_symlink():
raise ValueError("Invalid Home preparation completion marker")
if marker.exists():
# Completion is published last; a crash during cleanup must not undo success.
json.loads(marker.read_text(encoding="utf-8"))
_remove(transaction)
sync_directory(state)
else:
rollback(root, state)
def publish(root: Path, state: Path, existed: list[bool], check_cancelled) -> None:
"""Publish all staged outputs, rolling back any reported failure."""
transaction = state / TRANSACTION
# fsync file contents and directory entries before publishing the journal.
for path in transaction.rglob("*"):
if path.is_file():
with path.open("rb") as stream:
os.fsync(stream.fileno())
for path in sorted((p for p in transaction.rglob("*") if p.is_dir()), reverse=True):
sync_directory(path)
temporary = transaction / "journal.tmp"
write_json(temporary, Journal(existed=existed).model_dump())
os.replace(temporary, transaction / "journal.json")
sync_directory(transaction)
for index, path in enumerate(targets(root, state)):
check_cancelled()
if existed[index]:
os.replace(path, transaction / f"backup-{index}")
sync_directory(path.parent)
sync_directory(transaction)
os.replace(transaction / f"output-{index}", path)
sync_directory(path.parent)
sync_directory(transaction)

View file

@ -0,0 +1,94 @@
# ReMe Candidate Development Guide
Use this guide when improving the code prepared by Home. The initial candidate is
a starting implementation; its internal design may change while the fixed callers
and interfaces continue to work.
## Workspace and modification boundary
All paths below are relative to the outer ReMe workspace:
- `home/domain_spec.yaml` records the resolved experiment configuration. Read
`candidate.paths` for the editable paths, `runtime.plugins` for the selected
plugins, and `runtime.config` for the application configuration.
- `home/code/repo/` is the candidate Git repository. Paths in `candidate.paths`
are relative to this repository. It contains complete selected plugin projects,
including packaging and configuration, even if only their `src/` is editable.
- `home/code/runtime/` is the fixed ReMe source. Evaluation assembles an isolated
tree from this runtime and the candidate commit. Plugins are installed from the
candidate; the fixed runtime has no duplicate plugin projects.
- `home/datasets/` contains prepared training and debug inputs. Treat them as
fixed evaluation inputs. Do not encode benchmark answers or case-specific
shortcuts in candidate code, prompts, or resources.
- `home/metadata/preparation.json` records source and initial candidate commits,
fingerprints, and the installation plan.
Keep edits within `candidate.paths`. Inspect the fixed runtime, prepared inputs,
and evaluation artifacts as needed, but do not modify them to improve a score.
Preserve the benchmark-facing Jobs, especially `auto_memory` and
`agentic_answer`, their accepted inputs, and their response contracts. Inspect
the actual callers and configured Steps before changing an interface. Use
dependencies available in the evaluation environment.
## ReMe architecture
ReMe is local-first and file-native. User-owned workspace files are durable
source data; indexes, catalogs, graphs, and caches must remain rebuildable.
Keep candidate workspace access within ReMe's path safety boundary.
- A Component is application-scoped. Use it for shared stores, indexes, clients,
locks, and resources with startup/shutdown requirements.
- A Job orchestrates Steps. Each invocation has a fresh RuntimeContext, a shared
Response, and fresh Step instances. `RuntimeContext.data` carries inputs and
intermediate values between Steps in that invocation.
- Step fields and `self.kwargs` are invocation-scoped. They do not persist across
Job calls. Use workspace files for durable state and namespaced, synchronized
`ApplicationContext.metadata` for application-lifetime state.
- A typed `Ref` resolves dependencies from Step kwargs, then RuntimeContext, then
application Components. Reuse existing stores and indexes through this wiring.
- Plugin `plugin.yaml` exposes importable `module:Class` backends and configures
Jobs, Steps, and Components under `application_defaults`. Keep registration,
configuration, prompt variables, and implementation consistent.
Inspect the bundled base classes and nearby plugin code for current signatures
and lifecycle behavior. Candidate prompts, helper modules, memory formats, and
retrieval strategies can change within the configured editable scope.
## Evaluating a candidate
Commit every candidate change before calling `home_evaluate`. Modified tracked
files, staged changes, and untracked files in `home/code/repo/` reject evaluation.
The Job uses its current HEAD and records that commit; it accepts no `code_ref`.
Run the initial full evaluation before changing the initial candidate.
The host program can inject `case_ids` and a `query_ids` mapping into the Job.
Omitted case IDs select all prepared cases; omitted query selections use every
query in each selected case. Debug runs use `debug=true` and
check execution only, without judging answer correctness. They select prepared
debug cases and default to `fail_fast=true`; scored runs default to false. An
explicit `fail_fast` overrides the mode default. Debug cannot reuse construction.
Setting `reuse_construction_from` restores a compatible completed run's pre-query
memory tar, skips construction, and answers and scores again using current code.
The caller must establish construction equivalence. Every selected case needs a
valid donor tar; missing cases cause a preflight error. There is no `eval_only`
flag. Host-installed benchmark judge plugins score answers outside Docker and
outside the candidate repository.
See `validation-artifact-guide.md` for saved evidence and score interpretation.
## Candidate versions
Use the Home code Jobs to list, create, switch, merge and commit candidates in
`home/code/repo`. `init` is the protected preparation baseline. Branch operations
require clean code; merges stay uncommitted for inspection. Commit all intended
changes with `home_commit_candidate` before debug or scored evaluation. Mutations
are limited to prepared `candidate_paths`; keep repository metadata and runtime
outside the optimizer's write permissions.
`home_list_code_tree`, `home_read_code_file` and `home_search_code` inspect raw
committed files. Their `revision` defaults to current HEAD and returns a resolved
SHA. For diagnosis or version comparisons, the host should inject a fixed SHA
through `injected_job_kwargs` for each agent/Job alias, so subsequent checkout
changes cannot affect the evidence. See the Home plugin README for inputs and
output limits.

View file

@ -0,0 +1,254 @@
# Home Evaluation Artifact Guide
`home_evaluate` stores each run at
`home/evaluations/<branch>/<commit-prefix>/<run-id>/` inside the outer ReMe
workspace. Commit prefixes start at seven characters and extend on collision.
Run IDs increase within that commit directory; prior results are preserved.
Result compaction and summary fields follow the reviewed `dev_meta` format.
## Saved files
Paths in this table are relative to one run directory:
| Path | Meaning |
| --- | --- |
| `manifest.json` | Candidate commit, selected case/query IDs, dataset, execution mode, input/configuration fingerprints, host judge identity, options, timestamps and run status. Retained for reproducibility and rebuilding summaries; identity is not repeated in the summary. |
| `summary.json` | Rebuildable evidence from a completed execution. Its fields are described below. |
| `partial_summary.json` | Available evidence after cancellation or failure; not a completed baseline. |
| `failure.json` | Run-level failure details, when present. |
| `cases/<case-id>/memory_construction/result.json` | Ordered construction Job results with success, errors, metadata, answers, reported token usage and inner Job call counts. Replays record their source and snapshot checksum without charging construction again. |
| `cases/<case-id>/memory_construction/build.log` | Construction log. Replays inherit the donor log. |
| `cases/<case-id>/memory_construction/reme_workspace.tar.gz` | Checksum-verified workspace snapshot exported before answering; the donor for construction reuse. |
| `cases/<case-id>/memory_construction/reme_workspace/` | Extracted construction workspace, including memory files and session Git checkpoints when exported. |
| `cases/<case-id>/queries/result.json` | The sole authoritative query record: question, reference and actual answer, question type, score, compact answer/judge responses and failures. All returned answers are frozen here before extraction and host judging; scores are published progressively. |
| `cases/<case-id>/queries/<query-id>/answer.log` | Query execution log, when available. |
| Case/query `failure.json` | Failure stage and diagnostics, when present. |
| `cases/<case-id>/full.tar.gz` | Best-effort sandbox diagnostics for execution failures, when available. |
Successful exports retain query logs, then discard the temporary post-query
`evaluation.tar.gz`. There is no separate `execution.json`, `artifacts.json`,
`queries/answers.json` or per-query `result.json`. A failed extraction can leave
its archive for diagnosis. Exported files depend on execution progress; inspect
failure records before interpreting an absent workspace or log.
Query compaction removes reconstructable `sys_prompt` values, duplicate nested
answers/questions/errors, the enclosing Job's one call, judge token usage,
duplicate parsed judgements when raw judgements exist, and repeated judge inputs.
Distinct errors, raw judgements, rubric assessments and reported answer token
metrics remain. Unreported token backends are omitted.
Construction compaction removes checkpoint descriptors already available in Git,
one enclosing Job call, unreported token backends, and transcript links identified
by a sibling session ID. Recognized cumulative daily-index `notes` snapshots become
`notes_count`; unrelated plugin payloads remain. All-zero construction token
records become `0`. Detailed construction costs stay in this case artifact.
Small scalar containers are written on one line, following `dev_meta` formatting.
Structured scores in query results and summaries use at most four decimal places.
Reference answers, raw judge strings and non-score metrics retain their original values.
New run manifests omit the unused schema version, duplicate code and judge fingerprints,
`options.debug` (use `dataset_kind`/`mode`), and duplicate sampling policy (use the plan).
The remaining `fingerprints.dataset` and `fingerprints.construction_config` guard memory
reuse compatibility; `judge.fingerprint` identifies the trusted scoring implementation.
Legacy manifests remain readable. Workspace markers, preparation recovery journals and
sampling state retain versions used for validation, rollback and migration.
## Summary fields
The run summary contains only:
- `case_count`, `case_status_counts`, `construction_status_counts`, and
`failure_stage_case_counts` (construction/query; tolerated construction failures
still count as construction errors).
- `scoring_enabled`, `expected_query_count`, `scored_query_count`, and `query_status_counts`
(completed, candidate failure, infrastructure error, unscored).
- `mean_query_score`, `observed_mean_query_score`, `low_score_cases`,
`question_type_statistics`, and `cases`.
Each `cases[case_id]` contains construction `status`, `success_job_count` and
`fail_job_count`; `mean_score`; `average_query_total_token_usage`; and a `queries`
map. Each query entry contains only `question_type` and `score`. Failed queries
also appear in `query_errors`, keyed by query ID, with `failure_kind`
(`candidate_failure` or `infra_error`), `stage` (null if unavailable), and an
error excerpt limited to 500 characters plus a truncation marker. Full errors
remain in the phase artifacts. A null score alone does not identify a failure:
successful queries also remain unscored when `scoring_enabled` is false.
`stage`, `construction_error_count` and `query_error_count` appear only when
applicable. There is no redundant case status or per-case coverage accounting.
Questions, answers, full errors, Job details and construction token usage stay in
phase artifacts, outside the summary.
Token summaries contain only average candidate answer **total** tokens per model,
divided by all returned queries, including failures and queries without reported
usage. Zero and unknown token totals are omitted. Judge/construction costs and
input/output token breakdowns are not aggregated. Question-type statistics contain
query count, average of available scores (rounded to four decimals), and the same
answer token averages across cases. Missing/empty types form `unknown`.
Candidate failures count as zero in the expected-query score denominator.
Construction failures leave queries unassessed rather than inventing query
failures. Infrastructure, artifact, cleanup or judge errors make the comparable
run mean unavailable; `observed_mean_query_score` averages available scores only.
`low_score_cases` contains `{case_id: mean_score}` for cases with at least one
scored query below `1`, sorted by mean then case ID; unavailable case means are
omitted.
Run status and debug/scored mode are in the manifest. Debug runs skip judging;
run and case means are null, `scored_query_count` and the unscored count are zero,
while `expected_query_count` records execution coverage. They cannot measure
answer quality. Only compare completed scored runs after checking infrastructure
outcomes, expected coverage and the candidate selection.
To rebuild the appropriate summary from a run's manifest and phase results:
```python
from pathlib import Path
from reme_home.evaluation.artifacts import rebuild_summary
summary = rebuild_summary(Path("/absolute/path/to/run"))
```
New results omit the former `schema_version`, `status`, `execution_complete`,
`execution_mode`, `expected_case_count`, `expected_scored_query_count`,
`scoring_required`, `scoring_complete`, `valid_for_comparison`, `scored_count`,
`unscored_count`, and top-level `query_error_count` from summaries. Use manifest
identity/status/mode and the retained counts instead. The Job returns the run
summary without `cases`, plus `result_dir`, `commit_sha`, `execution_mode` and
execution `status`. Existing result directories are not rewritten automatically.
## Investigating a result
1. Read the manifest for candidate, selection, mode, judge and construction source.
2. Read the summary, then an affected case's `queries/result.json`. Compare the
answer, reference and judge assessment. Frozen answers survive judge failures.
3. Read `answer.log` for retrieval/answer execution; the host judge is outside it.
4. Read construction `result.json` and `build.log` for Job failures and costs.
5. Inspect the pre-query workspace and its Git history with read-only `git log`,
`git show` and `git diff` to trace session memory checkpoints. Memory/session
directory names depend on runtime configuration.
Failure records preserve structured exception types, HTTP status codes and cause
chains from `Response.metadata["reme.job_error"]`. Network/provider failures in
any construction Job or exception cause invalidate comparison. Failed Jobs without
structured exception details are treated as infrastructure errors. Construction
failure-rate tolerance applies only to classified candidate errors.
Construction reuse answers and judges again with the current candidate; it does
not reuse donor answers or scores.
## Evaluation history and adaptive sampling evidence
`home_evaluate sampling=true` enables adaptive selection; false preserves full or
explicit evaluation. Debug always bypasses history. `sampling_policy` is the
host policy input, distinct from the boolean toggle. The effective policy is
captured in the started plan, or `manifest.selection_plan` for untracked adaptive runs.
`record_history=true` (the default) includes a successful full baseline even when
`sampling=false`. `record_history=false` runs the same scoring and artifact pipeline
without registering results or advancing any optimization/sampling counters. The
choice is stored in `manifest.options.record_history` and respected by history
registration and rebuilding; a missing legacy setting means true. Debug always
skips history. The response reports effective participation as `record_history`.
An untracked adaptive run may read existing observations and saves its draw locally
in `manifest.selection_plan`, without a shared started plan or ledger round.
`home.evaluation_history` discovers successful results directly from run manifests
and summaries. No `history.json` index is generated or read; legacy copies are ignored.
Entries require completed scored training runs
with `success=true` (or equivalent legacy summary counts) and no infrastructure
errors. Low scores remain useful. Failed/cancelled/debug/partial runs are excluded;
a mixed run with an infrastructure failure contributes no quality observations.
Source manifests, summaries and query results remain authoritative.
`sandbox_started_at` marks the first successfully initialized sandbox. The
history-enabled manifest's `sampling.plan_path` points to its policy, seed, weights, selection
and started round under `home/sampling/plans/`. Pre-start failures have no marker
or plan, and additional workers do not create extra rounds. The derived sampling
cache separates started `round` and `successful_round`: only successful history
advances staleness/recheck timing and trains EWMA. Adaptive plans preserve their
probability clock in `selection_round`. Actual numeric scores alone count as
quality observations; missing/unjudged answers never become zero scores.
Missing sampling caches and `HomeEvaluationHistory.rebuild()` reconstruct
sampling observations from saved evidence. Result listings always scan the source artifacts. Rebuilding never modifies datasets,
candidate code or memory files. See the Home README for configuration, host
per-call policy overrides and enforced agent injection examples.
## Read-only evaluation inspection
Use `home_list_validation_results` to find runs and pass a returned
`validation_path` to `home_inspect_validation`. The catalog includes failed,
debug, running and untracked runs, not only successful optimization history.
Both Jobs page with `offset` and `limit` (default 20, maximum 100), returning
`row_count`, `rows`, `next_offset`. Catalog filters are `branch_name` and
`commit_sha`; a hexadecimal commit prefix must uniquely match among all commits
reachable from the specified branch, or the current branch when omitted.
Inspection without `case_id` returns case summaries; add `case_id` for query
summaries and `query_id` for bounded answer/judge fields. Each row's JSON pointer
is relative to `rows_source`. Failed construction Jobs have separate pagination
and pointers relative to `construction.source`. Query fields are limited to
2,000-character previews and errors to 600 characters. Follow returned source
paths for complete evidence. Missing files/results remain explicit, with null
scores, and partial summaries are used when full summaries are absent. Reads do
not rebuild summaries or modify evaluation history, sampling state or artifacts.
## Memory checkpoint tools
Start a failed-query investigation with
`home_trace_query_evidence(validation_path, case_id, query_id)`.
It returns compact question/answer/score/error previews, source evidence session
IDs, matching Git checkpoints, changed files and paths to complete evidence. It
does not call a model or include transcripts, diffs or judge-result bodies.
Dataset `evidence_session_ids` annotations take precedence; only when absent may
you supply inferred `evidence_session_ids`. Respect `evidence_source` and
`evidence_is_certain`: inferred or uncertain sessions are leads, not established
causes. No annotation or supplied IDs yields `evidence_source: unavailable`.
Missing query results are explicit and scores remain null. `memory_status`
distinguishes unavailable histories; a null session checkpoint means no matching
commit was found. Question/answer previews have a 2,000-character limit, errors
600, and session/changed-file lists 50 entries with omitted counts. Follow
`paths.source_case`, `paths.query_result` and `paths.query_artifacts` for details;
use the tools below for history and focused diffs. Both original transcripts and
derived memory may be searchable, so absence from a summary alone proves no
retrieval failure.
After locating one case, call `home_inspect_memory_history` with its
`validation_path` and `case_id` to page through exported memory checkpoints.
Optional `path` follows a literal memory-workspace-relative file or directory.
The response distinguishes a missing workspace, absent history and an empty
repository; invalid history fails explicitly. Session IDs are extracted only
from the established `session: <id>` commit-subject convention.
Use `home_compare_memory` with `base_ref` and optional `target_ref` (default
`HEAD`) to inspect how memory changed within that evaluation. Add
`target_validation_path` to compare the same case across evaluations; both
references then default to `HEAD`. References must resolve to commits reachable
from each memory HEAD, and hexadecimal prefixes must be unique within that
reachable history. Compare candidate identities and fingerprints before drawing
cross-evaluation conclusions.
Diffs describe committed tracked regular-file contents rather than dirty checkout
files, untracked files or mode-only changes. Binary changes are identified without
printing bytes. Narrow `path` when the 2,000-file, 20-MB snapshot, 5-MB file or
100,000-character diff limit is reached. Hashes cover the selected path scope.
These tools neither reconstruct missing exports nor modify Git/evaluation/history
artifacts. Raw logs and source JSON remain available through the existing result
inspection paths.
## Single-case diagnostic agent
Call `home_case_study(validation_path, case_id, focus=...)` to investigate exactly
one case. It returns a text report with evidence citations and uncertainty, using
fresh agent state, bounded iterations and advisory time reminders. There is no
parallel case dispatcher. Scope and evaluated code revision are host-injected
into the structured tools. AutoBash and Python share a mandatory read-only file
sandbox covering the selected case artifacts, source JSON, validation manifest,
home/docs and all home/code, plus required system runtimes. Neither executor has
host-writable roots or a case scratch directory. They allow analysis commands
and Python rather than a static command whitelist. The directly readable code
worktree can differ from the evaluated commit; use pinned code tools to verify
version-sensitive claims. No write/edit, validation or code-mutation tools are
exposed. Sandbox startup failure prevents model execution; it never falls back
to an unrestricted process. See the Home README for model, budget and platform
sandbox configuration.

View file

@ -0,0 +1,168 @@
"""Adaptive formal-validation sampling ported from dev_meta b53048e4.
Keep the source weighting, weighted ordering and fair-share query allocation.
EvaluationHistory owns persisted observations and started-validation plans.
"""
import json
import math
import random
import time
from typing import Any, Mapping
from pydantic import ConfigDict, Field
from .preparation.data.models import CaseSpec
from .preparation.spec import SpecModel
class SamplingPolicy(SpecModel):
"""Host-owned limits with the same defaults as dev_meta's validation.sampling."""
model_config = ConfigDict(frozen=True)
max_cases: int = Field(default=5, ge=1)
max_queries: int | None = Field(default=None, ge=1)
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
ewma_alpha: float = Field(default=0.3, gt=0, le=1, allow_inf_nan=False)
uniform_exploration: float = Field(default=0.2, ge=0, le=1, allow_inf_nan=False)
def create_sampling_plan(
cases: list[CaseSpec],
state: Mapping[str, Any],
policy: SamplingPolicy,
*,
eligible_query_ids: Mapping[str, list[str]] | None = None,
) -> dict:
"""Draw without replacement, without advancing state or writing an attempted plan."""
if eligible_query_ids is not None:
cases = [case for case in cases if case.case_id in eligible_query_ids]
if not cases:
raise ValueError("training-set contains no cases to sample")
round_id = int(state.get("round", 0)) + 1
seed = time.time_ns()
rng = random.Random(seed)
case_records = state.get("cases", {})
case_weights = _mixed_weights(
{case.case_id: case_records.get(case.case_id, {}) for case in cases},
round_id,
policy,
)
ordered_cases = _weighted_order(cases, case_weights, rng, lambda case: case.case_id)
query_limited = policy.max_queries is not None
remaining_queries = policy.max_queries
selected_cases: list[str] = []
selected_queries: dict[str, list[str]] = {}
query_weights: dict[str, dict[str, float]] = {}
query_records = state.get("queries", {})
for case in ordered_cases:
if len(selected_cases) >= policy.max_cases:
break
if remaining_queries is not None and remaining_queries <= 0:
break
query_ids = [query.query_id for query in case.queries]
if eligible_query_ids is not None:
allowed = set(eligible_query_ids.get(case.case_id, []))
query_ids = [query_id for query_id in query_ids if query_id in allowed]
if not query_ids:
continue
if query_limited:
weights = _mixed_weights(
{query_id: query_records.get(_query_key(case.case_id, query_id), {}) for query_id in query_ids},
round_id,
policy,
)
query_weights[case.case_id] = weights
ordered_queries = _weighted_order(query_ids, weights, rng, lambda value: value)
open_case_slots = max(policy.max_cases - len(selected_cases), 1)
fair_share = max(math.ceil(int(remaining_queries or 0) / open_case_slots), 1)
take = min(len(ordered_queries), fair_share, int(remaining_queries or 0))
query_ids = ordered_queries[:take]
if not query_ids:
continue
remaining_queries = int(remaining_queries or 0) - len(query_ids)
selected_cases.append(case.case_id)
selected_queries[case.case_id] = query_ids
if not selected_cases:
raise ValueError("sampling budget selected no executable case/query pairs")
return {
"mode": "sampled",
"round": round_id,
"seed": seed,
"policy": policy.model_dump(),
"case_ids": selected_cases,
"query_ids": selected_queries,
"case_draw_weights": case_weights,
"query_draw_weights": query_weights,
"eligibility": dict(eligible_query_ids) if eligible_query_ids is not None else None,
}
def _observe(record: dict[str, Any], loss: float, round_id: int, alpha: float) -> None:
previous = record.get("ewma_loss")
record["ewma_loss"] = loss if previous is None else alpha * loss + (1.0 - alpha) * float(previous)
record["observed_count"] += 1
record["last_valid_score_round"] = round_id
if loss > 0:
record["last_error_round"] = round_id
record["successful_observations_after_last_error"] = 0
elif record.get("last_error_round") is not None:
record["successful_observations_after_last_error"] += 1
def _adaptive_weight(record: Mapping[str, Any], round_id: int, policy: SamplingPolicy) -> float:
observed = int(record.get("observed_count", 0))
raw_risk = record.get("ewma_loss")
risk = float(raw_risk) if isinstance(raw_risk, (int, float)) and not isinstance(raw_risk, bool) else 0.5
last_selected = record.get("last_selected_round")
staleness = round_id if last_selected is None else max(round_id - int(last_selected), 0)
uncertainty = 1.0 / math.sqrt(observed + 1.0)
last_error = record.get("last_error_round")
successes = int(record.get("successful_observations_after_last_error", 0))
unrechecked_error = 0.0
if last_error is not None:
unrechecked_error = math.log1p(max(round_id - int(last_error), 0)) / (successes + 1.0)
priority = risk + 0.2 * math.log1p(staleness) + 0.2 * uncertainty + 0.1 * unrechecked_error
return math.exp(min(priority / policy.temperature, 50.0))
def _mixed_weights(
records: Mapping[str, Mapping[str, Any]],
round_id: int,
policy: SamplingPolicy,
) -> dict[str, float]:
"""Mix normalized adaptive weights with a true uniform exploration mass."""
adaptive = {key: _adaptive_weight(record, round_id, policy) for key, record in records.items()}
total = sum(adaptive.values())
uniform = 1.0 / len(adaptive)
return {
key: policy.uniform_exploration * uniform + (1.0 - policy.uniform_exploration) * value / total
for key, value in adaptive.items()
}
def _weighted_order(values: list[Any], weights: Mapping[str, float], rng: random.Random, key: Any) -> list[Any]:
return sorted(
values,
key=lambda value: rng.random() ** (1.0 / max(weights[key(value)], 1e-12)),
reverse=True,
)
def _empty_record() -> dict[str, Any]:
return {
"observed_count": 0,
"ewma_loss": None,
"last_selected_round": None,
"last_valid_score_round": None,
"last_error_round": None,
"successful_observations_after_last_error": 0,
"candidate_failure_count": 0,
"infra_error_count": 0,
}
def _query_key(case_id: str, query_id: str) -> str:
return json.dumps([case_id, query_id], ensure_ascii=False, separators=(",", ":"))

View file

@ -0,0 +1,202 @@
"""Initialize Home's layout inside an existing ReMe workspace."""
import json
import os
import tempfile
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
from typing import Literal
from uuid import UUID, uuid4
from pydantic import AwareDatetime, BaseModel, ConfigDict
from reme.components.base_component import BaseComponent
DIRECTORIES = (
"code/repo",
"code/runtime",
"datasets/training-set/cases",
"datasets/debug-cases/cases",
"docs",
"evaluations",
"sampling/plans",
"tmp",
"logs",
)
ROOT_DIRECTORIES = tuple(dict.fromkeys(path.split("/", maxsplit=1)[0] for path in DIRECTORIES))
class WorkspaceManifest(BaseModel):
"""Layout identity, independent of future experiment configuration."""
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1] = 1
workspace_id: UUID
created_at: AwareDatetime
def _validate_directory(root: Path, path: Path) -> None:
"""Reject links and non-directories anywhere along a managed path."""
relative = path.relative_to(root)
current = root
for part in relative.parts:
current = current / part
if current.is_symlink() or (current.exists() and not current.is_dir()):
raise ValueError(f"Home workspace requires a real directory: {current}")
@contextmanager
def _initialization_lock(path: Path):
"""Hold a process lock; the persistent lock file is not an ownership marker."""
if path.is_symlink() or (path.exists() and not path.is_file()):
raise ValueError(f"Invalid Home workspace lock: {path}")
with path.open("a+b") as stream:
if os.name == "nt":
import msvcrt # pylint: disable=import-outside-toplevel
stream.seek(0, os.SEEK_END)
if not stream.tell():
stream.write(b"\0")
stream.flush()
stream.seek(0)
def acquire():
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
def release():
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl # pylint: disable=import-outside-toplevel
def acquire():
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
def release():
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
try:
acquire()
except OSError as exc:
raise RuntimeError(f"Home workspace initialization is already in progress: {path}") from exc
try:
yield
finally:
stream.seek(0)
release()
def _publish_manifest(path: Path, manifest: WorkspaceManifest) -> None:
"""Publish the initialization marker only after the layout is complete."""
descriptor, name = tempfile.mkstemp(prefix=".workspace-", suffix=".tmp", dir=path.parent)
temporary = Path(name)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
stream.write(json.dumps(manifest.model_dump(mode="json"), indent=2) + "\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
if os.name != "nt":
directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
temporary.unlink(missing_ok=True)
class HomeWorkspace(BaseComponent):
"""Create Home directories at application startup, preserving ReMe data."""
component_type = "home.workspace"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.manifest: WorkspaceManifest | None = None
@property
def home_path(self) -> Path:
"""Root for every Home-owned output; future jobs/search must use this path."""
return self.workspace_path.resolve() / "home"
@property
def state_path(self) -> Path:
"""Keep Home markers, locks and recovery state separate from ReMe metadata."""
return self.home_path / "metadata"
async def _start(self) -> None:
if self.app_context is None:
raise RuntimeError("HomeWorkspace requires an ApplicationContext")
workspace = self.workspace_path.resolve()
root = self.home_path
config = self.app_context.app_config
metadata = workspace / config.metadata_dir
if (
Path(config.metadata_dir).is_absolute()
or metadata == workspace
or not metadata.is_relative_to(workspace)
or ".." in metadata.relative_to(workspace).parts
):
raise ValueError("Home metadata_dir must be a non-empty workspace-relative directory")
for name in (
config.metadata_dir,
config.session_dir,
config.mem_session_dir,
config.resource_dir,
config.daily_dir,
config.digest_dir,
):
if not name:
continue
configured = (workspace / name).resolve()
if configured.is_relative_to(root) or root.is_relative_to(configured):
raise ValueError(f"Home directory 'home' conflicts with ReMe directory {name!r}")
legacy = metadata / "home/workspace.json"
if legacy.exists() or legacy.is_symlink():
raise ValueError("Legacy Home workspace layout detected; use a new workspace")
state = self.state_path
_validate_directory(workspace, state)
state.mkdir(parents=True, exist_ok=True)
with _initialization_lock(state / "workspace.lock"):
from .preparation.transaction import recover_preparation # pylint: disable=import-outside-toplevel
recover_preparation(root, state)
self._initialize(root, state / "workspace.json")
def _initialize(self, root: Path, marker: Path) -> None:
if marker.is_symlink() or (marker.exists() and not marker.is_file()):
raise ValueError(f"Invalid Home workspace manifest: {marker}")
manifest = (
WorkspaceManifest.model_validate_json(marker.read_text(encoding="utf-8")) if marker.exists() else None
)
for relative in DIRECTORIES:
_validate_directory(root, root / relative)
if manifest is None:
for directory in ROOT_DIRECTORIES:
path = root / directory
if path.exists() and any(path.iterdir()):
raise ValueError(f"Refusing to initialize non-empty Home directory: {path}")
created: list[Path] = []
try:
for relative in DIRECTORIES:
current = root
for part in Path(relative).parts:
current = current / part
if not current.exists():
current.mkdir()
created.append(current)
if manifest is None:
manifest = WorkspaceManifest(workspace_id=uuid4(), created_at=datetime.now(timezone.utc))
_publish_manifest(marker, manifest)
self.manifest = manifest
except BaseException:
if not marker.exists():
for path in reversed(created):
try:
path.rmdir()
except OSError:
pass # Preserve files created concurrently; never recursively delete user data.
raise

View file

@ -0,0 +1,124 @@
"""Benchmark Home defaults and lifecycle wiring, without models or dataset writes."""
import hashlib
from importlib import import_module
from pathlib import Path
from unittest.mock import AsyncMock
import pytest
import yaml
from reme.schema import Response
from benchmark import home_runner
ROOT = Path(__file__).resolve().parents[3]
def test_preserved_lme_split_and_evidence():
"""Retain dev_meta's exact reviewed partition and annotations."""
directory = ROOT / "benchmark/longmemeval"
assert hashlib.sha256((directory / "home_evidence_sessions.jsonl").read_bytes()).hexdigest() == (
"35655c51210a96b595910923576fa97b93c81abd82e3cfcda801180007c23a7d"
)
workspace, spec = home_runner.load_config(directory / "config_home.yaml")
selected = spec["benchmark"]["case_ids"]
assert len(selected) == len(set(selected)) == 250
# Hash the original first-five-group sequence to preserve both IDs and order.
assert hashlib.sha256("\n".join(selected).encode()).hexdigest() == (
"f4bee54f5afe9e0b6c6be8f95c7e1a803a3c2ba8183a77c5f8188e1875818e67"
)
assert spec["runtime"]["plugins"] == ["lme"]
assert spec["candidate"]["paths"] == ["plugins/lme/src"]
assert workspace == directory / "workspaces/home-groups-1-5"
def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
"""Paths and selection do not depend on where Python was launched."""
monkeypatch.chdir(tmp_path)
workspace, spec = home_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
assert spec["benchmark"]["variant"] == "100K"
assert spec["benchmark"]["all_cases"] is True
assert spec["benchmark"]["case_ids"] is None
assert spec["benchmark"]["source"] == str(ROOT / "benchmark/beam/dataset/BEAM")
assert spec["runtime"]["source_repo"] == str(ROOT)
assert spec["runtime"]["plugins"] == ["beam"]
assert spec["candidate"]["paths"] == ["plugins/beam/src"]
assert workspace == ROOT / "benchmark/beam/workspaces/home-100K"
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, benchmark):
"""Both Python entry points invoke only home_prepare through a local application."""
module = import_module(f"benchmark.{benchmark}.run_home")
application = AsyncMock()
application.run_job.return_value = Response(answer={"dataset": {"case_count": 1}})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(home_runner, "Application", factory)
result = await module.run(workspace_dir=str(tmp_path / "experiment"))
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
assert observed["service"] == {"backend": "cli"}
assert observed["workspace_dir"] == str(tmp_path / "experiment")
assert observed["components"]["home.evaluation"]["default"]["sampling_policy"] == {
"max_cases": 5,
"max_queries": None,
"temperature": 1.0,
"ewma_alpha": 0.3,
"uniform_exploration": 0.2,
}
application.start.assert_awaited_once()
application.close.assert_awaited_once()
assert application.run_job.await_count == 1
assert application.run_job.call_args.args == ("home_prepare",)
assert result["workspace_dir"] == str(tmp_path / "experiment")
@pytest.mark.asyncio
async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
"""Partial policies merge typed defaults and never leak into preparation inputs."""
config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(home_runner, "Application", factory)
await home_runner.prepare(path)
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
for invalid in ({"max_cases": 0}, {"unknown": 1}, {"max_queries": "7"}):
config["sampling_policy"] = invalid
path.write_text(yaml.safe_dump(config))
with pytest.raises(ValueError):
await home_runner.prepare(path)
assert application.start.await_count == 1
assert application.run_job.await_count == 1
assert application.run_job.call_args.args == ("home_prepare",)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["startup", "preparation"])
async def test_runner_propagates_failure_and_closes(monkeypatch, failure):
"""A failed preparation is never printed or returned as success."""
application = AsyncMock()
if failure == "startup":
application.start.side_effect = RuntimeError("startup failed")
else:
application.run_job.return_value = Response(success=False, answer="already prepared")
monkeypatch.setattr(home_runner, "Application", lambda **kwargs: application)
with pytest.raises(RuntimeError):
await home_runner.prepare(ROOT / "benchmark/beam/config_home.yaml")
application.close.assert_awaited_once()

View file

@ -0,0 +1,351 @@
"""Home budgets exercised through AgentScope's real reply and event loop."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
import asyncio
from types import SimpleNamespace
import pytest
from agentscope.agent import Agent, ReActConfig
from agentscope.agent._config import ModelConfig
from agentscope.event import ModelCallEndEvent, ReplyEndEvent, ReplyEndReason
from agentscope.message import AssistantMsg, HintBlock, TextBlock, ThinkingBlock, ToolCallBlock, UserMsg
from agentscope.model import ChatResponse, ChatUsage, FinishedReason
from agentscope.tool import FunctionTool, Toolkit
from reme_home.agent import BoundedReActAgent, RunTimeBudgetTimer
class RecordingModel:
model = "isolated-test"
context_size = 10000
formatter = SimpleNamespace(supported_input_media_types=[])
def __init__(self, *, final=None, stream=False):
self.calls = []
self.final = final
self.stream = stream
async def count_tokens(self, messages, tools):
return 10
async def __call__(self, **kwargs):
self.calls.append(kwargs)
if kwargs["tool_choice"] is not None and kwargs["tool_choice"].mode == "none":
if isinstance(self.final, BaseException):
raise self.final
response = self.final or ChatResponse(content=[TextBlock(text="final report")], is_last=True)
if self.stream:
async def chunks():
yield ChatResponse(content=[TextBlock(text="final ")], is_last=False)
yield response
return chunks()
return response
return ChatResponse(
content=[ThinkingBlock(thinking="investigating")],
is_last=True,
usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
)
def hints(agent, source):
return [
block
for message in agent.state.context
for block in message.content
if isinstance(block, HintBlock) and block.source == source
]
def make_agent(model=None, timer=None, max_iters=2):
return BoundedReActAgent(
name="investigator",
system_prompt="Investigate the evidence.",
model=model or RecordingModel(),
react_config=ReActConfig(max_iters=max_iters, interruption_raise_cancelled_error=True),
model_config=ModelConfig(max_retries=0),
run_time_budget_timer=timer,
)
@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
def test_timer_rejects_invalid_budget(budget):
with pytest.raises(ValueError, match="positive"):
RunTimeBudgetTimer(budget, agent_label="diagnostic")
def test_timer_thresholds_restart_and_skipped_thresholds():
now = [0.0]
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
assert timer.next_reminder() is None
timer.start()
for percentage, persistent in [(59, None), (80, False), (90, True), (100, False), (110, True)]:
now[0] = percentage * 60
reminder = timer.next_reminder()
if persistent is None:
assert reminder is None
else:
assert reminder.persistent is persistent
assert "does not terminate" in reminder.content
assert timer.next_reminder() is None
timer.record_context_compression()
timer.stop()
assert timer.next_reminder() is None
timer.start()
assert timer.context_compression_count == 0
assert timer.next_reminder() is None
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_native_loop_has_exactly_one_tool_free_finalization(stream):
usage = ChatUsage(input_tokens=5, output_tokens=7, time=0, cache_input_tokens=2)
model = RecordingModel(
final=ChatResponse(content=[TextBlock(text="report")], is_last=True, usage=usage),
stream=stream,
)
agent = make_agent(model)
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent.run_time_budget_timer = timer
items = [
item
async for item in agent.reply_stream(UserMsg(name="caller", content="Find the cause"), yield_final_msg=True)
]
assert len(model.calls) == 3
assert agent.state.cur_iter == 3
request = model.calls[-1]
assert request["tools"] == []
assert request["tool_choice"].mode == "none"
assert "Find the cause" in request["messages"][-1].get_text_content()
assert items[-1].get_text_content() == "report"
# Reminders must preserve cumulative reply usage as well as per-call events.
assert items[-1].usage.input_tokens == 9
assert items[-1].usage.output_tokens == 13
assert items[-1].usage.cache_input_tokens == 2
calls = [item for item in items if isinstance(item, ModelCallEndEvent)]
assert len(calls) == 3
assert sum(item.input_tokens for item in calls) == 9
assert sum(item.output_tokens for item in calls) == 13
ends = [item for item in items if isinstance(item, ReplyEndEvent)]
assert len(ends) == 1 and ends[0].finished_reason == ReplyEndReason.EXCEED_MAX_ITERS
assert not timer.active
@pytest.mark.asyncio
async def test_finalization_discards_unsolicited_tools():
executed = []
def forbidden():
executed.append(True)
model = RecordingModel(
final=ChatResponse(
content=[TextBlock(text="report"), ToolCallBlock(id="call", name="forbidden", input="{}")],
is_last=True,
),
)
agent = make_agent(model)
agent.toolkit = Toolkit(tools=[FunctionTool(forbidden)])
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "report"
assert not executed
assert not agent.state.get_unfinished_tool_calls(agent.name)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure",
[
RuntimeError("provider unavailable"),
ChatResponse(content=[], is_last=True),
ChatResponse(content=[], is_last=True, usage=ChatUsage(input_tokens=5, output_tokens=7, time=0)),
],
)
async def test_finalization_failure_returns_explicit_incomplete_output(failure):
model = RecordingModel(final=failure)
agent = make_agent(model)
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert "incomplete prior model output" in result.get_text_content()
assert len(model.calls) == 3
final_usage = failure.usage if isinstance(failure, ChatResponse) else None
assert result.usage.input_tokens == 4 + (final_usage.input_tokens if final_usage else 0)
assert result.usage.output_tokens == 6 + (final_usage.output_tokens if final_usage else 0)
@pytest.mark.asyncio
async def test_cancellation_stops_timer_without_fallback():
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(RecordingModel(final=asyncio.CancelledError()), timer)
with pytest.raises(asyncio.CancelledError):
await agent.reply(UserMsg(name="caller", content="Investigate"))
assert not timer.active
assert not any("incomplete prior model output" in (msg.get_text_content() or "") for msg in agent.state.context)
@pytest.mark.asyncio
async def test_interrupted_response_preserves_native_interruption():
model = RecordingModel(final=ChatResponse(content=[], is_last=True, finished_reason=FinishedReason.INTERRUPTED))
with pytest.raises(asyncio.CancelledError):
await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
@pytest.mark.asyncio
async def test_new_reply_resets_iteration_reminders():
model = RecordingModel()
agent = make_agent(model)
for task in ("first", "second"):
result = await agent.reply(UserMsg(name="caller", content=task))
assert result.get_text_content() == "final report"
assert agent._original_task_prompt == task
assert len(model.calls) == 6
reminders = hints(agent, "home_iteration_guard")
assert len(reminders) == 4
@pytest.mark.asyncio
async def test_normal_reply_does_not_force_finalization():
class ImmediateModel(RecordingModel):
async def __call__(self, **kwargs):
self.calls.append(kwargs)
return ChatResponse(content=[TextBlock(text="already sufficient")], is_last=True)
model = ImmediateModel()
result = await make_agent(model).reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "already sufficient"
assert len(model.calls) == 1
@pytest.mark.asyncio
async def test_final_context_trims_old_evidence_but_preserves_system_and_task():
class SmallContextModel(RecordingModel):
async def count_tokens(self, messages, tools):
return 10000 if len(messages) > 2 else 10
model = SmallContextModel()
agent = make_agent(model)
agent.state.cur_iter = 2
agent._original_task_prompt = "original task"
messages = [UserMsg(name="system", content="system prompt"), UserMsg(name="caller", content="old evidence")]
await agent._call_model(messages, tools=[{"name": "unused"}])
assert len(model.calls[-1]["messages"]) == 2
assert model.calls[-1]["messages"][0] is messages[0]
assert "original task" in model.calls[-1]["messages"][-1].get_text_content()
assert len(messages) == 2
@pytest.mark.asyncio
async def test_missing_stream_completion_returns_previous_text_without_retry():
class IncompleteStreamModel(RecordingModel):
async def __call__(self, **kwargs):
self.calls.append(kwargs)
async def chunks():
yield ChatResponse(content=[TextBlock(text="partial")], is_last=False)
return chunks()
model = IncompleteStreamModel()
agent = make_agent(model)
agent.state.cur_iter = 2
agent.state.context.append(AssistantMsg(name=agent.name, content="previous finding"))
response = await agent._call_model([], tools=[])
assert "incomplete prior model output" in response.content[0].text
assert "previous finding" in response.content[0].text
assert len(model.calls) == 1
@pytest.mark.asyncio
async def test_time_reminders_alternate_and_do_not_stop_investigation():
now = [0.0]
timer = RunTimeBudgetTimer(100, agent_label="diagnostic", clock=lambda: now[0])
model = RecordingModel()
agent = make_agent(model, timer, max_iters=100)
timer.start()
for percentage in (60, 75, 90, 100, 110):
now[0] = percentage * 60
_ = [item async for item in agent._reasoning()]
persistent = hints(agent, "home_run_time_guard")
assert len(persistent) == 2
assert "(75%" in persistent[0].hint
assert "(100%" in persistent[1].hint
assert len(model.calls) == 5
assert timer.active
@pytest.mark.asyncio
async def test_ephemeral_reminder_removed_when_reasoning_fails(monkeypatch):
now = [0.0]
timer = RunTimeBudgetTimer(1, agent_label="diagnostic", clock=lambda: now[0])
agent = make_agent(timer=timer)
timer.start()
now[0] = 40
async def fail(*args, **kwargs):
raise RuntimeError("failed")
yield # pylint: disable=unreachable
monkeypatch.setattr(Agent, "_reasoning", fail)
with pytest.raises(RuntimeError):
_ = [item async for item in agent._reasoning()]
assert not hints(agent, "home_run_time_guard")
@pytest.mark.asyncio
async def test_only_successful_context_replacement_counts_as_compression(monkeypatch):
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(timer=timer)
async def no_change(self):
pass
monkeypatch.setattr(Agent, "compress_context", no_change)
await agent.compress_context()
assert timer.context_compression_count == 0
async def change(self):
self.state.context = []
monkeypatch.setattr(Agent, "compress_context", change)
await agent.compress_context()
assert timer.context_compression_count == 1
class CompressionFailureModel(RecordingModel):
def __init__(self, failure, fail_after=2):
super().__init__()
self.failure = failure
self.fail_after = fail_after
async def count_tokens(self, messages, tools):
return 9000 if len(self.calls) >= self.fail_after and len(messages) > 2 else 10
async def generate_structured_output(self, **kwargs):
raise self.failure
@pytest.mark.asyncio
async def test_final_report_survives_real_compression_failure():
model = CompressionFailureModel(RuntimeError("compression provider unavailable"))
timer = RunTimeBudgetTimer(1, agent_label="diagnostic")
agent = make_agent(model, timer)
agent.context_config.compression_fallback_to_truncation = False
result = await agent.reply(UserMsg(name="caller", content="Investigate"))
assert result.get_text_content() == "final report"
assert len(model.calls) == 3
assert model.calls[-1]["tools"] == []
assert len(model.calls[-1]["messages"]) == 2
assert timer.context_compression_count == 0
assert not timer.active
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [RuntimeError("compression failed"), asyncio.CancelledError()])
async def test_nonfinal_compression_errors_and_cancellation_propagate(failure):
model = CompressionFailureModel(failure, fail_after=1 if isinstance(failure, RuntimeError) else 2)
agent = make_agent(model)
agent.context_config.compression_fallback_to_truncation = False
with pytest.raises(type(failure)):
await agent.reply(UserMsg(name="caller", content="Investigate"))
assert len(model.calls) == model.fail_after

View file

@ -0,0 +1,210 @@
"""Bounded ReMe agents use real Job tools, injection and persisted sessions."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
import json
from importlib.metadata import EntryPoint
from types import SimpleNamespace
import pytest
from agentscope.message import HintBlock, TextBlock, ToolCallBlock
from agentscope.model import ChatResponse, ChatUsage
from agentscope.tool import FunctionTool, Toolkit
from reme_home.agent import BoundedAgentWrapper, BoundedReActAgent, RunTimeBudgetTimer
from test_bounded_agent import RecordingModel
from reme.components.agent_wrapper import AsAgentWrapper
from reme.components.application_context import ApplicationContext
from reme.components.job import BaseJob
from reme.enumeration import ComponentEnum
from reme.plugin import resolve_plugin_runtime
from reme.steps.base_step import BaseStep
from reme.utils import global_counter_get
class CaptureStep(BaseStep):
async def execute(self):
self.app_context.metadata.setdefault("test.calls", []).append(dict(self.context.data))
self.context.response.answer = "captured"
class JobModel(RecordingModel):
def __init__(self, arguments):
super().__init__()
self.arguments = arguments
async def __call__(self, **kwargs):
self.calls.append(kwargs)
content = (
[ToolCallBlock(id="capture-call", name="capture", input=json.dumps(self.arguments))]
if len(self.calls) == 1
else [TextBlock(text="finished")]
)
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
@pytest.fixture(name="wrapper")
def bounded_wrapper(tmp_path):
context = ApplicationContext(workspace_dir=str(tmp_path))
result = BoundedAgentWrapper(
app_context=context,
name="investigator",
as_llm="",
time_budget_minutes=30,
react_config={"max_iters": 2},
)
result.as_llm = SimpleNamespace(model=RecordingModel())
return result
def test_plugin_registers_bounded_agent_only_when_enabled(monkeypatch):
entry = EntryPoint(name="home", value="reme_home", group="reme.plugins")
monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entry])
enabled = resolve_plugin_runtime({"plugins": ["home"]})
assert enabled.registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is BoundedAgentWrapper
assert resolve_plugin_runtime({}).registry.get(ComponentEnum.AGENT_WRAPPER, "home_bounded") is None
assert issubclass(BoundedAgentWrapper, AsAgentWrapper)
@pytest.mark.asyncio
async def test_each_build_owns_timer_and_honors_call_overrides(wrapper):
first, _ = await wrapper._build_agent("first", **wrapper._merged_kwargs({}))
second, _ = await wrapper._build_agent("second", **wrapper._merged_kwargs({"time_budget_minutes": 5}))
assert isinstance(first, BoundedReActAgent)
assert first.run_time_budget_timer.time_budget_minutes == 30
assert second.run_time_budget_timer.time_budget_minutes == 5
assert first.run_time_budget_timer is not second.run_time_budget_timer
assert first.react_config.max_iters == 2
no_timer, _ = await wrapper._build_agent("third", **wrapper._merged_kwargs({"time_budget_minutes": None}))
assert no_timer.run_time_budget_timer is None
@pytest.mark.asyncio
@pytest.mark.parametrize("conflict", [False, True])
async def test_real_job_tools_enforce_hidden_injected_arguments(wrapper, conflict):
context = wrapper.app_context
context.registry.register(CaptureStep, "test_capture")
job = BaseJob(
name="capture",
app_context=context,
parameters={
"type": "object",
"properties": {"visible": {"type": "string"}, "case_ids": {"type": "array"}},
"required": ["visible", "case_ids"],
},
steps=[{"backend": "test_capture"}],
)
context.jobs["capture"] = job
arguments = {"visible": "model value"}
if conflict:
arguments["case_ids"] = ["model-chosen"]
model = JobModel(arguments)
wrapper.as_llm.model = model
await job.start()
await wrapper.start()
try:
result = await wrapper.reply(
"Run the capture job",
job_tools=["capture"],
injected_job_kwargs={"case_ids": ["host-chosen"]},
tool_context_id="investigation-1",
)
schemas = json.dumps(model.calls[0]["tools"])
assert "case_ids" not in schemas
assert "tool_context_id" not in schemas
if conflict:
assert "test.calls" not in context.metadata
else:
assert context.metadata["test.calls"] == [
{
"visible": "model value",
"case_ids": ["host-chosen"],
"tool_context_id": "investigation-1",
},
]
assert result["result"] == "finished"
assert (wrapper.session_path / f"{result['session_id']}.jsonl").is_file()
resumed = await wrapper.reply("Continue", resume=result["session_id"])
assert resumed["session_id"] == result["session_id"]
finally:
await wrapper.close()
await job.close()
@pytest.mark.asyncio
async def test_wrapper_preserves_observed_task_in_forced_finalization(wrapper):
result = await wrapper.reply("Identify the original failure")
assert result["result"] == "final report"
messages = wrapper.as_llm.model.calls[-1]["messages"]
assert "Original task: Identify the original failure" in messages[-1].get_text_content()
assert result["session_id"]
@pytest.mark.asyncio
async def test_wrapper_streams_reme_chunks_and_persists_session(wrapper):
chunks = [chunk async for chunk in wrapper.reply_stream("Investigate")]
assert any(chunk.chunk == "final report" for chunk in chunks)
session_id = chunks[-1].session_id
assert session_id
assert (wrapper.session_path / f"{session_id}.jsonl").is_file()
@pytest.mark.asyncio
@pytest.mark.parametrize("timed", [False, True])
async def test_multiround_usage_and_tools_continue_past_soft_time_limit(wrapper, monkeypatch, timed):
now = [0.0]
timer = RunTimeBudgetTimer(1, agent_label="investigator", clock=lambda: now[0])
monkeypatch.setattr("reme_home.agent.wrapper.RunTimeBudgetTimer", lambda *args, **kwargs: timer)
executions = []
async def inspect_evidence():
executions.append(now[0])
return "evidence"
class MultiRoundModel(RecordingModel):
def __init__(self):
super().__init__()
self.time_reminders = []
async def __call__(self, **kwargs):
self.calls.append(kwargs)
self.time_reminders.extend(
block.hint
for message in kwargs["messages"]
for block in message.content
if isinstance(block, HintBlock) and block.source == "home_run_time_guard"
)
now[0] += 65
count = len(self.calls)
content = (
[ToolCallBlock(id=f"inspect-{count}", name="inspect_evidence", input="{}")]
if count < 3
else [TextBlock(text="finished")]
)
return ChatResponse(
content=content,
is_last=True,
usage=ChatUsage(input_tokens=10, output_tokens=5, time=0),
)
model = MultiRoundModel()
wrapper.as_llm.model = model
options = {
"react_config": {"max_iters": 4},
"time_budget_minutes": 1 if timed else None,
"toolkit": Toolkit(tools=[FunctionTool(inspect_evidence)]),
}
result = await wrapper.reply("Investigate", **options)
assert result["usage"] == {"input_tokens": 30, "output_tokens": 15, "total_tokens": 45}
assert result["last_message"]["finished_reason"] == "completed"
assert len(executions) == 2
assert executions[-1] > 60 # A tool really executes after the soft deadline.
assert len(model.calls) == 3
assert all(call["tools"] for call in model.calls)
assert bool(model.time_reminders) is timed
assert not timer.active
metadata = wrapper.app_context.metadata
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 45
resumed = await wrapper.reply("Continue", resume=result["session_id"], **options)
assert resumed["usage"]["total_tokens"] == 15
assert global_counter_get(metadata, ["__token_counter", wrapper.name, "total_tokens"]) == 60

View file

@ -0,0 +1,267 @@
"""Single case investigations use normal ReMe lifecycle and sandboxed executors."""
# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
# pylint: disable=unused-argument,protected-access
import asyncio
import json
import sys
from types import SimpleNamespace
import pytest
from agentscope.message import TextBlock, ToolCallBlock
from agentscope.model import ChatResponse, ChatUsage
from agentscope.tool import ExecResult
from reme_home.agent import case_study
from reme_home.evaluation.artifacts import write_json
from reme_home.preparation.code import git
from test_bounded_agent import RecordingModel
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from test_evaluation_inspection import write_run, snapshot
from reme.utils.local_backend import LocalProcessBackend
from reme.utils.file_sandbox import create_file_sandbox_backend
class RecordingSandbox:
"""Mock only the subprocess boundary; never claim this is an OS sandbox."""
os_name = "posix"
def __init__(self, **kwargs):
self.policy = kwargs
self.calls = []
async def exec_shell(self, command, *, cwd, timeout):
self.calls.append((command, cwd, timeout))
return ExecResult(exit_code=0, stdout=b"analysis\n", stderr=b"")
@pytest.fixture
def sandbox(monkeypatch):
instances = []
def create(**kwargs):
backend = RecordingSandbox(**kwargs)
instances.append(backend)
return backend
monkeypatch.setattr(case_study, "create_file_sandbox_backend", create)
return instances
@pytest.fixture
def case_run(prepared):
commit = git(prepared / "code/repo", "rev-parse", "HEAD")
source = sorted((prepared / "datasets/training-set/cases").glob("*.json"))[0]
case = json.loads(source.read_text())
case_id = case["case_id"]
query_id = case["queries"][0]["query_id"]
run = write_run(prepared, commit=commit, case_ids=[case_id], query_ids={case_id: [query_id]})
write_json(
run / f"cases/{case_id}/queries/result.json",
{
"queries": [{"query_id": query_id, "answer": "wrong", "score": 0}],
},
)
return run, case_id, query_id, commit, source
class InvestigationModel(RecordingModel):
def __init__(self, query_id):
super().__init__()
self.query_id = query_id
async def __call__(self, **kwargs):
self.calls.append(kwargs)
instructions = [
("home_trace_query_evidence", {"query_id": self.query_id}),
("home_read_code_file", {"path": "plugins/beam/src/reme_beam/__init__.py"}),
("python_execute", {"code": "print('analysis')"}),
("AutoBash", {"command": "for x in 1 2; do echo $x; done"}),
]
index = len(self.calls) - 1
if index < len(instructions):
name, arguments = instructions[index]
content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))]
else:
content = [TextBlock(text="Evidence-supported report; root cause remains uncertain.")]
return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
@pytest.mark.asyncio
async def test_single_run_real_agent_tools_scope_and_no_scratch(app, prepared, case_run, sandbox):
run, case_id, query_id, commit, source = case_run
wrapper = app.context.components["agent_wrapper"]["home_case_study"]
model = InvestigationModel(query_id)
wrapper.as_llm = SimpleNamespace(model=model)
(prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("value = 999\n")
before = snapshot(prepared)
response = await app.run_job(
"home_case_study",
validation_path=str(run),
case_id=case_id,
focus="Trace the wrong answer",
)
assert response.success, response.answer
assert "Evidence-supported report" in response.answer
assert response.metadata["candidate_commit"] == commit and response.metadata["usage"]["total_tokens"] == 25
assert response.metadata["session_id"]
assert "value = 1" in str(model.calls[2]["messages"])
assert "value = 999" not in str(model.calls[2]["messages"])
schemas = {item["function"]["name"]: item["function"]["parameters"] for item in model.calls[0]["tools"]}
assert set(schemas) == set(case_study.CASE_TOOLS + case_study.CODE_TOOLS) | {"AutoBash", "python_execute"}
assert "case_id" not in schemas["home_trace_query_evidence"]["properties"]
assert "validation_path" not in schemas["home_inspect_validation"]["properties"]
assert "revision" not in schemas["home_read_code_file"]["properties"]
assert "target_validation_path" not in schemas["home_compare_memory"]["properties"]
policy = sandbox[0].policy
assert policy["writable_paths"] == [] and policy["cwd"] == str(run / f"cases/{case_id}")
assert str(prepared / "code") in policy["read_only_paths"] and str(source) in policy["read_only_paths"]
assert str(prepared) not in policy["read_only_paths"]
assert len(sandbox[0].calls) == 3 # sandbox probe, Python, unrestricted Bash
assert sandbox[0].calls[1][0][:4] == [sys.executable, "-I", "-B", "-c"]
assert "for x in 1 2" in str(sandbox[0].calls[2][0])
assert snapshot(prepared) == before
# Each invocation is fresh, even through the same application-owned wrapper.
wrapper.as_llm = SimpleNamespace(model=InvestigationModel(query_id))
second = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id)
assert second.success and second.metadata["session_id"] != response.metadata["session_id"]
@pytest.mark.asyncio
async def test_scope_override_rejected_by_real_job_tool(app, prepared, case_run, sandbox):
run, case_id, _query_id, _commit, _source = case_run
wrapper = app.context.components["agent_wrapper"]["home_case_study"]
scope = case_study.prepare_scope(prepared, case_study.CaseStudyInput(validation_path=str(run), case_id=case_id))
toolkit = case_study.create_toolkit(wrapper, scope, RecordingSandbox())
schemas = await toolkit.get_tool_schemas()
assert len(schemas) == 9
tool = wrapper._make_tool(
app.context.jobs["home_inspect_validation"],
injected_job_kwargs={
"validation_path": str(run),
"case_id": case_id,
},
)
with pytest.raises(ValueError, match="cannot be provided"):
await tool._func(case_id="another-case")
@pytest.mark.asyncio
async def test_invalid_scope_and_sandbox_failure_precede_llm(app, prepared, case_run, sandbox, monkeypatch):
run, case_id, _query_id, _commit, _source = case_run
wrapper = app.context.components["agent_wrapper"]["home_case_study"]
model = InvestigationModel("unused")
wrapper.as_llm = SimpleNamespace(model=model)
response = await app.run_job("home_case_study", validation_path=str(run), case_id="../escape")
assert not response.success and not sandbox
response = await app.run_job(
"home_case_study",
validation_path=str(run),
case_id=case_id,
_allowed_paths=[str(run)],
)
assert not response.success and not sandbox
async def denied(self, *args, **kwargs):
return ExecResult(exit_code=1, stdout=b"", stderr=b"sandbox denied")
monkeypatch.setattr(RecordingSandbox, "exec_shell", denied)
response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id)
assert not response.success and "sandbox unavailable" in str(response.answer)
assert not model.calls
def unavailable(**kwargs):
raise RuntimeError("No sandbox backend")
monkeypatch.setattr(case_study, "create_file_sandbox_backend", unavailable)
response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id)
assert not response.success and "No sandbox backend" in str(response.answer) and not model.calls
@pytest.mark.asyncio
async def test_iteration_limit_returns_final_report(app, prepared, case_run, sandbox):
run, case_id, _query_id, _commit, _source = case_run
wrapper = app.context.components["agent_wrapper"]["home_case_study"]
assert wrapper.kwargs["react_config"]["max_iters"] == 60
assert wrapper.kwargs["time_budget_minutes"] == 30
wrapper.kwargs["react_config"] = {"max_iters": 1}
model = RecordingModel()
wrapper.as_llm = SimpleNamespace(model=model)
response = await app.run_job("home_case_study", validation_path=str(run), case_id=case_id)
assert response.success and response.answer == "final report"
assert model.calls[-1]["tools"] == []
@pytest.mark.asyncio
async def test_os_sandbox_read_scope_and_no_writes(tmp_path):
case = tmp_path / "case"
code = tmp_path / "code"
case.mkdir()
code.mkdir()
(code / "source.py").write_text("visible code")
outside = tmp_path / "secret.txt"
outside.write_text("outside scope")
backend = create_file_sandbox_backend(
cwd=str(case),
read_only_paths=[str(case), str(code)],
writable_paths=[],
memory_limit_mib=0,
)
try:
probe = await backend.exec_shell([sys.executable, "-I", "-B", "-c", "pass"], cwd=str(case), timeout=10)
except (OSError, RuntimeError) as error:
pytest.skip(f"OS sandbox unavailable on this host: {error}")
if not probe.ok():
pytest.skip("OS sandbox cannot start on this host: " + probe.stderr.decode(errors="replace"))
tool = case_study.python_tool(backend, str(case))
result = await tool._func(code=f"from pathlib import Path; print(Path({str(code / 'source.py')!r}).read_text())")
assert result["returncode"] == 0 and "visible code" in result["stdout"]
for script in (
f"open({str(outside)!r}).read()",
"open('written.txt', 'w').write('denied')",
f"open({str(code / 'source.py')!r}, 'w').write('changed')",
):
result = await tool._func(code=script)
assert result["returncode"] != 0
bash = case_study.AutoBash(cwd=str(case), backend=backend, is_read_only=False)
chunks = [chunk async for chunk in bash.call(f"cat '{code / 'source.py'}'")]
assert "visible code" in str(chunks)
chunks = [chunk async for chunk in bash.call("echo changed > written.txt")]
assert "Command failed" in str(chunks)
assert not (case / "written.txt").exists() and (code / "source.py").read_text() == "visible code"
@pytest.mark.asyncio
async def test_python_code_file_timeout_and_cancellation(tmp_path):
# Execution protocol uses real subprocesses; OS enforcement is tested separately.
tool = case_study.python_tool(LocalProcessBackend(), str(tmp_path))
result = await tool._func(code="print(1 + 2)")
assert result["stdout"].strip() == "3" and result["returncode"] == 0
script = tmp_path / "script.py"
script.write_text("import sys; print(sys.argv[1])")
result = await tool._func(file="script.py", args=["argument"])
assert result["stdout"].strip() == "argument"
result = await tool._func(code="import time; time.sleep(20)", timeout=0.05)
assert result["timed_out"]
task = asyncio.create_task(tool._func(code="import time; time.sleep(20)"))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
for arguments in (
{},
{"code": "pass", "file": "x"},
{"code": "pass", "args": []},
{"code": "pass", "timeout": float("nan")},
{"code": "pass", "timeout": True},
):
with pytest.raises(ValueError):
await tool._func(**arguments)

View file

@ -0,0 +1,316 @@
"""Candidate version Jobs exercised against real isolated Git repositories."""
# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name,unused-argument
import asyncio
from pathlib import Path
import subprocess
from threading import Event
import pytest
import pytest_asyncio
from reme_home.code import snapshot
from reme_home.code.models import ARGUMENTS
from reme_home.workspace import _initialization_lock
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
command,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
@pytest_asyncio.fixture
async def prepared(app, inputs):
spec, _ = inputs
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
repo = Path(app.config.workspace_dir) / "home/code/repo"
return app, repo, spec
async def run(app, job, **kwargs):
response = await app.run_job(job, **kwargs)
assert response.success, response.answer
return response.answer
async def create(app, name="candidate"):
return await run(app, "home_create_candidate_branch", branch_name=name)
async def commit(app, repo, text="value = 2\n"):
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text(text)
return await run(app, "home_commit_candidate", message="Improve candidate")
@pytest.mark.asyncio
async def test_branch_commit_diff_and_immutable_read(prepared):
app, repo, _ = prepared
initial = await run(app, "home_list_candidate_branches")
assert initial["current_branch"] == "init"
assert initial["candidate_branches"] == []
created = await create(app)
assert created["candidate_branch_count"] == 1
result = await commit(app, repo)
assert result["commit_sha"] != initial["commit_sha"]
assert (await run(app, "home_code_status"))["clean"]
diff = await run(app, "home_code_diff", base_revision="init", revision=result["commit_sha"])
assert "+value = 2" in diff["content"]
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("dirty = 3\n")
status = await run(app, "home_code_status")
assert status["changes"] == [{"status": " M", "path": "plugins/beam/src/reme_beam/__init__.py"}]
assert not status["clean"]
assert "+dirty = 3" in (await run(app, "home_code_diff"))["content"]
read = await run(
app,
"home_read_code_file",
path="plugins/beam/src/reme_beam/__init__.py",
revision=initial["commit_sha"],
)
assert read["content"] == "value = 1\n"
with pytest.raises(ValueError, match="clean"):
await app.context.components["home.code"]["default"].run("switch_branch", branch_name="init")
@pytest.mark.asyncio
async def test_host_pinned_job_tool_and_schema(prepared):
app, repo, _ = prepared
initial = command(repo, "rev-parse", "HEAD")
await create(app)
await commit(app, repo)
job = app.context.jobs["home_read_code_file"]
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"revision": initial})
assert "revision" not in tool.input_schema["properties"]
# Test the actual wrapper's merge contract through its executable FunctionTool.
response = await tool(path="plugins/beam/src/reme_beam/__init__.py")
assert "value = 1" in response.content[0].text
assert "value = 2" not in response.content[0].text
with pytest.raises(ValueError, match="injected"):
await tool(path="plugins/beam/src/reme_beam/__init__.py", revision="HEAD")
assert all(model.model_json_schema()["additionalProperties"] is False for model in ARGUMENTS.values())
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["init", "a/b", "../escape", "-bad", "a\\b", "@{-1}"])
async def test_invalid_candidate_branch(prepared, name):
app, _, _ = prepared
response = await app.run_job("home_create_candidate_branch", branch_name=name)
assert not response.success
assert (await run(app, "home_code_status"))["current_branch"] == "init"
@pytest.mark.asyncio
async def test_protected_init_and_scope(prepared):
app, repo, _ = prepared
response = await app.run_job("home_commit_candidate", message="not allowed")
assert not response.success
await create(app)
baseline = command(repo, "rev-parse", "HEAD")
readme = repo / "plugins/beam/README.md"
readme.write_text("outside declared editable roots")
response = await app.run_job("home_commit_candidate", message="outside scope")
assert not response.success
assert command(repo, "rev-parse", "HEAD") == baseline
assert not command(repo, "diff", "--cached", "--name-only")
readme.write_text("Plugin installation instructions\n")
target = repo / "plugins/beam/src/link"
target.symlink_to("/etc/passwd")
response = await app.run_job("home_commit_candidate", message="link")
assert not response.success
target.unlink()
result = await commit(app, repo)
command(repo, "update-ref", "refs/heads/init", result["commit_sha"])
response = await app.run_job("home_code_status")
assert not response.success
@pytest.mark.asyncio
async def test_merge_then_explicit_commit(prepared):
app, repo, _ = prepared
await create(app, "left")
left = await commit(app, repo)
await run(app, "home_switch_candidate_branch", branch_name="init")
await create(app, "right")
(repo / "plugins/beam/src/new.py").write_text("another mechanism\n")
right = await run(app, "home_commit_candidate", message="right")
merged = await run(app, "home_merge_candidate_branch", branch_name="left")
assert merged["status"] == "merged_not_committed"
assert merged["commit_sha"] == right["commit_sha"]
assert (await run(app, "home_code_status"))["merge_pending"]
response = await app.run_job("home_switch_candidate_branch", branch_name="init")
assert not response.success
merged = await run(app, "home_commit_candidate", message="Combine compatible improvements")
assert command(repo, "rev-list", "--parents", "-n", "1", "HEAD").split() == [
merged["commit_sha"],
right["commit_sha"],
left["commit_sha"],
]
@pytest.mark.asyncio
async def test_conflicting_merge_aborts_cleanly(prepared):
app, repo, _ = prepared
await create(app, "left")
await commit(app, repo, "left = 2\n")
await run(app, "home_switch_candidate_branch", branch_name="init")
await create(app, "right")
right = await commit(app, repo, "right = 2\n")
response = await app.run_job("home_merge_candidate_branch", branch_name="left")
assert not response.success
assert not (repo / ".git/MERGE_HEAD").exists()
state = await run(app, "home_code_status")
assert state["clean"] and state["commit_sha"] == right["commit_sha"]
@pytest.mark.asyncio
async def test_raw_snapshot_regex_paths_and_truncation(prepared):
app, repo, _ = prepared
await create(app)
folder = repo / "plugins/beam/src"
(folder / "space name.py").write_text("prefix\nhello\nworld\nsuffix\n" + "x\n" * 400)
(folder / ".gitattributes").write_text('"space name.py" export-ignore\n')
(folder / "binary").write_bytes(b"a\0b")
(folder / "carriage.py").write_bytes(b"first\rsecond\rthird")
(folder / "colon:name.py").write_text("hello")
# Non-portable paths fail closed instead of becoming Git pathspecs.
rejected = await app.run_job("home_commit_candidate", message="bad path")
assert not rejected.success
(folder / "colon:name.py").unlink()
result = await run(app, "home_commit_candidate", message="Navigation fixtures")
tree = await run(app, "home_list_code_tree", path="plugins/beam/src", max_results=1)
assert tree["revision"] == result["commit_sha"] and tree["truncated"]
found = await run(app, "home_search_code", pattern="hello\nworld", path="plugins/beam/src")
assert found["occurrence_count"] == 1
assert found["matches"][0]["start_line"] == 1
assert "hello" in found["matches"][0]["content"]
carriage = await run(app, "home_search_code", pattern="second", path="plugins/beam/src/carriage.py")
assert "> 2 | second" in carriage["matches"][0]["content"]
read = await run(app, "home_read_code_file", path="plugins/beam/src/space name.py")
assert read["end_line"] == 300 and read["truncated"]
for kwargs in [
{"path": "../escape"},
{"path": ".git/HEAD"},
{"path": "/etc/passwd"},
{"path": "plugins/beam/src/binary"},
{"path": "plugins/beam/src", "revision": "--help"},
]:
assert not (await app.run_job("home_read_code_file", **kwargs)).success
for pattern in ("[", "^", ""):
assert not (await app.run_job("home_search_code", pattern=pattern)).success
assert not (await app.run_job("home_list_code_tree", path=":(glob)**")).success
@pytest.mark.asyncio
async def test_lock_restrictions_and_unprepared(app):
assert not (await app.run_job("home_code_status")).success
assert not (await app.run_job("home_code_status", _allowed_paths=[])).success
component = app.context.components["home.code"]["default"]
with pytest.raises(ValueError, match="Unknown"):
await component.run("reset")
with pytest.raises(ValueError):
await component.run("create_branch", branch_name=1)
with _initialization_lock(component.workspace.state_path / "workspace.lock"):
assert not (await app.run_job("home_list_code_tree")).success
@pytest.mark.asyncio
async def test_cancel_waits_for_owned_git_thread(prepared, monkeypatch):
app, _, _ = prepared
component = app.context.components["home.code"]["default"]
entered, release, finished = Event(), Event(), Event()
original = component._run
def blocked(*args):
entered.set()
assert release.wait(5)
try:
return original(*args)
finally:
finished.set()
monkeypatch.setattr(component, "_run", blocked)
task = asyncio.create_task(component.run("status"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert finished.is_set() and not component._pending
@pytest.mark.asyncio
async def test_snapshot_size_and_regex_timeout(prepared, monkeypatch):
app, repo, _ = prepared
await create(app)
await commit(app, repo, 'x = "' + "a" * 10000 + '!"\n')
original = snapshot.subprocess.run
def short_timeout(*args, **kwargs):
if args[0][0] == snapshot.sys.executable:
kwargs["timeout"] = 0.05
return original(*args, **kwargs) # pylint: disable=subprocess-run-check
monkeypatch.setattr(snapshot.subprocess, "run", short_timeout)
with pytest.raises(subprocess.TimeoutExpired):
snapshot.search_code(repo, {}, pattern="(a+)+$", path="plugins/beam/src")
monkeypatch.setattr(snapshot.subprocess, "run", original)
monkeypatch.setattr(snapshot, "MAX_FILE_BYTES", 3)
response = await app.run_job("home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py")
assert not response.success
@pytest.mark.asyncio
@pytest.mark.parametrize("mutation", ["staged", "untracked"])
async def test_branch_operations_refuse_dirty_inputs(prepared, mutation):
app, repo, _ = prepared
if mutation == "staged":
target = repo / "plugins/beam/src/reme_beam/__init__.py"
target.write_text("staged = True\n")
command(repo, "add", ".")
else:
target = repo / "plugins/beam/src/untracked file.py"
target.write_text("untracked = True\n")
before = target.read_bytes()
assert not (await app.run_job("home_create_candidate_branch", branch_name="dirty")).success
assert target.read_bytes() == before
assert command(repo, "branch", "--show-current") == "init"
@pytest.mark.asyncio
async def test_exact_paths_rename_deletion_and_revision_pinning(prepared):
app, repo, _ = prepared
await create(app)
original = repo / "plugins/beam/src/reme_beam/__init__.py"
name = "plugins/beam/src/space name\nand newline.py"
target = repo / name
original.rename(target)
first = await run(app, "home_commit_candidate", message="Rename a file")
assert (await run(app, "home_read_code_file", path=name))["content"] == "value = 1\n"
target.write_text("value = 9\n")
pinned = await run(app, "home_commit_candidate", message="Improve renamed code")
target.unlink()
await run(app, "home_commit_candidate", message="Remove obsolete file")
await run(app, "home_switch_candidate_branch", branch_name="init")
read = await run(app, "home_read_code_file", path=name, revision=pinned["commit_sha"])
assert read["content"] == "value = 9\n"
assert name in (await run(app, "home_list_code_tree", revision=first["commit_sha"]))["paths"]
@pytest.mark.asyncio
async def test_reject_symlink_repository_and_detached_candidate(prepared):
app, repo, _ = prepared
await create(app)
commit_sha = (await commit(app, repo))["commit_sha"]
command(repo, "checkout", "--detach", commit_sha)
assert not (await app.run_job("home_commit_candidate", message="detached")).success
assert (await run(app, "home_read_code_file", path="plugins/beam/src/reme_beam/__init__.py"))["content"]
moved = repo.with_name("moved")
repo.rename(moved)
repo.symlink_to(moved, target_is_directory=True)
assert not (await app.run_job("home_code_status")).success

View file

@ -0,0 +1,54 @@
"""Debug derivation reads only selected cases and preserves deterministic output."""
from pathlib import Path
from reme_home.preparation.data.basic import derive_debug_cases, write_normalized_dataset
from reme_home.preparation.data.models import CaseSpec, QuerySpec, SessionSpec
def test_debug_derivation_reads_only_selected_cases(tmp_path, monkeypatch):
"""Unselected training files need not be loaded to derive the smoke dataset."""
training = tmp_path / "training"
output = tmp_path / "debug"
write_normalized_dataset(
training,
"beam",
"10M",
(
CaseSpec(
case_id=str(index),
sessions=[
SessionSpec(
session_id=f"s{session_index}",
messages=[{"role": "user", "content": str(message_index)} for message_index in range(4)],
)
for session_index in range(3)
],
queries=[QuerySpec(query_id="q", question="What?", golden_answer="answer")],
)
for index in range(30)
),
)
expected_indices = [0, 14, 29]
expected_paths = [training / "cases" / f"{index:06d}.json" for index in expected_indices]
reads = []
original_read = Path.read_text
def read_text(path, *args, **kwargs):
if path.parent == training / "cases":
assert path in expected_paths, f"Unselected training case read: {path}"
reads.append(path)
return original_read(path, *args, **kwargs)
monkeypatch.setattr(Path, "read_text", read_text)
manifest = derive_debug_cases(training, output, sessions_per_case=2, messages_per_session=2)
assert reads == expected_paths
assert manifest.case_ids == [f"debug-{index}" for index in expected_indices]
assert manifest.query_count == 3
assert not (output / "derivation.json").exists()
assert "schema_version" not in manifest.model_dump()
for path in sorted((output / "cases").glob("*.json")):
case = CaseSpec.model_validate_json(path.read_text())
assert [session.session_id for session in case.sessions] == ["s0", "s2"]
assert all([message["content"] for message in session.messages] == ["0", "1"] for session in case.sessions)

View file

@ -0,0 +1,753 @@
"""Real Home Jobs and pool scheduling with isolated fake Docker/model boundaries."""
# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument,protected-access
import asyncio
import io
import json
from pathlib import Path
import tarfile
from unittest.mock import AsyncMock
import pytest
import pytest_asyncio
from reme_home.evaluation import component as evaluation
from reme_home.evaluation.artifacts import inspect_archive
# Reuse the real Application, committed repository and tiny source-dataset fixtures.
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
command,
)
def archive(path, files):
path.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(path, "w:gz") as stream:
for name, content in files.items():
data = content.encode()
info = tarfile.TarInfo(name)
info.size = len(data)
stream.addfile(info, io.BytesIO(data))
return path
class FakeWorker:
def __init__(self, factory):
self.factory = factory
self.container_id = f"fake-{len(factory.workers)}"
self.reusable = True
self.case_id = None
self.queries = []
self.closed = False
async def reset_case(self, case_id):
self.case_id = case_id
self.factory.events.append((case_id, "reset"))
async def upload_workspace(self, source):
inspect_archive(Path(source), 1_000_000)
self.factory.events.append((self.case_id, "upload"))
async def run_build(self, jobs):
self.factory.events.append((self.case_id, "build", jobs))
await asyncio.sleep(0)
success = self.case_id != self.factory.fail_build
result = {"success": success}
if not success:
result["exception"] = {"type": "ValueError", "module": "builtins"}
return {"success": success, "jobs": [{"job": job.job, "result": dict(result)} for job in jobs]}
async def run_queries(self, queries):
self.queries = queries
self.factory.events.append((self.case_id, "answer"))
if self.factory.block is not None:
self.factory.entered.set()
await self.factory.block.wait()
return {
"success": True,
"queries": [
{
"query_id": q.query_id,
"answer": "hello",
"answer_result": {"success": True, "token_usage": {"answer": {"total_tokens": 2}}},
"error": None,
}
for q in queries
],
}
async def export_workspace(self, destination):
self.factory.events.append((self.case_id, "snapshot"))
return archive(Path(destination), {"daily/note.md": self.case_id, ".git/HEAD": "ref: refs/heads/main\n"})
async def export_build_log(self, destination):
Path(destination).parent.mkdir(parents=True, exist_ok=True)
Path(destination).write_text("session construction log", encoding="utf-8")
return Path(destination)
async def read_build_log(self):
return "failed build log"
async def export_evaluation(self, destination):
files = {
"reme_workspace/daily/note.md": "after-answer",
"manifest.json": "{}",
"queries/result.json": '{"queries": [{"query_id": "fact:1", "answer": "forged"}]}',
}
for query in self.queries:
files[f"queries/{query.query_id}/answer.log"] = "answer log"
if self.factory.bad_export:
files["../escaped"] = "invalid"
return archive(Path(destination), files)
async def export_full(self, destination):
return archive(Path(destination), {"build_log/build.log": "failed diagnostic"})
async def close(self):
self.closed = True
class FakeFactory:
instances = []
fail_build = None
bad_export = False
block = None
def __init__(self, candidate, **kwargs):
self.candidate = candidate
self.kwargs = kwargs
self.events = []
self.workers = []
self.entered = asyncio.Event()
self.instances.append(self)
async def create_cases(self, ids, **kwargs):
self.workers = [FakeWorker(self) for _ in ids]
if self.workers and self.kwargs.get("on_sandbox_started"):
self.kwargs["on_sandbox_started"]()
return self.workers
@pytest_asyncio.fixture(name="prepared")
async def prepared_home(app, inputs, monkeypatch):
spec, _ = inputs
source = Path(spec["benchmark"]["source"])
for path in source.glob("chats/*/*/probing_questions/probing_questions.json"):
raw = json.loads(path.read_text())
raw["fact"][0]["rubric"] = ["says hello"]
path.write_text(json.dumps(raw))
if "lme-judge" in app.config.plugins:
spec["benchmark"] = {"name": "longmemeval", "source": str(inputs[1]), "all_cases": True}
spec["runtime"]["plugins"] = []
spec["candidate"]["paths"] = ["reme/candidate.py"]
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
FakeFactory.instances = []
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", FakeFactory)
# Run the real judge Job and Step; mock only its model-facing wrapper.
wrapper = app.context.components["agent_wrapper"]["judge"]
monkeypatch.setattr(wrapper, "reply", AsyncMock(return_value={"result": '{"score": 0.5}'}))
return Path(app.config.workspace_dir) / "home"
@pytest.mark.asyncio
async def test_case_and_query_selection_are_host_injected(app, prepared):
from reme.components.agent_wrapper import AsAgentWrapper
job = app.context.jobs["home_evaluate"]
assert "case_ids" not in job.parameters["properties"]
assert "query_ids" not in job.parameters["properties"]
tool = AsAgentWrapper._make_tool(
job,
"ctx-1",
{"case_ids": ["1"], "query_ids": {"1": ["fact:1"]}},
)
assert "case_ids" not in tool.input_schema["properties"]
assert "query_ids" not in tool.input_schema["properties"]
await tool.call()
assert {event[0] for event in FakeFactory.instances[0].events} == {"1"}
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 1
@pytest.mark.asyncio
async def test_all_cases_and_artifacts_via_real_job(app, prepared):
response = await app.run_job("home_evaluate", concurrency=1)
assert response.success, response.answer
result = response.answer
assert result["case_count"] == result["scored_query_count"] == 2
assert result["mean_query_score"] == result["observed_mean_query_score"] == 0.5
run = Path(result["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
assert manifest["case_ids"] == ["2", "1"]
assert {"case_ids", "query_ids", "reuse_construction_from"}.isdisjoint(manifest["options"])
assert "judge_model" not in manifest
for case_id in ["1", "2"]:
folder = run / "cases" / case_id
assert (folder / "memory_construction/reme_workspace/daily/note.md").read_text() == case_id
assert (folder / "memory_construction/build.log").is_file()
assert (folder / "queries/fact:1/answer.log").is_file()
assert not (folder / "evaluation.tar.gz").exists()
assert not (folder / "execution.json").exists()
assert not (folder / "artifacts.json").exists()
assert not (folder / "queries/answers.json").exists()
assert not (folder / "failure.json").exists()
row = json.loads((folder / "queries/result.json").read_text())["queries"][0]
assert row["golden_answer"] == "hello" and row["score"] == 0.5
assert row["answer"] == "hello"
factory = FakeFactory.instances[0]
for case_id in ["1", "2"]:
events = [event[1] for event in factory.events if event[0] == case_id]
assert events.index("snapshot") < events.index("answer")
assert all(worker.closed for worker in factory.workers)
second = await app.run_job("home_evaluate", case_ids=["1"])
assert second.success and Path(second.answer["result_dir"]).name == "2"
@pytest.mark.asyncio
async def test_reuse_tar_runs_current_candidate_without_building(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success, baseline.answer
repo = prepared / "code/repo"
assert (await app.run_job("home_create_candidate_branch", branch_name="new-answer")).success
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 2\n")
command(repo, "add", ".")
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "new answer")
reused = await app.run_job("home_evaluate", case_ids=["1"], reuse_construction_from=baseline.answer["result_dir"])
assert reused.success, reused.answer
assert reused.answer["commit_sha"] != baseline.answer["commit_sha"]
assert reused.answer["execution_mode"] == "query_only_replay"
events = FakeFactory.instances[-1].events
assert ("1", "upload") in events
assert not any(event[1] == "build" for event in events)
run = Path(reused.answer["result_dir"])
memory = run / "cases/1/memory_construction"
construction = json.loads((memory / "result.json").read_text())
assert construction["status"] == "reused" and not construction["jobs"]
assert (memory / "reme_workspace/daily/note.md").read_text() == "1"
@pytest.mark.asyncio
@pytest.mark.parametrize("mutation", ["missing", "corrupt", "failure", "escape"])
async def test_reuse_preflight_rejects_entire_request(app, prepared, mutation):
baseline = await app.run_job("home_evaluate", case_ids=["1"])
assert baseline.success, baseline.answer
run = Path(baseline.answer["result_dir"])
kwargs = {"reuse_construction_from": str(run), "case_ids": ["1"]}
if mutation == "missing":
kwargs.pop("case_ids") # All training cases, not merely cases present in source.
elif mutation == "corrupt":
(run / "cases/1/memory_construction/reme_workspace.tar.gz").write_bytes(b"broken")
elif mutation == "failure":
result = run / "cases/1/memory_construction/result.json"
raw = json.loads(result.read_text())
raw["success"] = False
result.write_text(json.dumps(raw))
else:
kwargs["reuse_construction_from"] = "../escape"
count = len(FakeFactory.instances)
response = await app.run_job("home_evaluate", **kwargs)
assert not response.success
assert len(FakeFactory.instances) == count
assert len(list((prepared / "evaluations").glob("*/*/*/manifest.json"))) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"kwargs",
[
{"case_ids": []},
{"case_ids": ["1", "1"]},
{"case_ids": ["absent"]},
{"query_ids": {"1": ["absent"]}},
{"eval_only": True},
{"code_ref": "HEAD"},
{"code_ref": None},
{"judge_concurrency": 2},
{"dataset_kind": "debug-cases"},
{"debug": "true"},
{"debug": True, "reuse_construction_from": "evaluations/prior"},
{"debug": True, "case_ids": ["1"]},
{"_allowed_paths": []},
],
)
async def test_invalid_inputs_do_not_start_pool(app, prepared, kwargs):
response = await app.run_job("home_evaluate", **kwargs)
assert not response.success
assert not FakeFactory.instances
assert not list((prepared / "evaluations").iterdir())
@pytest.mark.asyncio
async def test_build_failure_retains_diagnostics_and_other_cases(app, prepared, monkeypatch):
monkeypatch.setattr(FakeFactory, "fail_build", "1")
response = await app.run_job("home_evaluate")
assert response.success, response.answer
run = Path(response.answer["result_dir"])
assert response.answer["scored_query_count"] == 1
assert response.answer["mean_query_score"] == 0.25
assert (run / "cases/1/full.tar.gz").is_file()
assert (run / "cases/1/memory_construction/build.log").is_file()
assert (run / "cases/1/failure.json").is_file()
summary = json.loads((run / "summary.json").read_text())
assert summary["cases"]["1"]["queries"] == {}
assert summary["query_status_counts"] == {
"completed": 1,
"candidate_failure": 0,
"infra_error": 0,
"unscored": 1,
}
@pytest.mark.asyncio
async def test_judge_failure_preserves_answers_and_fail_fast_result(app, prepared, monkeypatch):
monkeypatch.setattr(
app.context.components["agent_wrapper"]["judge"],
"reply",
AsyncMock(side_effect=RuntimeError("judge unavailable")),
)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
run = Path(response.answer["result_dir"])
assert response.answer["query_status_counts"]["unscored"] == 1
assert json.loads((run / "cases/1/queries/result.json").read_text())["queries"][0]["answer"] == "hello"
assert (run / "cases/1/queries/fact:1/failure.json").is_file()
failed = await app.run_job("home_evaluate", fail_fast=True)
assert not failed.success
assert list((prepared / "evaluations").glob("*/*/*/partial_summary.json"))
assert all(worker.closed for factory in FakeFactory.instances for worker in factory.workers)
@pytest.mark.asyncio
async def test_cancellation_closes_pool_and_retains_partial_run(app, prepared, monkeypatch):
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
task = asyncio.create_task(app.run_job("home_evaluate", concurrency=1))
while not FakeFactory.instances:
await asyncio.sleep(0.01)
factory = FakeFactory.instances[0]
await asyncio.wait_for(factory.entered.wait(), 5)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert all(worker.closed for worker in factory.workers)
assert list((prepared / "evaluations").glob("*/*/*/failure.json"))
manifest = json.loads(next((prepared / "evaluations").glob("*/*/*/manifest.json")).read_text())
assert manifest["status"] == "cancelled"
@pytest.mark.asyncio
async def test_debug_skips_judge_and_invalid_export_keeps_answer(app, prepared, monkeypatch):
response = await app.run_job("home_evaluate", debug=True)
assert response.success, response.answer
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
assert response.answer["scored_query_count"] == 0
monkeypatch.setattr(FakeFactory, "bad_export", True)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
run = Path(response.answer["result_dir"])
assert (run / "cases/1/queries/result.json").is_file()
assert response.answer["scored_query_count"] == 1
assert not list(prepared.rglob("escaped"))
assert "artifact" in (run / "cases/1/failure.json").read_text()
assert response.answer["mean_query_score"] is None
assert response.answer["case_status_counts"]["infra_error"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"debug,override,expected_fail_fast",
[(False, None, False), (True, None, True), (True, False, False), (False, True, True)],
)
async def test_mode_controls_data_scoring_and_fail_fast_defaults(app, prepared, debug, override, expected_fail_fast):
kwargs = {"debug": debug}
if override is not None:
kwargs["fail_fast"] = override
response = await app.run_job("home_evaluate", **kwargs)
assert response.success, response.answer
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
summary = json.loads((run / "summary.json").read_text())
assert manifest["options"]["fail_fast"] is expected_fail_fast
assert "debug" not in manifest["options"]
assert "schema_version" not in manifest
assert set(manifest["fingerprints"]) == {"dataset", "construction_config"}
assert "dataset_kind" not in manifest["options"]
assert manifest["dataset_kind"] == ("debug-cases" if debug else "training-set")
assert manifest["mode"] == ("debug" if debug else "scored")
assert set(summary["cases"]) == ({"debug-1", "debug-2"} if debug else {"1", "2"})
assert summary["expected_query_count"] == 2
assert summary["scored_query_count"] == (0 if debug else 2)
assert summary["scoring_enabled"] is not debug
assert manifest["status"] == "completed"
assert summary["mean_query_score"] == (None if debug else 0.5)
assert summary["observed_mean_query_score"] == (None if debug else 0.5)
assert summary["query_status_counts"]["unscored"] == 0
for case in summary["cases"].values():
assert case["mean_score"] == (None if debug else 0.5)
assert "query_errors" not in case
assert all("status" not in query for query in case["queries"].values())
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == (0 if debug else 2)
@pytest.mark.asyncio
@pytest.mark.parametrize("fail_fast", [True, False])
async def test_debug_empty_answers_fail_without_scoring(app, prepared, monkeypatch, fail_fast):
original = FakeWorker.run_queries
async def empty_answer(worker, queries):
result = await original(worker, queries)
for row in result["queries"]:
row["answer"] = " "
return result
monkeypatch.setattr(FakeWorker, "run_queries", empty_answer)
response = await app.run_job("home_evaluate", debug=True, fail_fast=fail_fast, case_ids=["debug-1"])
assert not response.success
runs = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
assert len(runs) == 1
run = runs[0].parent
summary = json.loads((run / ("partial_summary.json" if fail_fast else "summary.json")).read_text())
assert summary["mean_query_score"] is None
assert summary["observed_mean_query_score"] is None
assert summary["scored_query_count"] == 0
assert summary["query_status_counts"]["unscored"] == 0
assert summary["case_status_counts"]["candidate_failure"] == 1
assert summary["cases"]["debug-1"]["mean_score"] is None
assert (run / "cases/debug-1/queries/result.json").is_file()
assert app.context.components["agent_wrapper"]["judge"].reply.await_count == 0
@pytest.mark.asyncio
async def test_current_head_is_immutable_and_does_not_ship_judge(app, prepared):
repo = prepared / "code/repo"
assert (await app.run_job("home_create_candidate_branch", branch_name="changed")).success
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 3\n", encoding="utf-8")
command(repo, "add", ".")
command(repo, "-c", "user.name=Fixture", "-c", "user.email=test@localhost", "commit", "-m", "change")
current = command(repo, "rev-parse", "HEAD")
response = await app.run_job("home_evaluate", case_ids=["1"], query_ids={"1": ["fact:1"]})
assert response.success, response.answer
snapshot = FakeFactory.instances[-1].candidate.snapshot
with tarfile.open(fileobj=io.BytesIO(snapshot.archive), mode="r:gz") as source:
assert source.extractfile("plugins/beam/src/reme_beam/__init__.py").read() == b"value = 3\n"
assert "plugins/beam/pyproject.toml" in source.getnames()
assert not any("reme_home" in Path(name).parts or ".git" in Path(name).parts for name in source.getnames())
assert response.answer["commit_sha"] == current
@pytest.mark.asyncio
@pytest.mark.parametrize("mutation", ["modified", "staged", "untracked", "runtime", "dataset", "symlink"])
async def test_modified_inputs_are_rejected_before_docker(app, prepared, mutation):
if mutation in {"modified", "staged"}:
repo = prepared / "code/repo"
(repo / "plugins/beam/src/reme_beam/__init__.py").write_text("value = 3\n", encoding="utf-8")
if mutation == "staged":
command(repo, "add", ".")
elif mutation == "untracked":
(prepared / "code/repo/new.py").write_text("uncommitted", encoding="utf-8")
elif mutation == "runtime":
(prepared / "code/runtime/reme/__init__.py").write_text("modified", encoding="utf-8")
elif mutation == "dataset":
path = prepared / "datasets/training-set/cases/000000.json"
raw = json.loads(path.read_text())
raw["queries"][0]["question"] = "tampered"
path.write_text(json.dumps(raw), encoding="utf-8")
else:
folder = prepared / "evaluations"
folder.rmdir()
folder.symlink_to(prepared / "docs", target_is_directory=True)
before = {path.name: path.read_bytes() for path in (prepared / "evaluations").iterdir()}
response = await app.run_job("home_evaluate")
assert not response.success
assert not FakeFactory.instances
assert {path.name: path.read_bytes() for path in (prepared / "evaluations").iterdir()} == before
def test_lme_adapter_cutoff_and_memory_checkpoints():
from reme_home.preparation.data.models import CaseSpec, QuerySpec, SessionSpec
from reme_home.evaluation.adapters import build_jobs, evaluation_query, accepts_build
query = QuerySpec(
query_id="q",
question="when",
golden_answer="then",
metadata={"question_date": "2024/01/02 (Tue) 00:00"},
)
case = CaseSpec(
case_id="c",
queries=[query],
sessions=[
SessionSpec(session_id="future", messages=[{"created_at": "2024-01-03T00:00:00"}]),
SessionSpec(session_id="past", messages=[{"created_at": "2024-01-01T00:00:00"}]),
],
)
jobs = build_jobs("longmemeval", case)
assert [job.job for job in jobs] == ["auto_memory", "index_update"]
assert jobs[0].arguments["session_id"] == "past"
assert jobs[1].memory_checkpoint == "session: past"
assert evaluation_query("longmemeval", query).answer_arguments["query_time"] == "2024-01-02T00:00:00"
build = {"success": False, "jobs": [{"result": {"success": n > 0}} for n in range(100)]}
build["jobs"][0]["result"]["exception"] = {"type": "ValueError", "module": "builtins"}
assert accepts_build(build, 0.03)
assert not accepts_build(build, 0.01)
@pytest.mark.parametrize("kind", ["link", "duplicate", "oversize"])
def test_unsafe_archives_are_rejected_without_partial_extraction(tmp_path, kind):
path = tmp_path / "archive.tar.gz"
with tarfile.open(path, "w:gz") as stream:
item = tarfile.TarInfo("data")
item.size = 3
stream.addfile(item, io.BytesIO(b"abc"))
if kind == "link":
link = tarfile.TarInfo("outside")
link.type = tarfile.SYMTYPE
link.linkname = "../outside"
stream.addfile(link)
elif kind == "duplicate":
stream.addfile(item, io.BytesIO(b"abc"))
destination = tmp_path / "extract"
with pytest.raises(ValueError):
inspect_archive(path, 2 if kind == "oversize" else 10000, destination)
assert not destination.exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_lme_evaluation_calls_host_judge_job(app, prepared, monkeypatch):
wrapper = app.context.components["agent_wrapper"]["judge"]
monkeypatch.setattr(wrapper, "reply", AsyncMock(side_effect=[{"result": "maybe"}, {"result": "yes"}]))
response = await app.run_job("home_evaluate")
assert response.success, response.answer
assert response.answer["mean_query_score"] == 1
assert wrapper.reply.await_count == 2
run = Path(response.answer["result_dir"])
result = json.loads((run / "cases/q1/queries/result.json").read_text())["queries"][0]
assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "yes"
assert "answer_judgement" not in result["judge_result"]["metadata"]
assert result["score"] == 1
manifest = json.loads((run / "manifest.json").read_text())
assert manifest["judge"]["plugin"] == "lme-judge"
assert manifest["judge"]["model"] == "qwen3.7-max"
assert "api_key" not in json.dumps(manifest)
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_invalid_lme_judge_is_infra_error_after_one_retry(app, prepared):
wrapper = app.context.components["agent_wrapper"]["judge"]
wrapper.reply.return_value = {"result": "maybe"}
response = await app.run_job("home_evaluate")
assert not response.success
assert response.answer["mean_query_score"] is None
assert wrapper.reply.await_count == 2
run = Path(response.answer["result_dir"])
assert (run / "cases/q1/queries/result.json").is_file()
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", ["nonsense", '{"score": 2}', '{"score": 0.2}', '{"score": NaN}'])
async def test_invalid_beam_judge_is_infra_error(app, prepared, raw):
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": raw}
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
assert response.answer["mean_query_score"] is None
assert response.answer["case_status_counts"]["infra_error"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("plugins", [["home"], ["home", "beam-judge", "lme-judge"]])
async def test_startup_requires_exactly_one_judge(app, tmp_path, plugins):
from reme.application import Application
other = Application(
workspace_dir=str(tmp_path / "other"),
plugins=plugins,
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
log_to_console=False,
)
try:
with pytest.raises(ValueError, match="exactly one"):
await other.start()
finally:
await other.close()
@pytest.mark.asyncio
async def test_preparation_rejects_mismatched_judge_without_writes(app, inputs):
from test_preparation import snapshot
spec, lme = inputs
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
root = Path(app.config.workspace_dir)
before = snapshot(root)
response = await app.run_job("home_prepare", spec=spec)
assert not response.success
assert "lme-judge" in str(response.answer)
assert snapshot(root) == before
@pytest.mark.asyncio
async def test_prepared_workspace_rejects_wrong_judge_on_restart(app, prepared):
from reme.application import Application
other = Application(
workspace_dir=app.config.workspace_dir,
plugins=["home", "lme-judge"],
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
log_to_console=False,
)
try:
with pytest.raises(ValueError, match="requires 'beam-judge'"):
await other.start()
finally:
await other.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("override", [{"enabled": False}, {"steps": [{"backend": "home_prepare_step"}]}])
async def test_startup_rejects_disabled_or_replaced_judge_job(app, tmp_path, override):
from reme.application import Application
other = Application(
workspace_dir=str(tmp_path / "other"),
plugins=["home", "beam-judge"],
jobs={"answer_judge": override},
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,
log_to_console=False,
)
try:
with pytest.raises(ValueError, match="answer_judge"):
await other.start()
finally:
await other.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("phase", ["answer", "construction"])
@pytest.mark.parametrize("kind", ["timeout", "wrapped_timeout", "http_429", "http_503", "code", "legacy"])
async def test_real_job_errors_reach_worker_and_home_summary(app, prepared, monkeypatch, phase, kind):
from types import SimpleNamespace
import httpx
from deploy.benchmark_docker.worker import _run_job_on_app
from reme.components.job.base_job import BaseJob
async def failing_step(_context):
if kind == "timeout":
raise TimeoutError("model timed out")
if kind == "wrapped_timeout":
try:
raise TimeoutError("model timed out")
except TimeoutError as error:
raise RuntimeError("model invocation failed") from error
if kind.startswith("http_"):
request = httpx.Request("POST", "https://example.test")
response = httpx.Response(int(kind.split("_")[1]), request=request)
raise httpx.HTTPStatusError("model request failed", request=request, response=response)
raise ValueError("candidate bug")
candidate = BaseJob(name="candidate-error", app_context=app.context)
candidate._build_steps = lambda: [failing_step]
worker_app = SimpleNamespace(context=app.context, run_job=lambda _name, **kwargs: candidate(**kwargs))
result = await _run_job_on_app(worker_app, "candidate-error", {})
if kind == "legacy":
result.pop("exception")
result["metadata"] = {}
else:
assert result["exception"] == result["metadata"]["reme.job_error"]
result = json.loads(json.dumps(result)) # The real sandbox boundary is JSON.
if phase == "answer":
async def answer(worker, queries):
return {
"success": False,
"queries": [
{
"query_id": query.query_id,
"answer": result["answer"],
"answer_result": result,
"error": result["error"],
**({"exception": result["exception"]} if "exception" in result else {}),
}
for query in queries
],
}
monkeypatch.setattr(FakeWorker, "run_queries", answer)
else:
async def build(worker, jobs):
# Two failures are below the 3% threshold. An earlier code failure
# must not conceal a later infrastructure/unclassified failure.
return {
"success": False,
"jobs": [
{"result": {"success": False, "exception": {"type": "ValueError", "module": "builtins"}}},
{"result": result},
*[{"result": {"success": True}} for _ in range(98)],
],
}
monkeypatch.setattr(FakeWorker, "run_build", build)
response = await app.run_job("home_evaluate", case_ids=["1"])
code_failure = kind == "code"
assert response.success is code_failure
summary = response.answer
assert summary["case_status_counts"]["infra_error"] == (0 if code_failure else 1)
assert summary["mean_query_score"] == (
0.5 if code_failure and phase == "construction" else 0 if code_failure else None
)
run = Path(summary["result_dir"])
if not code_failure:
failure = json.loads((run / "cases/1/failure.json").read_text())
assert failure["failure_kind"] == "infra_error"
if kind != "legacy":
assert failure["exception"] == result["exception"]
assert all(worker.closed for worker in FakeFactory.instances[-1].workers)
@pytest.mark.asyncio
async def test_late_cleanup_failure_survives_without_duplicate_execution_record(app, prepared, monkeypatch):
async def close(worker):
worker.closed = True
raise RuntimeError("late cleanup failure")
monkeypatch.setattr(FakeWorker, "close", close)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
assert response.answer["mean_query_score"] is None
assert response.answer["observed_mean_query_score"] == 0.5
folder = Path(response.answer["result_dir"]) / "cases/1"
failure = json.loads((folder / "failure.json").read_text())
assert failure["stage"] == "cleanup" and failure["failure_kind"] == "infra_error"
assert "late cleanup failure" in failure["error"]
assert not (folder / "execution.json").exists()
@pytest.mark.asyncio
async def test_evaluation_reads_legacy_dataset_versions_without_rewriting_them(app, prepared):
paths = [prepared / "datasets/training-set/manifest.json", prepared / "metadata/preparation.json"]
for path in paths:
payload = json.loads(path.read_text())
payload["schema_version"] = 1
if path.name == "preparation.json":
payload["dataset"]["schema_version"] = 1
path.write_text(json.dumps(payload))
original = [path.read_bytes() for path in paths]
response = await app.run_job("home_evaluate")
assert response.success, response.answer
assert [path.read_bytes() for path in paths] == original
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
assert "schema_version" not in manifest and "construction_source" not in manifest
assert set(manifest["fingerprints"]) == {"dataset", "construction_config"}

View file

@ -0,0 +1,406 @@
"""Reviewed dev_meta result content, compact statistics and rebuildability."""
# pylint: disable=missing-function-docstring
import asyncio
from copy import deepcopy
import json
import pytest
from reme_home.evaluation.artifacts import average_tokens, read_json, rebuild_summary, summarize, write_json
from reme_home.evaluation.compaction import compact_construction, compact_query
from reme_home.preparation.data.models import CaseSpec, QuerySpec
def test_scores_are_rounded_without_changing_reference_text_or_token_metrics(tmp_path):
raw = {
"query_id": "q",
"question": "?",
"golden_answer": {"score": 0.123456789},
"answer": "0.123456789",
"score": 1 / 3,
"error": None,
"judge_result": {
"success": True,
"metadata": {
"llm_judge_score": 1 / 3,
"event_ordering": {"precision": 1 / 3, "f1": 2 / 3, "count": 2},
"raw_answer_judgement": "0.333333333",
},
},
}
result = compact_query(raw)
assert result["score"] == result["judge_result"]["metadata"]["llm_judge_score"] == 0.3333
assert raw["score"] == 1 / 3
assert result["judge_result"]["metadata"]["event_ordering"] == {"precision": 0.3333, "f1": 0.6667, "count": 2}
assert result["golden_answer"] == raw["golden_answer"]
assert result["answer"] == raw["answer"]
assert result["judge_result"]["metadata"]["raw_answer_judgement"] == "0.333333333"
case = CaseSpec(case_id="c", sessions=[], queries=[QuerySpec(query_id="q", question="?", golden_answer=None)])
write_json(tmp_path / "cases/c/memory_construction/result.json", {"status": "success", "jobs": []})
# Legacy long-precision query files also produce short-precision summaries.
write_json(tmp_path / "cases/c/queries/result.json", {"queries": [raw]})
summary = summarize(tmp_path, [case])
assert summary["mean_query_score"] == summary["observed_mean_query_score"] == 0.3333
assert summary["low_score_cases"] == [{"c": 0.3333}]
assert summary["cases"]["c"]["mean_score"] == summary["cases"]["c"]["queries"]["q"]["score"] == 0.3333
assert summary["question_type_statistics"]["unknown"]["average_score"] == 0.3333
@pytest.mark.parametrize("scoring_enabled", [True, False])
def test_summary_separates_query_errors_from_scores(tmp_path, scoring_enabled):
ids = ["zero", "unscored", "candidate", "infra"]
case = CaseSpec(
case_id="c",
sessions=[],
queries=[QuerySpec(query_id=value, question="?", golden_answer=None) for value in ids],
)
queries = [
{"query_id": "zero", "score": 0},
{"query_id": "unscored", "score": None},
{"query_id": "candidate", "score": None, "error": "bad answer", "failure_stage": "answer"},
{
"query_id": "infra",
"score": None,
"error": "x" * 600,
"failure_stage": "judge",
"infrastructure_error": True,
},
]
write_json(tmp_path / "cases/c/memory_construction/result.json", {"status": "success", "jobs": []})
source = tmp_path / "cases/c/queries/result.json"
write_json(source, {"queries": queries})
summary = summarize(tmp_path, [case], scoring_required=scoring_enabled)
assert summary["scoring_enabled"] is scoring_enabled
row = summary["cases"]["c"]
assert all("status" not in query for query in row["queries"].values())
assert row["queries"]["zero"]["score"] == 0
assert row["queries"]["unscored"]["score"] is None
assert row["query_errors"] == {
"candidate": {"failure_kind": "candidate_failure", "stage": "answer", "error": "bad answer"},
"infra": {"failure_kind": "infra_error", "stage": "judge", "error": "x" * 500 + "…"},
}
assert read_json(source) == {"queries": queries}
def test_query_compaction_preserves_evidence_without_duplicate_prompts_or_costs():
raw = {
"question": "What happened?",
"golden_answer": "An event.",
"answer": "The event happened.",
"score": 1.0,
"error": None,
"answer_result": {
"success": True,
"answer": "The event happened.",
"error": None,
"job_call_counts": {"agentic_answer": 2, "search": 3},
"token_usage": {
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
"empty": {"input_tokens": None, "output_tokens": None, "total_tokens": None},
},
"metadata": {
"query": "What happened?",
"response": "The event happened.",
"query_time": "2026-09-12T10:00:00",
"sys_prompt": "reconstructable prompt",
"nested": [{"sys_prompt": "nested prompt", "evidence": "keep"}],
},
},
"judge_result": {
"success": True,
"answer": "yes",
"job_call_counts": {"answer_judge": 1},
"token_usage": {"judge": {"total_tokens": 100}},
"metadata": {
"answer_judgement": "yes",
"raw_answer_judgement": "Yes, the answer is correct.",
"agent_answer": "The event happened.",
"golden_answer": "An event.",
"llm_response": "The event happened.",
"probing_question": "What happened?",
"query": "What happened?",
"question_type": "fact",
"rubric": ["An event"],
},
},
}
original = deepcopy(raw)
result = compact_query(raw)
assert raw == original
assert result["answer_result"] == {
"success": True,
"job_call_counts": {"agentic_answer": 1, "search": 3},
"token_usage": {"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}},
"metadata": {"query_time": "2026-09-12T10:00:00", "nested": [{"evidence": "keep"}]},
}
assert result["judge_result"] == {
"success": True,
"metadata": {"raw_answer_judgement": "Yes, the answer is correct."},
}
assert result["question"] == raw["question"]
assert result["golden_answer"] == raw["golden_answer"]
assert result["answer"] == raw["answer"]
def test_distinct_failure_details_and_failed_judge_response_are_retained():
raw = {
"answer": "partial answer",
"error": "judge failed",
"score": None,
"answer_result": {"answer": "different", "error": "answer failed"},
"judge_result": {"answer": "invalid judgement", "metadata": {"reason": "diagnostic"}},
"exception": {"type": "TimeoutError", "causes": [{"type": "ReadTimeout"}]},
}
assert compact_query(raw) == raw
def test_construction_compaction_preserves_jobs_and_only_drops_reconstructable_data():
raw = {
"success": True,
"jobs": [
{
"job": "auto_memory",
"memory_checkpoint": {"commit_sha": "abc", "message": "session: s1"},
"result": {
"success": True,
"job_call_counts": {"auto_memory": 2, "daily_list": 1},
"token_usage": {
"answer": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
"empty": {"total_tokens": None},
},
"metadata": {
"source_conversation": "[[session/dialog/s1.jsonl]]",
"details": [
{"session_id": "s1", "source_conversation": "[[session/dialog/s1.jsonl]]"},
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
],
"plugin_payload": {"index": {"notes": ["keep", "this"]}},
"index": {
"date": "2026-01-01",
"path": "custom/daily/2026-01-01.md",
"created": False,
"changed": True,
"notes": [{"text": "old"}, {"text": "new"}],
},
},
},
},
{
"job": "index_update",
"result": {
"job_call_counts": {"index_update": 1},
"token_usage": {"answer": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}},
},
},
{"job": "index_update", "result": {"token_usage": {"empty": {"total_tokens": None}}}},
],
}
original = deepcopy(raw)
result = compact_construction(raw)
assert raw == original
job = result["jobs"][0]
assert "memory_checkpoint" not in job
assert job["result"]["job_call_counts"] == {"auto_memory": 1, "daily_list": 1}
assert set(job["result"]["token_usage"]) == {"answer"}
metadata = job["result"]["metadata"]
assert metadata["source_conversation"] == "[[session/dialog/s1.jsonl]]"
assert metadata["details"] == [
{"session_id": "s1"},
{"session_id": "s2", "source_conversation": "[[session/dialog/other.jsonl]]"},
]
assert metadata["plugin_payload"] == {"index": {"notes": ["keep", "this"]}}
assert metadata["index"] == {
"date": "2026-01-01",
"path": "custom/daily/2026-01-01.md",
"created": False,
"changed": True,
"notes_count": 2,
}
assert result["jobs"][1]["result"] == {"token_usage": 0}
assert result["jobs"][2]["result"] == {}
def test_summary_matches_reviewed_dev_meta_fields_and_denominators(tmp_path):
cases = [
CaseSpec(
case_id=case_id,
sessions=[],
queries=[QuerySpec(query_id=query_id, question="?", golden_answer="a") for query_id in ids],
)
for case_id, ids in [("one", ["q1", "q2"]), ("two", ["q3"])]
]
write_json(
tmp_path / "cases/one/memory_construction/result.json",
{
"status": "partial_success",
"jobs": [{"result": {"success": True}}, {"result": {"success": False}}],
},
)
write_json(
tmp_path / "cases/one/queries/result.json",
{
"queries": [
{
"query_id": "q1",
"question_type": "fact",
"score": 0.5,
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}}},
},
{
"query_id": "q2",
"question_type": "fact",
"score": None,
"error": "candidate bug",
"answer_result": {},
},
],
},
)
write_json(tmp_path / "cases/two/memory_construction/result.json", {"status": "failure", "jobs": []})
write_json(
tmp_path / "cases/two/failure.json",
{"stage": "construction", "failure_kind": "candidate_failure", "error": "build failed"},
)
summary = summarize(tmp_path, cases)
assert summary == {
"case_count": 2,
"scoring_enabled": True,
"case_status_counts": {"completed": 0, "candidate_failure": 2, "infra_error": 0},
"construction_status_counts": {"success": 0, "partial_success": 1, "failure": 1},
"failure_stage_case_counts": {"construction": 2, "query": 1},
"expected_query_count": 3,
"scored_query_count": 1,
"query_status_counts": {"completed": 1, "candidate_failure": 1, "infra_error": 0, "unscored": 2},
"mean_query_score": 0.1667,
"observed_mean_query_score": 0.5,
"low_score_cases": [{"one": 0.25}],
"question_type_statistics": {
"fact": {"query_count": 2, "average_score": 0.5, "average_query_total_token_usage": {"answer": 6.0}},
},
"cases": {
"one": {
"construction": {"status": "partial_success", "success_job_count": 1, "fail_job_count": 1},
"mean_score": 0.25,
"average_query_total_token_usage": {"answer": 6.0},
"queries": {
"q1": {"question_type": "fact", "score": 0.5},
"q2": {"question_type": "fact", "score": None},
},
"query_errors": {"q2": {"failure_kind": "candidate_failure", "stage": None, "error": "candidate bug"}},
"construction_error_count": 1,
"query_error_count": 1,
},
"two": {
"construction": {"status": "failure", "success_job_count": 0, "fail_job_count": 0},
"mean_score": 0,
"average_query_total_token_usage": {},
"queries": {},
"stage": "construction",
},
},
}
write_json(
tmp_path / "manifest.json",
{
"status": "completed",
"mode": "scored",
"case_ids": ["one", "two"],
"query_ids": {"one": ["q1", "q2"], "two": ["q3"]},
},
)
assert rebuild_summary(tmp_path) == summary
assert read_json(tmp_path / "summary.json") == summary
failure_path = tmp_path / "cases/two/failure.json"
failure = read_json(failure_path)
failure["failure_kind"] = "infra_error"
write_json(failure_path, failure)
infra = rebuild_summary(tmp_path)
assert infra["mean_query_score"] is None
assert infra["observed_mean_query_score"] == 0.5
assert infra["cases"]["two"]["mean_score"] is None
assert infra["cases"]["one"]["mean_score"] == 0.25
def test_token_averages_exclude_zero_unknown_and_judge_cost_but_count_all_returned_queries():
assert average_tokens(
[
{
"answer_result": {"token_usage": {"answer": {"total_tokens": 12}, "zero": {"total_tokens": 0}}},
"judge_result": {"token_usage": {"judge": {"total_tokens": 100}}},
},
{"answer_result": {"token_usage": {"unknown": {"total_tokens": None}, "bad": {"total_tokens": True}}}},
{},
],
) == {"answer": 4.0}
def test_compact_json_round_trips_strings_and_keeps_small_leaves_inline(tmp_path):
value = {"result": {"token_usage": 0}, "unicode": ["你好", 'quote"\n'], "large": [1, 2, 3]}
path = tmp_path / "result.json"
write_json(path, value, compact_leaf_containers=True)
assert read_json(path) == value
assert '"result": {"token_usage": 0}' in path.read_text()
assert "\\u4f60" not in path.read_text()
with pytest.raises(ValueError):
write_json(path, {"score": float("nan")}, compact_leaf_containers=True)
assert json.loads(path.read_text()) == value
@pytest.mark.asyncio
@pytest.mark.parametrize("stop", [RuntimeError("judge unavailable"), asyncio.CancelledError()])
async def test_judge_interruption_preserves_all_frozen_answers_and_compacts_each_response_once(tmp_path, stop):
from types import SimpleNamespace
from unittest.mock import AsyncMock
from deploy.benchmark_docker.pool import CaseResult
from reme_home.evaluation.component import HomeEvaluation
from reme_home.evaluation.inputs import EvaluationSpec
case = CaseSpec(
case_id="case",
sessions=[],
queries=[QuerySpec(query_id=f"q{i}", question="?", golden_answer="a") for i in range(2)],
)
answers = [
{
"query_id": query.query_id,
"answer": "frozen answer",
"error": None,
"answer_result": {"answer": "frozen answer", "job_call_counts": {"agentic_answer": 2}},
}
for query in case.queries
]
original = deepcopy(answers)
result = CaseResult("case", build={"success": True, "jobs": []}, queries={"queries": answers})
spec = EvaluationSpec(fail_fast=True)
await HomeEvaluation._persist_case(tmp_path, case, result, None, spec) # pylint: disable=protected-access
judge = SimpleNamespace(evaluate=AsyncMock(side_effect=stop))
with pytest.raises(type(stop)):
await HomeEvaluation._score_case( # pylint: disable=protected-access
tmp_path,
case,
"beam",
judge,
asyncio.Semaphore(1),
spec,
answers,
)
rows = read_json(tmp_path / "cases/case/queries/result.json")["queries"]
assert [row["answer"] for row in rows] == ["frozen answer", "frozen answer"]
assert [row["answer_result"]["job_call_counts"] for row in rows] == [{"agentic_answer": 1}] * 2
assert answers == original
assert not (tmp_path / "cases/case/queries/answers.json").exists()
def test_rebuild_rejects_missing_completed_answers_without_replacing_summary(tmp_path):
write_json(
tmp_path / "manifest.json",
{"status": "completed", "case_ids": ["case"], "query_ids": {"case": ["q"]}},
)
write_json(tmp_path / "cases/case/memory_construction/result.json", {"status": "success", "jobs": []})
write_json(tmp_path / "summary.json", {"preserve": "existing summary"})
with pytest.raises(ValueError, match="incomplete query results"):
rebuild_summary(tmp_path)
assert read_json(tmp_path / "summary.json") == {"preserve": "existing summary"}

View file

@ -0,0 +1,367 @@
"""Successful-history ownership and evaluation parameter precedence through real Jobs."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import json
from pathlib import Path
import pytest
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
from reme_home.sampling import SamplingPolicy, create_sampling_plan
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
def history(app):
return app.context.components["home.evaluation_history"]["default"]
def manifest(response):
return json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
@pytest.mark.asyncio
async def test_history_catalog_keeps_successful_low_scores_and_deduplicates(app, prepared):
manager = history(app)
assert isinstance(manager, HomeEvaluationHistory) and manager.is_started
assert manager.list_results() == []
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 0}'}
response = await app.run_job("home_evaluate")
assert response.success, response.answer
results = manager.list_results()
assert len(results) == 1 and isinstance(results[0], EvaluationRecord)
assert results[0].mean_query_score == 0 and results[0].scored_query_count == 2
assert manager.get_result(results[0].run_key) == results[0]
run = Path(response.answer["result_dir"])
before = manager.sampling_state()
assert manager.record(run) == results[0]
assert len(manager.list_results()) == 1 and manager.sampling_state() == before
assert not (prepared / "evaluations/history.json").exists()
# A stale or corrupt legacy index cannot hide authoritative results.
legacy = prepared / "evaluations/history.json"
legacy.write_text("obsolete index")
assert manager.list_results() == results
assert manager.rebuild() == results
assert legacy.read_text() == "obsolete index"
debug = await app.run_job("home_evaluate", sampling=True, debug=True)
assert debug.success and manager.record(Path(debug.answer["result_dir"])) is None
assert manager.list_results() == results
with pytest.raises(KeyError):
manager.get_result("unknown")
@pytest.mark.asyncio
async def test_failed_run_does_not_change_next_probabilities_or_successful_history(app, prepared, monkeypatch):
baseline = await app.run_job("home_evaluate")
assert baseline.success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
policy = SamplingPolicy()
from reme_home.preparation.data.models import CaseSpec
cases = [
CaseSpec.model_validate_json(path.read_text())
for path in sorted(
(prepared / "datasets/training-set/cases").glob("*.json"),
)
]
monkeypatch.setattr("reme_home.sampling.time.time_ns", lambda: 5678)
before = create_sampling_plan(cases, observations, policy)
original = FakeFactory.create_cases
async def create(factory, *args, **kwargs):
workers = await original(factory, *args, **kwargs)
for worker in workers:
build = worker.run_build
async def fail_one(jobs, worker=worker, build=build):
if worker.case_id == "2":
raise ConnectionError("only one case failed")
return await build(jobs)
worker.run_build = fail_one
return workers
monkeypatch.setattr(FakeFactory, "create_cases", create)
failed = await app.run_job("home_evaluate", concurrency=1)
assert not failed.success
assert failed.answer["scored_query_count"] == 1 # Its healthy case is also excluded from feedback.
assert manager.record(Path(failed.answer["result_dir"])) is None
assert manager.list_results() == records and manager.sampling_state() == observations
assert create_sampling_plan(cases, manager.sampling_state(), policy) == before
assert read_state(prepared)["round"] == 2
assert read_state(prepared)["successful_round"] == 1
assert manager.rebuild() == records
assert manager.sampling_state() == observations
@pytest.mark.asyncio
async def test_history_and_observations_rebuild_without_touching_artifacts(app, prepared):
baseline = await app.run_job("home_evaluate")
sampled = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
assert baseline.success and sampled.success
manager = history(app)
expected, observations = manager.list_results(), manager.sampling_state()
source = Path(sampled.answer["result_dir"]) / "summary.json"
original = source.read_bytes()
(prepared / "sampling/state.json").unlink()
assert manager.list_results() == expected
assert manager.sampling_state() == observations
assert manager.rebuild() == expected
assert source.read_bytes() == original
assert not (prepared / "evaluations/history.json").exists()
assert (prepared / "sampling/state.json").is_file()
@pytest.mark.asyncio
async def test_call_policy_merges_component_defaults_without_mutating_them(app, prepared):
evaluator = app.context.components["home.evaluation"]["default"]
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=2, temperature=0.7, ewma_alpha=0.6)
response = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_cases": 1})
assert response.success, response.answer
options = manifest(response)["options"]
assert options["sampling"] is True
assert "sampling_policy" not in options
plan = json.loads((prepared / manifest(response)["sampling"]["plan_path"]).read_text())
assert plan["policy"] == {
"max_cases": 1,
"max_queries": 2,
"temperature": 0.7,
"ewma_alpha": 0.6,
"uniform_exploration": 0.2,
}
assert response.answer["case_count"] == 1
following = await app.run_job("home_evaluate", sampling=True)
assert following.success and following.answer["case_count"] == 2
assert evaluator.sampling_policy.max_cases == 2
plan = json.loads((prepared / manifest(following)["sampling"]["plan_path"]).read_text())
assert plan["policy"]["max_cases"] == 2
for kwargs in (
{"sampling": "true"},
{"sampling": {"max_cases": 1}},
{"record_history": "false"},
{"sampling_policy": {"max_cases": 0}},
{"sampling_policy": {"unknown": 1}},
):
assert not (await app.run_job("home_evaluate", **kwargs)).success
assert len(history(app).list_results()) == 2
@pytest.mark.asyncio
async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared):
job = app.context.jobs["home_evaluate"]
assert "home_validate" not in app.context.jobs
job.kwargs.update(sampling=True, sampling_policy={"max_cases": 1})
default = await app.run_job("home_evaluate")
assert default.success and default.answer["case_count"] == 1
explicit = await app.run_job("home_evaluate", sampling=False)
assert explicit.success and explicit.answer["case_count"] == 2
tool = AsAgentWrapper._make_tool(
job,
injected_job_kwargs={
"sampling": True,
"sampling_policy": {"max_cases": 1, "max_queries": 1},
},
)
assert "sampling" not in tool.input_schema["properties"]
await tool.call()
records = history(app).list_results()
assert records[-1].case_count == records[-1].scored_query_count == 1
assert len(records) == 3
with pytest.raises(ValueError, match="injected"):
await tool.call(sampling=False)
with pytest.raises(ValueError, match="injected"):
await tool.call(sampling_policy={"max_cases": 100})
assert len(history(app).list_results()) == 3
@pytest.mark.asyncio
@pytest.mark.parametrize("baseline", [False, True])
async def test_standalone_full_evaluation_preserves_history_and_rebuild_excludes_it(app, prepared, baseline):
manager = history(app)
if baseline:
assert (await app.run_job("home_evaluate")).success
records, observations = manager.list_results(), manager.sampling_state()
cache_paths = [prepared / name for name in ("sampling/state.json",)]
caches = [path.read_bytes() if path.exists() else None for path in cache_paths]
response = await app.run_job("home_evaluate", sampling=False, record_history=False)
assert response.success, response.answer
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
assert response.answer["mean_query_score"] == 0.5 and response.answer["record_history"] is False
run = Path(response.answer["result_dir"])
payload = manifest(response)
assert payload["status"] == "completed" and payload["success"] is True
assert payload["options"]["record_history"] is False and payload["sandbox_started_at"]
assert "sampling" not in payload and "selection_plan" not in payload
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
assert manager.record_started(run, SamplingPolicy()) == {}
assert manager.record(run) is None
assert [path.read_bytes() if path.exists() else None for path in cache_paths] == caches
assert manager.list_results() == records and manager.sampling_state() == observations
assert manager.rebuild() == records and manager.sampling_state() == observations
following = await app.run_job("home_evaluate")
assert following.success and following.answer["record_history"] is True
state = read_state(prepared)
assert state["round"] == state["successful_round"] == 1 + int(baseline)
# Legacy manifests without the new flag keep the previous opt-in behavior.
tracked_run = Path(following.answer["result_dir"])
tracked_manifest = manifest(following)
tracked_manifest["options"].pop("record_history")
(tracked_run / "manifest.json").write_text(json.dumps(tracked_manifest))
assert len(manager.rebuild()) == 1 + int(baseline)
@pytest.mark.asyncio
async def test_untracked_full_evaluation_does_not_consult_history(app, prepared, monkeypatch):
manager = history(app)
def fail(*args, **kwargs):
raise AssertionError("standalone full evaluation must not access history")
for name in ("sampling_state", "record_started", "record", "invalidate"):
monkeypatch.setattr(manager, name, fail)
response = await app.run_job("home_evaluate", record_history=False)
assert response.success, response.answer
assert response.answer["case_count"] == response.answer["scored_query_count"] == 2
@pytest.mark.asyncio
async def test_sampling_and_history_are_independent_and_selection_evidence_is_local(app, prepared):
assert (await app.run_job("home_evaluate")).success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
response = await app.run_job(
"home_evaluate",
sampling=True,
record_history=False,
sampling_policy={"max_cases": 1},
)
assert response.success and response.answer["case_count"] == 1
payload = manifest(response)
assert "sampling" not in payload
assert payload["selection_plan"]["case_ids"] == payload["case_ids"]
assert payload["selection_plan"]["seed"] and payload["selection_plan"]["policy"]["max_cases"] == 1
assert manager.rebuild() == records and manager.sampling_state() == observations
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["infra", "exception"])
async def test_untracked_failures_preserve_existing_history(app, prepared, monkeypatch, failure):
assert (await app.run_job("home_evaluate")).success
manager = history(app)
records, observations = manager.list_results(), manager.sampling_state()
paths = [prepared / name for name in ("sampling/state.json",)]
caches = [path.read_bytes() for path in paths]
if failure == "infra":
monkeypatch.setattr(FakeFactory, "bad_export", True)
else:
original = FakeFactory.create_cases
async def fail(factory, *args, **kwargs):
await original(factory, *args, **kwargs)
raise ConnectionError("pool failed after first initialized sandbox")
monkeypatch.setattr(FakeFactory, "create_cases", fail)
response = await app.run_job("home_evaluate", record_history=False)
assert not response.success
assert [path.read_bytes() for path in paths] == caches
assert manager.rebuild() == records and manager.sampling_state() == observations
assert read_state(prepared)["round"] == read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
async def test_history_job_defaults_and_agent_injection_are_enforced(app, prepared):
job = app.context.jobs["home_evaluate"]
assert job.parameters["properties"]["record_history"]["default"] is True
job.kwargs["record_history"] = False
standalone = await app.run_job("home_evaluate")
assert standalone.success and not history(app).list_results()
baseline = await app.run_job("home_evaluate", sampling=False, record_history=True)
assert baseline.success and baseline.answer["case_count"] == 2
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"record_history": True})
assert "record_history" not in tool.input_schema["properties"]
await tool.call()
assert len(history(app).list_results()) == 2
with pytest.raises(ValueError, match="injected"):
await tool.call(record_history=False)
@pytest.mark.asyncio
async def test_legacy_cache_rebuild_and_explicit_null_query_budget(app, prepared):
evaluator = app.context.components["home.evaluation"]["default"]
evaluator.sampling_policy = SamplingPolicy(max_cases=2, max_queries=1)
result = await app.run_job("home_evaluate", sampling=True, sampling_policy={"max_queries": None})
assert result.success and result.answer["scored_query_count"] == 2
plan = json.loads((prepared / manifest(result)["sampling"]["plan_path"]).read_text())
assert plan["policy"]["max_queries"] is None
assert evaluator.sampling_policy.max_queries == 1
manager = history(app)
observations = manager.sampling_state()
legacy = read_state(prepared)
legacy["schema_version"] = 1
legacy.pop("successful_round")
legacy["cases"]["1"]["ewma_loss"] = 999
(prepared / "sampling/state.json").write_text(json.dumps(legacy))
assert manager.sampling_state() == observations
manager.rebuild()
assert read_state(prepared)["schema_version"] == 2
assert read_state(prepared)["successful_round"] == 1
@pytest.mark.asyncio
async def test_cancellation_after_durable_completion_keeps_history_consistent(app, prepared, monkeypatch):
import asyncio
from threading import Event
manager = history(app)
original = manager.record
entered, release = Event(), Event()
def blocked(run):
entered.set()
assert release.wait(5)
return original(run)
monkeypatch.setattr(manager, "record", blocked)
task = asyncio.create_task(app.run_job("home_evaluate"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
records = manager.list_results()
assert len(records) == 1
run = prepared / "evaluations" / records[0].run_key
completed = json.loads((run / "manifest.json").read_text())
assert completed["status"] == "completed" and completed["success"] is True
assert manager.rebuild() == records
@pytest.mark.asyncio
async def test_registration_failure_invalidates_only_caches(app, prepared, monkeypatch):
manager = history(app)
original = manager.record
def fail(*args):
raise OSError("sampling observations could not be saved")
monkeypatch.setattr(manager, "record", fail)
response = await app.run_job("home_evaluate")
assert not response.success
runs = list((prepared / "evaluations").glob("*/*/*/manifest.json"))
assert len(runs) == 1
run = runs[0].parent
failed = json.loads(runs[0].read_text())
assert failed["status"] == "failed" and failed["success"] is False
assert (run / "summary.json").is_file() and (run / "cases/1/queries/result.json").is_file()
assert not (prepared / "sampling/state.json").exists()
assert read_state(prepared)["round"] == 1 and read_state(prepared)["successful_round"] == 0
monkeypatch.setattr(manager, "record", original)
assert manager.list_results() == [] and not manager.sampling_state()["cases"]

View file

@ -0,0 +1,312 @@
"""Read-only result Jobs against real Home applications and isolated Git history."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import asyncio
from pathlib import Path
from threading import Event
import pytest
from reme_home.evaluation import inspection_step as inspection
from reme_home.evaluation.artifacts import write_json
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from test_preparation import command
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
async def call(app, name, **kwargs):
response = await app.run_job(name, **kwargs)
assert response.success, response.answer
return response.answer
def snapshot(home):
return {path.relative_to(home): path.read_bytes() for path in home.rglob("*") if path.is_file()}
def write_run(home, run_id="1", *, branch="candidate", commit="a" * 40, **updates):
run = home / "evaluations" / branch / commit[:7] / run_id
manifest = {
"branch_name": branch,
"commit_sha": commit,
"validation_id": run_id,
"case_ids": ["case~a", "missing-case"],
"query_ids": {"case~a": ["q1", "q2"], "missing-case": ["q3"]},
"mode": "scored",
"dataset_kind": "training-set",
"status": "failed",
"started_at": f"2026-01-01T00:00:{int(run_id):02d}+00:00",
"options": {"record_history": False},
**updates,
}
write_json(run / "manifest.json", manifest)
return run
@pytest.mark.asyncio
async def test_real_evaluation_catalog_and_case_query_navigation_are_read_only(app, prepared):
scored = await call(app, "home_evaluate", record_history=False)
debug = await call(app, "home_evaluate", debug=True)
before = snapshot(prepared)
page = await call(app, "home_list_validation_results", limit=1)
assert page["row_count"] == 2 and page["next_offset"] == 1
assert page["rows"][0]["mode"] == "debug" and page["rows"][0]["record_history"] is False
last = await call(app, "home_list_validation_results", offset=1, limit=1)
assert last["next_offset"] is None and last["rows"][0]["validation_path"] == scored["result_dir"]
assert last["rows"][0]["totals"]["scored_query_count"] == 2
run = Path(scored["result_dir"])
result = await call(app, "home_inspect_validation", validation_path=run.relative_to(prepared).as_posix(), limit=1)
assert result["row_count"] == 2 and result["next_offset"] == 1
case = result["rows"][0]
assert case["mean_score"] == 0.5 and case["pointer"].startswith("/cases/")
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id=case["case_id"])
query = result["rows"][0]
assert query["score"] == 0.5 and query["pointer"] == "/queries/0"
assert "fields" not in query and result["construction"]["failed_jobs"]["rows"] == []
result = await call(
app,
"home_inspect_validation",
validation_path=str(run),
case_id=case["case_id"],
query_id=query["query_id"],
)
fields = result["rows"][0]["fields"]
assert fields["answer"] == "hello" and fields["score"] == 0.5 and "judge_result" in fields
assert result["query_missing"] is False and result["answer_log"].endswith("answer.log")
assert snapshot(prepared) == before
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
assert debug["result_dir"] != scored["result_dir"]
@pytest.mark.asyncio
async def test_partial_missing_and_corrupt_runs_are_visible_without_rebuilding(app, prepared):
run = write_run(prepared)
write_json(
run / "partial_summary.json",
{
"case_count": 1,
"cases": {
"case~a": {
"mean_score": 0.0,
"construction": {"status": "partial_success"},
"construction_error_count": 2,
},
},
},
)
write_json(run / "failure.json", {"error": "failure" * 300})
missing = write_run(prepared, "2", mode="debug", status="running")
corrupt = write_run(prepared, "3")
(corrupt / "manifest.json").write_text("invalid JSON")
before = snapshot(prepared)
result = await call(app, "home_list_validation_results")
assert result["row_count"] == 3
assert sum("read_error" in row for row in result["rows"]) == 1
row = next(row for row in result["rows"] if row["validation_path"] == str(run))
assert row["summary_path"].endswith("partial_summary.json") and "truncated" in row["failure"]
result = await call(app, "home_inspect_validation", validation_path=str(run))
assert result["rows"][0]["case_id"] == "missing-case" and result["rows"][0]["mean_score"] is None
assert result["rows"][1]["pointer"] == "/cases/case~0a"
result = await call(app, "home_inspect_validation", validation_path=str(missing), case_id="case~a", query_id="q1")
assert result["query_missing"] and result["rows"][0]["score"] is None
assert len(result["missing_files"]) == 2
result = await call(app, "home_inspect_validation", validation_path=str(missing))
assert result["summary_missing"] and result["rows_source"] is None and result["row_count"] == 2
assert snapshot(prepared) == before and not (missing / "partial_summary.json").exists()
@pytest.mark.asyncio
async def test_failure_and_query_paging_previews_and_source_coordinates(app, prepared):
run = write_run(prepared)
folder = run / "cases/case~a"
write_json(
folder / "memory_construction/result.json",
{
"success": False,
"jobs": [{"job": f"job-{index}", "result": {"success": False, "error": "x" * 900}} for index in range(3)],
},
)
write_json(
folder / "queries/result.json",
{
"queries": [
{
"query_id": "q2",
"score": 0,
"answer": "a" * 3000,
"judge_result": {"success": True},
"golden_answer": ["rubric"],
"answer_result": None,
},
{"query_id": "q1", "score": None, "error": "e" * 900},
],
},
)
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", limit=1)
assert result["rows"][0]["query_id"] == "q1" and result["rows"][0]["pointer"] == "/queries/1"
assert result["next_offset"] == 1 and "truncated" in result["rows"][0]["error"]
failed_jobs = result["construction"]["failed_jobs"]
assert failed_jobs["row_count"] == 3 and failed_jobs["next_offset"] == 1
assert failed_jobs["rows"][0]["pointer"] == "/jobs/0/result"
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", offset=2, limit=1)
assert result["rows"] == [] and result["construction"]["failed_jobs"]["rows"][0]["job"] == "job-2"
result = await call(app, "home_inspect_validation", validation_path=str(run), case_id="case~a", query_id="q2")
assert result["rows"][0]["pointer"] == "/queries/0"
assert "truncated" in result["rows"][0]["fields"]["answer"]
assert result["rows"][0]["fields"]["golden_answer"] == ["rubric"]
assert result["rows"][0]["score"] == 0
@pytest.mark.asyncio
async def test_prefix_resolves_on_branch_history_not_only_evaluated_commits(app, prepared):
repo = prepared / "code/repo"
initial = command(repo, "rev-parse", "HEAD")
await call(app, "home_create_candidate_branch", branch_name="candidate")
write_run(prepared, commit=initial)
write_run(prepared, branch="other", commit=initial)
result = await call(app, "home_list_validation_results", commit_sha=initial[:7].upper())
assert result["branch_name"] == "candidate" and result["commit_sha"] == initial
assert result["row_count"] == 1
# An unevaluated reachable commit can make a short prefix ambiguous.
tree = command(repo, "rev-parse", "HEAD^{tree}")
for index in range(500):
sha = command(
repo,
"-c",
"user.name=Home",
"-c",
"user.email=home@local",
"commit-tree",
tree,
"-p",
initial,
"-m",
f"unevaluated {index}",
)
if sha[0] == initial[0]:
break
else:
pytest.fail("Could not construct a prefix collision")
command(repo, "update-ref", "refs/heads/candidate", sha)
response = await app.run_job("home_list_validation_results", commit_sha=initial[0])
assert not response.success and "2 matches" in response.answer
# The very same prefix remains unique on init, despite the other Git object.
result = await call(app, "home_list_validation_results", branch_name="init", commit_sha=initial[0])
assert result["commit_sha"] == initial and result["row_count"] == 0
result = await call(app, "home_list_validation_results", commit_sha=sha)
assert result["commit_sha"] == sha and result["rows"] == []
absent = "f" * len(initial)
assert not (await app.run_job("home_list_validation_results", commit_sha=absent)).success
@pytest.mark.asyncio
async def test_branch_filters_and_deleted_historical_branch_do_not_require_git(app, prepared):
write_run(prepared, branch="removed")
write_run(prepared, "2", branch="candidate")
result = await call(app, "home_list_validation_results", branch_name="removed")
assert result["row_count"] == 1 and result["rows"][0]["branch_name"] == "removed"
response = await app.run_job("home_list_validation_results", branch_name="removed", commit_sha="a")
assert not response.success
@pytest.mark.asyncio
@pytest.mark.parametrize("corruption", ["summary", "options"])
async def test_commit_filtered_catalog_reports_corrupt_run_with_identity(app, prepared, corruption):
sha = command(prepared / "code/repo", "rev-parse", "HEAD")
run = write_run(prepared, branch="init", commit=sha, options=None if corruption == "options" else {})
if corruption == "summary":
write_json(run / "summary.json", [])
result = await call(app, "home_list_validation_results", commit_sha=sha[:7])
assert result["row_count"] == 1
assert result["rows"][0]["commit_sha"] == sha and "read_error" in result["rows"][0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"job,kwargs",
[
("home_list_validation_results", {"offset": -1}),
("home_list_validation_results", {"limit": 0}),
("home_list_validation_results", {"limit": 101}),
("home_list_validation_results", {"limit": True}),
("home_list_validation_results", {"offset": "1"}),
("home_list_validation_results", {"commit_sha": "HEAD"}),
("home_list_validation_results", {"commit_sha": ""}),
("home_list_validation_results", {"branch_name": "../init"}),
("home_inspect_validation", {}),
("home_inspect_validation", {"validation_path": "evaluations"}),
("home_inspect_validation", {"validation_path": "../outside"}),
("home_inspect_validation", {"validation_path": "/tmp/outside"}),
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "query_id": "q"}),
("home_inspect_validation", {"validation_path": "evaluations/a/b/c", "case_id": "a/b"}),
("home_list_validation_results", {"_allowed_paths": []}),
],
)
async def test_invalid_input_fails_closed(app, prepared, job, kwargs):
assert not (await app.run_job(job, **kwargs)).success
@pytest.mark.asyncio
async def test_unknown_ids_and_symlink_artifacts_are_rejected(app, prepared, tmp_path):
run = write_run(prepared)
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="other")).success
assert not (
await app.run_job(
"home_inspect_validation",
validation_path=str(run),
case_id="case~a",
query_id="other",
)
).success
secret = tmp_path / "secret.json"
secret.write_text('{"secret": "must not be read"}')
(run / "summary.json").symlink_to(secret)
response = await app.run_job("home_inspect_validation", validation_path=str(run))
assert not response.success and "must not be read" not in response.answer
result = await call(app, "home_list_validation_results")
assert "read_error" in result["rows"][0] and "secret" not in str(result["rows"])
(run / "summary.json").unlink()
(run / "cases").symlink_to(tmp_path, target_is_directory=True)
assert not (await app.run_job("home_inspect_validation", validation_path=str(run), case_id="case~a")).success
@pytest.mark.asyncio
async def test_hidden_case_scope_is_enforced_by_agent_job_injection(app, prepared):
run = write_run(prepared)
job = app.context.jobs["home_inspect_validation"]
tool = AsAgentWrapper._make_tool(job, injected_job_kwargs={"validation_path": str(run), "case_id": "case~a"})
assert "validation_path" not in tool.input_schema["properties"] and "case_id" not in tool.input_schema["properties"]
await tool.call(query_id="q1")
with pytest.raises(ValueError, match="injected"):
await tool.call(case_id="missing-case")
with pytest.raises(ValueError, match="injected"):
await tool.call(validation_path="evaluations/other/commit/1")
@pytest.mark.asyncio
async def test_cancelled_job_drains_read_thread(app, prepared, monkeypatch):
entered, release, finished = Event(), Event(), Event()
def read(*args):
entered.set()
assert release.wait(5)
finished.set()
return {}
monkeypatch.setattr(inspection, "_list_results", read)
task = asyncio.create_task(app.run_job("home_list_validation_results"))
assert await asyncio.to_thread(entered.wait, 5)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await task
assert finished.is_set()

View file

@ -0,0 +1,230 @@
"""Memory evidence Jobs against isolated real Git repositories."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import shutil
import pytest
from pydantic import ValidationError
from reme_home.evaluation import memory_inspection_step as memory
from reme_home.preparation.code import git
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
from test_evaluation_inspection import call, snapshot, write_run
@pytest.fixture
def evidence(tmp_path):
home = tmp_path / "home"
run = write_run(home, case_ids=["case1"])
repo = run / "cases/case1/memory_construction/reme_workspace"
repo.mkdir(parents=True)
git(repo, "init")
git(repo, "config", "user.name", "test")
git(repo, "config", "user.email", "test@localhost")
(repo / "memory").mkdir()
(repo / "memory/note.md").write_text("old fact\n")
(repo / "removed.txt").write_text("removed\n")
first = commit(repo, "session: session1")
(repo / "memory/note.md").write_text("new fact\n")
(repo / "removed.txt").unlink()
(repo / "added.bin").write_bytes(b"\x00\xff")
second = commit(repo, "session: session2")
return home, run, repo, first, second
def commit(repo, subject):
git(repo, "add", "--all")
git(repo, "commit", "-m", subject)
return git(repo, "rev-parse", "HEAD")
def history(run, **kwargs):
return memory.MemoryHistoryInput(validation_path=str(run), case_id="case1", **kwargs)
def comparison(run, **kwargs):
return memory.CompareMemoryInput(validation_path=str(run), case_id="case1", **kwargs)
def test_history_paging_filter_and_no_writes(evidence):
home, run, repo, first, second = evidence
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
before = snapshot(home)
result = memory.inspect_history(home, history(run, limit=1))
assert result["head"] == second and result["row_count"] == 2
assert result["rows"][0]["session_id"] == "session2" and result["next_offset"] == 1
result = memory.inspect_history(home, history(run, offset=1, limit=1))
assert result["rows"][0]["commit_sha"] == first and result["next_offset"] is None
assert memory.inspect_history(home, history(run, path="added.bin"))["row_count"] == 1
assert memory.inspect_history(home, history(run, path="memory/*"))["rows"] == []
assert snapshot(home) == before
def test_same_repository_diff_and_unique_prefix(evidence):
home, run, repo, first, second = evidence
(repo / "memory/note.md").write_text("dirty checkout ignored\n")
before = snapshot(home)
result = memory.compare_memory(home, comparison(run, base_ref=first[:7].upper()))
assert result["base_commit"] == first and result["target_commit"] == second
assert result["added"] == ["added.bin"] and result["removed"] == ["removed.txt"]
assert result["modified"] == ["memory/note.md"]
assert "-old fact" in result["diff"] and "+new fact" in result["diff"]
assert "Binary files differ" in result["diff"] and not result["truncated"]
assert "dirty checkout" not in result["diff"]
assert memory.compare_memory(home, comparison(run, base_ref="HEAD~1", path="memory"))["added"] == []
assert snapshot(home) == before
def test_cross_evaluation_final_snapshots(evidence):
home, run, _repo, _first, second = evidence
other = write_run(home, "2", case_ids=["case1"], commit="b" * 40)
target = other / "cases/case1/memory_construction/reme_workspace"
target.mkdir(parents=True)
git(target, "init")
git(target, "config", "user.name", "test")
git(target, "config", "user.email", "test@localhost")
(target / "target.txt").write_text("target\n")
target_head = commit(target, "new history")
before = snapshot(home)
result = memory.compare_memory(home, comparison(run, target_validation_path=str(other)))
assert result["base_commit"] == second and result["target_commit"] == target_head
assert result["added"] == ["target.txt"]
assert result["removed"] == ["added.bin", "memory/note.md"]
assert result["base_tree_sha256"] != result["target_tree_sha256"]
assert snapshot(home) == before
@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"])
def test_missing_and_empty_history(tmp_path, state):
home = tmp_path / "home"
run = write_run(home, case_ids=["case1"])
repo = run / "cases/case1/memory_construction/reme_workspace"
if state != "workspace_missing":
repo.mkdir(parents=True)
if state == "history_empty":
git(repo, "init")
result = memory.inspect_history(home, history(run))
assert result["status"] == state and result["rows"] == [] and not result["has_git_history"]
with pytest.raises(ValueError, match="requires memory commits"):
memory.compare_memory(home, comparison(run, base_ref="HEAD"))
@pytest.mark.parametrize(
"kwargs",
[
{"offset": -1},
{"limit": 101},
{"limit": True},
{"case_id": "../bad"},
{"path": "../escape"},
{"path": ".git/config"},
],
)
def test_input_validation(kwargs):
with pytest.raises(ValidationError):
memory.MemoryHistoryInput.model_validate({"validation_path": "a", "case_id": "case1", **kwargs})
def test_comparison_requires_base():
with pytest.raises(ValidationError, match="base_ref is required"):
comparison("a")
@pytest.mark.parametrize("kind", ["gitfile", "symlink", "alternates", "corrupt"])
def test_invalid_or_external_git_storage(evidence, tmp_path, kind):
home, run, repo, _first, _second = evidence
if kind == "gitfile":
(repo / ".git").rename(repo / "saved-git")
(repo / ".git").write_text("gitdir: saved-git\n")
elif kind == "symlink":
(repo / ".git/objects/external").symlink_to(tmp_path)
elif kind == "alternates":
(repo / ".git/objects/info/alternates").write_text(str(tmp_path))
else:
(repo / ".git/HEAD").write_text("broken\n")
with pytest.raises((ValueError, memory.subprocess.CalledProcessError)):
memory.inspect_history(home, history(run))
def test_scope_and_limits(evidence, monkeypatch):
home, run, _repo, first, _second = evidence
with pytest.raises(ValueError, match="case_id"):
memory.inspect_history(home, memory.MemoryHistoryInput(validation_path=str(run), case_id="other"))
with pytest.raises(ValueError):
memory.inspect_history(home, history(home))
monkeypatch.setattr(memory, "MAX_OUTPUT", 20)
result = memory.compare_memory(home, comparison(run, base_ref=first))
assert result["truncated"] and len(result["diff"]) <= 20
monkeypatch.setattr(memory, "MAX_TREE_FILES", 1)
with pytest.raises(ValueError, match="2000 files"):
memory.compare_memory(home, comparison(run, base_ref=first))
def test_ambiguous_and_unreachable_revisions(evidence):
home, run, repo, first, second = evidence
# Publish real commits until two reachable SHAs share a one-character prefix.
previous = second
seen = {first[0]: first, second[0]: second}
for index in range(100):
sha = git(repo, "commit-tree", "HEAD^{tree}", "-p", previous, "-m", str(index))
previous = sha
if sha[0] in seen:
prefix = sha[0]
break
seen[sha[0]] = sha
git(repo, "update-ref", "HEAD", previous)
with pytest.raises(ValueError, match="matches reachable"):
memory.compare_memory(home, comparison(run, base_ref=prefix))
with pytest.raises(ValueError, match="0 matches"):
memory.compare_memory(home, comparison(run, base_ref="f" * 40))
detached = git(repo, "commit-tree", "HEAD^{tree}", "-m", "unreachable")
with pytest.raises(memory.subprocess.CalledProcessError):
memory.compare_memory(home, comparison(run, base_ref=detached + "~0"))
@pytest.mark.asyncio
async def test_registered_jobs_and_restricted_scope(app, prepared, evidence):
run = write_run(prepared, case_ids=["case1"])
result = await call(app, "home_inspect_memory_history", validation_path=str(run), case_id="case1")
assert result["status"] == "workspace_missing"
response = await app.run_job("home_compare_memory", validation_path=str(run), case_id="case1")
assert not response.success and "base_ref" in str(response.answer)
response = await app.run_job(
"home_inspect_memory_history",
validation_path=str(run),
case_id="case1",
_allowed_paths=[str(run)],
)
assert not response.success and "unrestricted" in str(response.answer)
_home, _run, repository, first, second = evidence
destination = run / "cases/case1/memory_construction/reme_workspace"
shutil.copytree(repository, destination)
result = await call(app, "home_compare_memory", validation_path=str(run), case_id="case1", base_ref=first)
assert result["target_commit"] == second and result["modified"] == ["memory/note.md"]
def test_unterminated_lines_and_snapshot_byte_limit(evidence, monkeypatch):
home, run, repo, _first, second = evidence
(repo / "memory/note.md").write_text("unterminated")
commit(repo, "ordinary checkpoint")
result = memory.compare_memory(home, comparison(run, base_ref=second, path="memory"))
assert "+unterminated\n\\ No newline at end of file\n" in result["diff"]
assert memory.inspect_history(home, history(run))["rows"][0]["session_id"] is None
monkeypatch.setattr(memory, "MAX_TREE_BYTES", 1)
with pytest.raises(ValueError, match="20 MB"):
memory.compare_memory(home, comparison(run, base_ref=second))
def test_tracked_symlinks_are_not_followed(evidence):
home, run, repo, _first, second = evidence
(repo / "memory/link").symlink_to("/etc/passwd")
commit(repo, "link")
with pytest.raises(ValueError, match="regular files"):
memory.compare_memory(home, comparison(run, base_ref=second))

View file

@ -0,0 +1,589 @@
"""One-time preparation through the real application lifecycle and Git objects."""
import asyncio
import copy
from threading import Event
from importlib.metadata import EntryPoint
from importlib.resources import files as resource_files
import json
from pathlib import Path
import subprocess
import shutil
import pytest
import pytest_asyncio
import yaml
from reme_home.preparation.spec import PreparationSpec
from reme_home.preparation.code import tree_fingerprint
from reme_home.preparation.transaction import OUTPUTS, TRANSACTION, recover_preparation
from reme_home.workspace import _initialization_lock
from reme.application import Application
def command(root, *args):
"""Git fixtures never depend on a developer's identity or hooks."""
return subprocess.check_output(
["git", "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false", "-C", str(root), *args],
text=True,
).strip()
@pytest.fixture(name="inputs")
def preparation_inputs(tmp_path):
"""A committed tiny ReMe project plus both benchmark formats."""
repository = tmp_path / "source"
repository.mkdir()
files = {
"pyproject.toml": '[project]\nname="fixture-reme"\nversion="0.0.1"\n',
"reme/__init__.py": '"""Fixture."""\n',
"reme/config/benchmark.yaml": "jobs: {}\n",
"reme/candidate.py": "baseline = 1\n",
"plugins/beam/pyproject.toml": '[project.entry-points."reme.plugins"]\nbeam="reme_beam"\n',
"plugins/beam/src/reme_beam/plugin.yaml": "backends: {}\n",
"plugins/beam/src/reme_beam/__init__.py": "value = 1\n",
"plugins/beam/README.md": "Plugin installation instructions\n",
"plugins/beam/config/default.yaml": "enabled: true\n",
}
for name, value in files.items():
path = repository / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(value)
command(repository, "init", "--initial-branch=main", "--template=")
command(repository, "add", ".")
command(repository, "-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", "commit", "-m", "fixture")
revision = command(repository, "rev-parse", "HEAD")
beam = tmp_path / "beam"
for case_id in ("1", "2"):
directory = beam / "chats/100K" / case_id
(directory / "probing_questions").mkdir(parents=True)
(directory / "chat.json").write_text(
json.dumps(
[
{
"batch_number": 1,
"time_anchor": "January-1-2024",
"turns": [[{"id": 1, "role": "user", "content": "hello"}]],
},
],
),
)
(directory / "probing_questions/probing_questions.json").write_text(
json.dumps(
{
"fact": [{"question": "What?", "ideal_answer": "hello", "source_chat_ids": [1]}],
},
),
)
lme = tmp_path / "lme.json"
lme.write_text(
json.dumps(
[
{
"question_id": "q1",
"question": "What?",
"answer": "hello",
"question_type": "single-session-user",
"question_date": "2024/01/02 (Tue) 00:00",
"haystack_dates": ["2024/01/01 (Mon) 00:00"],
"haystack_session_ids": ["s1"],
"haystack_sessions": [[{"role": "user", "content": "hello"}]],
},
],
),
)
spec = {
"benchmark": {"name": "beam", "source": str(beam), "variant": "100K", "case_ids": ["2", "1"]},
"runtime": {"source_repo": str(repository), "source_ref": revision, "plugins": ["beam"]},
"candidate": {"paths": ["plugins/beam/src"]},
}
return spec, lme
@pytest_asyncio.fixture(name="app")
async def home_application(tmp_path, monkeypatch, request):
"""Enable Home by entry point, as an installed plugin would be enabled."""
judge_name = getattr(request, "param", "beam-judge")
for name in ("beam-judge", "lme-judge"):
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / name / "src"))
entries = {
name: EntryPoint(name=name, value=value, group="reme.plugins")
for name, value in [("home", "reme_home"), ("beam-judge", "judge_beam"), ("lme-judge", "judge_lme")]
}
monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entries[name]])
monkeypatch.setenv("LLM_API_KEY", "test")
application = Application(
workspace_dir=str(tmp_path / "workspace"),
plugins=["home", judge_name],
service={"backend": "cli"},
enable_logo=False,
log_to_console=False,
log_to_file=False,
)
await application.start()
try:
yield application
finally:
await application.close()
def snapshot(root):
"""Compare durable paths, bytes and modes, ignoring atime/mtime bookkeeping."""
return {
path.relative_to(root).as_posix(): (path.stat().st_mode, path.read_bytes() if path.is_file() else None)
for path in root.rglob("*")
}
@pytest.mark.asyncio
async def test_python_job_prepares_once(app, inputs):
"""Dict input prepares selected cases and only committed source bytes."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
(Path(spec["runtime"]["source_repo"]) / "plugins/beam/src/reme_beam/__init__.py").write_text("dirty = True")
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
assert response.answer["dataset"]["case_ids"] == ["2", "1"]
assert response.answer["dataset"]["query_count"] == 2
assert (root / "home/code/repo/plugins/beam/src/reme_beam/__init__.py").read_text() == "value = 1\n"
case = json.loads((root / "home/datasets/training-set/cases/000000.json").read_text())
assert case["sessions"][0]["messages"][0]["id"] == "1"
assert case["queries"][0]["metadata"]["evidence_session_ids"] == ["beam_100K_2_batch1"]
assert not (root / "home/datasets/debug-cases/derivation.json").exists()
docs = root / "home/docs"
resources = resource_files("reme_home").joinpath("runtime_docs")
assert sorted(path.name for path in docs.iterdir()) == ["architecture-guide.md", "validation-artifact-guide.md"]
for path in docs.iterdir():
assert path.read_bytes() == resources.joinpath(path.name).read_bytes()
assert command(root / "home/code/repo", "branch", "--show-current") == "init"
assert not command(root / "home/code/repo", "status", "--porcelain")
before = snapshot(root)
again = await app.run_job("home_prepare", spec=spec)
assert not again.success
assert "already prepared" in str(again.answer)
assert snapshot(root) == before
@pytest.mark.asyncio
@pytest.mark.parametrize(
"candidate_path",
["plugins/beam/src", "plugins/beam", "plugins/beam/src/reme_beam/__init__.py"],
)
async def test_complete_plugins_live_only_in_candidate(app, inputs, tmp_path, candidate_path):
"""An editable subset still yields an installable plugin and a Git baseline."""
spec, _ = inputs
spec["candidate"]["paths"] = [candidate_path, "reme/candidate.py"]
source = Path(spec["runtime"]["source_repo"])
(source / "plugins/beam/README.md").write_text("uncommitted documentation")
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
root = Path(app.config.workspace_dir)
runtime, candidate = root / "home/code/runtime", root / "home/code/repo"
assert not (runtime / "plugins").exists()
for relative in (
"pyproject.toml",
"README.md",
"config/default.yaml",
"src/reme_beam/plugin.yaml",
"src/reme_beam/__init__.py",
):
name = f"plugins/beam/{relative}"
committed = command(source, "show", f"{spec['runtime']['source_ref']}:{name}")
assert (candidate / name).read_text().strip() == committed
assert command(candidate, "show", f"init:{name}") == committed
assert (candidate / "reme/candidate.py").read_bytes() == (runtime / "reme/candidate.py").read_bytes()
code = response.answer["code"]
assert code["candidate_paths"] == spec["candidate"]["paths"]
assert code["runtime_fingerprint"] == tree_fingerprint(runtime)
assert code["candidate_fingerprint"] == tree_fingerprint(candidate)
# Future evaluators combine these trees; plugin installation uses repo bytes.
evaluation = tmp_path / "evaluation"
shutil.copytree(runtime, evaluation)
shutil.copytree(candidate, evaluation, dirs_exist_ok=True, ignore=shutil.ignore_patterns(".git"))
for project in code["install_projects"]:
assert (evaluation / project / "pyproject.toml").is_file()
assert not command(candidate, "status", "--porcelain")
@pytest.mark.asyncio
@pytest.mark.parametrize("spec_path", ["prepare.yaml", "home/prepare.yaml"])
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_yaml_lme_core_candidate(app, inputs, spec_path):
"""Workspace YAML and core candidates use the same job as Python dictionaries."""
spec, lme = inputs
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
spec["runtime"]["plugins"] = []
spec["candidate"]["paths"] = ["reme/candidate.py"]
spec["debug"] = {"enabled": False}
root = Path(app.config.workspace_dir)
(root / spec_path).write_text(yaml.safe_dump(spec))
response = await app.run_job("home_prepare", spec_path=spec_path)
assert response.success, response.answer
domain = yaml.safe_load((root / "home/domain_spec.yaml").read_text())
assert domain["benchmark"]["case_ids"] == ["q1"]
assert not domain["benchmark"]["all_cases"]
assert (root / "home/code/repo/reme/candidate.py").is_file()
case = json.loads((root / "home/datasets/training-set/cases/000000.json").read_text())
assert case["queries"][0]["metadata"]["question_date"] == "2024/01/02 (Tue) 00:00"
assert case["sessions"][0]["messages"][0]["created_at"] == "2024-01-01T00:00:00"
@pytest.mark.asyncio
async def test_relative_inputs_still_resolve_against_outer_workspace(app, inputs):
"""Nesting outputs must not change existing source-path configuration."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
shutil.copytree(spec["benchmark"]["source"], root / "inputs/beam")
shutil.copytree(spec["runtime"]["source_repo"], root / "inputs/source")
spec["benchmark"]["source"] = "inputs/beam"
spec["runtime"]["source_repo"] = "inputs/source"
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
domain = yaml.safe_load((root / "home/domain_spec.yaml").read_text())
assert domain["benchmark"]["source"] == str(root / "inputs/beam")
assert domain["runtime"]["source_repo"] == str(root / "inputs/source")
assert not (root / "code").exists()
assert not (root / "datasets").exists()
assert not (root / "metadata/home").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure",
[
"convert",
"code",
"docs",
"absent-outputs",
*[f"publish-{i}" for i in range(len(OUTPUTS) + 1)],
*[f"after-{i}" for i in range(len(OUTPUTS) + 1)],
],
)
async def test_failures_restore_original_workspace_and_allow_retry(app, inputs, monkeypatch, failure):
"""Failures during conversion, code export, and every publication phase roll back."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
(root / "daily/keep.md").write_text("user memory")
if failure == "absent-outputs":
for relative in ("code/runtime", "code/repo", "datasets/training-set", "datasets/debug-cases", "docs"):
shutil.rmtree(root / "home" / relative)
failure = "after-5"
before = snapshot(root)
with monkeypatch.context() as patch:
if failure == "convert":
patch.setattr(
"reme_home.preparation.component.prepare_dataset",
lambda *a, **kw: (_ for _ in ()).throw(OSError("test")),
)
elif failure == "code":
patch.setattr(
"reme_home.preparation.component.prepare_code",
lambda *a, **kw: (_ for _ in ()).throw(OSError("test")),
)
elif failure == "docs":
original_read = Path.read_bytes
def read_bytes(path):
if path.name == "validation-artifact-guide.md":
raise FileNotFoundError("missing packaged document")
return original_read(path)
patch.setattr(Path, "read_bytes", read_bytes)
else:
original = __import__("os").replace
def replace(source, destination):
if Path(source).name == failure.replace("publish-", "output-"):
raise OSError("test publication failure")
result = original(source, destination)
if failure.startswith("after-") and Path(source).name == failure.replace("after-", "output-"):
raise OSError("test failure after rename")
return result
patch.setattr("reme_home.preparation.transaction.os.replace", replace)
response = await app.run_job("home_prepare", spec=spec)
assert not response.success
assert snapshot(root) == before
assert (await app.run_job("home_prepare", spec=spec)).success
@pytest.mark.asyncio
async def test_preparation_preserves_existing_docs(app, inputs):
"""Existing user documentation is a conflict, never overwritten or removed."""
root = Path(app.config.workspace_dir)
(root / "home/docs/architecture-guide.md").write_text("user-owned documentation")
before = snapshot(root)
response = await app.run_job("home_prepare", spec=inputs[0])
assert not response.success
assert "not empty" in str(response.answer)
assert snapshot(root) == before
@pytest.mark.asyncio
async def test_legacy_preparation_journal_recovery_preserves_docs(app):
"""Version 1 recovery uses the original output indices and leaves docs alone."""
root = Path(app.config.workspace_dir) / "home"
state = root / "metadata"
(root / "docs/keep.md").write_text("existing documentation")
before = snapshot(root)
transaction = state / TRANSACTION
transaction.mkdir()
(transaction / "journal.json").write_text(
json.dumps({"schema_version": 1, "existed": [True, True, True, True, False, False]}),
)
(root / "code/runtime").rename(transaction / "backup-0")
(root / "code/runtime").mkdir()
(root / "code/runtime/partial.py").write_text("partial runtime")
(root / "code/repo").rename(transaction / "backup-1")
(transaction / "output-1").mkdir()
recover_preparation(root, state)
assert snapshot(root) == before
@pytest.mark.asyncio
async def test_interrupted_publication_recovers_at_startup(app, inputs, monkeypatch):
"""A durable journal survives process interruption and is recovered on startup."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
before = snapshot(root)
preparation = app.context.components["home.preparation"]["default"]
with monkeypatch.context() as patch:
original = __import__("os").replace
def replace(source, destination):
if Path(source).name == "output-3":
raise RuntimeError("simulated crash")
return original(source, destination)
patch.setattr("reme_home.preparation.transaction.os.replace", replace)
patch.setattr("reme_home.preparation.component.rollback", lambda *args: None)
with pytest.raises(RuntimeError, match="simulated crash"):
await preparation.prepare(spec=spec)
workspace = app.context.components["home.workspace"]["default"]
await workspace.close()
await workspace.start()
assert snapshot(root) == before
assert (await app.run_job("home_prepare", spec=spec)).success
@pytest.mark.asyncio
async def test_success_survives_cleanup_failure_and_restart(app, inputs, monkeypatch):
"""Committed completion is retained when cleanup was interrupted."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
original = __import__("shutil").rmtree
with monkeypatch.context() as patch:
def remove(path, *args, **kwargs):
if Path(path).name == TRANSACTION:
raise OSError("cleanup")
return original(path, *args, **kwargs)
patch.setattr("reme_home.preparation.component.shutil.rmtree", remove)
response = await app.run_job("home_prepare", spec=spec)
assert response.success
assert (root / "home/metadata" / TRANSACTION).exists()
workspace = app.context.components["home.workspace"]["default"]
await workspace.close()
await workspace.start()
assert not (root / "home/metadata" / TRANSACTION).exists()
assert not (await app.run_job("home_prepare", spec=spec)).success
@pytest.mark.asyncio
async def test_conflicts_and_lock_preserve_user_content(app, inputs):
"""Competing jobs and user-owned output directories cannot be overwritten."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
before = snapshot(root)
with _initialization_lock(root / "home/metadata/workspace.lock"):
response = await app.run_job("home_prepare", spec=spec)
assert not response.success
assert snapshot(root) == before
(root / "home/code/repo/keep.py").write_text("keep")
before = snapshot(root)
assert not (await app.run_job("home_prepare", spec=spec)).success
assert snapshot(root) == before
@pytest.mark.asyncio
@pytest.mark.parametrize(
"kwargs",
[
{},
{"spec": {}, "spec_path": "prepare.yaml"},
{"spec_path": "../outside.yaml"},
{"spec_path": "~/prepare.yaml"},
{"spec": {}, "_allowed_paths": []},
],
)
async def test_bad_job_inputs_do_not_mutate_workspace(app, kwargs):
"""Path escape and restricted invocation fail closed."""
root = Path(app.config.workspace_dir)
before = snapshot(root)
assert not (await app.run_job("home_prepare", **kwargs)).success
assert snapshot(root) == before
@pytest.mark.parametrize(
"change",
[
{"case_ids": []},
{"case_ids": ["1", "1"]},
{"case_ids": [1]},
{"case_ids": None},
{"all_cases": True},
{"variant": "bad"},
{"evidence_source": "annotations.jsonl"},
],
)
def test_strict_case_selection(inputs, change):
"""Do not silently select everything or accept misspelled dataset options."""
spec, _ = inputs
spec = copy.deepcopy(spec)
spec["benchmark"].update(change)
with pytest.raises(ValueError):
PreparationSpec.model_validate(spec)
@pytest.mark.asyncio
@pytest.mark.parametrize("mutation", ["unknown-case", "unknown-plugin", "bad-ref", "bad-candidate", "symlink"])
async def test_invalid_sources_roll_back(app, inputs, mutation):
"""Input and revision failures leave no partially prepared dataset."""
spec, _ = inputs
if mutation == "unknown-case":
spec["benchmark"]["case_ids"] = ["missing"]
elif mutation == "unknown-plugin":
spec["runtime"]["plugins"] = ["missing"]
elif mutation == "bad-ref":
spec["runtime"]["source_ref"] = "--bad-ref"
elif mutation == "bad-candidate":
spec["candidate"]["paths"] = ["../outside"]
else:
repository = Path(spec["runtime"]["source_repo"])
(repository / "reme/link").symlink_to("__init__.py")
command(repository, "add", ".")
command(repository, "-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", "commit", "-m", "link")
spec["runtime"]["source_ref"] = "HEAD"
root = Path(app.config.workspace_dir)
before = snapshot(root)
assert not (await app.run_job("home_prepare", spec=spec)).success
assert snapshot(root) == before
def test_bad_recovery_journal_does_not_remove_outside_paths(tmp_path):
"""Recovery accepts no path names from disk."""
state = tmp_path / "home/metadata"
transaction = state / TRANSACTION
transaction.mkdir(parents=True)
(transaction / "journal.json").write_text('{"existed": [], "paths": ["../../outside"]}')
with pytest.raises(ValueError):
recover_preparation(tmp_path / "home", state)
@pytest.mark.asyncio
async def test_cancellation_waits_for_rollback(app, inputs, monkeypatch):
"""The event loop stays available, and cancellation waits for the worker to restore state."""
spec, _ = inputs
root = Path(app.config.workspace_dir)
before = snapshot(root)
component = app.context.components["home.preparation"]["default"]
stage = component._stage # pylint: disable=protected-access
entered = Event()
proceed = Event()
def blocked(*args):
entered.set()
if not proceed.wait(10):
raise RuntimeError("test worker timed out")
return stage(*args)
monkeypatch.setattr(component, "_stage", blocked)
task = asyncio.create_task(app.run_job("home_prepare", spec=spec))
try:
assert await asyncio.to_thread(entered.wait, 10)
task.cancel()
await asyncio.sleep(0)
assert not task.done()
proceed.set()
with pytest.raises(asyncio.CancelledError):
await task
finally:
proceed.set()
await asyncio.gather(task, return_exceptions=True)
assert snapshot(root) == before
monkeypatch.setattr(component, "_stage", stage)
assert (await app.run_job("home_prepare", spec=spec)).success
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_duplicate_lme_ids_fail_without_mutation(app, inputs):
"""Upstream duplicate IDs cannot silently replace selected cases."""
spec, lme = inputs
items = json.loads(lme.read_text())
lme.write_text(json.dumps(items + items))
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
root = Path(app.config.workspace_dir)
before = snapshot(root)
response = await app.run_job("home_prepare", spec=spec)
assert not response.success
assert "Duplicate LongMemEval" in str(response.answer)
assert snapshot(root) == before
@pytest.mark.asyncio
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
async def test_lme_evidence_and_time_cutoff_debug_cases(app, inputs):
"""Evidence survives conversion and debug data obeys question-time cutoffs."""
spec, lme = inputs
items = json.loads(lme.read_text())
item = items[0]
item["haystack_dates"].append("2024/01/03 (Wed) 00:00")
item["haystack_session_ids"].append("future")
item["haystack_sessions"].append([{"role": "user", "content": "future information"}])
lme.write_text(json.dumps(items))
evidence = lme.with_suffix(".jsonl")
evidence.write_text(json.dumps({"case_id": "q1", "evidence_sessions": ["s1"], "is_certain": True}) + "\n")
spec["benchmark"] = {
"name": "longmemeval",
"source": str(lme),
"case_ids": ["q1"],
"evidence_source": str(evidence),
}
response = await app.run_job("home_prepare", spec=spec)
assert response.success, response.answer
root = Path(app.config.workspace_dir)
case = json.loads((root / "home/datasets/training-set/cases/000000.json").read_text())
assert case["queries"][0]["metadata"]["evidence_session_ids"] == ["s1"]
debug = json.loads((root / "home/datasets/debug-cases/cases/000000.json").read_text())
assert [session["session_id"] for session in debug["sessions"]] == ["s1"]
@pytest.mark.asyncio
async def test_concurrent_preparation_rejects_second_writer(app, inputs, monkeypatch):
"""Two active Jobs share the process lock rather than racing to publish."""
spec, _ = inputs
component = app.context.components["home.preparation"]["default"]
stage = component._stage # pylint: disable=protected-access
entered = Event()
proceed = Event()
def blocked(*args):
entered.set()
if not proceed.wait(10):
raise RuntimeError("test worker timed out")
return stage(*args)
monkeypatch.setattr(component, "_stage", blocked)
first = asyncio.create_task(app.run_job("home_prepare", spec=spec))
try:
assert await asyncio.to_thread(entered.wait, 10)
second = await app.run_job("home_prepare", spec=spec)
assert not second.success
assert "already in progress" in str(second.answer)
proceed.set()
assert (await first).success
finally:
proceed.set()
await asyncio.gather(first, return_exceptions=True)

View file

@ -0,0 +1,221 @@
"""Compact evidence tracing against real, isolated evaluation and Git artifacts."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import json
import shutil
import pytest
from pydantic import ValidationError
from reme_home.evaluation import evidence_step as evidence_tools
from reme_home.evaluation.artifacts import write_json
from reme_home.preparation.code import git
from test_memory_inspection import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
commit,
evidence,
home_application,
preparation_inputs,
prepared_home,
)
from test_evaluation_inspection import call, snapshot, write_run
@pytest.fixture
def query_evidence(evidence): # noqa: F811
home, run, repo, first, second = evidence
manifest = json.loads((run / "manifest.json").read_text())
manifest["query_ids"] = {"case1": ["q1"]}
write_json(run / "manifest.json", manifest)
case = {
"case_id": "case1",
"sessions": [{"session_id": "session1", "messages": [{"content": "source transcript"}]}],
"queries": [
{
"query_id": "q1",
"question": "What is the fact?",
"golden_answer": "old fact",
"metadata": {"evidence_session_ids": ["session1", "session1"], "evidence_is_certain": False},
},
],
}
source = home / "datasets/training-set/cases/000000.json"
write_json(source, case)
write_json(
run / "cases/case1/queries/result.json",
{"queries": [{"query_id": "q1", "answer": "new fact", "score": 0, "judge_result": {"large": "hidden"}}]},
)
return home, run, repo, first, second, source, case
def trace(home, run, **kwargs):
spec = evidence_tools.TraceQueryEvidenceInput(validation_path=str(run), case_id="case1", query_id="q1", **kwargs)
return evidence_tools.trace_query_evidence(home, spec)
def test_compact_annotation_root_checkpoint_and_read_only(query_evidence):
home, run, repo, first, _second, source, _case = query_evidence
(repo / "memory/note.md").write_text("dirty ignored")
before = snapshot(home)
# Independent annotations take precedence, even over nonexistent inferred IDs.
result = trace(home, run, evidence_session_ids=["unknown"])
assert result["query"] == {
"question": "What is the fact?",
"expected": "old fact",
"actual": "new fact",
"score": 0,
"error": None,
"result_missing": False,
}
assert result["evidence_source"] == "independent_annotation"
assert result["evidence_is_certain"] is False and result["memory_status"] == "available"
assert result["sessions"] == [
{"id": "session1", "checkpoint": first, "changed_files": ["memory/note.md", "removed.txt"]},
]
assert result["paths"]["source_case"] == str(source)
assert result["paths"]["memory_repository"] == str(repo)
assert "source transcript" not in json.dumps(result) and "judge_result" not in json.dumps(result)
assert snapshot(home) == before
def test_inference_missing_checkpoint_and_debug_dataset(query_evidence):
home, run, _repo, _first, second, source, case = query_evidence
case["sessions"] += [{"session_id": "session2"}, {"session_id": "not-constructed"}]
case["queries"][0]["metadata"] = {}
debug_source = home / "datasets/debug-cases/cases/000001.json"
write_json(debug_source, case)
source.unlink()
manifest = json.loads((run / "manifest.json").read_text())
manifest["dataset_kind"] = "debug-cases"
write_json(run / "manifest.json", manifest)
result = trace(home, run, evidence_session_ids=["session2", "not-constructed", "session2"])
assert result["evidence_source"] == "diagnostician_inference"
assert result["paths"]["source_case"] == str(debug_source)
assert result["sessions"][0]["checkpoint"] == second
assert result["sessions"][0]["changed_files"] == ["added.bin", "memory/note.md", "removed.txt"]
assert result["sessions"][1] == {"id": "not-constructed", "checkpoint": None, "changed_files": []}
assert trace(home, run)["evidence_source"] == "unavailable"
with pytest.raises(ValueError, match="not present"):
trace(home, run, evidence_session_ids=["unknown"])
@pytest.mark.parametrize("annotation", [[], [""], "session1", [123]])
def test_empty_and_invalid_annotations(query_evidence, annotation):
home, run, _repo, _first, _second, source, case = query_evidence
case["queries"][0]["metadata"]["evidence_session_ids"] = annotation
write_json(source, case)
if not annotation:
result = trace(home, run, evidence_session_ids=["session1"])
assert not result["sessions"] and result["evidence_source"] == "independent_annotation"
else:
with pytest.raises(ValidationError):
trace(home, run)
@pytest.mark.parametrize("state", ["workspace_missing", "history_missing", "history_empty"])
def test_missing_history_and_results(query_evidence, state):
home, run, repo, _first, _second, _source, _case = query_evidence
shutil.rmtree(repo if state == "workspace_missing" else repo / ".git")
if state == "history_empty":
git(repo, "init")
(run / "cases/case1/queries/result.json").unlink()
result = trace(home, run)
assert result["memory_status"] == state and result["sessions"][0]["checkpoint"] is None
assert result["query"]["result_missing"] and result["query"]["score"] is None
assert result["query"]["actual"] is None and result["query"]["expected"] == "old fact"
def test_large_fields_and_lists_are_bounded(query_evidence):
home, run, repo, _first, _second, source, case = query_evidence
for index in range(55):
(repo / f"file-{index:03}").write_text("fact")
checkpoint = commit(repo, "session: session1")
case["sessions"] = [{"session_id": f"s{index}"} for index in range(55)] + case["sessions"]
case["queries"][0]["metadata"]["evidence_session_ids"] = ["session1"] + [f"s{i}" for i in range(55)]
case["queries"][0]["question"] = "x" * 100_000
write_json(source, case)
result = trace(home, run)
assert len(result["sessions"]) == 50 and result["omitted_session_count"] == 6
assert result["sessions"][0]["checkpoint"] == checkpoint
assert len(result["sessions"][0]["changed_files"]) == 50
assert result["sessions"][0]["omitted_changed_file_count"] == 5
assert len(result["query"]["question"]) < 2100
assert len(json.dumps(result)) < 15_000
def test_reject_unselected_queries_and_external_sources(query_evidence, tmp_path):
home, run, _repo, _first, _second, source, _case = query_evidence
manifest = json.loads((run / "manifest.json").read_text())
manifest["query_ids"] = {"case1": []}
write_json(run / "manifest.json", manifest)
with pytest.raises(ValueError, match="not selected"):
trace(home, run)
manifest["query_ids"] = {"case1": ["q1"]}
write_json(run / "manifest.json", manifest)
source.unlink()
source.symlink_to(tmp_path / "external.json")
with pytest.raises(ValueError, match="Symbolic link"):
trace(home, run)
source.unlink()
with pytest.raises(ValueError, match="case is missing"):
trace(home, run)
def test_missing_query_invalid_dataset_and_corrupt_history(query_evidence):
home, run, repo, _first, _second, source, case = query_evidence
case["queries"] = []
write_json(source, case)
with pytest.raises(ValueError, match="has no query"):
trace(home, run)
manifest = json.loads((run / "manifest.json").read_text())
manifest["dataset_kind"] = "../outside"
write_json(run / "manifest.json", manifest)
with pytest.raises(ValueError, match="dataset_kind"):
trace(home, run)
manifest["dataset_kind"] = "training-set"
write_json(run / "manifest.json", manifest)
case["queries"] = [{"query_id": "q1"}]
write_json(source, case)
(repo / ".git/objects/info/alternates").write_text("/external/objects")
with pytest.raises(ValueError, match="indirect"):
trace(home, run)
def test_run_and_case_scope(query_evidence):
home, run, _repo, _first, _second, _source, _case = query_evidence
with pytest.raises(ValueError, match="case_id"):
evidence_tools.trace_query_evidence(
home,
evidence_tools.TraceQueryEvidenceInput(validation_path=str(run), case_id="unknown", query_id="q1"),
)
with pytest.raises(ValueError, match="escapes"):
trace(home, home.parent / "outside")
@pytest.mark.parametrize("kwargs", [{"query_id": "../q"}, {"case_id": "../c"}, {"evidence_session_ids": [""]}])
def test_invalid_inputs(kwargs):
with pytest.raises(ValidationError):
evidence_tools.TraceQueryEvidenceInput.model_validate(
{"validation_path": "run", "case_id": "case1", "query_id": "q1", **kwargs},
)
@pytest.mark.asyncio
async def test_registered_job_and_restricted_scope(app, prepared, query_evidence):
home, source_run, _repo, _first, _second, _source, _case = query_evidence
run = write_run(prepared, case_ids=["case1"], query_ids={"case1": ["q1"]})
shutil.copytree(home / "datasets/training-set", prepared / "datasets/training-set", dirs_exist_ok=True)
shutil.copytree(source_run / "cases", run / "cases")
before = snapshot(prepared)
result = await call(app, "home_trace_query_evidence", validation_path=str(run), case_id="case1", query_id="q1")
assert result["query"]["score"] == 0 and result["sessions"][0]["checkpoint"]
response = await app.run_job(
"home_trace_query_evidence",
validation_path=str(run),
case_id="case1",
query_id="q1",
_allowed_paths=[str(run)],
)
assert not response.success and "unrestricted" in str(response.answer)
assert snapshot(prepared) == before

View file

@ -0,0 +1,412 @@
"""Adaptive sampling parity and actual-start accounting through real Home Jobs."""
# pylint: disable=missing-function-docstring,redefined-outer-name,unused-argument,protected-access
import asyncio
import copy
import json
from pathlib import Path
import pytest
from reme_home import sampling
from reme_home.evaluation import history as history_module
from reme_home.preparation.data.models import CaseSpec, QuerySpec
from reme_home.workspace import _initialization_lock
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
FakeFactory,
home_application,
preparation_inputs,
prepared_home,
)
def cases():
return [
CaseSpec(
case_id=key,
sessions=[],
queries=[
QuerySpec(query_id=f"query-{key}-{index}", question="?", golden_answer="answer") for index in range(4)
],
)
for key in "abc"
]
def test_plan_matches_dev_meta_golden_draw(monkeypatch):
# Golden output computed with dev_meta b53048e4 sampling.py, seed=123456.
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
state = {
"round": 2,
"cases": {
"a": {
"observed_count": 2,
"ewma_loss": 0.9,
"last_selected_round": 2,
"last_error_round": 1,
},
},
"queries": {},
}
before = copy.deepcopy(state)
policy = sampling.SamplingPolicy(max_cases=2, max_queries=3, temperature=0.8)
plan = sampling.create_sampling_plan(cases(), state, policy)
assert plan["case_ids"] == ["a", "b"]
assert plan["query_ids"] == {"a": ["query-a-2", "query-a-0"], "b": ["query-b-3"]}
assert plan["case_draw_weights"] == pytest.approx(
{
"a": 0.40033106216673214,
"b": 0.29983446891663396,
"c": 0.29983446891663396,
},
)
assert plan["query_draw_weights"]["a"] == {f"query-a-{index}": 0.25 for index in range(4)}
assert state == before
assert sampling.create_sampling_plan(cases(), state, policy) == plan
def test_replay_eligibility_and_unrestricted_next_draw(monkeypatch):
monkeypatch.setattr(sampling.time, "time_ns", lambda: 100)
policy = sampling.SamplingPolicy(max_cases=3, max_queries=3)
eligible = {"b": ["query-b-1", "query-b-3"], "c": ["query-c-2"]}
plan = sampling.create_sampling_plan(cases(), {}, policy, eligible_query_ids=eligible)
assert set(plan["case_ids"]) == {"b", "c"}
assert all(set(values) <= set(eligible[key]) for key, values in plan["query_ids"].items())
full = sampling.create_sampling_plan(cases(), {}, policy)
assert set(full["case_draw_weights"]) == {"a", "b", "c"}
assert full["eligibility"] is None
@pytest.mark.parametrize(
"options",
[
{"max_cases": 0},
{"max_queries": 0},
{"max_cases": True},
{"max_queries": True},
{"temperature": 0},
{"temperature": float("nan")},
{"temperature": float("inf")},
{"ewma_alpha": 0},
{"ewma_alpha": 1.1},
{"uniform_exploration": -0.1},
{"uniform_exploration": 1.1},
{"unexpected": 1},
],
)
def test_policy_rejects_invalid_limits(options):
with pytest.raises(ValueError):
sampling.SamplingPolicy(**options)
def test_uniform_mass_and_unobserved_priority():
records = {
"easy": {"ewma_loss": 0.0, "observed_count": 100, "last_selected_round": 10},
"hard": {"ewma_loss": 1.0, "observed_count": 1, "last_selected_round": 1, "last_error_round": 1},
"unseen": {},
}
weights = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=0.3))
assert sum(weights.values()) == pytest.approx(1)
assert all(weight >= 0.1 for weight in weights.values())
assert weights["hard"] > weights["easy"] and weights["unseen"] > weights["easy"]
uniform = sampling._mixed_weights(records, 11, sampling.SamplingPolicy(uniform_exploration=1))
assert set(uniform.values()) == {1 / 3}
def state(home):
return history_module.read_state(home)
@pytest.mark.asyncio
async def test_baseline_then_adaptive_job_updates_only_selected_cases(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success, baseline.answer
assert state(prepared)["round"] == 1
assert all(record["ewma_loss"] == 0.5 for record in state(prepared)["cases"].values())
evaluation = app.context.components["home.evaluation"]["default"]
evaluation.sampling_policy = sampling.SamplingPolicy(max_cases=1, max_queries=1, ewma_alpha=0.5)
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": '{"score": 1}'}
before = copy.deepcopy(state(prepared))
response = await app.run_job("home_evaluate", sampling=True)
assert response.success, response.answer
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
selected = manifest["case_ids"][0]
unselected = ({"1", "2"} - {selected}).pop()
assert manifest["case_ids"] == plan["case_ids"] and manifest["query_ids"] == plan["query_ids"]
assert plan["mode"] == "sampled" and plan["round"] == 2
assert response.answer["case_count"] == response.answer["scored_query_count"] == 1
after = state(prepared)
assert after["cases"][selected]["ewma_loss"] == 0.25
assert after["cases"][selected]["observed_count"] == 2
assert after["cases"][selected]["successful_observations_after_last_error"] == 1
assert after["cases"][unselected] == before["cases"][unselected]
evaluation._finish_sampling(run)
assert state(prepared) == after # A summary is consumed once, including after reopening.
assert "home_validate" not in app.context.jobs
assert app.context.jobs["home_evaluate"].parameters["properties"]["sampling"]["type"] == "boolean"
assert "sampling_policy" not in app.context.jobs["home_evaluate"].parameters["properties"]
assert "case_ids" not in app.context.jobs["home_evaluate"].parameters["properties"]
@pytest.mark.asyncio
async def test_debug_never_reads_or_changes_sampling_state(app, prepared):
before = prepared / "sampling/state.json"
for enabled in (False, True):
response = await app.run_job("home_evaluate", sampling=enabled, debug=True)
assert response.success, response.answer
assert not before.exists()
assert not list((prepared / "sampling/plans").rglob("*.json"))
# Corrupt sampling state cannot prevent an unrelated debug smoke test.
before.write_text("corrupt cached state")
response = await app.run_job("home_evaluate", sampling=True, debug=True)
assert response.success and before.read_text() == "corrupt cached state"
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
assert "sandbox_started_at" not in manifest and "sampling" not in manifest
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["dirty", "factory_constructor", "startup", "selection", "replay"])
async def test_prestart_failure_never_counts(app, prepared, monkeypatch, failure):
await app.run_job("home_evaluate")
previous = (prepared / "sampling/state.json").read_bytes()
plans = sorted((prepared / "sampling/plans").rglob("*.json"))
options = {}
if failure == "dirty":
(prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("dirty")
elif failure == "selection":
options["case_ids"] = ["1"]
elif failure == "replay":
options["reuse_construction_from"] = "evaluations/not-a-run"
elif failure == "factory_constructor":
def fail(*args, **kwargs):
raise RuntimeError("factory never created")
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", fail)
else:
async def fail(*args, **kwargs):
raise RuntimeError("no sandbox initialized")
monkeypatch.setattr(FakeFactory, "create_cases", fail)
response = await app.run_job("home_evaluate", sampling=True, **options)
assert not response.success
assert (prepared / "sampling/state.json").read_bytes() == previous
assert sorted((prepared / "sampling/plans").rglob("*.json")) == plans
@pytest.mark.asyncio
async def test_started_infrastructure_failure_is_not_a_quality_zero(app, prepared, monkeypatch):
async def fail(*args, **kwargs):
raise ConnectionError("construction provider unavailable")
monkeypatch.setattr("test_evaluation.FakeWorker.run_build", fail)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert not response.success
observed = state(prepared)
assert observed["round"] == 1
assert not observed["cases"] and not observed["queries"]
assert app.context.components["home.evaluation_history"]["default"].list_results() == []
@pytest.mark.asyncio
async def test_started_candidate_failure_does_not_fabricate_a_score(app, prepared, monkeypatch):
factory = FakeFactory.create_cases
async def create(self, *args, **kwargs):
workers = await factory(self, *args, **kwargs)
async def empty(_queries):
return {"success": True, "queries": [{"query_id": "fact:1", "answer": "", "error": None}]}
for worker in workers:
worker.run_queries = empty
return workers
monkeypatch.setattr(FakeFactory, "create_cases", create)
response = await app.run_job("home_evaluate", case_ids=["1"])
assert response.success, response.answer
observed = state(prepared)
assert observed["round"] == 1
record = observed["cases"]["1"]
assert record["candidate_failure_count"] == 1
assert record["observed_count"] == 0 and record["ewma_loss"] is None
query = observed["queries"][sampling._query_key("1", "fact:1")]
assert query["candidate_failure_count"] == 1 and query["observed_count"] == 0
@pytest.mark.asyncio
async def test_started_cancellation_retains_round_but_no_partial_quality(app, prepared, monkeypatch):
monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
task = asyncio.create_task(app.run_job("home_evaluate", sampling=True, concurrency=1))
for _ in range(200):
if FakeFactory.instances and FakeFactory.instances[0].entered.is_set():
break
await asyncio.sleep(0.01)
assert FakeFactory.instances[0].entered.is_set()
assert state(prepared)["round"] == 1
concurrent = await app.run_job("home_evaluate", sampling=True)
assert not concurrent.success # Formal validations follow dev_meta's serialization boundary.
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
observed = state(prepared)
assert observed["round"] == 1 and not observed["cases"] and not observed["queries"]
assert not observed["processed_validations"]
with _initialization_lock(prepared / "metadata/validation.lock"):
pass
@pytest.mark.asyncio
async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
baseline = await app.run_job("home_evaluate", case_ids=["1"])
assert baseline.success, baseline.answer
app.context.components["home.evaluation"]["default"].sampling_policy = sampling.SamplingPolicy(max_cases=1)
response = await app.run_job("home_evaluate", sampling=True, reuse_construction_from=baseline.answer["result_dir"])
assert response.success, response.answer
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
assert manifest["case_ids"] == ["1"]
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert plan["eligibility"] == {"1": ["fact:1"]}
# The next full rebuild must see the entire training set again.
response = await app.run_job("home_evaluate", sampling=True)
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None
@pytest.mark.asyncio
async def test_sampling_cache_rebuild_and_new_round_after_reopen(app, prepared):
baseline = await app.run_job("home_evaluate")
assert baseline.success
expected = state(prepared)
(prepared / "sampling/state.json").unlink()
assert state(prepared) == expected
assert not (prepared / "sampling/state.json").exists() # Rebuilding reads never claim a new round.
with _initialization_lock(prepared / "metadata/workspace.lock"):
assert history_module.rebuild_sampling_state(prepared) == expected
result = await app.run_job("home_evaluate", sampling=True)
assert result.success, result.answer
assert state(prepared)["round"] == 2
@pytest.mark.asyncio
async def test_fail_fast_started_round_without_summary_is_not_quality_feedback(app, prepared):
app.context.components["agent_wrapper"]["judge"].reply.return_value = {"result": "invalid judge output"}
response = await app.run_job("home_evaluate", sampling=True, fail_fast=True)
assert not response.success
observed = state(prepared)
assert observed["round"] == 1
assert not observed["processed_validations"] and not observed["cases"]
assert not observed["queries"]
(prepared / "sampling/state.json").unlink()
assert state(prepared) == observed
@pytest.mark.asyncio
async def test_partial_pool_start_failure_counts_started_round(app, prepared, monkeypatch):
original = FakeFactory.create_cases
async def fail_after_first_start(factory, *args, **kwargs):
await original(factory, *args, **kwargs)
raise ConnectionError("a later worker failed after first initialized")
monkeypatch.setattr(FakeFactory, "create_cases", fail_after_first_start)
result = await app.run_job("home_evaluate", sampling=True)
assert not result.success
observed = state(prepared)
assert observed["round"] == 1 and not observed["processed_validations"]
assert not observed["cases"] and not observed["queries"]
@pytest.mark.asyncio
async def test_started_callback_is_idempotent(app, prepared):
response = await app.run_job("home_evaluate", sampling=True)
assert response.success
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
original = state(prepared)
component = app.context.components["home.evaluation"]["default"]
component._sampling_started(run, manifest, None)
assert state(prepared) == original
@pytest.mark.parametrize("missing", [1, 2])
def test_replay_eligibility_uses_dev_meta_five_percent_threshold(tmp_path, missing):
from reme_home.evaluation.inputs import EvaluationSpec, replay_sampling_eligibility
from reme_home.evaluation.artifacts import write_json
home = tmp_path / "home"
source = home / "evaluations/init/commit/1"
selected = [
CaseSpec(
case_id=f"case-{index}",
sessions=[],
queries=[
QuerySpec(query_id="q", question="?", golden_answer="answer"),
],
)
for index in range(20)
]
write_json(
source / "manifest.json",
{
"dataset_kind": "training-set",
"mode": "scored",
"status": "completed",
"case_ids": [case.case_id for case in selected],
},
)
write_json(source / "summary.json", {})
for case in selected[missing:]:
folder = source / "cases" / case.case_id
write_json(folder / "memory_construction/result.json", {"success": True})
(folder / "memory_construction/reme_workspace.tar.gz").write_bytes(b"fixture")
write_json(folder / "queries/result.json", {"queries": [{"answer": "answer", "error": None}]})
spec = EvaluationSpec(reuse_construction_from=str(source))
if missing == 1:
eligible = replay_sampling_eligibility(home, spec, selected)
assert len(eligible) == 19 and "case-0" not in eligible
else:
with pytest.raises(ValueError, match="5%"):
replay_sampling_eligibility(home, spec, selected)
@pytest.mark.asyncio
async def test_query_budget_and_draw_order_reach_actual_execution(app, inputs, monkeypatch):
from unittest.mock import AsyncMock
source = Path(inputs[0]["benchmark"]["source"])
for path in source.glob("chats/*/*/probing_questions/probing_questions.json"):
payload = json.loads(path.read_text())
payload["fact"] = [
dict(payload["fact"][0], question=f"question {index}", rubric=["says hello"]) for index in range(4)
]
path.write_text(json.dumps(payload))
prepared = await app.run_job("home_prepare", spec=inputs[0])
assert prepared.success
monkeypatch.setattr("reme_home.evaluation.component.DockerReMeSandboxFactory", FakeFactory)
monkeypatch.setattr(
app.context.components["agent_wrapper"]["judge"],
"reply",
AsyncMock(return_value={"result": '{"score": 0.5}'}),
)
monkeypatch.setattr(sampling.time, "time_ns", lambda: 123456)
component = app.context.components["home.evaluation"]["default"]
component.sampling_policy = sampling.SamplingPolicy(max_cases=2, max_queries=3)
response = await app.run_job("home_evaluate", sampling=True, concurrency=1)
assert response.success, response.answer
run = Path(response.answer["result_dir"])
manifest = json.loads((run / "manifest.json").read_text())
home = Path(app.config.workspace_dir) / "home"
plan = json.loads((home / manifest["sampling"]["plan_path"]).read_text())
assert manifest["query_ids"] == plan["query_ids"]
assert len(manifest["case_ids"]) == 2 and sum(map(len, manifest["query_ids"].values())) == 3
worker = FakeFactory.instances[-1].workers[0]
assert [query.query_id for query in worker.queries] == manifest["query_ids"][worker.case_id]
for case_id, query_ids in manifest["query_ids"].items():
result = json.loads((run / "cases" / case_id / "queries/result.json").read_text())
assert [row["query_id"] for row in result["queries"]] == query_ids
assert response.answer["scored_query_count"] == 3

View file

@ -0,0 +1,209 @@
"""Home layout initialization through the application and component lifecycle."""
import json
from importlib.metadata import EntryPoint
from pathlib import Path
import pytest
from reme_home.workspace import DIRECTORIES, HomeWorkspace, _initialization_lock
from reme.application import Application
from reme.components.application_context import ApplicationContext
def component(root, **config):
"""Build an isolated component without starting model or service clients."""
return HomeWorkspace(app_context=ApplicationContext(workspace_dir=str(root), **config))
@pytest.mark.asyncio
async def test_application_start_initializes_enabled_home(tmp_path, monkeypatch):
"""The installed plugin contributes its component without changing core."""
entry = EntryPoint(name="home", value="reme_home", group="reme.plugins")
monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[2] / "beam-judge/src"))
judge = EntryPoint(name="beam-judge", value="judge_beam", group="reme.plugins")
monkeypatch.setattr("reme.plugin.find_entry_points", lambda group, name: [entry if name == "home" else judge])
monkeypatch.setenv("LLM_API_KEY", "test")
app = Application(
workspace_dir=str(tmp_path),
plugins=["home", "beam-judge"],
service={"backend": "cli"},
enable_logo=False,
log_to_console=False,
log_to_file=False,
)
assert not (tmp_path / "home/code").exists()
try:
await app.start()
assert all((tmp_path / "home" / path).is_dir() for path in DIRECTORIES)
assert (tmp_path / "session").is_dir()
assert (tmp_path / "home/metadata/workspace.json").is_file()
assert not (tmp_path / "code").exists()
assert not (tmp_path / "metadata/home").exists()
workspace = app.context.components["home.workspace"]["default"]
assert workspace.home_path == tmp_path / "home"
assert workspace.state_path == tmp_path / "home/metadata"
assert app.context.components["home.workspace"]["default"].manifest is not None
finally:
await app.close()
@pytest.mark.asyncio
async def test_application_without_home_keeps_standard_layout(tmp_path):
"""Ordinary ReMe startup does not acquire Home directories."""
app = Application(
workspace_dir=str(tmp_path),
service={"backend": "cli"},
enable_logo=False,
log_to_console=False,
log_to_file=False,
)
try:
await app.start()
assert not (tmp_path / "home/code").exists()
finally:
await app.close()
@pytest.mark.asyncio
async def test_empty_roots_and_existing_memory_are_preserved(tmp_path):
"""First initialization accepts empty Home roots alongside user memory."""
(tmp_path / "daily").mkdir()
note = tmp_path / "daily/note.md"
note.write_text("owned memory", encoding="utf-8")
(tmp_path / "code").mkdir()
unrelated = tmp_path / "code/user.py"
unrelated.write_text("user-owned code", encoding="utf-8")
(tmp_path / "home/code").mkdir(parents=True)
workspace = component(tmp_path, metadata_dir="state")
await workspace.start()
await workspace.close()
marker = tmp_path / "home/metadata/workspace.json"
before = marker.read_bytes()
(tmp_path / "home/code/repo/candidate.py").write_text("candidate", encoding="utf-8")
(tmp_path / "home/sampling/plans").rmdir()
reopened = component(tmp_path, metadata_dir="state")
await reopened.start()
await reopened.close()
assert marker.read_bytes() == before
assert note.read_text(encoding="utf-8") == "owned memory"
assert unrelated.read_text(encoding="utf-8") == "user-owned code"
assert (tmp_path / "home/code/repo/candidate.py").read_text(encoding="utf-8") == "candidate"
assert (tmp_path / "home/sampling/plans").is_dir()
@pytest.mark.asyncio
@pytest.mark.parametrize("relative", ["code/note.txt", "code/repo", "docs/guide.md"])
async def test_first_initialization_rejects_nonempty_roots(tmp_path, relative):
"""Even an empty child directory makes a top-level directory nonempty."""
target = tmp_path / "home" / relative
target.parent.mkdir(parents=True)
if relative.endswith("repo"):
target.mkdir()
else:
target.write_text("keep", encoding="utf-8")
with pytest.raises(ValueError, match="non-empty Home directory"):
await component(tmp_path).start()
assert target.exists()
assert not (tmp_path / "home/metadata/workspace.json").exists()
assert not (tmp_path / "home/datasets").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize("relative", ["code", "metadata", "metadata/workspace.json", "."])
async def test_rejects_file_conflicts(tmp_path, relative):
"""Conflicting files are retained and reported."""
target = tmp_path / "home" / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text("keep", encoding="utf-8")
with pytest.raises((ValueError, OSError)):
await component(tmp_path).start()
assert target.read_text(encoding="utf-8") == "keep"
@pytest.mark.asyncio
@pytest.mark.parametrize("relative", ["code", "metadata", "metadata/workspace.json", "."])
async def test_rejects_symlinks(tmp_path, relative):
"""Managed paths must not follow links, even to another internal path."""
target = tmp_path / "home" / relative
target.parent.mkdir(parents=True, exist_ok=True)
destination = tmp_path / "destination"
destination.mkdir()
target.symlink_to(destination, target_is_directory=True)
with pytest.raises(ValueError):
await component(tmp_path).start()
assert not list(destination.iterdir())
@pytest.mark.asyncio
@pytest.mark.parametrize("content", ["broken json", '{"schema_version": 2}', '{"schema_version": 1}'])
async def test_rejects_invalid_manifest(tmp_path, content):
"""A broken or unsupported manifest never resets workspace identity."""
marker = tmp_path / "home/metadata/workspace.json"
marker.parent.mkdir(parents=True)
marker.write_text(content, encoding="utf-8")
with pytest.raises(ValueError):
await component(tmp_path).start()
assert marker.read_text(encoding="utf-8") == content
assert not (tmp_path / "home/code").exists()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"config",
[{"daily_dir": "home/code"}, {"session_dir": "home/docs/dialog"}, {"metadata_dir": "home"}, {"metadata_dir": ".."}],
)
async def test_rejects_configuration_collisions(tmp_path, config):
"""ReMe data directories cannot overlap Home's reserved layout."""
with pytest.raises(ValueError):
await component(tmp_path, **config).start()
assert not (tmp_path / "home/code").exists()
@pytest.mark.asyncio
async def test_publication_failure_rolls_back_only_created_directories(tmp_path, monkeypatch):
"""Failed initialization can be retried without touching pre-existing paths."""
(tmp_path / "home/code").mkdir(parents=True)
(tmp_path / "keep.txt").write_text("keep", encoding="utf-8")
workspace = component(tmp_path)
def fail_replace(*_args):
raise OSError("disk error")
with monkeypatch.context() as patch:
patch.setattr("reme_home.workspace.os.replace", fail_replace)
with pytest.raises(OSError, match="disk error"):
await workspace.start()
assert (tmp_path / "home/code").is_dir()
assert not list((tmp_path / "home/code").iterdir())
assert not (tmp_path / "home/datasets").exists()
assert (tmp_path / "keep.txt").read_text(encoding="utf-8") == "keep"
assert not list((tmp_path / "home/metadata").glob("*.tmp"))
await workspace.start()
await workspace.close()
assert json.loads((tmp_path / "home/metadata/workspace.json").read_text())["schema_version"] == 1
@pytest.mark.asyncio
async def test_initialization_lock_prevents_competing_writer(tmp_path):
"""A competing initializer fails explicitly and can retry after release."""
state = tmp_path / "home/metadata"
state.mkdir(parents=True)
with _initialization_lock(state / "workspace.lock"):
with pytest.raises(RuntimeError, match="already in progress"):
await component(tmp_path).start()
workspace = component(tmp_path)
await workspace.start()
await workspace.close()
@pytest.mark.asyncio
async def test_legacy_layout_requires_new_workspace(tmp_path):
"""Never silently adopt or move a previously initialized flat workspace."""
marker = tmp_path / "metadata/home/workspace.json"
marker.parent.mkdir(parents=True)
marker.write_text('{"legacy": true}', encoding="utf-8")
with pytest.raises(ValueError, match="Legacy Home workspace layout"):
await component(tmp_path).start()
assert marker.read_text(encoding="utf-8") == '{"legacy": true}'
assert not (tmp_path / "home").exists()

View file

@ -1,3 +1,5 @@
# ReMe LongMemEval Judge
Trusted LongMemEval scoring plugin. Install it alongside `plugins/lme` for a full benchmark run. Meta-ReMe candidate bundles must not include this plugin.
Unrecognized yes/no output is retried once. If both outputs are invalid, the Job fails instead of returning a score.

View file

@ -55,13 +55,15 @@ class LmeAnswerJudgeStep(BaseStep):
golden_answer=golden_answer,
agent_answer=agent_answer,
)
result = await self.agent_wrapper.reply(
user_prompt,
system_prompt=self.prompt_format(judge_prompt_key),
)
raw_answer = (result.get("result") or "").strip()
answer = self._normalize_judgement(raw_answer)
system_prompt = self.prompt_format(judge_prompt_key)
for _ in range(2): # One initial attempt and at most one invalid-output retry.
result = await self.agent_wrapper.reply(user_prompt, system_prompt=system_prompt)
raw_answer = (result.get("result") or "").strip()
answer = self._normalize_judgement(raw_answer)
if answer in {"yes", "no"}:
break
else:
raise ValueError(f"LongMemEval judge returned an invalid yes/no judgement after 2 attempts: {raw_answer!r}")
self.logger.info(f"[{self.name}] answer judgement: {answer}")
self.context["answer_judgement"] = answer

View file

@ -299,6 +299,11 @@ class AsAgentWrapper(BaseAgentWrapper):
"""Resolve configured skill names to AgentScope local skill directories."""
return [str(path) for path in self._resolve_project_skills(skills).values()]
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> Agent:
"""Construct the SDK agent while subclasses reuse ReMe's tool and session wiring."""
del options
return Agent(**kwargs)
async def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]:
"""Build an Agent instance from kwargs. Returns (agent, processed_inputs)."""
model = self.as_llm.model if self.as_llm else None
@ -327,7 +332,8 @@ class AsAgentWrapper(BaseAgentWrapper):
perm_mode = PermissionMode(kwargs.get("permission_mode", "bypass"))
state = await self._load_state(kwargs, perm_mode)
agent = Agent(
agent = self._create_agent(
options=kwargs,
name=self.name,
system_prompt=system_prompt,
model=model,

View file

@ -13,25 +13,45 @@ if TYPE_CHECKING:
from ...steps import BaseStep
def _describe_exception(exc: BaseException) -> str:
"""Render an exception and its causes without dropping empty messages."""
parts: list[str] = []
def _exception_chain(exc: BaseException):
"""Walk explicit causes or unsuppressed contexts without following cycles."""
current: BaseException | None = exc
seen: set[int] = set()
while current is not None and id(current) not in seen:
seen.add(id(current))
message = str(current).strip()
name = type(current).__name__
parts.append(f"{name}: {message}" if message else name)
yield current
if current.__cause__ is not None:
current = current.__cause__
elif not current.__suppress_context__:
current = current.__context__
else:
current = None
def _describe_exception(exc: BaseException) -> str:
"""Render an exception and its causes without dropping empty messages."""
parts = []
for current in _exception_chain(exc):
message = str(current).strip()
name = type(current).__name__
parts.append(f"{name}: {message}" if message else name)
return " <- ".join(parts)
def _exception_details(exc: BaseException) -> dict:
"""Expose classification fields only, without requests, headers or bodies."""
chain = []
for current in _exception_chain(exc):
details = {"type": type(current).__name__, "module": type(current).__module__}
status = getattr(current, "status_code", None)
if not isinstance(status, int):
status = getattr(getattr(current, "response", None), "status_code", None)
if isinstance(status, int) and not isinstance(status, bool):
details["status_code"] = status
chain.append(details)
return {**chain[0], "causes": chain[1:]}
@R.register("base")
class BaseJob(BaseComponent):
"""Job that executes steps sequentially and returns a Response."""
@ -103,4 +123,5 @@ class BaseJob(BaseComponent):
self.logger.exception(f"Failed to execute job: {e}")
context.response.success = False
context.response.answer = _describe_exception(e)
context.response.metadata["reme.job_error"] = _exception_details(e)
return context.response

View file

@ -10,6 +10,8 @@ class Response(BaseModel):
``answer`` is the primary tool result and must be sufficient for the next LLM action.
``metadata`` is auxiliary request context for programmatic clients and diagnostics.
Base jobs record caught exception types, modules, HTTP status codes and causes
under ``metadata["reme.job_error"]`` while retaining their error text in ``answer``.
"""
model_config = ConfigDict(extra="allow")

View file

@ -225,3 +225,55 @@ async def test_real_docker_prebuilt_image(base_image, tmp_path):
assert not _container_exists(container_name)
finally:
_docker("image", "rm", image)
@pytest.mark.asyncio
async def test_real_docker_preserves_caught_job_errors(base_image, tmp_path):
"""Structured errors survive BaseJob capture and the container JSON boundary."""
source = _candidate_source(tmp_path)
(source / "reme/steps/smoke_failure.py").write_text(
"from reme.steps.base_step import BaseStep\n"
"from reme.components.component_registry import R\n"
'@R.register("smoke_failure_step")\n'
"class SmokeFailureStep(BaseStep):\n"
" async def execute(self):\n"
' if self.context.get("wrapped"):\n'
" try:\n"
' raise TimeoutError("model timed out")\n'
" except TimeoutError as error:\n"
' raise RuntimeError("model call failed") from error\n'
' raise ValueError("candidate error")\n',
encoding="utf-8",
)
with (source / "reme/steps/__init__.py").open("a", encoding="utf-8") as stream:
stream.write("\nfrom . import smoke_failure\n")
(source / "smoke.yaml").write_text(
SMOKE_CONFIG + "\n smoke_failure:\n backend: base\n steps:\n - backend: smoke_failure_step\n",
encoding="utf-8",
)
factory = DockerReMeSandboxFactory(
SourceCandidate(SourceSnapshot.from_directory(source), base_image=base_image),
config="/workspace/candidate/smoke.yaml",
)
async with await factory.create_case("error-smoke") as case:
build = await case.run_build([JobRequest("smoke_failure", {"wrapped": True})])
assert not build["success"]
error = build["jobs"][0]["result"]["exception"]
assert error["type"] == "RuntimeError"
assert error["causes"] == [{"type": "TimeoutError", "module": "builtins"}]
queries = await case.run_queries(
[
EvaluationQuery("code-error", "fail", None, answer_job="smoke_failure"),
EvaluationQuery(
"infra-error",
"fail",
None,
answer_job="smoke_failure",
answer_arguments={"wrapped": True},
),
],
)
assert not queries["success"]
for row, name in zip(queries["queries"], ["ValueError", "RuntimeError"], strict=True):
assert row["exception"]["type"] == name
assert row["exception"] == row["answer_result"]["metadata"]["reme.job_error"]

View file

@ -774,7 +774,7 @@ class FakeBackend:
output = json.dumps(
{
"reme": "/workspace/candidate/reme/__init__.py",
"agentscope": "2.0.9",
"agentscope": "2.0.10",
"plugins": {"lme": "/workspace/candidate/plugins/lme/src/reme_lme/__init__.py"},
},
).encode()

View file

@ -0,0 +1,84 @@
"""Observe the first initialized sandbox without counting failed startup attempts."""
import asyncio
from unittest.mock import AsyncMock
import pytest
from deploy.benchmark_docker import DockerReMeSandboxFactory, ImageCandidate
from deploy.benchmark_docker import docker_sandbox
@pytest.mark.asyncio
async def test_start_callback_only_once_after_initialization(monkeypatch):
"""Initialization is the boundary, including replacement workers and partial pool failure."""
initialized, reported = [], []
async def initialize(case):
await asyncio.sleep(0)
initialized.append(case.case_id)
if case.case_id == "bad":
raise RuntimeError("startup failed")
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
workspace = AsyncMock()
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=lambda *_: workspace,
on_sandbox_started=lambda: reported.append(list(initialized)),
)
with pytest.raises(RuntimeError, match="startup failed"):
await factory.create_case("bad")
assert not reported
await factory.create_case("good")
assert reported == [["bad", "good"]]
await factory.create_case("replacement")
assert len(reported) == 1
@pytest.mark.asyncio
async def test_start_callback_failure_closes_initialized_resource(monkeypatch):
"""Persisting a start marker must not orphan an already-created workspace on failure."""
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", AsyncMock())
workspace = AsyncMock()
def fail():
raise OSError("start marker unavailable")
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=lambda *_: workspace,
on_sandbox_started=fail,
)
with pytest.raises(OSError, match="start marker unavailable"):
await factory.create_case("created")
workspace.close.assert_awaited_once()
assert not factory._sandbox_started_reported # pylint: disable=protected-access
@pytest.mark.asyncio
async def test_partial_pool_start_reports_first_success_before_failure(monkeypatch):
"""An initialized worker counts even if a subsequent worker prevents pool startup."""
workspaces, started = [], []
def workspace_builder(*_args):
workspace = AsyncMock()
workspaces.append(workspace)
return workspace
async def initialize(case):
if case.case_id == "bad":
raise ConnectionError("second worker failed")
monkeypatch.setattr(docker_sandbox.DockerReMeSandbox, "initialize", initialize)
factory = DockerReMeSandboxFactory(
ImageCandidate("fixture"),
workspace_builder=workspace_builder,
on_sandbox_started=lambda: started.append(True),
)
with pytest.raises(ConnectionError, match="second worker failed"):
await factory.create_cases(["good", "bad"], concurrency=1)
assert started == [True]
assert len(workspaces) == 2
for workspace in workspaces:
workspace.close.assert_awaited_once()

View file

@ -0,0 +1,54 @@
"""Host and candidate version guards match the supported AgentScope runtime."""
from types import SimpleNamespace
from unittest.mock import AsyncMock
import json
import pytest
from deploy.benchmark_docker import ImageCandidate
from deploy.benchmark_docker.docker_sandbox import (
DockerReMeSandbox,
SandboxCommandError,
_default_workspace_builder,
)
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10"])
def test_host_version_guard(monkeypatch, version):
"""The configured dependency can create a workspace; stale hosts fail early."""
pytest.importorskip("aiodocker")
monkeypatch.setattr("agentscope.__version__", version)
if version == "2.0.9":
with pytest.raises(RuntimeError, match="requires agentscope 2.0.10"):
_default_workspace_builder("version-test", "test-image", {})
else:
workspace = _default_workspace_builder("version-test", "test-image", {})
assert workspace.base_image == "test-image"
@pytest.mark.asyncio
@pytest.mark.parametrize("version", ["2.0.9", "2.0.10"])
async def test_candidate_version_guard(monkeypatch, version):
"""Candidate validation rejects stale images even on a supported host."""
sandbox = DockerReMeSandbox(
case_id="version-test",
container_id="version-test",
candidate=ImageCandidate("test-image"),
workspace=None,
config="benchmark.yaml",
plugins=(),
command_timeout=30,
environment_names=[],
)
validation = {"agentscope": version, "plugins": {}}
monkeypatch.setattr(
sandbox,
"_exec_checked",
AsyncMock(return_value=SimpleNamespace(stdout=json.dumps(validation).encode())),
)
if version == "2.0.9":
with pytest.raises(SandboxCommandError, match="requires agentscope 2.0.10"):
await sandbox._validate_candidate("python") # pylint: disable=protected-access
else:
await sandbox._validate_candidate("python") # pylint: disable=protected-access

View file

@ -0,0 +1,60 @@
"""Incremental benchmark persistence and build-policy hooks without Docker."""
# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument
import pytest
from deploy.benchmark_docker import BenchmarkDockerPool, CasePlan, EvaluationQuery, JobRequest
from .test_benchmark_docker import FakeCase, FakeFactory
@pytest.mark.asyncio
async def test_pool_incremental_callback_and_partial_build_policy(tmp_path):
events = []
class PartialCase(FakeCase):
async def run_build(self, jobs):
await super().run_build(jobs)
return {"success": False, "jobs": [{"result": {"success": False}}, {"result": {"success": True}}]}
class Factory(FakeFactory):
async def create_cases(self, case_ids, **kwargs):
return [PartialCase(name, self.events) for name in case_ids]
completed = []
async def finished(plan, result):
completed.append(result)
events.append((plan.case_id, "persist"))
plans = [
CasePlan(
"case",
build_jobs=(JobRequest("auto_memory"),),
queries=(EvaluationQuery("q", "question", "answer"),),
),
]
result = await BenchmarkDockerPool(Factory(events), concurrency=1).run(
plans,
on_case_finished=finished,
accept_build=lambda build: len(build["jobs"]) == 2,
fail_fast=True,
)
assert result == completed
assert ("case", "query", 1) in events
assert ("case", "persist") in events
assert ("pool-0", "close") in events
@pytest.mark.asyncio
async def test_pool_callback_failure_still_closes_workers():
events = []
async def fail(plan, result):
raise RuntimeError("persistence failed")
with pytest.raises(RuntimeError, match="persistence failed"):
await BenchmarkDockerPool(FakeFactory(events), concurrency=1).run(
[CasePlan("case", build_jobs=(JobRequest("auto_memory"),))],
on_case_finished=fail,
)
assert ("pool-0", "close") in events

View file

@ -489,3 +489,49 @@ def test_application_start_failure_propagates_and_closes_started_components():
assert not app._started_components
asyncio.run(run())
@pytest.mark.asyncio
async def test_job_error_metadata_preserves_status_and_causes_without_request_data():
import httpx
request = httpx.Request("POST", "https://example.test", headers={"Authorization": "secret"})
response = httpx.Response(429, request=request, text="private response body")
async def failing_step(context):
context.response.metadata["earlier_step"] = "preserved"
try:
raise httpx.HTTPStatusError("provider unavailable", request=request, response=response)
except httpx.HTTPStatusError as error:
raise RuntimeError("wrapped") from error
job = BaseJob(name="structured-error")
job._build_steps = lambda: [failing_step]
result = await job()
details = result.metadata["reme.job_error"]
assert not result.success
assert result.answer.startswith("RuntimeError: wrapped <- HTTPStatusError:")
assert details["type"] == "RuntimeError"
assert details["causes"] == [{"type": "HTTPStatusError", "module": "httpx", "status_code": 429}]
assert result.metadata["earlier_step"] == "preserved"
assert "secret" not in str(details) and "private response body" not in str(details)
result.model_dump_json()
job._build_steps = list
following = await job()
assert following.success
assert "reme.job_error" not in following.metadata
def test_job_error_metadata_handles_cyclic_and_suppressed_causes():
from reme.components.job.base_job import _exception_details
error = RuntimeError("outer")
cause = TimeoutError("inner")
error.__cause__ = cause
cause.__cause__ = error
assert _exception_details(error)["causes"] == [{"type": "TimeoutError", "module": "builtins"}]
error.__cause__ = None
error.__context__ = cause
error.__suppress_context__ = True
assert _exception_details(error)["causes"] == []

View file

@ -0,0 +1,61 @@
"""LongMemEval judging retries invalid outputs without treating them as scores."""
# pylint: disable=missing-function-docstring
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from reme.components.runtime_context import RuntimeContext
@pytest.fixture(name="judge")
def fixture_judge(monkeypatch):
root = Path(__file__).resolve().parents[2] / "plugins/lme-judge/src"
monkeypatch.syspath_prepend(str(root))
from judge_lme.llm_judge import LmeAnswerJudgeStep
step = LmeAnswerJudgeStep()
step.context = RuntimeContext.from_context(None, query="When?", agent_answer="Today", golden_answer="Today")
step.agent_wrapper = SimpleNamespace(reply=AsyncMock())
return step
@pytest.mark.asyncio
@pytest.mark.parametrize("raw,expected", [("YES", "yes"), (" No, incorrect.", "no")])
async def test_valid_judgement_does_not_retry(judge, raw, expected):
judge.agent_wrapper.reply.return_value = {"result": raw}
response = await judge.execute()
assert response.success
assert response.answer == expected
assert response.metadata["raw_answer_judgement"] == raw.strip()
assert judge.agent_wrapper.reply.await_count == 1
@pytest.mark.asyncio
async def test_invalid_output_retries_once_and_recovers(judge):
judge.agent_wrapper.reply.side_effect = [{"result": "mostly correct"}, {"result": "yes"}]
response = await judge.execute()
assert response.success and response.answer == "yes"
assert response.metadata["raw_answer_judgement"] == "yes"
calls = judge.agent_wrapper.reply.await_args_list
assert len(calls) == 2 and calls[0] == calls[1]
@pytest.mark.asyncio
@pytest.mark.parametrize("raw", ["", "maybe", "yesterday", "correct"])
async def test_two_invalid_outputs_fail_without_a_score(judge, raw):
judge.agent_wrapper.reply.return_value = {"result": raw}
with pytest.raises(ValueError, match="invalid yes/no judgement after 2 attempts"):
await judge.execute()
assert judge.agent_wrapper.reply.await_count == 2
assert "answer_judgement" not in judge.context.response.metadata
@pytest.mark.asyncio
async def test_transport_errors_are_not_invalid_output_retries(judge):
judge.agent_wrapper.reply.side_effect = TimeoutError("provider")
with pytest.raises(TimeoutError):
await judge.execute()
assert judge.agent_wrapper.reply.await_count == 1