mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
feat(home): add standalone benchmark test evaluation and fix compression recovery
This commit is contained in:
parent
abf36eceec
commit
0e4ec5d287
15 changed files with 854 additions and 15 deletions
|
|
@ -192,3 +192,39 @@ from benchmark.beam.run_home import run
|
|||
|
||||
result = await run(workspace_dir="/path/to/new-workspace")
|
||||
```
|
||||
|
||||
|
||||
## Evaluate a candidate on an independent test set
|
||||
|
||||
The default evaluates every 100K case, matching the previous standalone evaluation. This is an independent
|
||||
workspace, not a disjoint held-out split. Select another `variant`, or replace `all_cases: true` with
|
||||
`case_ids: [...]`, in [config_home_eval.yaml](config_home_eval.yaml) for a different test set.
|
||||
|
||||
The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home
|
||||
optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source
|
||||
repository or change the optimizer's checkout. Paths in the configuration are repository-relative.
|
||||
|
||||
Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`):
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
|
||||
# Terminal 1:
|
||||
reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]'
|
||||
# Terminal 2:
|
||||
reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false
|
||||
```
|
||||
|
||||
### Local one-shot test evaluation
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA
|
||||
# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
|
||||
```
|
||||
|
||||
Python callers can use `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`.
|
||||
The runner rebuilds construction and scores every selected query with sampling and history recording disabled.
|
||||
It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required
|
||||
for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance
|
||||
in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`.
|
||||
Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a
|
||||
new test workspace for another candidate or selection. Source and test workspaces must not overlap.
|
||||
|
|
|
|||
|
|
@ -172,3 +172,38 @@ python benchmark/beam/run_home.py --optimize
|
|||
不加 `--optimize` 时仍只准备。
|
||||
|
||||
默认选择 BEAM **100K 全部 case**。
|
||||
|
||||
|
||||
## 独立测试集评测
|
||||
|
||||
默认在独立工作区评测 **100K 全部 case**,与旧独立评测入口一致,不代表与训练集互斥的留出集。
|
||||
可在 [config_home_eval.yaml](config_home_eval.yaml) 更换 `variant`,或把 `all_cases: true`
|
||||
替换为 `case_ids: [...]`,指定测试数据。
|
||||
|
||||
入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA,
|
||||
并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。
|
||||
配置内路径均相对仓库根目录。
|
||||
|
||||
先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`):
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
|
||||
# 终端 1:
|
||||
reme start workspace_dir=benchmark/beam/workspaces/home-test-100K 'plugins=["home","beam-judge"]'
|
||||
# 终端 2:
|
||||
reme home_evaluate workspace_dir=benchmark/beam/workspaces/home-test-100K record_history=false
|
||||
```
|
||||
|
||||
### 本地一次性测试评测
|
||||
|
||||
```bash
|
||||
python benchmark/beam/run_home_eval.py --commit CANDIDATE_SHA
|
||||
# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
|
||||
```
|
||||
|
||||
Python 可调用 `await benchmark.beam.run_home_eval.run(commit="CANDIDATE_SHA")`。
|
||||
入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker
|
||||
评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`,
|
||||
manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。
|
||||
相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。
|
||||
测试工作区与来源工作区必须互不包含。
|
||||
|
|
|
|||
13
benchmark/beam/config_home_eval.yaml
Normal file
13
benchmark/beam/config_home_eval.yaml
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# Independent full-dataset evaluation; paths are relative to the ReMe repository.
|
||||
source_workspace: benchmark/beam/workspaces/home-100K
|
||||
test_workspace: benchmark/beam/workspaces/home-test-100K
|
||||
code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit.
|
||||
benchmark:
|
||||
name: beam
|
||||
source: benchmark/beam/dataset/BEAM
|
||||
variant: 100K
|
||||
all_cases: true # Full 100K evaluation; not a disjoint held-out split.
|
||||
evaluation:
|
||||
concurrency: 4
|
||||
timeout_seconds: 7200
|
||||
record_history: false
|
||||
21
benchmark/beam/run_home_eval.py
Normal file
21
benchmark/beam/run_home_eval.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Evaluate a BEAM Home candidate in an independent test workspace."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
|
||||
|
||||
|
||||
async def run(config_path=DEFAULT_CONFIG, **kwargs):
|
||||
"""Score the selected BEAM commit without changing its optimization workspace."""
|
||||
return await run_evaluation(config_path, expected_benchmark="beam", **kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main("beam", DEFAULT_CONFIG)
|
||||
207
benchmark/home_eval.py
Normal file
207
benchmark/home_eval.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
"""Standalone test evaluation shared by the BEAM and LongMemEval entry points."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
import yaml
|
||||
|
||||
from reme_home.code.repository import resolve_revision, validate_repository
|
||||
from reme_home.evaluation.artifacts import contained, read_json
|
||||
from reme_home.evaluation.component import blocking
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
|
||||
from reme_home.judge import JUDGE_PLUGINS
|
||||
from reme_home.preparation.code import tree_fingerprint
|
||||
from reme_home.preparation.data.models import fingerprint
|
||||
from reme_home.preparation.evaluation_source import EvaluationSource
|
||||
from reme_home.preparation.spec import BenchmarkSpec, DebugSpec, SpecModel
|
||||
|
||||
from reme.application import Application
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class TestRunConfig(SpecModel):
|
||||
"""Explicit dataset selection and execution options for one pinned candidate."""
|
||||
|
||||
source_workspace: str = Field(min_length=1)
|
||||
test_workspace: str = Field(min_length=1)
|
||||
code_commit: str = Field(min_length=4)
|
||||
benchmark: BenchmarkSpec
|
||||
exclude_training_cases: bool = False
|
||||
evaluation: EvaluationSpec = Field(default_factory=lambda: EvaluationSpec(record_history=False))
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_test_mode(self):
|
||||
"""Test runs score every prepared query and never consume optimizer budgets."""
|
||||
options = self.evaluation
|
||||
if any(
|
||||
(
|
||||
options.debug,
|
||||
options.sampling,
|
||||
options.record_history,
|
||||
options.max_validations is not None,
|
||||
options.sampling_policy is not None,
|
||||
options.case_ids is not None,
|
||||
options.query_ids is not None,
|
||||
options.reuse_construction_from is not None,
|
||||
),
|
||||
):
|
||||
raise ValueError(
|
||||
"Test evaluation requires full scoring with record_history=false; select cases in benchmark",
|
||||
)
|
||||
if self.exclude_training_cases and (self.benchmark.name != "longmemeval" or not self.benchmark.all_cases):
|
||||
raise ValueError("exclude_training_cases requires LongMemEval with all_cases=true")
|
||||
return self
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
"""Resolve runner paths independently of the launching working directory."""
|
||||
if value.startswith("~"):
|
||||
raise ValueError("Use absolute or project-relative paths, not home-relative paths")
|
||||
return (project_root / value).resolve()
|
||||
|
||||
|
||||
def load_request(config_path, *, commit=None, source_workspace=None, test_workspace=None, project_root=PROJECT_ROOT):
|
||||
"""Validate all source inputs before starting an application or writing test data."""
|
||||
raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
|
||||
for key, value in (
|
||||
("code_commit", commit),
|
||||
("source_workspace", source_workspace),
|
||||
("test_workspace", test_workspace),
|
||||
):
|
||||
if value is not None:
|
||||
raw[key] = value
|
||||
# Execution settings can be partial while retaining the test-only history default.
|
||||
raw["evaluation"] = {"record_history": False, **raw.get("evaluation", {})}
|
||||
config = TestRunConfig.model_validate(raw)
|
||||
source_path = project_path(config.source_workspace, project_root)
|
||||
target = project_path(config.test_workspace, project_root)
|
||||
if target.is_relative_to(source_path) or source_path.is_relative_to(target):
|
||||
raise ValueError("Test and source workspaces must be separate, non-nested directories")
|
||||
if target.exists() and (
|
||||
not target.is_dir() or (any(target.iterdir()) and not (target / "home/metadata/workspace.json").is_file())
|
||||
):
|
||||
raise ValueError("Use an empty test workspace or a compatible prepared Home test workspace")
|
||||
source = EvaluationSource.load(source_path, config.code_commit)
|
||||
benchmark = config.benchmark.model_copy(deep=True)
|
||||
if benchmark.name != source.spec.benchmark.name:
|
||||
raise ValueError("Test benchmark must match the source experiment")
|
||||
benchmark.source = str(project_path(benchmark.source, project_root).resolve(strict=True))
|
||||
if benchmark.evidence_source is not None:
|
||||
benchmark.evidence_source = str(project_path(benchmark.evidence_source, project_root).resolve(strict=True))
|
||||
if config.exclude_training_cases:
|
||||
items = json.loads(Path(benchmark.source).read_text(encoding="utf-8"))
|
||||
ids = [str(item["question_id"]) for item in items]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("Duplicate LongMemEval question IDs")
|
||||
training = set(source.record["dataset"]["case_ids"])
|
||||
if training - set(ids):
|
||||
raise ValueError("Test dataset is missing training IDs; provide the original full dataset")
|
||||
benchmark.case_ids = [case_id for case_id in ids if case_id not in training]
|
||||
benchmark.all_cases = False
|
||||
benchmark = BenchmarkSpec.model_validate(benchmark.model_dump())
|
||||
spec = source.spec.model_copy(deep=True)
|
||||
spec.benchmark = benchmark
|
||||
spec.debug = DebugSpec(enabled=False)
|
||||
return config, target, source, spec
|
||||
|
||||
|
||||
def existing_preparation(home, source, spec):
|
||||
"""Reopen only the same request with unchanged code, runtime and normalized data."""
|
||||
marker = contained(home, "metadata/preparation.json")
|
||||
if not marker.exists():
|
||||
return None
|
||||
record = read_json(marker)
|
||||
expected = {**source.provenance, "request_fingerprint": fingerprint(spec.model_dump(mode="json"))}
|
||||
if record.get("test_evaluation") != expected:
|
||||
raise ValueError("Test workspace belongs to a different candidate or dataset; use a new workspace")
|
||||
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
|
||||
if fingerprint(domain) != record["config_fingerprint"]:
|
||||
raise ValueError("Test preparation configuration has changed")
|
||||
repository, _ = validate_repository(home, record)
|
||||
if (
|
||||
resolve_revision(repository, "HEAD") != record["code"]["initial_commit"]
|
||||
or tree_fingerprint(repository) != record["code"]["candidate_fingerprint"]
|
||||
or tree_fingerprint(contained(home, "code/runtime", directory=True)) != source.provenance["runtime_fingerprint"]
|
||||
):
|
||||
raise ValueError("Prepared test code or runtime has changed")
|
||||
load_inputs(home, EvaluationSpec(record_history=False))
|
||||
return record
|
||||
|
||||
|
||||
async def run(
|
||||
config_path,
|
||||
*,
|
||||
commit=None,
|
||||
source_workspace=None,
|
||||
test_workspace=None,
|
||||
prepare_only=False,
|
||||
expected_benchmark=None,
|
||||
):
|
||||
"""Prepare transactionally, then score through the normal application lifecycle."""
|
||||
config, target, source, spec = await blocking(
|
||||
load_request,
|
||||
config_path,
|
||||
commit=commit,
|
||||
source_workspace=source_workspace,
|
||||
test_workspace=test_workspace,
|
||||
)
|
||||
if expected_benchmark is not None and config.benchmark.name != expected_benchmark:
|
||||
raise ValueError(f"This entry point requires benchmark.name={expected_benchmark}")
|
||||
app = Application(
|
||||
workspace_dir=str(target),
|
||||
plugins=["home", JUDGE_PLUGINS[config.benchmark.name]],
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
try:
|
||||
await app.start()
|
||||
home = target / "home"
|
||||
prepared = await blocking(existing_preparation, home, source, spec)
|
||||
if prepared is None:
|
||||
preparation = app.context.components["home.preparation"]["default"]
|
||||
prepared = await preparation.prepare(spec=spec.model_dump(mode="json"), code_source=source)
|
||||
result = {
|
||||
"test_workspace": str(target),
|
||||
**source.provenance,
|
||||
"evaluation_commit_sha": prepared["code"]["initial_commit"],
|
||||
"case_count": prepared["dataset"]["case_count"],
|
||||
"query_count": prepared["dataset"]["query_count"],
|
||||
}
|
||||
if not prepare_only:
|
||||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
**config.evaluation.model_dump(mode="json", exclude_unset=True),
|
||||
)
|
||||
if not response.success:
|
||||
raise RuntimeError(response.answer)
|
||||
result["evaluation"] = response.answer
|
||||
return result
|
||||
finally:
|
||||
await app.close()
|
||||
|
||||
|
||||
def main(benchmark, default_config):
|
||||
"""Explicit local one-shot entry point, with preparation-only service setup."""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, default=default_config)
|
||||
parser.add_argument("--commit", help="Candidate commit SHA or unambiguous hexadecimal prefix")
|
||||
parser.add_argument("--source-workspace", help="Prepared optimization workspace")
|
||||
parser.add_argument("--test-workspace", help="Independent test output workspace")
|
||||
parser.add_argument("--prepare-only", action="store_true", help="Prepare for a later home_evaluate service call")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(
|
||||
run(
|
||||
args.config,
|
||||
commit=args.commit,
|
||||
source_workspace=args.source_workspace,
|
||||
test_workspace=args.test_workspace,
|
||||
prepare_only=args.prepare_only,
|
||||
expected_benchmark=benchmark,
|
||||
),
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
|
@ -174,3 +174,40 @@ out. The runner reads the IDs directly from YAML without loading group files.
|
|||
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl) is a byte-for-byte
|
||||
copy of that revision's `dataset/longmemeval_evidence_sessions.jsonl`;
|
||||
reviewed evidence-session annotations are preserved during normalization.
|
||||
|
||||
|
||||
## Evaluate a candidate on an independent test set
|
||||
|
||||
The default excludes the source experiment's actual training IDs from the full dataset. With the default
|
||||
250-case training workspace this selects held-out groups 6–10. For an explicit selection, set
|
||||
`exclude_training_cases: false` and replace `all_cases: true` with `case_ids: [...]` in
|
||||
[config_home_eval.yaml](config_home_eval.yaml). An empty held-out selection fails before preparation.
|
||||
|
||||
The entry point imports a full candidate SHA or unambiguous hexadecimal prefix from an existing Home
|
||||
optimization workspace and copies its prepared fixed runtime. It does not require the original ReMe source
|
||||
repository or change the optimizer's checkout. Paths in the configuration are repository-relative.
|
||||
|
||||
Prepare first, then use the service from two terminals (replace `CANDIDATE_SHA`):
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
|
||||
# Terminal 1:
|
||||
reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]'
|
||||
# Terminal 2:
|
||||
reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false
|
||||
```
|
||||
|
||||
### Local one-shot test evaluation
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA
|
||||
# Optional overrides: --config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
|
||||
```
|
||||
|
||||
Python callers can use `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`.
|
||||
The runner rebuilds construction and scores every selected query with sampling and history recording disabled.
|
||||
It uses the existing Docker evaluation runtime and host judge configuration; model credentials are required
|
||||
for evaluation. Results stay in the test workspace's `home/evaluations/`, with original candidate provenance
|
||||
in each manifest's `test_evaluation`. The exported snapshot has its own local `evaluation_commit_sha`.
|
||||
Rerunning the same request reopens unchanged prepared inputs and creates another evaluation result. Use a
|
||||
new test workspace for another candidate or selection. Source and test workspaces must not overlap.
|
||||
|
|
|
|||
|
|
@ -147,3 +147,39 @@ ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchma
|
|||
运行时无需读取分组文件,group_6–group_10 保持为留出集。
|
||||
独立审阅的证据标注保存在
|
||||
[home_evidence_sessions.jsonl](home_evidence_sessions.jsonl),来源见英文说明。
|
||||
|
||||
|
||||
## 独立测试集评测
|
||||
|
||||
默认从完整数据中排除来源工作区实际使用的训练 ID;使用默认 250 个训练 case 时,测试集为
|
||||
**group_6–group_10,共 250 个 case**。若要显式选取测试 case,可在
|
||||
[config_home_eval.yaml](config_home_eval.yaml) 设置 `exclude_training_cases: false`,
|
||||
并把 `all_cases: true` 替换为 `case_ids: [...]`。留出集为空时会直接报错。
|
||||
|
||||
入口从已有 Home 优化工作区导出指定候选提交,接受完整 SHA 或无歧义的十六进制短 SHA,
|
||||
并复制该实验已准备的固定 runtime。无需原始 ReMe 源码仓库,也不会切换或修改优化工作区。
|
||||
配置内路径均相对仓库根目录。
|
||||
|
||||
先准备测试工作区,再分别在两个终端启动服务和调用评测(替换 `CANDIDATE_SHA`):
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA --prepare-only
|
||||
# 终端 1:
|
||||
reme start workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out 'plugins=["home","lme-judge"]'
|
||||
# 终端 2:
|
||||
reme home_evaluate workspace_dir=benchmark/longmemeval/workspaces/home-test-held-out record_history=false
|
||||
```
|
||||
|
||||
### 本地一次性测试评测
|
||||
|
||||
```bash
|
||||
python benchmark/longmemeval/run_home_eval.py --commit CANDIDATE_SHA
|
||||
# 可选:--config /path/to/test.yaml --source-workspace /path/to/search --test-workspace /path/to/test
|
||||
```
|
||||
|
||||
Python 可调用 `await benchmark.longmemeval.run_home_eval.run(commit="CANDIDATE_SHA")`。
|
||||
入口完整重建记忆并对所选 case 的全部 query 评分,关闭采样与优化历史记录,使用现有 Docker
|
||||
评测环境和宿主 judge 配置;实际评测需要模型凭证。结果保存在测试工作区的 `home/evaluations/`,
|
||||
manifest 的 `test_evaluation` 记录来源候选,导出后的本地提交另记为 `evaluation_commit_sha`。
|
||||
相同请求可复用未改动的准备结果并新增一次评测;更换候选或数据选择时需使用新测试工作区。
|
||||
测试工作区与来源工作区必须互不包含。
|
||||
|
|
|
|||
14
benchmark/longmemeval/config_home_eval.yaml
Normal file
14
benchmark/longmemeval/config_home_eval.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# Evaluate the complement of the source experiment's training IDs in the full dataset.
|
||||
# With the default training workspace this selects held-out groups 6-10 (250 cases).
|
||||
source_workspace: benchmark/longmemeval/workspaces/home-groups-1-5
|
||||
test_workspace: benchmark/longmemeval/workspaces/home-test-held-out
|
||||
code_commit: "" # Required: set a candidate SHA/prefix, or pass --commit.
|
||||
exclude_training_cases: true
|
||||
benchmark:
|
||||
name: longmemeval
|
||||
source: benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json
|
||||
all_cases: true
|
||||
evaluation:
|
||||
concurrency: 4
|
||||
timeout_seconds: 7200
|
||||
record_history: false
|
||||
21
benchmark/longmemeval/run_home_eval.py
Normal file
21
benchmark/longmemeval/run_home_eval.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Evaluate a LongMemEval Home candidate on an independent test dataset."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from benchmark.home_eval import main, run as run_evaluation # noqa: E402 # pylint: disable=wrong-import-position
|
||||
|
||||
DEFAULT_CONFIG = Path(__file__).with_name("config_home_eval.yaml")
|
||||
|
||||
|
||||
async def run(config_path=DEFAULT_CONFIG, **kwargs):
|
||||
"""Score the selected LongMemEval commit in its own Home workspace."""
|
||||
return await run_evaluation(config_path, expected_benchmark="longmemeval", **kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main("longmemeval", DEFAULT_CONFIG)
|
||||
|
|
@ -5,7 +5,7 @@ import logging
|
|||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from agentscope.exception import ToolJSONDecodeError
|
||||
from agentscope.exception import StructuredOutputError, ToolJSONDecodeError
|
||||
from agentscope.message import TextBlock
|
||||
import jsonschema
|
||||
import openai
|
||||
|
|
@ -67,7 +67,16 @@ class RecoverableCompressionAgent(BoundedReActAgent):
|
|||
"""Match dev_meta's model/output failures, excluding uncompressible prompts."""
|
||||
if isinstance(exc, RuntimeError):
|
||||
return not str(exc).startswith("The system prompt")
|
||||
return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError))
|
||||
return isinstance(
|
||||
exc,
|
||||
(
|
||||
StructuredOutputError,
|
||||
ToolJSONDecodeError,
|
||||
jsonschema.ValidationError,
|
||||
ValidationError,
|
||||
openai.OpenAIError,
|
||||
),
|
||||
)
|
||||
|
||||
async def _fallback_compress_context(self, cause, context_ratio, config):
|
||||
"""Persist older messages before replacing state; never call a second model."""
|
||||
|
|
|
|||
|
|
@ -291,6 +291,8 @@ class HomeEvaluation(BaseComponent):
|
|||
}
|
||||
if source is not None:
|
||||
manifest["construction_source"] = source
|
||||
if "test_evaluation" in record:
|
||||
manifest["test_evaluation"] = record["test_evaluation"]
|
||||
if admission is not None:
|
||||
manifest["candidate_gate"] = admission
|
||||
if reuse is not None:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from reme.components.base_component import BaseComponent
|
|||
from reme.steps.base_step import BaseStep, Ref
|
||||
|
||||
from .code import prepare_code
|
||||
from .evaluation_source import EvaluationSource
|
||||
from .data import prepare_dataset
|
||||
from .data.basic import derive_debug_cases
|
||||
from .data.models import fingerprint
|
||||
|
|
@ -58,10 +59,18 @@ class HomePreparation(BaseComponent):
|
|||
if self._pending:
|
||||
await asyncio.gather(*self._pending, return_exceptions=True)
|
||||
|
||||
async def prepare(self, *, spec: dict | None = None, spec_path: str | None = None) -> dict:
|
||||
async def prepare(
|
||||
self,
|
||||
*,
|
||||
spec: dict | None = None,
|
||||
spec_path: str | None = None,
|
||||
code_source: EvaluationSource | None = None,
|
||||
) -> dict:
|
||||
"""Keep blocking data/Git work off the event loop and await rollback on cancellation."""
|
||||
stop = Event()
|
||||
task = asyncio.create_task(asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop))
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(self._prepare, spec=spec, spec_path=spec_path, stop=stop, code_source=code_source),
|
||||
)
|
||||
self._pending.add(task)
|
||||
cancelled = False
|
||||
try:
|
||||
|
|
@ -80,7 +89,7 @@ class HomePreparation(BaseComponent):
|
|||
finally:
|
||||
self._pending.discard(task)
|
||||
|
||||
def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event) -> dict:
|
||||
def _prepare(self, *, spec: dict | None, spec_path: str | None, stop: Event, code_source=None) -> dict:
|
||||
"""Prepare a started application's workspace from exactly one input form.
|
||||
|
||||
Local YAML paths must stay inside the workspace. Explicit dataset/repository
|
||||
|
|
@ -108,19 +117,23 @@ class HomePreparation(BaseComponent):
|
|||
self.judge.require_benchmark(config.benchmark.name)
|
||||
# Resolve explicitly supplied source inputs before creating transaction state.
|
||||
source = _source_path(workspace, config.benchmark.source)
|
||||
repository = _source_path(workspace, config.runtime.source_repo)
|
||||
repository = _source_path(workspace, config.runtime.source_repo) if code_source is None else None
|
||||
evidence = (
|
||||
_source_path(workspace, config.benchmark.evidence_source) if config.benchmark.evidence_source else None
|
||||
)
|
||||
config.benchmark.source = str(source)
|
||||
config.runtime.source_repo = str(repository)
|
||||
if repository is not None:
|
||||
config.runtime.source_repo = str(repository)
|
||||
config.benchmark.evidence_source = str(evidence) if evidence else None
|
||||
existed = check_targets(root, state)
|
||||
transaction = state / TRANSACTION
|
||||
transaction.mkdir()
|
||||
try:
|
||||
sync_directory(state)
|
||||
result = self._stage(config, transaction, source, repository, evidence)
|
||||
if code_source is None:
|
||||
result = self._stage(config, transaction, source, repository, evidence)
|
||||
else:
|
||||
result = self._stage(config, transaction, source, repository, evidence, code_source=code_source)
|
||||
|
||||
def check_cancelled():
|
||||
if stop.is_set():
|
||||
|
|
@ -142,7 +155,8 @@ class HomePreparation(BaseComponent):
|
|||
return result
|
||||
|
||||
@staticmethod
|
||||
def _stage(config, transaction, source, repository, evidence) -> dict:
|
||||
def _stage(config, transaction, source, repository, evidence, *, code_source=None) -> dict:
|
||||
request_fingerprint = fingerprint(config.model_dump(mode="json"))
|
||||
dataset = prepare_dataset(
|
||||
transaction / "output-2",
|
||||
config.benchmark.name,
|
||||
|
|
@ -163,7 +177,11 @@ class HomePreparation(BaseComponent):
|
|||
)
|
||||
else:
|
||||
(transaction / "output-3/cases").mkdir(parents=True)
|
||||
code = prepare_code(config, repository, transaction / "output-0", transaction / "output-1")
|
||||
code = (
|
||||
prepare_code(config, repository, transaction / "output-0", transaction / "output-1")
|
||||
if code_source is None
|
||||
else code_source.install(transaction / "output-0", transaction / "output-1")
|
||||
)
|
||||
config.runtime.source_ref = code["source_commit"]
|
||||
domain = config.model_dump(mode="json")
|
||||
(transaction / "output-4").write_text(yaml.safe_dump(domain, sort_keys=False), encoding="utf-8")
|
||||
|
|
@ -179,6 +197,11 @@ class HomePreparation(BaseComponent):
|
|||
"dataset": dataset.model_dump(mode="json"),
|
||||
"code": code,
|
||||
}
|
||||
if code_source is not None:
|
||||
result["test_evaluation"] = {
|
||||
**code_source.provenance,
|
||||
"request_fingerprint": request_fingerprint,
|
||||
}
|
||||
write_json(transaction / "output-6", result)
|
||||
return result
|
||||
|
||||
|
|
|
|||
87
plugins/home/src/reme_home/preparation/evaluation_source.py
Normal file
87
plugins/home/src/reme_home/preparation/evaluation_source.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
"""Read-only import of a pinned Home candidate and its fixed runtime."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
|
||||
import yaml
|
||||
|
||||
from ..code.repository import resolve_revision, validate_repository, validate_tree
|
||||
from ..code.snapshot import export_blob
|
||||
from ..evaluation.artifacts import contained, read_json
|
||||
from .code import git, tree_fingerprint
|
||||
from .data.models import fingerprint
|
||||
from .spec import PreparationSpec, relative_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvaluationSource:
|
||||
"""Host-owned source identity; imports never change the source checkout."""
|
||||
|
||||
home: Path
|
||||
commit: str
|
||||
spec: PreparationSpec
|
||||
record: dict
|
||||
|
||||
@classmethod
|
||||
def load(cls, workspace: Path, commit: str):
|
||||
"""Require a prepared workspace and an unambiguous commit SHA or prefix."""
|
||||
if re.fullmatch(r"[0-9a-fA-F]{4,64}", commit) is None:
|
||||
raise ValueError("code_commit must be a full commit SHA or hexadecimal prefix")
|
||||
home = contained(workspace, "home", directory=True)
|
||||
record = read_json(contained(home, "metadata/preparation.json"))
|
||||
domain = yaml.safe_load(contained(home, "domain_spec.yaml").read_text(encoding="utf-8"))
|
||||
if fingerprint(domain) != record["config_fingerprint"]:
|
||||
raise ValueError("Source preparation configuration has changed")
|
||||
spec = PreparationSpec.model_validate(domain)
|
||||
repository, _ = validate_repository(home, record)
|
||||
revision = resolve_revision(repository, commit)
|
||||
git(repository, "merge-base", "--is-ancestor", record["code"]["initial_commit"], revision)
|
||||
validate_tree(repository, record, revision)
|
||||
runtime = contained(home, "code/runtime", directory=True)
|
||||
if tree_fingerprint(runtime) != record["code"]["runtime_fingerprint"]:
|
||||
raise ValueError("Source fixed runtime has changed since preparation")
|
||||
return cls(home, revision, spec, record)
|
||||
|
||||
@property
|
||||
def provenance(self):
|
||||
"""Stable identity checked when reopening a standalone test workspace."""
|
||||
return {
|
||||
"source_workspace": str(self.home.parent),
|
||||
"source_preparation_id": self.record["preparation_id"],
|
||||
"source_commit_sha": self.commit,
|
||||
"runtime_fingerprint": self.record["code"]["runtime_fingerprint"],
|
||||
}
|
||||
|
||||
def install(self, runtime: Path, candidate: Path) -> dict:
|
||||
"""Copy fixed sources and export regular Git blobs, preserving executable bits."""
|
||||
source = contained(self.home, "code/runtime", directory=True)
|
||||
expected = self.record["code"]["runtime_fingerprint"]
|
||||
if tree_fingerprint(source) != expected:
|
||||
raise ValueError("Source fixed runtime changed before import")
|
||||
shutil.copytree(source, runtime)
|
||||
if tree_fingerprint(runtime) != expected or tree_fingerprint(source) != expected:
|
||||
raise ValueError("Source fixed runtime changed during import")
|
||||
repository = contained(self.home, "code/repo", directory=True)
|
||||
candidate.mkdir()
|
||||
entries = git(repository, "ls-tree", "-r", "-z", "--full-tree", self.commit, strip=False)
|
||||
for entry in filter(None, entries.split("\0")):
|
||||
header, name = entry.split("\t", 1)
|
||||
mode, kind, oid = header.split()
|
||||
path = Path(relative_path(name))
|
||||
if mode not in {"100644", "100755"} or kind != "blob" or ".git" in path.parts:
|
||||
raise ValueError(f"Candidate must contain regular files only: {name}")
|
||||
destination = candidate / path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
export_blob(repository, oid, destination)
|
||||
destination.chmod(0o755 if mode == "100755" else 0o644)
|
||||
# The isolated repository's init is the selected snapshot, not the optimizer baseline.
|
||||
git(candidate, "init", "--initial-branch=init", "--template=")
|
||||
git(candidate, "add", "--force", "--all")
|
||||
git(candidate, "-c", "user.name=Home", "-c", "user.email=home@localhost", "commit", "-m", "Test candidate")
|
||||
return {
|
||||
**self.record["code"],
|
||||
"initial_commit": resolve_revision(candidate, "HEAD"),
|
||||
"candidate_fingerprint": tree_fingerprint(candidate),
|
||||
}
|
||||
283
plugins/home/tests/test_home_eval.py
Normal file
283
plugins/home/tests/test_home_eval.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
"""Standalone test workspaces use pinned sources and isolated evaluation lifecycles."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,unused-argument,protected-access,redefined-outer-name
|
||||
|
||||
import copy
|
||||
from importlib import import_module
|
||||
import json
|
||||
from pathlib import Path
|
||||
from subprocess import CalledProcessError
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from reme_home.evaluation.artifacts import read_json
|
||||
from reme_home.evaluation.inputs import EvaluationSpec, load_inputs
|
||||
from reme_home.preparation.code import tree_fingerprint
|
||||
from reme_home.preparation.evaluation_source import EvaluationSource
|
||||
from reme_home.evaluation import component as evaluation
|
||||
|
||||
from test_evaluation import FakeFactory
|
||||
|
||||
from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
command,
|
||||
home_application,
|
||||
preparation_inputs,
|
||||
snapshot,
|
||||
)
|
||||
|
||||
from benchmark import home_eval
|
||||
from reme.schema import Response
|
||||
from reme.components.agent_wrapper import AsAgentWrapper
|
||||
|
||||
|
||||
def write_config(tmp_path, app, spec, commit):
|
||||
config = {
|
||||
"source_workspace": str(app.config.workspace_dir),
|
||||
"test_workspace": str(tmp_path / "test-workspace"),
|
||||
"code_commit": commit,
|
||||
"benchmark": copy.deepcopy(spec["benchmark"]),
|
||||
"evaluation": {"concurrency": 2, "timeout_seconds": 60},
|
||||
}
|
||||
path = tmp_path / "test.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path, config
|
||||
|
||||
|
||||
async def prepare_source(app, inputs):
|
||||
spec, _ = inputs
|
||||
response = await app.run_job("home_prepare", spec=spec)
|
||||
assert response.success, response.answer
|
||||
home = Path(app.config.workspace_dir) / "home"
|
||||
return home, response.answer["code"]["initial_commit"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_historical_commit_ignores_dirty_checkout_and_missing_original_repo(app, inputs, tmp_path):
|
||||
home, _ = await prepare_source(app, inputs)
|
||||
repo = home / "code/repo"
|
||||
command(repo, "switch", "-c", "candidate")
|
||||
file = repo / "plugins/beam/src/reme_beam/__init__.py"
|
||||
file.write_text("value = 2\n")
|
||||
file.chmod(0o755)
|
||||
command(repo, "add", ".")
|
||||
command(repo, "-c", "user.name=Test", "-c", "user.email=test@localhost", "commit", "-m", "candidate")
|
||||
selected = command(repo, "rev-parse", "HEAD")
|
||||
file.write_text("value = 3\n")
|
||||
(repo / "untracked.txt").write_text("must not export")
|
||||
spec, _ = inputs
|
||||
original = Path(spec["runtime"]["source_repo"])
|
||||
original.rename(tmp_path / "original-unavailable")
|
||||
path, _ = write_config(tmp_path, app, spec, selected[:10])
|
||||
before = snapshot(home)
|
||||
|
||||
result = await home_eval.run(path, prepare_only=True)
|
||||
|
||||
assert result["source_commit_sha"] == selected
|
||||
target = Path(result["test_workspace"]) / "home"
|
||||
imported = target / "code/repo/plugins/beam/src/reme_beam/__init__.py"
|
||||
assert imported.read_text() == "value = 2\n"
|
||||
assert imported.stat().st_mode & 0o111
|
||||
assert not (target / "code/repo/untracked.txt").exists()
|
||||
assert command(target / "code/repo", "branch", "--show-current") == "init"
|
||||
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(home / "code/runtime")
|
||||
assert read_json(target / "metadata/preparation.json")["test_evaluation"]["source_commit_sha"] == selected
|
||||
assert not (target / "datasets/debug-cases/manifest.json").exists()
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reopen_scores_all_queries_without_optimizer_history_and_closes(app, inputs, tmp_path, monkeypatch):
|
||||
home, commit = await prepare_source(app, inputs)
|
||||
path, _ = write_config(tmp_path, app, inputs[0], commit)
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
applications = []
|
||||
factory = home_eval.Application
|
||||
|
||||
def tracked_application(**kwargs):
|
||||
application = factory(**kwargs)
|
||||
application.close = AsyncMock(wraps=application.close)
|
||||
|
||||
async def evaluate(name, **options):
|
||||
assert name == "home_evaluate"
|
||||
assert options["record_history"] is False
|
||||
assert options["concurrency"] == 2
|
||||
assert not options.get("sampling") and not options.get("debug")
|
||||
record, _, cases = load_inputs(Path(kwargs["workspace_dir"]) / "home", EvaluationSpec(**options))
|
||||
assert len(cases) == record["dataset"]["case_count"] == 2
|
||||
assert sum(len(case.queries) for case in cases) == 2
|
||||
return Response(answer={"result_dir": "test-results"})
|
||||
|
||||
application.run_job = AsyncMock(side_effect=evaluate)
|
||||
applications.append(application)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(home_eval, "Application", tracked_application)
|
||||
result = await home_eval.run(path)
|
||||
assert result["evaluation"] == {"result_dir": "test-results"}
|
||||
applications[-1].close.assert_awaited_once()
|
||||
assert not list((home / "evaluations").rglob("manifest.json"))
|
||||
assert not list((home / "sampling").rglob("*.json"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("app", ["beam-judge", "lme-judge"], indirect=True)
|
||||
async def test_full_evaluation_records_source_provenance_with_fake_docker(app, inputs, tmp_path, monkeypatch):
|
||||
spec, lme = inputs
|
||||
if "lme-judge" in app.config.plugins:
|
||||
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
|
||||
else:
|
||||
for file in Path(spec["benchmark"]["source"]).glob("chats/*/*/probing_questions/probing_questions.json"):
|
||||
data = read_json(file)
|
||||
data["fact"][0]["rubric"] = ["says hello"]
|
||||
file.write_text(json.dumps(data))
|
||||
home, commit = await prepare_source(app, inputs)
|
||||
path, _ = write_config(tmp_path, app, spec, commit)
|
||||
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", FakeFactory)
|
||||
answer = "yes" if spec["benchmark"]["name"] == "longmemeval" else '{"score": 0.5}'
|
||||
monkeypatch.setattr(AsAgentWrapper, "reply", AsyncMock(return_value={"result": answer}))
|
||||
before = snapshot(home)
|
||||
result = await home_eval.run(path)
|
||||
run = Path(result["evaluation"]["result_dir"])
|
||||
manifest = read_json(run / "manifest.json")
|
||||
assert manifest["test_evaluation"]["source_commit_sha"] == commit
|
||||
assert manifest["commit_sha"] == result["evaluation_commit_sha"]
|
||||
assert manifest["execution_mode"] == "full_validation"
|
||||
assert manifest["options"]["record_history"] is False
|
||||
assert result["evaluation"]["scored_query_count"] == result["query_count"]
|
||||
assert result["evaluation"]["mean_query_score"] == (1.0 if answer == "yes" else 0.5)
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluation_failure_closes_application_and_preserves_preparation(app, inputs, tmp_path, monkeypatch):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
factory = home_eval.Application
|
||||
applications = []
|
||||
|
||||
def failing_application(**kwargs):
|
||||
application = factory(**kwargs)
|
||||
application.run_job = AsyncMock(return_value=Response(success=False, answer="Infrastructure failure"))
|
||||
application.close = AsyncMock(wraps=application.close)
|
||||
applications.append(application)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(home_eval, "Application", failing_application)
|
||||
with pytest.raises(RuntimeError, match="Infrastructure failure"):
|
||||
await home_eval.run(path)
|
||||
applications[-1].close.assert_awaited_once()
|
||||
assert (Path(config["test_workspace"]) / "home/metadata/preparation.json").is_file()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("app", ["lme-judge"], indirect=True)
|
||||
async def test_lme_default_selection_excludes_actual_training_cases(app, inputs, tmp_path):
|
||||
spec, lme = inputs
|
||||
original = json.loads(lme.read_text())
|
||||
held_out = {**original[0], "question_id": "q2"}
|
||||
lme.write_text(json.dumps([*original, held_out]))
|
||||
spec["benchmark"] = {"name": "longmemeval", "source": str(lme), "case_ids": ["q1"]}
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, spec, commit)
|
||||
config["benchmark"] = {"name": "longmemeval", "source": str(lme), "all_cases": True}
|
||||
config["exclude_training_cases"] = True
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
result = await home_eval.run(path, prepare_only=True, expected_benchmark="longmemeval")
|
||||
target = Path(result["test_workspace"]) / "home"
|
||||
_, dataset, cases = load_inputs(target, EvaluationSpec(record_history=False))
|
||||
assert dataset.case_ids == ["q2"]
|
||||
assert [case.case_id for case in cases] == ["q2"]
|
||||
assert result["case_count"] == result["query_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["candidate", "dataset", "runtime", "test_code", "test_data"])
|
||||
async def test_reopen_rejects_changed_identity_or_artifacts(app, inputs, tmp_path, change):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
result = await home_eval.run(path, prepare_only=True)
|
||||
home = Path(result["test_workspace"]) / "home"
|
||||
if change == "candidate":
|
||||
config["code_commit"] = "1234567890"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
elif change == "dataset":
|
||||
config["benchmark"]["case_ids"] = ["1"]
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
elif change == "runtime":
|
||||
(home / "code/runtime/reme/__init__.py").write_text("tampered")
|
||||
elif change == "test_code":
|
||||
(home / "code/repo/plugins/beam/src/reme_beam/__init__.py").write_text("tampered")
|
||||
else:
|
||||
file = next((home / "datasets/training-set/cases").glob("*.json"))
|
||||
case = read_json(file)
|
||||
case["queries"][0]["question"] = "tampered"
|
||||
file.write_text(json.dumps(case))
|
||||
before = snapshot(home)
|
||||
with pytest.raises((ValueError, CalledProcessError)):
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
assert snapshot(home) == before
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("location", ["same", "child", "parent"])
|
||||
async def test_rejects_overlapping_workspaces_before_start(app, inputs, tmp_path, monkeypatch, location):
|
||||
_, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
source = Path(app.config.workspace_dir)
|
||||
target = {"same": source, "child": source / "test", "parent": source.parent}[location]
|
||||
config["test_workspace"] = str(target)
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
factory = AsyncMock()
|
||||
monkeypatch.setattr(home_eval, "Application", factory)
|
||||
with pytest.raises(ValueError, match="non-nested"):
|
||||
await home_eval.run(path)
|
||||
factory.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_runtime_drift_rolls_back_and_allows_retry(app, inputs, tmp_path, monkeypatch):
|
||||
source_home, commit = await prepare_source(app, inputs)
|
||||
path, config = write_config(tmp_path, app, inputs[0], commit)
|
||||
install = EvaluationSource.install
|
||||
|
||||
def fail_install(self, runtime, candidate):
|
||||
runtime.mkdir()
|
||||
(runtime / "partial").write_text("partial import")
|
||||
raise ValueError("Source fixed runtime changed during import")
|
||||
|
||||
monkeypatch.setattr(EvaluationSource, "install", fail_install)
|
||||
with pytest.raises(ValueError, match="during import"):
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
target = Path(config["test_workspace"]) / "home"
|
||||
assert not (target / "metadata/preparation.json").exists()
|
||||
assert not list((target / "code/runtime").iterdir())
|
||||
assert not list((target / "code/repo").iterdir())
|
||||
monkeypatch.setattr(EvaluationSource, "install", install)
|
||||
await home_eval.run(path, prepare_only=True)
|
||||
assert tree_fingerprint(target / "code/runtime") == tree_fingerprint(source_home / "code/runtime")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeypatch):
|
||||
module = import_module(f"benchmark.{benchmark}.run_home_eval")
|
||||
runner = AsyncMock(return_value={"status": "completed"})
|
||||
monkeypatch.setattr(module, "run_evaluation", runner)
|
||||
assert await module.run(commit="abcdef0") == {"status": "completed"}
|
||||
runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}])
|
||||
def test_test_runner_rejects_optimizer_modes(options):
|
||||
with pytest.raises(ValueError):
|
||||
home_eval.TestRunConfig.model_validate(
|
||||
{
|
||||
"source_workspace": "/source",
|
||||
"test_workspace": "/test",
|
||||
"code_commit": "abcdef0",
|
||||
"benchmark": {"name": "beam", "source": "/data", "variant": "100K", "all_cases": True},
|
||||
"evaluation": {"record_history": False, **options},
|
||||
},
|
||||
)
|
||||
|
|
@ -7,6 +7,7 @@ import asyncio
|
|||
import json
|
||||
|
||||
from agentscope.agent import Agent, ContextConfig
|
||||
from agentscope.exception import StructuredOutputError
|
||||
from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage
|
||||
from agentscope.state import AgentState
|
||||
import jsonschema
|
||||
|
|
@ -45,12 +46,19 @@ def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError])
|
||||
@pytest.mark.parametrize(
|
||||
("trigger", "ratio", "expected"),
|
||||
[(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)],
|
||||
)
|
||||
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected):
|
||||
agent = make_agent(tmp_path, ratio, trigger)
|
||||
async def test_real_compression_failure_uses_dev_meta_emergency_threshold(
|
||||
tmp_path,
|
||||
trigger,
|
||||
ratio,
|
||||
expected,
|
||||
failure_type,
|
||||
):
|
||||
agent = make_agent(tmp_path, ratio, trigger, failure=failure_type("Invalid structured output"))
|
||||
context_before, summary_before = agent.state.context, agent.state.summary
|
||||
await agent.compress_context()
|
||||
assert agent.validation_cycle_timer.context_compression_count == int(expected)
|
||||
|
|
@ -63,10 +71,16 @@ async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_pa
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure_type", [RuntimeError, StructuredOutputError])
|
||||
@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]])
|
||||
@pytest.mark.parametrize("reserve_ratio", [0.2, 0])
|
||||
async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio):
|
||||
agent = make_agent(tmp_path, summary=summary)
|
||||
async def test_offload_retains_full_messages_summary_usage_and_recent_context(
|
||||
tmp_path,
|
||||
summary,
|
||||
reserve_ratio,
|
||||
failure_type,
|
||||
):
|
||||
agent = make_agent(tmp_path, summary=summary, failure=failure_type("Invalid structured output"))
|
||||
agent.context_config.reserve_ratio = reserve_ratio
|
||||
agent.state.context[-1] = AssistantMsg(
|
||||
id=agent.state.reply_id,
|
||||
|
|
@ -87,7 +101,7 @@ async def test_offload_retains_full_messages_summary_usage_and_recent_context(tm
|
|||
assert data["session_id"] == agent.state.session_id
|
||||
assert data["context_ratio"] == 0.9
|
||||
assert data["emergency_ratio"] == 0.9
|
||||
assert data["failure"]["type"] == "RuntimeError"
|
||||
assert data["failure"]["type"] == failure_type.__name__
|
||||
text = agent.state.summary
|
||||
if isinstance(text, list):
|
||||
text = "\n".join(block.text for block in text)
|
||||
|
|
@ -117,6 +131,7 @@ async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failu
|
|||
("failure", "expected"),
|
||||
[
|
||||
(RuntimeError("provider failed"), True),
|
||||
(StructuredOutputError("invalid summary"), True),
|
||||
(RuntimeError("The system prompt exceeds threshold"), False),
|
||||
(jsonschema.ValidationError("invalid summary"), True),
|
||||
(openai.OpenAIError("provider unavailable"), True),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue