diff --git a/.gitignore b/.gitignore index 98939eb0..6e5e64c3 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,6 @@ benchmark/results/ # integration tests outputs tests/integration/logs/ daily/ + +#meta-reme +meta-reme/bundles/ \ No newline at end of file diff --git a/meta-reme/build_bundle.yaml b/meta-reme/build_bundle.yaml new file mode 100644 index 00000000..5070dc35 --- /dev/null +++ b/meta-reme/build_bundle.yaml @@ -0,0 +1,88 @@ +version: 1 + +# Paths are relative to the full ReMe repository. Directories are expanded in +# sorted order by bundle_builder.py; caches and compiled Python files are never +# copied. +common: + include: + - LICENSE + - README.md + - pyproject.toml + - reme/__init__.py + - reme/application.py + - reme/constants.py + - reme/reme.py + - reme/components + - reme/enumeration + - reme/schema + - reme/utils + - reme/config/__init__.py + - reme/config/config_parser.py + - reme/steps/base_step.py + - reme/steps/common + - reme/steps/evolve + - reme/steps/file_io + - reme/steps/index + - reme/steps/transfer + step_packages: + - common + - evolve + - file_io + - index + - transfer + validation: + imports: + - reme + - reme.application + required_steps: + - auto_memory_step + - search_step + - search_v2_step + forbidden_paths: + - reme/steps/cookbook + +targets: + default: + include: + - reme/config/default.yaml + validation: + imports: + - reme.steps.evolve + - reme.steps.index + required_steps: + - auto_memory_cc_step + - auto_resource_step + forbidden_paths: + - reme/steps/benchmark + + lme: + include: + - reme/config/lme.yaml + - reme/steps/benchmark/base + - reme/steps/benchmark/lme + benchmark_package: lme + validation: + imports: + - reme.steps.benchmark.lme + required_steps: + - lme_auto_memory_step + - lme_agentic_answer_step + - lme_answer_judge_step + forbidden_paths: + - reme/steps/benchmark/beam + + beam: + include: + - reme/config/beam.yaml + - reme/steps/benchmark/base + - reme/steps/benchmark/beam + benchmark_package: beam + validation: + imports: + - reme.steps.benchmark.beam + required_steps: + - beam_auto_memory_step + - beam_agentic_answer_step + - beam_rubric_judge_step + forbidden_paths: + - reme/steps/benchmark/lme diff --git a/meta-reme/bundle_builder.py b/meta-reme/bundle_builder.py new file mode 100644 index 00000000..572ff635 --- /dev/null +++ b/meta-reme/bundle_builder.py @@ -0,0 +1,249 @@ +"""Build deterministic, benchmark-specific ReMe source bundles. + +The file allowlists and validation contracts live in ``build_bundle.yaml``. +This module deliberately copies from an allowlist instead of copying the full +repository and pruning it afterwards. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from typing import Any, Iterable, Mapping + +import yaml + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_SPEC = SCRIPT_DIR / "build_bundle.yaml" +DEFAULT_SOURCE_REPO = SCRIPT_DIR.parent +TARGETS = ("default", "lme", "beam") +_IGNORED_PARTS = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache"} +_IGNORED_SUFFIXES = {".pyc", ".pyo"} + + +class BundleBuildError(RuntimeError): + """Raised when a bundle specification or generated bundle is invalid.""" + + +def _load_spec(spec_path: Path) -> Mapping[str, Any]: + if not spec_path.is_file(): + raise BundleBuildError(f"Bundle specification does not exist: {spec_path}") + try: + loaded = yaml.safe_load(spec_path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise BundleBuildError(f"Invalid YAML in {spec_path}: {exc}") from exc + if not isinstance(loaded, dict): + raise BundleBuildError(f"Bundle specification must be a mapping: {spec_path}") + return loaded + + +def _mapping(value: Any, field: str) -> Mapping[str, Any]: + if not isinstance(value, dict): + raise BundleBuildError(f"'{field}' must be a mapping") + return value + + +def _string_list(value: Any, field: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value): + raise BundleBuildError(f"'{field}' must be a list of non-empty strings") + return value + + +def _target_config(spec: Mapping[str, Any], target: str) -> tuple[Mapping[str, Any], Mapping[str, Any]]: + targets = _mapping(spec.get("targets"), "targets") + if target not in targets: + available = ", ".join(sorted(str(name) for name in targets)) + raise ValueError(f"Unknown bundle target {target!r}; expected one of: {available}") + common = _mapping(spec.get("common", {}), "common") + target_spec = _mapping(targets[target], f"targets.{target}") + return common, target_spec + + +def _resolve_source_path(source_repo: Path, relative: str) -> Path: + candidate = Path(relative) + if candidate.is_absolute() or ".." in candidate.parts: + raise BundleBuildError(f"Bundle path must be a safe repository-relative path: {relative!r}") + resolved = (source_repo / candidate).resolve() + try: + resolved.relative_to(source_repo) + except ValueError as exc: + raise BundleBuildError(f"Bundle path escapes the source repository: {relative!r}") from exc + if not resolved.exists(): + raise BundleBuildError(f"Bundle source path does not exist: {relative}") + return resolved + + +def _iter_files(source_repo: Path, entries: Iterable[str]) -> list[tuple[Path, Path]]: + files: dict[Path, Path] = {} + for entry in entries: + source = _resolve_source_path(source_repo, entry) + candidates = [source] if source.is_file() else sorted(path for path in source.rglob("*") if path.is_file()) + for candidate in candidates: + relative = candidate.relative_to(source_repo) + if any(part in _IGNORED_PARTS for part in relative.parts) or candidate.suffix in _IGNORED_SUFFIXES: + continue + if candidate.is_symlink(): + raise BundleBuildError(f"Symbolic links are not allowed in a bundle: {relative}") + files[relative] = candidate + return [(relative, files[relative]) for relative in sorted(files)] + + +def _write_step_initializers(bundle_root: Path, step_packages: list[str], benchmark_package: str | None) -> None: + steps_dir = bundle_root / "reme" / "steps" + imports = list(step_packages) + if benchmark_package: + imports.insert(0, "benchmark") + import_line = f"from . import {', '.join(imports)}\n" if imports else "" + exported = "\n".join(f' "{name}",' for name in imports) + (steps_dir / "__init__.py").write_text( + f'"""Steps included in this generated bundle."""\n\n{import_line}from .base_step import BaseStep\n\n' + f'__all__ = [\n "BaseStep",\n{exported}\n]\n', + encoding="utf-8", + ) + + if benchmark_package: + benchmark_dir = steps_dir / "benchmark" + benchmark_dir.mkdir(parents=True, exist_ok=True) + class_names = { + "lme": "LmeAgenticAnswerStep, LmeAnswerJudgeStep, LmeAutoMemoryStep", + "beam": "BeamAgenticAnswerStep, BeamRubricJudgeStep, BeamAutoMemoryStep", + } + names = class_names.get(benchmark_package) + if names is None: + raise BundleBuildError(f"Unsupported benchmark package: {benchmark_package}") + exported_names = "\n".join(f' "{name.strip()}",' for name in names.split(",")) + (benchmark_dir / "__init__.py").write_text( + '"""Benchmark steps included in this generated bundle."""\n\n' + f"from . import base, {benchmark_package}\n" + "from .base import BaseAgenticAnswerStep\n" + f"from .{benchmark_package} import {names}\n\n" + f'__all__ = [\n "BaseAgenticAnswerStep",\n{exported_names}\n "base",\n' + f' "{benchmark_package}",\n]\n', + encoding="utf-8", + ) + + +def _merged_validation(common: Mapping[str, Any], target_spec: Mapping[str, Any]) -> dict[str, list[str]]: + common_validation = _mapping(common.get("validation", {}), "common.validation") + target_validation = _mapping(target_spec.get("validation", {}), "target.validation") + result: dict[str, list[str]] = {} + for field in ("imports", "required_steps", "forbidden_paths"): + result[field] = _string_list(common_validation.get(field), f"common.validation.{field}") + _string_list( + target_validation.get(field), + f"target.validation.{field}", + ) + return result + + +def _validate_bundle(bundle_root: Path, validation: Mapping[str, list[str]]) -> None: + for relative in validation["forbidden_paths"]: + if (bundle_root / relative).exists(): + raise BundleBuildError(f"Forbidden path was included in bundle: {relative}") + + check = ( + "import importlib, sys\n" + f"sys.path.insert(0, {str(bundle_root)!r})\n" + "from reme.components import R\n" + "from reme.enumeration import ComponentEnum\n" + f"imports = {validation['imports']!r}\n" + f"required = {validation['required_steps']!r}\n" + "for name in imports: importlib.import_module(name)\n" + "registered = R.get_all(ComponentEnum.STEP)\n" + "missing = [name for name in required if name not in registered]\n" + "if missing: raise RuntimeError(f'Missing registered steps: {missing}')\n" + ) + environment = os.environ.copy() + environment["PYTHONPATH"] = str(bundle_root) + result = subprocess.run( + [sys.executable, "-I", "-B", "-c", check], + cwd=bundle_root, + env=environment, + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + raise BundleBuildError(f"Generated bundle failed import/registration validation:\n{detail}") + + +def build_bundle( + target: str, + output_dir: Path, + source_repo: Path = DEFAULT_SOURCE_REPO, + *, + spec_path: Path = DEFAULT_SPEC, + validate: bool = True, +) -> Path: + """Build one target below ``output_dir/reme`` and return that path.""" + + source_repo = Path(source_repo).resolve() + output_dir = Path(output_dir).resolve() + spec = _load_spec(Path(spec_path).resolve()) + common, target_spec = _target_config(spec, target) + entries = _string_list(common.get("include"), "common.include") + _string_list( + target_spec.get("include"), + f"targets.{target}.include", + ) + step_packages = _string_list(common.get("step_packages"), "common.step_packages") + benchmark_package = target_spec.get("benchmark_package") + if benchmark_package is not None and not isinstance(benchmark_package, str): + raise BundleBuildError(f"'targets.{target}.benchmark_package' must be a string") + files = _iter_files(source_repo, entries) + validation = _merged_validation(common, target_spec) + + output_dir.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{target}-bundle-", dir=output_dir)) + staged_root = temporary / "reme" + staged_root.mkdir() + try: + for relative, source in files: + destination = staged_root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + _write_step_initializers(staged_root, step_packages, benchmark_package) + if validate: + _validate_bundle(staged_root, validation) + + destination_root = output_dir / "reme" + if destination_root.exists(): + if destination_root.is_symlink() or not destination_root.is_dir(): + raise BundleBuildError(f"Refusing to replace non-directory bundle target: {destination_root}") + shutil.rmtree(destination_root) + staged_root.replace(destination_root) + return destination_root + finally: + shutil.rmtree(temporary, ignore_errors=True) + + +def main(argv: list[str] | None = None) -> int: + """Build all bundles from the command line.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-repo", type=Path, default=DEFAULT_SOURCE_REPO) + parser.add_argument("--output-dir", type=Path, default=SCRIPT_DIR / "bundles") + parser.add_argument("--spec", type=Path, default=DEFAULT_SPEC) + parser.add_argument("--no-validate", action="store_true", help="skip import and step registration checks") + args = parser.parse_args(argv) + + for target in TARGETS: + bundle = build_bundle( + target, + args.output_dir / target, + args.source_repo, + spec_path=args.spec, + validate=not args.no_validate, + ) + file_count = sum(1 for path in bundle.rglob("*") if path.is_file()) + print(f"{target}: {bundle} ({file_count} files)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/meta-reme/design.md b/meta-reme/design.md new file mode 100644 index 00000000..6a2808c3 --- /dev/null +++ b/meta-reme/design.md @@ -0,0 +1,228 @@ +# Meta-ReMe 实现方案设计 + +## 1. 建设目标 + +Meta-ReMe 是面向 ReMe auto-memory 机制的自动搜索系统。给定某个 benchmark 的 search set,系统通过“weakness mining → harness proposal → harness validation”循环产生多个候选 harness,以 search set 上所有 query 的平均 score 作为唯一优化目标,选出最佳候选,最终在隔离的 test set 上进行一次正式评测。 + +Harness 定义为 auto-memory 相关代码、Prompt 和配置。ReMe 的检索、回答、Judge、基础组件及 benchmark 逻辑保持固定。每个 benchmark 对应独立 domain spec,因此 LongMemEval、BEAM 等数据集分别搜索自己的最优 harness。 + +## 2. 总体架构 + +Meta-ReMe 放置于 `meta-reme/`,入口为: + +```bash +python meta-reme/run.py \ + --dataset longmemeval \ + --workspace /path/to/workspace \ + --config meta-reme/configs/longmemeval.yaml +``` + +主要模块如下: + +```text +meta-reme/ + run.py + models.py + config.py + workspace.py + git_manager.py + scope_guard.py + search_loop.py + agent.py + evaluator.py + result_store.py + bundle_builder.py + datasets/ + base.py + longmemeval.py + beam.py +``` + +`run.py` 负责初始化或恢复搜索;`search_loop.py` 负责外循环;`agent.py` 管理 weakness mining 和 proposal agent;`git_manager.py` 管理分支、提交和 worktree;`evaluator.py` 调用 sandbox;`result_store.py` 持久化结果;`scope_guard.py` 执行修改白名单检查。 + +## 3. Domain Spec + +每个 benchmark 提供一个 `domain_spec.yaml`,内容包括: + +- search set 的来源、格式和 fingerprint; +- case、session、query 的转换方法; +- benchmark runner 和评分器; +- baseline bundle 的文件清单; +- auto-memory 正式修改白名单; +- debug 阶段允许使用的临时目录; +- sandbox 镜像、超时、并发和重试策略; +- proposer 模型、搜索预算和 top-K 数量; +- 唯一目标 `mean_query_score`。 + +平均分定义为所有 query score 的算术平均。候选造成的超时、空回答或运行错误记为 0;Docker、网络等基础设施故障按固定次数重试,仍失败则记为 `infra_error`,不直接参与候选比较。token、耗时和改动大小只作为诊断信息。 + +## 4. Workspace 设计 + +一次搜索的全部状态保存在用户指定 workspace: + +```text +workspace/ + domain_spec.yaml + run.json + events.jsonl + leaderboard.json + code/ + repo/ + worktrees/ + datasets/ + search/ + manifest.json + cases/ + harnesses/ + registry.jsonl + / + manifest.json + summary.md + weaknesses/ + proposals/ + evaluations/ + / + search.json + debug/ + cases/ + logs/ +``` + +search set 被转换为统一 JSON 格式复制到 workspace,并设置为只读。Test set 不复制到该目录,也不向 agent 暴露;只有最佳 harness 冻结后,外部 evaluator 才加载 test set。 + +`run.json` 保存运行状态、baseline、当前轮次、预算和最佳候选。`events.jsonl` 采用 append-only 方式记录 proposal、commit、debug、validation、失败和 selection 等事件,使搜索可以在进程中断后恢复。 + +## 5. Baseline Bundle + +`bundle_builder.py` 从完整 ReMe 仓库构建 benchmark 所需的最小代码仓库,包括 application、schema、registry、必要组件、auto-memory、search、answer、judge、配置和 packaging 文件。DingTalk、cookbook 等无关代码不进入 bundle。 + +Bundle 不能依赖人工随意删除文件,而应由 domain spec 中的文件清单确定性生成。生成后执行 import、组件注册、安装和固定 smoke case 检查,并验证精简版本与完整 ReMe baseline 行为一致。随后在 workspace 中初始化 Git,创建不可变 baseline commit。 + +### 5.1 Bundle Target + +每个 bundle 由一个 target 名称标识,对应 ReMe 仓库中已有的配置文件和 benchmark 步骤: + +| target | 配置文件 | benchmark 步骤 | 用途 | +| --- | --- | --- | --- | +| `default` | `reme/config/default.yaml` | 无 benchmark 步骤 | 通用最小 ReMe,不含任何 benchmark 专用代码 | +| `lme` | `reme/config/lme.yaml` | `reme/steps/benchmark/lme/` | LongMemEval 评测专用 | +| `beam` | `reme/config/beam.yaml` | `reme/steps/benchmark/beam/` | BEAM 评测专用 | + +`default` bundle 只包含 ReMe 核心(application、schema、registry、auto-memory、search、answer 等通用组件),不引入 `reme/steps/benchmark/` 下的任何步骤;`lme` 和 `beam` bundle 在 `default` 基础上额外包含对应 benchmark 子包及其 auto-memory、agentic-answer、judge 步骤和配置。三类 bundle 共享核心代码,仅在 benchmark 专用部分不同。 + +### 5.2 调用方式 + +`bundle_builder.py` 支持两种调用方式:独立运行与模块导入。 + +**独立运行**(`python meta-reme/bundle_builder.py`):一次性构建全部三类 bundle,输出到脚本同目录下的 `bundles/` 文件夹,目录结构如下: + +```text +meta-reme/ + bundle_builder.py + bundles/ + default/ + reme/ # default bundle 生成的最小代码仓库 + lme/ + reme/ # lme bundle 生成的最小代码仓库 + beam/ + reme/ # beam bundle 生成的最小代码仓库 +``` + +每个 target 子目录下统一为 `reme/` 文件夹,结构与完整 ReMe 仓库中的 `reme/` 保持一致,可直接作为独立代码仓库使用。独立运行主要用于本地预生成、调试和校验 bundle 内容,产物可被后续搜索流程直接引用,避免每次运行重复构建。 + +**模块导入**(被 `run.py`、`search_loop.py` 等调用):按指定 target 生成单个 bundle 到调用方指定的输出目录,供 workspace 初始化和 Git baseline commit 使用。接口签名如下: + +```python +from meta_reme.bundle_builder import build_bundle + +build_bundle( + target: str, # "default" | "lme" | "beam" + output_dir: Path, # bundle 写入目录,其下会生成 reme/ 子目录 + source_repo: Path, # 完整 ReMe 仓库根目录,默认为项目根 +) -> Path # 返回生成的 reme/ 目录路径 +``` + +模块模式不写入 `bundles/`,而是完全由调用方控制输出位置,以便将 bundle 直接放置到 workspace 的 `code/repo/` 中并就地初始化 Git。无论哪种方式,构建逻辑均由 domain spec 中的文件清单驱动,保证产物确定且可复现。 + +## 6. Harness 与 Git 管理 + +Branch 表示一条可继续演化的 proposal 线,commit SHA 表示一个不可变 harness。Agent 可以: + +- 从任意历史 commit fork; +- 在独立 Git worktree 中修改; +- merge 多个历史分支; +- 多次 commit 和运行 debug case; +- 自主决定何时冻结候选。 + +禁止 force push、改写已登记 commit、删除受保护分支。每个正式候选通过 commit SHA、源码 snapshot SHA256 和配置 fingerprint 唯一标识。 + +Agent 具有较大的 Git 主动权,但所有操作通过 `git_manager.py` 执行,以防止覆盖历史结果。Merge 后必须重新比较 candidate 与 baseline 的完整文件树,而不是只检查最后一次 commit。 + +## 7. 修改权限 + +系统区分 debug scope 和 harness scope。 + +Debug 阶段允许 Agent 在 `meta_debug/` 或指定临时测试目录中增加诊断脚本、测试代码和辅助文件,也允许产生多个中间 commit。此类版本只能运行 debug evaluation,不能进入排行榜。 + +正式冻结时,仅允许修改 domain spec 声明的 auto-memory Python、Prompt、配置节点和必要注册文件。配置文件采用语义级检查,例如只允许修改 `jobs.auto_memory`,不能因为整个 `lme.yaml` 被放行而修改 search、answer 或 judge。正式 candidate 必须删除 debug 文件,并从干净 commit 重新执行 scope、import 和 smoke 检查。 + +## 8. Agent 工作方式 + +首版可以将 weakness mining agent 与 proposal agent 合并为一个自治 agent,使其能够连续完成分析、选父节点、修改、debug 和提交。为了保证过程可审计,每轮仍需生成两个结构化文件: + +- `weaknesses/.json`:问题模式、证据 query、可能原因、回归风险; +- `proposals/.json`:选择的 parent、merge 决策、修改假设、预期改善目标。 + +Agent 可以读取最新 harness、当前 top-K、任意历史候选、代码 diff、search dataset、运行 artifacts 和总结文件。最佳候选仍由编排器根据平均分决定,不能由 agent 自行宣布。 + +## 9. Sandbox 执行与 Debug + +所有代码运行必须进入 sandbox,包括正式 validation 和单 case debug。运行阶段包括: + +```text +prepare → install → ingest → index → answer → export +``` + +Agent 可以自主选择完整 case、指定 session、session prefix 或少量 query 进行 debug。部分 session 结果只用于排查语法、配置、记忆写入和运行错误,不进入正式评分。 + +现有 sandbox 需要补充失败导出能力:即使在安装、初始化或 ingest 阶段失败,也应 best-effort 导出 `failure.json`、stdout、stderr、Python traceback、action log、已完成阶段和宿主异常。每次运行结束后,宿主下载原始压缩包,并安全解压到: + +```text +evaluations///cases//attempt-/ +``` + +解压过程需要拒绝绝对路径、`../`、symlink、设备文件和异常大的压缩包,同时保存 tar 包和 SHA256,确保 artifacts 可验证、可复查。 + +## 10. Validation 与结果记录 + +正式 validation 必须从 frozen commit 创建全新 SourceCandidate。不同 harness、case 之间使用独立 runtime workspace;同一 case 的多个 query 应从相同的 post-ingestion memory snapshot 启动,避免 query 顺序造成状态污染。 + +每个 harness 的 `search.json` 保存: + +- commit、parents、changed paths 和 scope 结果; +- dataset、代码、配置、模型和镜像 fingerprint; +- 总平均分、query 数、失败数和运行状态; +- 每个 case/session 的处理结果; +- 每个 query 的问题、golden answer、模型回答、score、Judge 结论、异常、token 和耗时; +- artifacts 路径及 hash。 + +`summary.md` 由 LLM 根据结构化结果生成,说明该 harness 针对的问题、修改内容、相对 parent/baseline 的改善与退化、代表性 case 和后续建议。JSON 是事实来源,summary 仅用于快速阅读。 + +## 11. 搜索与最终测试 + +系统首先评测 baseline,然后循环执行: + +```text +读取历史与 top-K +→ weakness mining +→ fork/merge +→ 修改和 sandbox debug +→ 冻结 commit +→ 白名单及接口检查 +→ 完整 search validation +→ 更新 registry、summary 和 leaderboard +``` + +达到 proposal 数、完整 validation 数、token 或时间预算后,系统选择 search set 平均分最高的候选。完全同分时保留较早的 incumbent,不引入其他优化指标。 + +最终 winner 必须从 commit SHA 重新构建,并在完整 search set 上进行 clean replay。只有 replay 成功后才加载 test set,分别评测 baseline 和 winner,生成最终报告。整个过程中 test 结果不反馈给 agent,从而保证 test set 只用于衡量泛化效果。 diff --git a/tests/integration/test_bundle_sandbox_benchmarks.py b/tests/integration/test_bundle_sandbox_benchmarks.py new file mode 100644 index 00000000..97fb353d --- /dev/null +++ b/tests/integration/test_bundle_sandbox_benchmarks.py @@ -0,0 +1,205 @@ +"""End-to-end bundle-builder coverage against the first benchmark cases. + +This is an opt-in integration test: it requires the benchmark datasets, a +running Docker daemon, the ReMe sandbox base image, and real model credentials. +The test intentionally invokes the builder through its CLI before constructing +either ``SourceCandidate`` so it covers the same artifacts users generate. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +from types import ModuleType +from typing import Any, Iterable + +from dotenv import load_dotenv +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +# pylint: disable=wrong-import-position +from sandbox import DockerReMeSandboxFactory, SourceCandidate, SourceSnapshot # noqa: E402 + +BUNDLE_BUILDER = PROJECT_ROOT / "meta-reme/bundle_builder.py" +LME_DATASET = PROJECT_ROOT / "benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json" +DEFAULT_BEAM_DATASET = PROJECT_ROOT / "benchmark/datasets/BEAM" +SANDBOX_ENV_NAMES = ( + "LLM_API_KEY", + "LLM_BASE_URL", + "LLM_BACKEND", + "LLM_MODEL_NAME", + "BENCH_MODEL_NAME", + "JUDGE_MODEL_NAME", + "EMBEDDING_API_KEY", + "EMBEDDING_BASE_URL", + "EMBEDDING_BACKEND", + "EMBEDDING_MODEL_NAME", +) + + +def _load_benchmark_module(name: str, relative_path: str) -> ModuleType: + """Load a runner without requiring ``benchmark`` to be a Python package.""" + path = PROJECT_ROOT / relative_path + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load benchmark runner: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +BEAM_RUNNER = _load_benchmark_module("reme_beam_runner", "benchmark/beam/run.py") +LME_RUNNER = _load_benchmark_module("reme_lme_runner", "benchmark/longmemeval/run.py") + + +def _sandbox_environment() -> dict[str, str]: + load_dotenv(PROJECT_ROOT / ".env", override=False) + missing = [name for name in ("LLM_API_KEY", "EMBEDDING_API_KEY") if not os.environ.get(name)] + if missing: + pytest.skip(f"missing sandbox integration credentials: {', '.join(missing)}") + return {name: os.environ[name] for name in SANDBOX_ENV_NAMES if os.environ.get(name)} + + +def _beam_dataset_root() -> Path: + configured = os.environ.get("BEAM_DATASET_ROOT") + return Path(configured).expanduser().resolve() if configured else DEFAULT_BEAM_DATASET + + +def _load_beam_questions(path: Path) -> list[tuple[str, str, list[str]]]: + """Flatten BEAM's question-type mapping while preserving dataset order.""" + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise AssertionError(f"BEAM probing questions must be an object: {path}") + questions: list[tuple[str, str, list[str]]] = [] + for question_type, entries in payload.items(): + assert isinstance(entries, list), f"BEAM question group {question_type!r} must be a list" + for entry in entries: + assert isinstance(entry, dict) and isinstance(entry.get("question"), str) + rubric = entry.get("rubric", []) + assert isinstance(rubric, list) and all(isinstance(value, str) for value in rubric) + questions.append((str(question_type), entry["question"], rubric)) + return questions + + +async def _ingest_sessions(case: Any, sessions: Iterable[dict[str, Any]]) -> int: + count = 0 + for session in sessions: + result = await case.ingest_session( + session_id=session["session_id"], + messages=session["messages"], + date=session["date"], + ) + assert result.success, result.error or result.answer + await case.commit_memory_history(f"session: {session['session_id']}") + count += 1 + return count + + +async def _run_first_lme_case(bundle_root: Path, env: dict[str, str], artifact_dir: Path) -> None: + dataset = json.loads(LME_DATASET.read_text(encoding="utf-8")) + assert isinstance(dataset, list) and dataset, f"LongMemEval dataset is empty: {LME_DATASET}" + item = dataset[0] + question_time = LME_RUNNER.parse_haystack_date(item["question_date"]) + sessions = [ + { + "session_id": session_id, + "messages": LME_RUNNER.format_messages_for_reme(messages, session_time), + "date": session_time.strftime("%Y-%m-%d"), + } + for _, session_time, session_id, messages in LME_RUNNER.sessions_sorted_by_time(item) + if session_time <= question_time + ] + + candidate = SourceCandidate(SourceSnapshot.from_directory(bundle_root)) + factory = DockerReMeSandboxFactory(candidate, env=env, config="lme.yaml", command_timeout=1800.0) + case = await factory.create_case(f"bundle-lme-{item['question_id']}") + try: + assert await _ingest_sessions(case, sessions) == len(sessions) + digest = await case.run_job("digest_update") + assert digest.success, digest.error or digest.answer + answer = await case.answer(query=item["question"], query_time=LME_RUNNER.to_iso(question_time)) + assert answer.success and answer.answer, answer.error or answer.answer + judgment = await case.judge( + query=item["question"], + agent_answer=answer.answer, + golden_answer=item["answer"], + question_type=item["question_type"], + ) + assert judgment.success, judgment.error or judgment.answer + await case.export(artifact_dir / "lme-first-case.tar.gz") + finally: + await case.close() + + +async def _run_first_beam_100k_case( + bundle_root: Path, + beam_root: Path, + env: dict[str, str], + artifact_dir: Path, +) -> None: + case_ids = BEAM_RUNNER.get_available_cases(beam_root, "100K") + assert case_ids, f"BEAM 100K contains no cases: {beam_root}" + case_id = case_ids[0] + case_dir = beam_root / "chats" / "100K" / case_id + sessions = BEAM_RUNNER.load_beam_chat(case_dir / "chat.json", "100K", case_id) + questions = _load_beam_questions(case_dir / "probing_questions/probing_questions.json") + assert sessions and questions + + candidate = SourceCandidate(SourceSnapshot.from_directory(bundle_root)) + factory = DockerReMeSandboxFactory(candidate, env=env, config="beam.yaml", command_timeout=1800.0) + case = await factory.create_case(f"bundle-beam-100k-{case_id}") + try: + assert await _ingest_sessions(case, sessions) == len(sessions) + digest = await case.run_job("digest_update") + assert digest.success, digest.error or digest.answer + for question_type, question, rubric in questions: + answer = await case.answer(query=question) + assert answer.success and answer.answer, answer.error or answer.answer + judgment = await case.run_job( + "answer_judge", + { + "llm_response": answer.answer, + "rubric": rubric, + "probing_question": question, + "question_type": question_type, + }, + ) + assert judgment.success, judgment.error or judgment.answer + await case.export(artifact_dir / "beam-100k-first-case.tar.gz") + finally: + await case.close() + + +@pytest.mark.asyncio +async def test_generated_bundles_run_first_lme_and_beam_100k_cases(tmp_path: Path) -> None: + """Build via CLI, then evaluate the first complete case of both datasets.""" + output_dir = tmp_path / "bundles" + subprocess.run( + [sys.executable, str(BUNDLE_BUILDER), "--output-dir", str(output_dir)], + cwd=PROJECT_ROOT, + check=True, + ) + lme_bundle = output_dir / "lme/reme" + beam_bundle = output_dir / "beam/reme" + assert lme_bundle.is_dir() and beam_bundle.is_dir() + + env = _sandbox_environment() + if not LME_DATASET.is_file(): + pytest.skip(f"LongMemEval dataset is unavailable: {LME_DATASET}") + beam_root = _beam_dataset_root() + if not (beam_root / "chats/100K").is_dir(): + pytest.skip(f"BEAM 100K dataset is unavailable: {beam_root}") + + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + await _run_first_lme_case(lme_bundle, env, artifact_dir) + await _run_first_beam_100k_case(beam_bundle, beam_root, env, artifact_dir) + assert (artifact_dir / "lme-first-case.tar.gz").stat().st_size > 0 + assert (artifact_dir / "beam-100k-first-case.tar.gz").stat().st_size > 0