feat(meta-reme): add deterministic bundle builder and design doc

Add the meta-reme subsystem that generates minimal, benchmark-specific
ReMe source bundles from an allowlist. The bundle_builder.py reads
build_bundle.yaml to produce three targets (default, lme, beam), each
validated by import/step-registration checks before being staged.

Also include the design document describing the search-driven harness
optimization loop and a sandbox integration test exercising the first
LongMemEval and BEAM 100K cases against generated bundles.

.gitignore

@@ -76,3 +76,6 @@ benchmark/results/
 # integration tests outputs
 tests/integration/logs/
 daily/
+
+#meta-reme
+meta-reme/bundles/

meta-reme/build_bundle.yaml

@@ -0,0 +1,88 @@
+version: 1
+
+# Paths are relative to the full ReMe repository.  Directories are expanded in
+# sorted order by bundle_builder.py; caches and compiled Python files are never
+# copied.
+common:
+  include:
+    - LICENSE
+    - README.md
+    - pyproject.toml
+    - reme/__init__.py
+    - reme/application.py
+    - reme/constants.py
+    - reme/reme.py
+    - reme/components
+    - reme/enumeration
+    - reme/schema
+    - reme/utils
+    - reme/config/__init__.py
+    - reme/config/config_parser.py
+    - reme/steps/base_step.py
+    - reme/steps/common
+    - reme/steps/evolve
+    - reme/steps/file_io
+    - reme/steps/index
+    - reme/steps/transfer
+  step_packages:
+    - common
+    - evolve
+    - file_io
+    - index
+    - transfer
+  validation:
+    imports:
+      - reme
+      - reme.application
+    required_steps:
+      - auto_memory_step
+      - search_step
+      - search_v2_step
+    forbidden_paths:
+      - reme/steps/cookbook
+
+targets:
+  default:
+    include:
+      - reme/config/default.yaml
+    validation:
+      imports:
+        - reme.steps.evolve
+        - reme.steps.index
+      required_steps:
+        - auto_memory_cc_step
+        - auto_resource_step
+      forbidden_paths:
+        - reme/steps/benchmark
+
+  lme:
+    include:
+      - reme/config/lme.yaml
+      - reme/steps/benchmark/base
+      - reme/steps/benchmark/lme
+    benchmark_package: lme
+    validation:
+      imports:
+        - reme.steps.benchmark.lme
+      required_steps:
+        - lme_auto_memory_step
+        - lme_agentic_answer_step
+        - lme_answer_judge_step
+      forbidden_paths:
+        - reme/steps/benchmark/beam
+
+  beam:
+    include:
+      - reme/config/beam.yaml
+      - reme/steps/benchmark/base
+      - reme/steps/benchmark/beam
+    benchmark_package: beam
+    validation:
+      imports:
+        - reme.steps.benchmark.beam
+      required_steps:
+        - beam_auto_memory_step
+        - beam_agentic_answer_step
+        - beam_rubric_judge_step
+      forbidden_paths:
+        - reme/steps/benchmark/lme

meta-reme/bundle_builder.py

@@ -0,0 +1,249 @@
+"""Build deterministic, benchmark-specific ReMe source bundles.
+
+The file allowlists and validation contracts live in ``build_bundle.yaml``.
+This module deliberately copies from an allowlist instead of copying the full
+repository and pruning it afterwards.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Any, Iterable, Mapping
+
+import yaml
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+DEFAULT_SPEC = SCRIPT_DIR / "build_bundle.yaml"
+DEFAULT_SOURCE_REPO = SCRIPT_DIR.parent
+TARGETS = ("default", "lme", "beam")
+_IGNORED_PARTS = {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache"}
+_IGNORED_SUFFIXES = {".pyc", ".pyo"}
+
+
+class BundleBuildError(RuntimeError):
+    """Raised when a bundle specification or generated bundle is invalid."""
+
+
+def _load_spec(spec_path: Path) -> Mapping[str, Any]:
+    if not spec_path.is_file():
+        raise BundleBuildError(f"Bundle specification does not exist: {spec_path}")
+    try:
+        loaded = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise BundleBuildError(f"Invalid YAML in {spec_path}: {exc}") from exc
+    if not isinstance(loaded, dict):
+        raise BundleBuildError(f"Bundle specification must be a mapping: {spec_path}")
+    return loaded
+
+
+def _mapping(value: Any, field: str) -> Mapping[str, Any]:
+    if not isinstance(value, dict):
+        raise BundleBuildError(f"'{field}' must be a mapping")
+    return value
+
+
+def _string_list(value: Any, field: str) -> list[str]:
+    if value is None:
+        return []
+    if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value):
+        raise BundleBuildError(f"'{field}' must be a list of non-empty strings")
+    return value
+
+
+def _target_config(spec: Mapping[str, Any], target: str) -> tuple[Mapping[str, Any], Mapping[str, Any]]:
+    targets = _mapping(spec.get("targets"), "targets")
+    if target not in targets:
+        available = ", ".join(sorted(str(name) for name in targets))
+        raise ValueError(f"Unknown bundle target {target!r}; expected one of: {available}")
+    common = _mapping(spec.get("common", {}), "common")
+    target_spec = _mapping(targets[target], f"targets.{target}")
+    return common, target_spec
+
+
+def _resolve_source_path(source_repo: Path, relative: str) -> Path:
+    candidate = Path(relative)
+    if candidate.is_absolute() or ".." in candidate.parts:
+        raise BundleBuildError(f"Bundle path must be a safe repository-relative path: {relative!r}")
+    resolved = (source_repo / candidate).resolve()
+    try:
+        resolved.relative_to(source_repo)
+    except ValueError as exc:
+        raise BundleBuildError(f"Bundle path escapes the source repository: {relative!r}") from exc
+    if not resolved.exists():
+        raise BundleBuildError(f"Bundle source path does not exist: {relative}")
+    return resolved
+
+
+def _iter_files(source_repo: Path, entries: Iterable[str]) -> list[tuple[Path, Path]]:
+    files: dict[Path, Path] = {}
+    for entry in entries:
+        source = _resolve_source_path(source_repo, entry)
+        candidates = [source] if source.is_file() else sorted(path for path in source.rglob("*") if path.is_file())
+        for candidate in candidates:
+            relative = candidate.relative_to(source_repo)
+            if any(part in _IGNORED_PARTS for part in relative.parts) or candidate.suffix in _IGNORED_SUFFIXES:
+                continue
+            if candidate.is_symlink():
+                raise BundleBuildError(f"Symbolic links are not allowed in a bundle: {relative}")
+            files[relative] = candidate
+    return [(relative, files[relative]) for relative in sorted(files)]
+
+
+def _write_step_initializers(bundle_root: Path, step_packages: list[str], benchmark_package: str | None) -> None:
+    steps_dir = bundle_root / "reme" / "steps"
+    imports = list(step_packages)
+    if benchmark_package:
+        imports.insert(0, "benchmark")
+    import_line = f"from . import {', '.join(imports)}\n" if imports else ""
+    exported = "\n".join(f'    "{name}",' for name in imports)
+    (steps_dir / "__init__.py").write_text(
+        f'"""Steps included in this generated bundle."""\n\n{import_line}from .base_step import BaseStep\n\n'
+        f'__all__ = [\n    "BaseStep",\n{exported}\n]\n',
+        encoding="utf-8",
+    )
+
+    if benchmark_package:
+        benchmark_dir = steps_dir / "benchmark"
+        benchmark_dir.mkdir(parents=True, exist_ok=True)
+        class_names = {
+            "lme": "LmeAgenticAnswerStep, LmeAnswerJudgeStep, LmeAutoMemoryStep",
+            "beam": "BeamAgenticAnswerStep, BeamRubricJudgeStep, BeamAutoMemoryStep",
+        }
+        names = class_names.get(benchmark_package)
+        if names is None:
+            raise BundleBuildError(f"Unsupported benchmark package: {benchmark_package}")
+        exported_names = "\n".join(f'    "{name.strip()}",' for name in names.split(","))
+        (benchmark_dir / "__init__.py").write_text(
+            '"""Benchmark steps included in this generated bundle."""\n\n'
+            f"from . import base, {benchmark_package}\n"
+            "from .base import BaseAgenticAnswerStep\n"
+            f"from .{benchmark_package} import {names}\n\n"
+            f'__all__ = [\n    "BaseAgenticAnswerStep",\n{exported_names}\n    "base",\n'
+            f'    "{benchmark_package}",\n]\n',
+            encoding="utf-8",
+        )
+
+
+def _merged_validation(common: Mapping[str, Any], target_spec: Mapping[str, Any]) -> dict[str, list[str]]:
+    common_validation = _mapping(common.get("validation", {}), "common.validation")
+    target_validation = _mapping(target_spec.get("validation", {}), "target.validation")
+    result: dict[str, list[str]] = {}
+    for field in ("imports", "required_steps", "forbidden_paths"):
+        result[field] = _string_list(common_validation.get(field), f"common.validation.{field}") + _string_list(
+            target_validation.get(field),
+            f"target.validation.{field}",
+        )
+    return result
+
+
+def _validate_bundle(bundle_root: Path, validation: Mapping[str, list[str]]) -> None:
+    for relative in validation["forbidden_paths"]:
+        if (bundle_root / relative).exists():
+            raise BundleBuildError(f"Forbidden path was included in bundle: {relative}")
+
+    check = (
+        "import importlib, sys\n"
+        f"sys.path.insert(0, {str(bundle_root)!r})\n"
+        "from reme.components import R\n"
+        "from reme.enumeration import ComponentEnum\n"
+        f"imports = {validation['imports']!r}\n"
+        f"required = {validation['required_steps']!r}\n"
+        "for name in imports: importlib.import_module(name)\n"
+        "registered = R.get_all(ComponentEnum.STEP)\n"
+        "missing = [name for name in required if name not in registered]\n"
+        "if missing: raise RuntimeError(f'Missing registered steps: {missing}')\n"
+    )
+    environment = os.environ.copy()
+    environment["PYTHONPATH"] = str(bundle_root)
+    result = subprocess.run(
+        [sys.executable, "-I", "-B", "-c", check],
+        cwd=bundle_root,
+        env=environment,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode:
+        detail = (result.stderr or result.stdout).strip()
+        raise BundleBuildError(f"Generated bundle failed import/registration validation:\n{detail}")
+
+
+def build_bundle(
+    target: str,
+    output_dir: Path,
+    source_repo: Path = DEFAULT_SOURCE_REPO,
+    *,
+    spec_path: Path = DEFAULT_SPEC,
+    validate: bool = True,
+) -> Path:
+    """Build one target below ``output_dir/reme`` and return that path."""
+
+    source_repo = Path(source_repo).resolve()
+    output_dir = Path(output_dir).resolve()
+    spec = _load_spec(Path(spec_path).resolve())
+    common, target_spec = _target_config(spec, target)
+    entries = _string_list(common.get("include"), "common.include") + _string_list(
+        target_spec.get("include"),
+        f"targets.{target}.include",
+    )
+    step_packages = _string_list(common.get("step_packages"), "common.step_packages")
+    benchmark_package = target_spec.get("benchmark_package")
+    if benchmark_package is not None and not isinstance(benchmark_package, str):
+        raise BundleBuildError(f"'targets.{target}.benchmark_package' must be a string")
+    files = _iter_files(source_repo, entries)
+    validation = _merged_validation(common, target_spec)
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    temporary = Path(tempfile.mkdtemp(prefix=f".{target}-bundle-", dir=output_dir))
+    staged_root = temporary / "reme"
+    staged_root.mkdir()
+    try:
+        for relative, source in files:
+            destination = staged_root / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source, destination)
+        _write_step_initializers(staged_root, step_packages, benchmark_package)
+        if validate:
+            _validate_bundle(staged_root, validation)
+
+        destination_root = output_dir / "reme"
+        if destination_root.exists():
+            if destination_root.is_symlink() or not destination_root.is_dir():
+                raise BundleBuildError(f"Refusing to replace non-directory bundle target: {destination_root}")
+            shutil.rmtree(destination_root)
+        staged_root.replace(destination_root)
+        return destination_root
+    finally:
+        shutil.rmtree(temporary, ignore_errors=True)
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Build all bundles from the command line."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source-repo", type=Path, default=DEFAULT_SOURCE_REPO)
+    parser.add_argument("--output-dir", type=Path, default=SCRIPT_DIR / "bundles")
+    parser.add_argument("--spec", type=Path, default=DEFAULT_SPEC)
+    parser.add_argument("--no-validate", action="store_true", help="skip import and step registration checks")
+    args = parser.parse_args(argv)
+
+    for target in TARGETS:
+        bundle = build_bundle(
+            target,
+            args.output_dir / target,
+            args.source_repo,
+            spec_path=args.spec,
+            validate=not args.no_validate,
+        )
+        file_count = sum(1 for path in bundle.rglob("*") if path.is_file())
+        print(f"{target}: {bundle} ({file_count} files)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

meta-reme/design.md

@@ -0,0 +1,228 @@
+# Meta-ReMe 实现方案设计
+
+## 1. 建设目标
+
+Meta-ReMe 是面向 ReMe auto-memory 机制的自动搜索系统。给定某个 benchmark 的 search set，系统通过“weakness mining → harness proposal → harness validation”循环产生多个候选 harness，以 search set 上所有 query 的平均 score 作为唯一优化目标，选出最佳候选，最终在隔离的 test set 上进行一次正式评测。
+
+Harness 定义为 auto-memory 相关代码、Prompt 和配置。ReMe 的检索、回答、Judge、基础组件及 benchmark 逻辑保持固定。每个 benchmark 对应独立 domain spec，因此 LongMemEval、BEAM 等数据集分别搜索自己的最优 harness。
+
+## 2. 总体架构
+
+Meta-ReMe 放置于 `meta-reme/`，入口为：
+
+```bash
+python meta-reme/run.py \
+  --dataset longmemeval \
+  --workspace /path/to/workspace \
+  --config meta-reme/configs/longmemeval.yaml
+```
+
+主要模块如下：
+
+```text
+meta-reme/
+  run.py
+  models.py
+  config.py
+  workspace.py
+  git_manager.py
+  scope_guard.py
+  search_loop.py
+  agent.py
+  evaluator.py
+  result_store.py
+  bundle_builder.py
+  datasets/
+    base.py
+    longmemeval.py
+    beam.py
+```
+
+`run.py` 负责初始化或恢复搜索；`search_loop.py` 负责外循环；`agent.py` 管理 weakness mining 和 proposal agent；`git_manager.py` 管理分支、提交和 worktree；`evaluator.py` 调用 sandbox；`result_store.py` 持久化结果；`scope_guard.py` 执行修改白名单检查。
+
+## 3. Domain Spec
+
+每个 benchmark 提供一个 `domain_spec.yaml`，内容包括：
+
+- search set 的来源、格式和 fingerprint；
+- case、session、query 的转换方法；
+- benchmark runner 和评分器；
+- baseline bundle 的文件清单；
+- auto-memory 正式修改白名单；
+- debug 阶段允许使用的临时目录；
+- sandbox 镜像、超时、并发和重试策略；
+- proposer 模型、搜索预算和 top-K 数量；
+- 唯一目标 `mean_query_score`。
+
+平均分定义为所有 query score 的算术平均。候选造成的超时、空回答或运行错误记为 0；Docker、网络等基础设施故障按固定次数重试，仍失败则记为 `infra_error`，不直接参与候选比较。token、耗时和改动大小只作为诊断信息。
+
+## 4. Workspace 设计
+
+一次搜索的全部状态保存在用户指定 workspace：
+
+```text
+workspace/
+  domain_spec.yaml
+  run.json
+  events.jsonl
+  leaderboard.json
+  code/
+    repo/
+    worktrees/
+  datasets/
+    search/
+      manifest.json
+      cases/
+  harnesses/
+    registry.jsonl
+    <commit_sha>/
+      manifest.json
+      summary.md
+  weaknesses/
+  proposals/
+  evaluations/
+    <commit_sha>/
+      search.json
+      debug/
+      cases/
+  logs/
+```
+
+search set 被转换为统一 JSON 格式复制到 workspace，并设置为只读。Test set 不复制到该目录，也不向 agent 暴露；只有最佳 harness 冻结后，外部 evaluator 才加载 test set。
+
+`run.json` 保存运行状态、baseline、当前轮次、预算和最佳候选。`events.jsonl` 采用 append-only 方式记录 proposal、commit、debug、validation、失败和 selection 等事件，使搜索可以在进程中断后恢复。
+
+## 5. Baseline Bundle
+
+`bundle_builder.py` 从完整 ReMe 仓库构建 benchmark 所需的最小代码仓库，包括 application、schema、registry、必要组件、auto-memory、search、answer、judge、配置和 packaging 文件。DingTalk、cookbook 等无关代码不进入 bundle。
+
+Bundle 不能依赖人工随意删除文件，而应由 domain spec 中的文件清单确定性生成。生成后执行 import、组件注册、安装和固定 smoke case 检查，并验证精简版本与完整 ReMe baseline 行为一致。随后在 workspace 中初始化 Git，创建不可变 baseline commit。
+
+### 5.1 Bundle Target
+
+每个 bundle 由一个 target 名称标识，对应 ReMe 仓库中已有的配置文件和 benchmark 步骤：
+
+| target | 配置文件 | benchmark 步骤 | 用途 |
+| --- | --- | --- | --- |
+| `default` | `reme/config/default.yaml` | 无 benchmark 步骤 | 通用最小 ReMe，不含任何 benchmark 专用代码 |
+| `lme` | `reme/config/lme.yaml` | `reme/steps/benchmark/lme/` | LongMemEval 评测专用 |
+| `beam` | `reme/config/beam.yaml` | `reme/steps/benchmark/beam/` | BEAM 评测专用 |
+
+`default` bundle 只包含 ReMe 核心（application、schema、registry、auto-memory、search、answer 等通用组件），不引入 `reme/steps/benchmark/` 下的任何步骤；`lme` 和 `beam` bundle 在 `default` 基础上额外包含对应 benchmark 子包及其 auto-memory、agentic-answer、judge 步骤和配置。三类 bundle 共享核心代码，仅在 benchmark 专用部分不同。
+
+### 5.2 调用方式
+
+`bundle_builder.py` 支持两种调用方式：独立运行与模块导入。
+
+**独立运行**（`python meta-reme/bundle_builder.py`）：一次性构建全部三类 bundle，输出到脚本同目录下的 `bundles/` 文件夹，目录结构如下：
+
+```text
+meta-reme/
+  bundle_builder.py
+  bundles/
+    default/
+      reme/          # default bundle 生成的最小代码仓库
+    lme/
+      reme/          # lme bundle 生成的最小代码仓库
+    beam/
+      reme/          # beam bundle 生成的最小代码仓库
+```
+
+每个 target 子目录下统一为 `reme/` 文件夹，结构与完整 ReMe 仓库中的 `reme/` 保持一致，可直接作为独立代码仓库使用。独立运行主要用于本地预生成、调试和校验 bundle 内容，产物可被后续搜索流程直接引用，避免每次运行重复构建。
+
+**模块导入**（被 `run.py`、`search_loop.py` 等调用）：按指定 target 生成单个 bundle 到调用方指定的输出目录，供 workspace 初始化和 Git baseline commit 使用。接口签名如下：
+
+```python
+from meta_reme.bundle_builder import build_bundle
+
+build_bundle(
+    target: str,        # "default" | "lme" | "beam"
+    output_dir: Path,   # bundle 写入目录，其下会生成 reme/ 子目录
+    source_repo: Path,  # 完整 ReMe 仓库根目录，默认为项目根
+) -> Path               # 返回生成的 reme/ 目录路径
+```
+
+模块模式不写入 `bundles/`，而是完全由调用方控制输出位置，以便将 bundle 直接放置到 workspace 的 `code/repo/` 中并就地初始化 Git。无论哪种方式，构建逻辑均由 domain spec 中的文件清单驱动，保证产物确定且可复现。
+
+## 6. Harness 与 Git 管理
+
+Branch 表示一条可继续演化的 proposal 线，commit SHA 表示一个不可变 harness。Agent 可以：
+
+- 从任意历史 commit fork；
+- 在独立 Git worktree 中修改；
+- merge 多个历史分支；
+- 多次 commit 和运行 debug case；
+- 自主决定何时冻结候选。
+
+禁止 force push、改写已登记 commit、删除受保护分支。每个正式候选通过 commit SHA、源码 snapshot SHA256 和配置 fingerprint 唯一标识。
+
+Agent 具有较大的 Git 主动权，但所有操作通过 `git_manager.py` 执行，以防止覆盖历史结果。Merge 后必须重新比较 candidate 与 baseline 的完整文件树，而不是只检查最后一次 commit。
+
+## 7. 修改权限
+
+系统区分 debug scope 和 harness scope。
+
+Debug 阶段允许 Agent 在 `meta_debug/` 或指定临时测试目录中增加诊断脚本、测试代码和辅助文件，也允许产生多个中间 commit。此类版本只能运行 debug evaluation，不能进入排行榜。
+
+正式冻结时，仅允许修改 domain spec 声明的 auto-memory Python、Prompt、配置节点和必要注册文件。配置文件采用语义级检查，例如只允许修改 `jobs.auto_memory`，不能因为整个 `lme.yaml` 被放行而修改 search、answer 或 judge。正式 candidate 必须删除 debug 文件，并从干净 commit 重新执行 scope、import 和 smoke 检查。
+
+## 8. Agent 工作方式
+
+首版可以将 weakness mining agent 与 proposal agent 合并为一个自治 agent，使其能够连续完成分析、选父节点、修改、debug 和提交。为了保证过程可审计，每轮仍需生成两个结构化文件：
+
+- `weaknesses/<round>.json`：问题模式、证据 query、可能原因、回归风险；
+- `proposals/<proposal>.json`：选择的 parent、merge 决策、修改假设、预期改善目标。
+
+Agent 可以读取最新 harness、当前 top-K、任意历史候选、代码 diff、search dataset、运行 artifacts 和总结文件。最佳候选仍由编排器根据平均分决定，不能由 agent 自行宣布。
+
+## 9. Sandbox 执行与 Debug
+
+所有代码运行必须进入 sandbox，包括正式 validation 和单 case debug。运行阶段包括：
+
+```text
+prepare → install → ingest → index → answer → export
+```
+
+Agent 可以自主选择完整 case、指定 session、session prefix 或少量 query 进行 debug。部分 session 结果只用于排查语法、配置、记忆写入和运行错误，不进入正式评分。
+
+现有 sandbox 需要补充失败导出能力：即使在安装、初始化或 ingest 阶段失败，也应 best-effort 导出 `failure.json`、stdout、stderr、Python traceback、action log、已完成阶段和宿主异常。每次运行结束后，宿主下载原始压缩包，并安全解压到：
+
+```text
+evaluations/<commit>/<mode>/cases/<case>/attempt-<n>/
+```
+
+解压过程需要拒绝绝对路径、`../`、symlink、设备文件和异常大的压缩包，同时保存 tar 包和 SHA256，确保 artifacts 可验证、可复查。
+
+## 10. Validation 与结果记录
+
+正式 validation 必须从 frozen commit 创建全新 SourceCandidate。不同 harness、case 之间使用独立 runtime workspace；同一 case 的多个 query 应从相同的 post-ingestion memory snapshot 启动，避免 query 顺序造成状态污染。
+
+每个 harness 的 `search.json` 保存：
+
+- commit、parents、changed paths 和 scope 结果；
+- dataset、代码、配置、模型和镜像 fingerprint；
+- 总平均分、query 数、失败数和运行状态；
+- 每个 case/session 的处理结果；
+- 每个 query 的问题、golden answer、模型回答、score、Judge 结论、异常、token 和耗时；
+- artifacts 路径及 hash。
+
+`summary.md` 由 LLM 根据结构化结果生成，说明该 harness 针对的问题、修改内容、相对 parent/baseline 的改善与退化、代表性 case 和后续建议。JSON 是事实来源，summary 仅用于快速阅读。
+
+## 11. 搜索与最终测试
+
+系统首先评测 baseline，然后循环执行：
+
+```text
+读取历史与 top-K
+→ weakness mining
+→ fork/merge
+→ 修改和 sandbox debug
+→ 冻结 commit
+→ 白名单及接口检查
+→ 完整 search validation
+→ 更新 registry、summary 和 leaderboard
+```
+
+达到 proposal 数、完整 validation 数、token 或时间预算后，系统选择 search set 平均分最高的候选。完全同分时保留较早的 incumbent，不引入其他优化指标。
+
+最终 winner 必须从 commit SHA 重新构建，并在完整 search set 上进行 clean replay。只有 replay 成功后才加载 test set，分别评测 baseline 和 winner，生成最终报告。整个过程中 test 结果不反馈给 agent，从而保证 test set 只用于衡量泛化效果。

tests/integration/test_bundle_sandbox_benchmarks.py

@@ -0,0 +1,205 @@
+"""End-to-end bundle-builder coverage against the first benchmark cases.
+
+This is an opt-in integration test: it requires the benchmark datasets, a
+running Docker daemon, the ReMe sandbox base image, and real model credentials.
+The test intentionally invokes the builder through its CLI before constructing
+either ``SourceCandidate`` so it covers the same artifacts users generate.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+from types import ModuleType
+from typing import Any, Iterable
+
+from dotenv import load_dotenv
+import pytest
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# pylint: disable=wrong-import-position
+from sandbox import DockerReMeSandboxFactory, SourceCandidate, SourceSnapshot  # noqa: E402
+
+BUNDLE_BUILDER = PROJECT_ROOT / "meta-reme/bundle_builder.py"
+LME_DATASET = PROJECT_ROOT / "benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json"
+DEFAULT_BEAM_DATASET = PROJECT_ROOT / "benchmark/datasets/BEAM"
+SANDBOX_ENV_NAMES = (
+    "LLM_API_KEY",
+    "LLM_BASE_URL",
+    "LLM_BACKEND",
+    "LLM_MODEL_NAME",
+    "BENCH_MODEL_NAME",
+    "JUDGE_MODEL_NAME",
+    "EMBEDDING_API_KEY",
+    "EMBEDDING_BASE_URL",
+    "EMBEDDING_BACKEND",
+    "EMBEDDING_MODEL_NAME",
+)
+
+
+def _load_benchmark_module(name: str, relative_path: str) -> ModuleType:
+    """Load a runner without requiring ``benchmark`` to be a Python package."""
+    path = PROJECT_ROOT / relative_path
+    spec = importlib.util.spec_from_file_location(name, path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"cannot load benchmark runner: {path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+BEAM_RUNNER = _load_benchmark_module("reme_beam_runner", "benchmark/beam/run.py")
+LME_RUNNER = _load_benchmark_module("reme_lme_runner", "benchmark/longmemeval/run.py")
+
+
+def _sandbox_environment() -> dict[str, str]:
+    load_dotenv(PROJECT_ROOT / ".env", override=False)
+    missing = [name for name in ("LLM_API_KEY", "EMBEDDING_API_KEY") if not os.environ.get(name)]
+    if missing:
+        pytest.skip(f"missing sandbox integration credentials: {', '.join(missing)}")
+    return {name: os.environ[name] for name in SANDBOX_ENV_NAMES if os.environ.get(name)}
+
+
+def _beam_dataset_root() -> Path:
+    configured = os.environ.get("BEAM_DATASET_ROOT")
+    return Path(configured).expanduser().resolve() if configured else DEFAULT_BEAM_DATASET
+
+
+def _load_beam_questions(path: Path) -> list[tuple[str, str, list[str]]]:
+    """Flatten BEAM's question-type mapping while preserving dataset order."""
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise AssertionError(f"BEAM probing questions must be an object: {path}")
+    questions: list[tuple[str, str, list[str]]] = []
+    for question_type, entries in payload.items():
+        assert isinstance(entries, list), f"BEAM question group {question_type!r} must be a list"
+        for entry in entries:
+            assert isinstance(entry, dict) and isinstance(entry.get("question"), str)
+            rubric = entry.get("rubric", [])
+            assert isinstance(rubric, list) and all(isinstance(value, str) for value in rubric)
+            questions.append((str(question_type), entry["question"], rubric))
+    return questions
+
+
+async def _ingest_sessions(case: Any, sessions: Iterable[dict[str, Any]]) -> int:
+    count = 0
+    for session in sessions:
+        result = await case.ingest_session(
+            session_id=session["session_id"],
+            messages=session["messages"],
+            date=session["date"],
+        )
+        assert result.success, result.error or result.answer
+        await case.commit_memory_history(f"session: {session['session_id']}")
+        count += 1
+    return count
+
+
+async def _run_first_lme_case(bundle_root: Path, env: dict[str, str], artifact_dir: Path) -> None:
+    dataset = json.loads(LME_DATASET.read_text(encoding="utf-8"))
+    assert isinstance(dataset, list) and dataset, f"LongMemEval dataset is empty: {LME_DATASET}"
+    item = dataset[0]
+    question_time = LME_RUNNER.parse_haystack_date(item["question_date"])
+    sessions = [
+        {
+            "session_id": session_id,
+            "messages": LME_RUNNER.format_messages_for_reme(messages, session_time),
+            "date": session_time.strftime("%Y-%m-%d"),
+        }
+        for _, session_time, session_id, messages in LME_RUNNER.sessions_sorted_by_time(item)
+        if session_time <= question_time
+    ]
+
+    candidate = SourceCandidate(SourceSnapshot.from_directory(bundle_root))
+    factory = DockerReMeSandboxFactory(candidate, env=env, config="lme.yaml", command_timeout=1800.0)
+    case = await factory.create_case(f"bundle-lme-{item['question_id']}")
+    try:
+        assert await _ingest_sessions(case, sessions) == len(sessions)
+        digest = await case.run_job("digest_update")
+        assert digest.success, digest.error or digest.answer
+        answer = await case.answer(query=item["question"], query_time=LME_RUNNER.to_iso(question_time))
+        assert answer.success and answer.answer, answer.error or answer.answer
+        judgment = await case.judge(
+            query=item["question"],
+            agent_answer=answer.answer,
+            golden_answer=item["answer"],
+            question_type=item["question_type"],
+        )
+        assert judgment.success, judgment.error or judgment.answer
+        await case.export(artifact_dir / "lme-first-case.tar.gz")
+    finally:
+        await case.close()
+
+
+async def _run_first_beam_100k_case(
+    bundle_root: Path,
+    beam_root: Path,
+    env: dict[str, str],
+    artifact_dir: Path,
+) -> None:
+    case_ids = BEAM_RUNNER.get_available_cases(beam_root, "100K")
+    assert case_ids, f"BEAM 100K contains no cases: {beam_root}"
+    case_id = case_ids[0]
+    case_dir = beam_root / "chats" / "100K" / case_id
+    sessions = BEAM_RUNNER.load_beam_chat(case_dir / "chat.json", "100K", case_id)
+    questions = _load_beam_questions(case_dir / "probing_questions/probing_questions.json")
+    assert sessions and questions
+
+    candidate = SourceCandidate(SourceSnapshot.from_directory(bundle_root))
+    factory = DockerReMeSandboxFactory(candidate, env=env, config="beam.yaml", command_timeout=1800.0)
+    case = await factory.create_case(f"bundle-beam-100k-{case_id}")
+    try:
+        assert await _ingest_sessions(case, sessions) == len(sessions)
+        digest = await case.run_job("digest_update")
+        assert digest.success, digest.error or digest.answer
+        for question_type, question, rubric in questions:
+            answer = await case.answer(query=question)
+            assert answer.success and answer.answer, answer.error or answer.answer
+            judgment = await case.run_job(
+                "answer_judge",
+                {
+                    "llm_response": answer.answer,
+                    "rubric": rubric,
+                    "probing_question": question,
+                    "question_type": question_type,
+                },
+            )
+            assert judgment.success, judgment.error or judgment.answer
+        await case.export(artifact_dir / "beam-100k-first-case.tar.gz")
+    finally:
+        await case.close()
+
+
+@pytest.mark.asyncio
+async def test_generated_bundles_run_first_lme_and_beam_100k_cases(tmp_path: Path) -> None:
+    """Build via CLI, then evaluate the first complete case of both datasets."""
+    output_dir = tmp_path / "bundles"
+    subprocess.run(
+        [sys.executable, str(BUNDLE_BUILDER), "--output-dir", str(output_dir)],
+        cwd=PROJECT_ROOT,
+        check=True,
+    )
+    lme_bundle = output_dir / "lme/reme"
+    beam_bundle = output_dir / "beam/reme"
+    assert lme_bundle.is_dir() and beam_bundle.is_dir()
+
+    env = _sandbox_environment()
+    if not LME_DATASET.is_file():
+        pytest.skip(f"LongMemEval dataset is unavailable: {LME_DATASET}")
+    beam_root = _beam_dataset_root()
+    if not (beam_root / "chats/100K").is_dir():
+        pytest.skip(f"BEAM 100K dataset is unavailable: {beam_root}")
+
+    artifact_dir = tmp_path / "artifacts"
+    artifact_dir.mkdir()
+    await _run_first_lme_case(lme_bundle, env, artifact_dir)
+    await _run_first_beam_100k_case(beam_bundle, beam_root, env, artifact_dir)
+    assert (artifact_dir / "lme-first-case.tar.gz").stat().st_size > 0
+    assert (artifact_dir / "beam-100k-first-case.tar.gz").stat().st_size > 0