From f2e1e9dd41bc5be478438d5ee4bc8c61d0f55e95 Mon Sep 17 00:00:00 2001 From: sa-buc Date: Fri, 31 Jul 2026 17:57:53 +0800 Subject: [PATCH] chore(benchmark): isolate dataset/workspaces/results per benchmark - Move shared benchmark/{datasets,memory_workspaces,results} into per-benchmark subdirs benchmark//{dataset,workspaces,results} - Update beam/longmemeval config.yaml and run.py path defaults - Relocate longmemeval download.py to benchmark/longmemeval/ (downloads into dataset/ subdir); inline dataset download docs into README - Update .gitignore: benchmark/*/{dataset,workspaces,results}/ - Move result-{beam,longmemeval}.md to benchmark/results_md/ and drop result- prefix; update README links - Fix stale path refs in llm_judge.py and logs/demo_search_format.py --- .gitignore | 15 +++---- benchmark/README.md | 35 ++++++++++------ benchmark/README_ZH.md | 32 +++++++++------ benchmark/beam/config.yaml | 6 +-- benchmark/beam/run.py | 8 ++-- benchmark/datasets/README.md | 40 ------------------- benchmark/datasets/README_EN.md | 40 ------------------- benchmark/longmemeval/config.yaml | 6 +-- .../{datasets => }/longmemeval/download.py | 8 ++-- benchmark/longmemeval/run.py | 4 +- .../{result-beam.md => results_md/beam.md} | 0 .../longmemeval.md} | 0 reme/steps/benchmark/beam/llm_judge.py | 2 +- 13 files changed, 65 insertions(+), 131 deletions(-) delete mode 100644 benchmark/datasets/README.md delete mode 100644 benchmark/datasets/README_EN.md rename benchmark/{datasets => }/longmemeval/download.py (86%) rename benchmark/{result-beam.md => results_md/beam.md} (100%) rename benchmark/{result-longmemeval.md => results_md/longmemeval.md} (100%) diff --git a/.gitignore b/.gitignore index 98939eb0..d9c98bac 100644 --- a/.gitignore +++ b/.gitignore @@ -61,17 +61,14 @@ datasets/ # Claude Code skills (local only) .claude/skills/ -# Memory workspaces (keep dirs, ignore contents) -benchmark/memory_workspaces/ +# Benchmark memory workspaces (created on demand by run.py via mkdir) +benchmark/*/workspaces/ -# LongMemEval raw data (download via download.py) -benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json +# Benchmark datasets (LongMemEval via download.py, BEAM via git clone) +benchmark/*/dataset/ -# BEAM dataset (clone via git) -benchmark/datasets/BEAM/ - -# Benchmark outputs -benchmark/results/ +# Benchmark outputs (created on demand by run.py via mkdir) +benchmark/*/results/ # integration tests outputs tests/integration/logs/ diff --git a/benchmark/README.md b/benchmark/README.md index bbbc101a..930a5ec1 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -34,22 +34,31 @@ Model names and component wiring live in the ReMe configs referenced by each ben ## 2. Download Datasets -See [`datasets/README_EN.md`](datasets/README_EN.md) for full details. +Each benchmark keeps its own data under its directory: +`benchmark//dataset` (input data), `benchmark//workspaces` +(per-item memory workspaces), and `benchmark//results` (evaluation outputs). +All three are excluded from Git. -**LongMemEval** (downloaded from a HuggingFace mirror): +**LongMemEval** — ReMe uses only the **cleaned-S** split, hosted on HuggingFace: +[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2) +(the script downloads via the hf-mirror.com mirror; to use a different mirror, +modify `BASE_URL` in `download.py`): ```bash -cd benchmark/datasets/longmemeval -python download.py # downloads the cleaned-S dataset; skips if already present +cd benchmark/longmemeval +python download.py # saves dataset/longmemeval_s_reme_cleaned.json; skips if already present ``` -**BEAM** (public repository, cloned into `benchmark/datasets/`): +**BEAM** (public repository, cloned into `benchmark/beam/dataset/`): ```bash -cd benchmark/datasets +cd benchmark/beam/dataset git clone https://github.com/mohammadtavakoli78/BEAM.git ``` +After cloning, `benchmark/beam/dataset/BEAM/` should contain `chats/`, `src/`, +`topics/` and other subdirectories. + ## 3. Run LongMemEval From the repository root: @@ -78,12 +87,12 @@ python benchmark/longmemeval/run.py --eval_only # reuse existing w | `dataset.path` | Dataset file to evaluate (e.g. `longmemeval_s_reme_cleaned.json`); ground truth is included. | | `dataset.start_index` / `num_items` | Slice of items to evaluate. | | `dataset.question_types` | Filter by question type; empty = all. | -| `dataset.workspace_root` | Per-item workspace root (`benchmark/memory_workspaces/longmemeval-s`). | +| `dataset.workspace_root` | Per-item workspace root (`benchmark/longmemeval/workspaces/longmemeval-s`). | | `evaluation.num_workers` | `0` = auto (cpu-2), `1` = sequential, `>1` = parallel. | | `evaluation.filter_future_sessions` | Only ingest sessions with timestamp ≤ `question_date`. | | `reme.config` | ReMe config used (`lme.yaml`). | | `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | Dream triggering behavior. | -| `output.dir` | Results directory (`benchmark/results/longmemeval`). | +| `output.dir` | Results directory (`benchmark/longmemeval/results`). | ## 4. Run BEAM @@ -107,14 +116,14 @@ python benchmark/beam/run.py --eval_only # reuse existing workspac | Key | Meaning | | --- | --- | -| `dataset.beam_root` | BEAM dataset root (`benchmark/datasets/BEAM`). | +| `dataset.beam_root` | BEAM dataset root (`benchmark/beam/dataset/BEAM`). | | `dataset.chat_size` | Variant to run: `100K` / `500K` / `1M` / `10M`. | | `dataset.case_ids` | Specific cases (e.g. `["1","2"]`); empty = all cases. | | `dataset.start_index` / `num_items` | Case pagination (`num_items` `0` = all). | -| `dataset.workspace_root` | Per-case workspace root (`benchmark/memory_workspaces/beam`). | +| `dataset.workspace_root` | Per-case workspace root (`benchmark/beam/workspaces/beam`). | | `evaluation.num_workers` | `0` = auto, `1` = sequential, `>1` = parallel. | | `reme.config` | ReMe config used (`beam.yaml`). | -| `output.dir` | Results directory (`benchmark/results/beam`). | +| `output.dir` | Results directory (`benchmark/beam/results`). | ## 5. Outputs & Logs @@ -141,5 +150,5 @@ The script gracefully sends `SIGTERM` to the whole process tree, then escalates Recorded evaluation results are available in: -- [`result-longmemeval.md`](./result-longmemeval.md) -- [`result-beam.md`](./result-beam.md) +- [`longmemeval.md`](./results_md/longmemeval.md) +- [`beam.md`](./results_md/beam.md) diff --git a/benchmark/README_ZH.md b/benchmark/README_ZH.md index c384a01c..cf4c4bbb 100644 --- a/benchmark/README_ZH.md +++ b/benchmark/README_ZH.md @@ -32,22 +32,28 @@ EMBEDDING_BASE_URL=... ## 2. 下载数据集 -完整说明见 [`datasets/README.md`](datasets/README.md)。 +每个基准的数据都存放在各自目录下：`benchmark//dataset`（输入数据）、 +`benchmark//workspaces`（按条目隔离的记忆工作区）、 +`benchmark//results`（评测输出）。三者均不纳入 Git 版本管理。 -**LongMemEval**（从 HuggingFace 镜像下载）： +**LongMemEval** —— ReMe 仅使用 **cleaned-S** 版本，数据托管在 HuggingFace： +[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2) +（下载脚本经 hf-mirror.com 镜像源获取，如需更换源请修改 `download.py` 中的 `BASE_URL`）： ```bash -cd benchmark/datasets/longmemeval -python download.py # 下载 cleaned-S 数据文件，已存在则自动跳过 +cd benchmark/longmemeval +python download.py # 保存为 dataset/longmemeval_s_reme_cleaned.json，已存在则自动跳过 ``` -**BEAM**（公开仓库，clone 到 `benchmark/datasets/` 下）： +**BEAM**（公开仓库，clone 到 `benchmark/beam/dataset/` 下）： ```bash -cd benchmark/datasets +cd benchmark/beam/dataset git clone https://github.com/mohammadtavakoli78/BEAM.git ``` +clone 完成后，`benchmark/beam/dataset/BEAM/` 目录下应包含 `chats/`、`src/`、`topics/` 等子目录。 + ## 3. 运行 LongMemEval 在仓库根目录执行： @@ -76,12 +82,12 @@ python benchmark/longmemeval/run.py --eval_only # 复用已有工 | `dataset.path` | 待评测的数据集文件（如 `longmemeval_s_reme_cleaned.json`），已包含 ground truth。 | | `dataset.start_index` / `num_items` | 评测条目的切片范围。 | | `dataset.question_types` | 按问题类型过滤，空表示全部。 | -| `dataset.workspace_root` | 条目工作区根目录（`benchmark/memory_workspaces/longmemeval-s`）。 | +| `dataset.workspace_root` | 条目工作区根目录（`benchmark/longmemeval/workspaces/longmemeval-s`）。 | | `evaluation.num_workers` | `0` = 自动（cpu-2），`1` = 串行，`>1` = 并行。 | | `evaluation.filter_future_sessions` | 仅摄入时间戳 ≤ `question_date` 的会话。 | | `reme.config` | 使用的 ReMe 配置（`lme.yaml`）。 | | `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | dream 触发行为。 | -| `output.dir` | 结果目录（`benchmark/results/longmemeval`）。 | +| `output.dir` | 结果目录（`benchmark/longmemeval/results`）。 | ## 4. 运行 BEAM @@ -105,14 +111,14 @@ python benchmark/beam/run.py --eval_only # 复用已有工作区 | 配置项 | 含义 | | --- | --- | -| `dataset.beam_root` | BEAM 数据集根目录（`benchmark/datasets/BEAM`）。 | +| `dataset.beam_root` | BEAM 数据集根目录（`benchmark/beam/dataset/BEAM`）。 | | `dataset.chat_size` | 运行的变体：`100K` / `500K` / `1M` / `10M`。 | | `dataset.case_ids` | 指定 case（如 `["1","2"]`），空表示全部。 | | `dataset.start_index` / `num_items` | case 分页（`num_items` 为 `0` 表示全部）。 | -| `dataset.workspace_root` | case 工作区根目录（`benchmark/memory_workspaces/beam`）。 | +| `dataset.workspace_root` | case 工作区根目录（`benchmark/beam/workspaces/beam`）。 | | `evaluation.num_workers` | `0` = 自动，`1` = 串行，`>1` = 并行。 | | `reme.config` | 使用的 ReMe 配置（`beam.yaml`）。 | -| `output.dir` | 结果目录（`benchmark/results/beam`）。 | +| `output.dir` | 结果目录（`benchmark/beam/results`）。 | ## 5. 输出与日志 @@ -138,5 +144,5 @@ bash benchmark/kill.sh 已记录的评测结果见： -- [`result-longmemeval.md`](./result-longmemeval.md) -- [`result-beam.md`](./result-beam.md) +- [`longmemeval.md`](./results_md/longmemeval.md) +- [`beam.md`](./results_md/beam.md) diff --git a/benchmark/beam/config.yaml b/benchmark/beam/config.yaml index 32e7620b..66e63d04 100644 --- a/benchmark/beam/config.yaml +++ b/benchmark/beam/config.yaml @@ -2,12 +2,12 @@ # This file controls what/how to evaluate. dataset: - beam_root: "benchmark/datasets/BEAM" # BEAM dataset root + beam_root: "benchmark/beam/dataset/BEAM" # BEAM dataset root chat_size: "1M" # 100K | 500K | 1M | 10M (dataset variant) case_ids: [] # empty = all cases; or ["1", "2", "3"] start_index: 0 # first case index (for pagination) num_items: 0 # 0 = all cases; >0 = limit - workspace_root: "benchmark/memory_workspaces/beam" # workspace root for case workspaces + workspace_root: "benchmark/beam/workspaces/beam" # workspace root for case workspaces evaluation: num_workers: 18 # 0 = auto; 1 = sequential; >1 = parallel (per-case) @@ -16,7 +16,7 @@ reme: config: "beam.yaml" # reme config (in reme/config/) output: - dir: "benchmark/results/beam" + dir: "benchmark/beam/results" log_dir: "logs" # log directory (relative to project root) log_prefix: "beam" # benchmark name used in log filenames log_to_console: true diff --git a/benchmark/beam/run.py b/benchmark/beam/run.py index 45c74f10..cc965a82 100644 --- a/benchmark/beam/run.py +++ b/benchmark/beam/run.py @@ -33,7 +33,7 @@ _PROJECT_ROOT = Path(__file__).parent.parent.parent load_dotenv(_PROJECT_ROOT / ".env") # Workspace root — read from config.yaml (dataset.workspace_root) -_WORKSPACE_ROOT_DEFAULT = "benchmark/memory_workspaces/beam" +_WORKSPACE_ROOT_DEFAULT = "benchmark/beam/workspaces/beam" # --------------------------------------------------------------------------- # Logging @@ -310,7 +310,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False dataset_cfg = eval_config["dataset"] chat_size = dataset_cfg["chat_size"] - beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/datasets/BEAM") + beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/beam/dataset/BEAM") chat_path = beam_root / "chats" / chat_size / case_id / "chat.json" probing_questions_path = beam_root / "chats" / chat_size / case_id / "probing_questions" / "probing_questions.json" @@ -544,7 +544,7 @@ def main( # pylint: disable=too-many-statements setup_logging(log_level, reme_log_level, log_dir=log_dir_abs) dataset_cfg = eval_config["dataset"] chat_size = dataset_cfg["chat_size"] - beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/datasets/BEAM") + beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/beam/dataset/BEAM") # Determine which cases to run case_ids = dataset_cfg.get("case_ids") or [] @@ -576,7 +576,7 @@ def main( # pylint: disable=too-many-statements logger.info(f"Using {num_workers} worker(s)") # Create output directory - output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/results/beam") + output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/beam/results") output_dir.mkdir(parents=True, exist_ok=True) # Create workspace root directory diff --git a/benchmark/datasets/README.md b/benchmark/datasets/README.md deleted file mode 100644 index 3a772a9d..00000000 --- a/benchmark/datasets/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# 数据集下载说明 - -本目录包含 ReMe 评测所需的数据集。部分数据集体积较大，不纳入 Git 版本管理，需要手动下载。 - -## LongMemEval（cleaned-S） - -ReMe 仅使用 LongMemEval 的 **cleaned-S** 版本，数据托管在 HuggingFace： -[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2) -（下载脚本经 hf-mirror.com 镜像源获取）。 - -按以下步骤下载： - -```bash -cd benchmark/datasets/longmemeval - -# 下载 cleaned-S 数据文件（已存在则自动跳过） -python download.py -``` - -下载完成后，目录下应包含以下文件： - -| 文件名 | 说明 | -| --- | --- | -| `longmemeval_s_reme_cleaned.json` | cleaned-S 数据集，已包含 ground truth 字段 | -| `download.py` | 下载脚本（已随仓库提供） | - -> **注意**：下载脚本使用 hf-mirror.com 镜像源，如需更换源请修改 `download.py` 中的 `BASE_URL`。 - -下载完成后即可参照 [`benchmark/README_ZH.md`](../README_ZH.md) 运行 LongMemEval 评测。 - -## BEAM - -BEAM 数据集为公开仓库，直接 clone 到 `benchmark/datasets/` 目录下即可： - -```bash -cd benchmark/datasets -git clone https://github.com/mohammadtavakoli78/BEAM.git -``` - -clone 完成后，`benchmark/datasets/BEAM/` 目录下应包含 `chats/`、`src/`、`topics/` 等子目录。 diff --git a/benchmark/datasets/README_EN.md b/benchmark/datasets/README_EN.md deleted file mode 100644 index 24a7bab0..00000000 --- a/benchmark/datasets/README_EN.md +++ /dev/null @@ -1,40 +0,0 @@ -# Dataset Download Guide - -This directory contains datasets required for ReMe evaluation. Some datasets are large and excluded from Git version control — they must be downloaded manually. - -## LongMemEval (cleaned-S) - -ReMe uses only the **cleaned-S** split of LongMemEval, hosted on HuggingFace: -[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2) -(the script downloads via the hf-mirror.com mirror). - -Download it with: - -```bash -cd benchmark/datasets/longmemeval - -# Download the cleaned-S data file (skipped automatically if it already exists) -python download.py -``` - -After downloading, the directory should contain: - -| File | Description | -| --- | --- | -| `longmemeval_s_reme_cleaned.json` | cleaned-S dataset with ground truth fields included | -| `download.py` | Download script (included in repo) | - -> **Note**: The download script uses hf-mirror.com by default. To use a different mirror, modify `BASE_URL` in `download.py`. - -Once the download completes, follow [`benchmark/README.md`](../README.md) to run the LongMemEval evaluation. - -## BEAM - -BEAM is a public repository. Clone it directly into the `benchmark/datasets/` directory: - -```bash -cd benchmark/datasets -git clone https://github.com/mohammadtavakoli78/BEAM.git -``` - -After cloning, `benchmark/datasets/BEAM/` should contain `chats/`, `src/`, `topics/` and other subdirectories. diff --git a/benchmark/longmemeval/config.yaml b/benchmark/longmemeval/config.yaml index ecc06f52..d79b41ea 100644 --- a/benchmark/longmemeval/config.yaml +++ b/benchmark/longmemeval/config.yaml @@ -2,12 +2,12 @@ # This file controls what/how to evaluate. dataset: - path: "benchmark/datasets/longmemeval/longmemeval_s_reme_cleaned.json" + path: "benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json" start_index: 0 # first item index num_items: 500 # how many items to evaluate (starting from start_index) max_sessions: 0 # 0 = all sessions; >0 = limit sessions per item for testing question_types: [] # filter by question_type; empty list = no filtering (all types) - workspace_root: "benchmark/memory_workspaces/longmemeval-s" # workspace root for item workspaces + workspace_root: "benchmark/longmemeval/workspaces/longmemeval-s" # workspace root for item workspaces evaluation: # LLM-as-judge uses the 'judge' as_llm component defined in lme.yaml @@ -25,7 +25,7 @@ reme: dream_max_units: 5 output: - dir: "benchmark/results/longmemeval" + dir: "benchmark/longmemeval/results" log_dir: "logs" # log directory (relative to project root) log_prefix: "longmemeval" # benchmark name used in log filenames log_to_console: true diff --git a/benchmark/datasets/longmemeval/download.py b/benchmark/longmemeval/download.py similarity index 86% rename from benchmark/datasets/longmemeval/download.py rename to benchmark/longmemeval/download.py index d6754640..115bb989 100644 --- a/benchmark/datasets/longmemeval/download.py +++ b/benchmark/longmemeval/download.py @@ -3,8 +3,9 @@ Source: https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2 (downloaded via the hf-mirror.com mirror for reliability). -The file ``longmemeval_s_reme_cleaned.json`` is saved under this directory using the same -name as on the remote (``benchmark/longmemeval/config.yaml`` points to it). +The file ``longmemeval_s_reme_cleaned.json`` is saved under ``dataset/`` next to this +script using the same name as on the remote (``benchmark/longmemeval/config.yaml`` +points to it). Usage: python download.py # download cleaned-S (skip if it already exists) @@ -15,7 +16,7 @@ import sys import urllib.request BASE_URL = "https://hf-mirror.com/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2/resolve/main" -TARGET_DIR = os.path.dirname(os.path.abspath(__file__)) +TARGET_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dataset") # Files to download (saved with the same name as on the remote). FILES = [ @@ -59,6 +60,7 @@ def _progress(block_num, block_size, total_size): if __name__ == "__main__": + os.makedirs(TARGET_DIR, exist_ok=True) print(f"Downloading LongMemEval cleaned-S dataset to: {TARGET_DIR}\n") for fname in FILES: download_file(fname) diff --git a/benchmark/longmemeval/run.py b/benchmark/longmemeval/run.py index f974b3a5..910eb274 100644 --- a/benchmark/longmemeval/run.py +++ b/benchmark/longmemeval/run.py @@ -32,7 +32,7 @@ _PROJECT_ROOT = Path(__file__).parent.parent.parent load_dotenv(_PROJECT_ROOT / ".env") # Workspace root for evaluation items — read from config.yaml (dataset.workspace_root) -_WORKSPACE_ROOT_DEFAULT = "benchmark/memory_workspaces/longmemeval-s" +_WORKSPACE_ROOT_DEFAULT = "benchmark/longmemeval/workspaces/longmemeval-s" # --------------------------------------------------------------------------- # Logging @@ -573,7 +573,7 @@ def main( logger.info(f"Using {num_workers} worker(s)") # Create output directory - output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/results/longmemeval") + output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/longmemeval/results") output_dir.mkdir(parents=True, exist_ok=True) # Create workspace root directory diff --git a/benchmark/result-beam.md b/benchmark/results_md/beam.md similarity index 100% rename from benchmark/result-beam.md rename to benchmark/results_md/beam.md diff --git a/benchmark/result-longmemeval.md b/benchmark/results_md/longmemeval.md similarity index 100% rename from benchmark/result-longmemeval.md rename to benchmark/results_md/longmemeval.md diff --git a/reme/steps/benchmark/beam/llm_judge.py b/reme/steps/benchmark/beam/llm_judge.py index bf51f345..879c82e3 100644 --- a/reme/steps/benchmark/beam/llm_judge.py +++ b/reme/steps/benchmark/beam/llm_judge.py @@ -14,7 +14,7 @@ A ``semantic`` alignment path is also available which uses ReMe's configured ``as_embedding`` model (replacing BEAM's sentence_transformers). This replicates the evaluation logic from -``benchmark/datasets/BEAM/src/evaluation/compute_metrics.py``. +``benchmark/beam/dataset/BEAM/src/evaluation/compute_metrics.py``. """ import json