From 65971bafe3221ae66e75f410324841f0c5582af5 Mon Sep 17 00:00:00 2001 From: zouyingcao <57442064+zouyingcao@users.noreply.github.com> Date: Fri, 6 Mar 2026 16:11:39 +0800 Subject: [PATCH] Update: check the code&docs for evaluation on bfcl&appworld (#141) * fix: df.columns bug * fix: await for asynchronous method * update: docs for bfcl&appworld quickstart * update: benchmark/bfcl for new version quickstart * slightly revise bfcl cookbook * update for pre-commit * handle boolean flags in split_into_trainval.py * fix typo in faq.md --- benchmark/bfcl/default_ids.py | 206 ++++++++++++++++++ benchmark/bfcl/init_task_memory_pool.py | 27 +-- benchmark/bfcl/local_file_to_library.py | 30 --- benchmark/bfcl/requirements.txt | 3 +- benchmark/bfcl/run_bfcl.py | 2 +- benchmark/bfcl/run_exp_statistic.py | 2 +- benchmark/bfcl/split_into_trainval.py | 49 ++++- docs/cookbook/appworld/quickstart.md | 38 ++-- docs/cookbook/bfcl/quickstart.md | 122 ++++++----- docs/cookbook/faq.md | 11 +- reme/config/service.yaml | 3 +- .../summary/comparative_extraction.py | 10 +- 12 files changed, 352 insertions(+), 151 deletions(-) create mode 100644 benchmark/bfcl/default_ids.py delete mode 100644 benchmark/bfcl/local_file_to_library.py diff --git a/benchmark/bfcl/default_ids.py b/benchmark/bfcl/default_ids.py new file mode 100644 index 00000000..43f4065e --- /dev/null +++ b/benchmark/bfcl/default_ids.py @@ -0,0 +1,206 @@ +# pylint: disable=C0114 +DEFAULT_TRAIN_IDS: set[str] = { + "multi_turn_base_102", + "multi_turn_base_107", + "multi_turn_base_110", + "multi_turn_base_114", + "multi_turn_base_115", + "multi_turn_base_118", + "multi_turn_base_122", + "multi_turn_base_123", + "multi_turn_base_128", + "multi_turn_base_13", + "multi_turn_base_130", + "multi_turn_base_132", + "multi_turn_base_133", + "multi_turn_base_143", + "multi_turn_base_144", + "multi_turn_base_146", + "multi_turn_base_15", + "multi_turn_base_158", + "multi_turn_base_169", + "multi_turn_base_17", + "multi_turn_base_172", + "multi_turn_base_176", + "multi_turn_base_182", + "multi_turn_base_187", + "multi_turn_base_197", + "multi_turn_base_199", + "multi_turn_base_22", + "multi_turn_base_23", + "multi_turn_base_24", + "multi_turn_base_36", + "multi_turn_base_40", + "multi_turn_base_44", + "multi_turn_base_47", + "multi_turn_base_48", + "multi_turn_base_5", + "multi_turn_base_51", + "multi_turn_base_59", + "multi_turn_base_63", + "multi_turn_base_65", + "multi_turn_base_66", + "multi_turn_base_67", + "multi_turn_base_68", + "multi_turn_base_70", + "multi_turn_base_75", + "multi_turn_base_77", + "multi_turn_base_78", + "multi_turn_base_79", + "multi_turn_base_81", + "multi_turn_base_83", + "multi_turn_base_93", +} + +DEFAULT_VAL_IDS: set[str] = { + "multi_turn_base_0", + "multi_turn_base_1", + "multi_turn_base_10", + "multi_turn_base_100", + "multi_turn_base_101", + "multi_turn_base_103", + "multi_turn_base_104", + "multi_turn_base_105", + "multi_turn_base_106", + "multi_turn_base_108", + "multi_turn_base_109", + "multi_turn_base_11", + "multi_turn_base_111", + "multi_turn_base_112", + "multi_turn_base_113", + "multi_turn_base_116", + "multi_turn_base_117", + "multi_turn_base_119", + "multi_turn_base_12", + "multi_turn_base_120", + "multi_turn_base_121", + "multi_turn_base_124", + "multi_turn_base_125", + "multi_turn_base_126", + "multi_turn_base_127", + "multi_turn_base_129", + "multi_turn_base_131", + "multi_turn_base_134", + "multi_turn_base_135", + "multi_turn_base_136", + "multi_turn_base_137", + "multi_turn_base_138", + "multi_turn_base_139", + "multi_turn_base_14", + "multi_turn_base_140", + "multi_turn_base_141", + "multi_turn_base_142", + "multi_turn_base_145", + "multi_turn_base_147", + "multi_turn_base_148", + "multi_turn_base_149", + "multi_turn_base_150", + "multi_turn_base_151", + "multi_turn_base_152", + "multi_turn_base_153", + "multi_turn_base_154", + "multi_turn_base_155", + "multi_turn_base_156", + "multi_turn_base_157", + "multi_turn_base_159", + "multi_turn_base_16", + "multi_turn_base_160", + "multi_turn_base_161", + "multi_turn_base_162", + "multi_turn_base_163", + "multi_turn_base_164", + "multi_turn_base_165", + "multi_turn_base_166", + "multi_turn_base_167", + "multi_turn_base_168", + "multi_turn_base_170", + "multi_turn_base_171", + "multi_turn_base_173", + "multi_turn_base_174", + "multi_turn_base_175", + "multi_turn_base_177", + "multi_turn_base_178", + "multi_turn_base_179", + "multi_turn_base_18", + "multi_turn_base_180", + "multi_turn_base_181", + "multi_turn_base_183", + "multi_turn_base_184", + "multi_turn_base_185", + "multi_turn_base_186", + "multi_turn_base_188", + "multi_turn_base_189", + "multi_turn_base_19", + "multi_turn_base_190", + "multi_turn_base_191", + "multi_turn_base_192", + "multi_turn_base_193", + "multi_turn_base_194", + "multi_turn_base_195", + "multi_turn_base_196", + "multi_turn_base_198", + "multi_turn_base_2", + "multi_turn_base_20", + "multi_turn_base_21", + "multi_turn_base_25", + "multi_turn_base_26", + "multi_turn_base_27", + "multi_turn_base_28", + "multi_turn_base_29", + "multi_turn_base_3", + "multi_turn_base_30", + "multi_turn_base_31", + "multi_turn_base_32", + "multi_turn_base_33", + "multi_turn_base_34", + "multi_turn_base_35", + "multi_turn_base_37", + "multi_turn_base_38", + "multi_turn_base_39", + "multi_turn_base_4", + "multi_turn_base_41", + "multi_turn_base_42", + "multi_turn_base_43", + "multi_turn_base_45", + "multi_turn_base_46", + "multi_turn_base_49", + "multi_turn_base_50", + "multi_turn_base_52", + "multi_turn_base_53", + "multi_turn_base_54", + "multi_turn_base_55", + "multi_turn_base_56", + "multi_turn_base_57", + "multi_turn_base_58", + "multi_turn_base_6", + "multi_turn_base_60", + "multi_turn_base_61", + "multi_turn_base_62", + "multi_turn_base_64", + "multi_turn_base_69", + "multi_turn_base_7", + "multi_turn_base_71", + "multi_turn_base_72", + "multi_turn_base_73", + "multi_turn_base_74", + "multi_turn_base_76", + "multi_turn_base_8", + "multi_turn_base_80", + "multi_turn_base_82", + "multi_turn_base_84", + "multi_turn_base_85", + "multi_turn_base_86", + "multi_turn_base_87", + "multi_turn_base_88", + "multi_turn_base_89", + "multi_turn_base_9", + "multi_turn_base_90", + "multi_turn_base_91", + "multi_turn_base_92", + "multi_turn_base_94", + "multi_turn_base_95", + "multi_turn_base_96", + "multi_turn_base_97", + "multi_turn_base_98", + "multi_turn_base_99", +} diff --git a/benchmark/bfcl/init_task_memory_pool.py b/benchmark/bfcl/init_task_memory_pool.py index bc38046f..2a4fcc87 100644 --- a/benchmark/bfcl/init_task_memory_pool.py +++ b/benchmark/bfcl/init_task_memory_pool.py @@ -114,6 +114,9 @@ def post_to_summarizer(trajectories: List[Any], service_url: str) -> Dict[str, A request_data = { "trajectories": trajectory_dicts, + "success_threshold": 1.0, + "enable_soft_comparison": True, + "validation_threshold": 0.5, } try: @@ -156,6 +159,9 @@ def process_trajectories_with_threads( results.append(result) if "memory_list" in result["metadata"]: print(f'✅ Group {group_index} processed: {result["metadata"].get("memory_list", 0)}') + memory_list = result["metadata"].get("memory_list", []) + response = requests.post(url=f"{service_url}/add_task_memory", json={"memory_list": memory_list}) + response.raise_for_status() else: print(f"❌ Group {group_index} processed: error") except Exception as e: @@ -174,7 +180,7 @@ def main(): """Main function to convert JSONL to memories using ReMe service.""" parser = argparse.ArgumentParser(description="Convert JSONL to memories using ReMe service") parser.add_argument("--jsonl_file", type=str, required=True, help="Path to the JSONL file") - parser.add_argument("--service_url", type=str, default="http://localhost:8001", help="ReMe service URL") + parser.add_argument("--service_url", type=str, default="http://localhost:8002", help="ReMe service URL") parser.add_argument("--output_file", type=str, help="Output file to save results (optional)") parser.add_argument("--n_threads", type=int, default=4, help="Number of threads for processing") @@ -226,21 +232,4 @@ def main(): if __name__ == "__main__": - import sys - - if len(sys.argv) > 1: - main() - else: - print("Running in compatibility mode...") - with open("exp_result/qwen3-8b/with_think/bfcl-multi-turn-base-train_wo-exp.jsonl", "r") as f: - data = [json.loads(line) for line in f] - - grouped_trajectories = group_trajectories_by_task_id(data) - print(f"Total groups: {len(grouped_trajectories)}") - - results = process_trajectories_with_threads( - grouped_trajectories, - "http://localhost:8001", - n_threads=4, - ) - print(f"Processed {len(results)} groups") + main() diff --git a/benchmark/bfcl/local_file_to_library.py b/benchmark/bfcl/local_file_to_library.py deleted file mode 100644 index a2d9ec15..00000000 --- a/benchmark/bfcl/local_file_to_library.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Load the library data and convert them to the new format""" - -import json - -with open("../../file_vector_store/bfcl_test.jsonl", "r", encoding="utf-8") as f: - bfcl = [json.loads(line) for line in f] - -new_bfcl = [] -for exp in bfcl: - new_exp = {} - new_exp["workspace_id"] = exp["workspace_id"] - new_exp["memory_id"] = exp["unique_id"] - new_exp["memory_type"] = exp["metadata"]["memory_type"] - - new_exp["when_to_use"] = exp["content"] - new_exp["content"] = exp["metadata"]["content"] - new_exp["score"] = exp["metadata"]["score"] - - new_exp["time_created"] = exp["metadata"]["time_created"] - new_exp["time_modified"] = exp["metadata"]["time_modified"] - new_exp["author"] = exp["metadata"]["author"] - - new_exp["metadata"] = exp["metadata"]["metadata"] - new_exp["metadata"]["utility"] = 0 - new_exp["metadata"]["freq"] = 0 - - new_bfcl.append(new_exp) - -with open("../../library/bfcl_test.jsonl", "w", encoding="utf-8") as f: - f.writelines(json.dumps(item, ensure_ascii=False) + "\n" for item in new_bfcl) diff --git a/benchmark/bfcl/requirements.txt b/benchmark/bfcl/requirements.txt index 86ebcb1f..445bf2b3 100644 --- a/benchmark/bfcl/requirements.txt +++ b/benchmark/bfcl/requirements.txt @@ -2,4 +2,5 @@ jinja2 loguru openai ray -pandas \ No newline at end of file +pandas +soundfile \ No newline at end of file diff --git a/benchmark/bfcl/run_bfcl.py b/benchmark/bfcl/run_bfcl.py index c01071ce..6ea8c325 100644 --- a/benchmark/bfcl/run_bfcl.py +++ b/benchmark/bfcl/run_bfcl.py @@ -131,7 +131,7 @@ def main(): run_agent( max_workers=max_workers, model_name=model_name, - dataset_name="bfcl-multi-turn-base", + dataset_name="bfcl-multi-turn-base-val", experiment_suffix="w-fixed-memory", data_path="data/multiturn_data_base_val.jsonl", answer_path=Path("data/possible_answer"), diff --git a/benchmark/bfcl/run_exp_statistic.py b/benchmark/bfcl/run_exp_statistic.py index 9eb9b3c8..18efcc8d 100644 --- a/benchmark/bfcl/run_exp_statistic.py +++ b/benchmark/bfcl/run_exp_statistic.py @@ -141,7 +141,7 @@ def run_exp_statistic(): # Sort columns by the number in column name (best@8, best@4, best@2, best@1) # best_columns = [col for col in df.columns if col.startswith('best@')] - best_columns = df.columns + best_columns = list(df.columns) best_columns.sort(key=lambda x: x, reverse=False) df = df[best_columns] diff --git a/benchmark/bfcl/split_into_trainval.py b/benchmark/bfcl/split_into_trainval.py index 82155855..e217def7 100644 --- a/benchmark/bfcl/split_into_trainval.py +++ b/benchmark/bfcl/split_into_trainval.py @@ -4,16 +4,46 @@ import argparse import json import random +from default_ids import DEFAULT_TRAIN_IDS, DEFAULT_VAL_IDS -def split_jsonl(input_file, train_file, val_file, ratio=0.8): + +def split_jsonl( + input_file: str, + train_file: str, + val_file: str, + ratio: float = 0.75, + random_split: bool = False, +) -> None: """Split the JSONL file into train and validation sets.""" with open(input_file, "r", encoding="utf-8") as f: data = [json.loads(line) for line in f] - random.shuffle(data) - split_idx = int(len(data) * ratio) - train_data = data[:split_idx] - val_data = data[split_idx:] + if random_split: + random.shuffle(data) + split_idx = int(len(data) * ratio) + train_data = data[:split_idx] + val_data = data[split_idx:] + else: + train_data = [] + val_data = [] + unknown_ids: list[str] = [] + for obj in data: + if "id" not in obj: + raise ValueError(f"Missing 'id' field in input file: {input_file}") + obj_id = str(obj["id"]) + if obj_id in DEFAULT_TRAIN_IDS: + train_data.append(obj) + elif obj_id in DEFAULT_VAL_IDS: + val_data.append(obj) + else: + unknown_ids.append(obj_id) + + if len(train_data) + len(val_data) != len(data): + missing = len(data) - (len(train_data) + len(val_data)) + examples = ", ".join(unknown_ids) if unknown_ids else "(none)" + raise ValueError( + f"{missing} samples in {input_file} not found in train_ref/val_ref id sets. Examples: {examples}", + ) with open(train_file, "w", encoding="utf-8") as f: for item in train_data: @@ -29,6 +59,11 @@ if __name__ == "__main__": parser.add_argument("--train", required=True, help="Path to output train file") parser.add_argument("--val", required=True, help="Path to output validation file") parser.add_argument("--ratio", type=float, default=0.5, help="Train ratio (default: 0.8)") - + parser.add_argument( + "--random", + action="store_true", + help="Whether to randomly split input into train/val. " + "If false, split strictly by default train/val id sets (see default_ids.py).", + ) args = parser.parse_args() - split_jsonl(args.input, args.train, args.val, args.ratio) + split_jsonl(args.input, args.train, args.val, args.ratio, args.random) diff --git a/docs/cookbook/appworld/quickstart.md b/docs/cookbook/appworld/quickstart.md index 78f6a1a8..45ea0d78 100644 --- a/docs/cookbook/appworld/quickstart.md +++ b/docs/cookbook/appworld/quickstart.md @@ -9,7 +9,7 @@ This guide helps you quickly set up and run AppWorld experiments with ReMe integ ```bash git clone https://github.com/agentscope-ai/ReMe.git -cd ReMe/cookbook/appworld +cd ReMe/benchmark/appworld ``` ### 2. Appworld Environment Setup @@ -56,26 +56,16 @@ pip install . Launch the ReMe service to enable memory library functionality: ```bash -reme \ +reme2 \ backend=http \ http.port=8002 \ - llm.default.model_name=qwen-max-latest \ - embedding_model.default.model_name=text-embedding-v4 \ - vector_store.default.backend=elasticsearch + llms.default.model_name=qwen3-8b \ + embedding_models.default.model_name=text-embedding-v4 \ + vector_stores.default.backend=es \ + vector_stores.default.collection_name=appworld \ + vector_stores.default.hosts=http://xx.yy.zz.mm:nn ``` -add memories for appworld: -```bash -curl -X POST "http://0.0.0.0:8002/vector_store" \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "appworld", - "action": "load", - "path": "./docs/library" - }' -``` -Now you have loaded the ReMe memory library to enable memory-based agent! - ### 4. Common Issues **AppWorld data not found**: Ensure `appworld download data` completed successfully @@ -95,21 +85,21 @@ python run_appworld.py ``` **What this does:** -- Runs AppWorld tasks on the development dataset +- Runs AppWorld tasks on the test-normal set - Compares agent performance with ReMe memory (`use_memory=True`) vs without memory - Uses multiple workers for parallel processing - Runs each task multiple times for statistical significance - Results are automatically saved to `./exp_result/` directory **Configuration options in `run_appworld.py`:** -- `max_workers`: Number of parallel workers (default: 8) -- `num_runs`: Number of times each task is repeated (default: 1) +- `max_workers`: Number of parallel workers (default: 16) +- `num_runs`: Number of times each task is repeated (default: 4) - `batch_size`: Number of concurrent tasks per batch (default: 8) - `num_trials`: Maximum number of self-reflections, failure-aware reflection mechanism is triggered when num_trials>1 (default: 1) -- `model_name`: Task execution model -- `use_memory`: Whether to use ReMe memory library -- `use_memory_addition`: Whether to enable selective addition -- `use_memory_deletion`: Whether to enable utility-based deletion +- `model_name`: Task execution model (default: "qwen3-8b") +- `use_memory`: Whether to use ReMe memory library (default: True) +- `use_memory_addition`: Whether to enable selective addition (default: False) +- `use_memory_deletion`: Whether to enable utility-based deletion (default: False) ### 2. View Experiment Results diff --git a/docs/cookbook/bfcl/quickstart.md b/docs/cookbook/bfcl/quickstart.md index c814b0ac..c80ef75f 100644 --- a/docs/cookbook/bfcl/quickstart.md +++ b/docs/cookbook/bfcl/quickstart.md @@ -7,99 +7,98 @@ This guide helps you quickly set up and run BFCL experiments with ReMe integrati ### 1. BFCL installation -#### clone the repository +#### Clone the repository ```bash +cd ReMe/benchmark/bfcl git clone https://github.com/ShishirPatil/gorilla.git +cd gorilla +git checkout ea13468 ``` #### Change directory to the `berkeley-function-call-leaderboard` ```bash -cd gorilla/berkeley-function-call-leaderboard +cd berkeley-function-call-leaderboard ``` #### Install the package in editable mode ```bash -conda create -n bfcl-env python==3.12 -conda activate bfcl-env pip install -e . +cd ../.. pip install -r requirements.txt ``` #### Move the dataset to the data folder under bfcl ```bash -cp -r bfcl_eval/data {/path/to/bfcl/data} +cp -r gorilla/berkeley-function-call-leaderboard/bfcl_eval/data ./ ``` -**Note**: The original BFCL data is designed as a benchmark dataset and does not have a train/validation split, you can use ``split_into_trainval.py`` to split JSONL file into train and validation sets. +#### Preprocess the data to get the suitable data format +```bash +python preprocess.py +``` -### 2. Collect agent trajectories on training data set - -Run the main experiment script to collect agent trajectories on training data set without task memory(`use_memory=False`): +**Note**: The original BFCL data is designed as a benchmark dataset and does not have a train/validation split, you can use ``split_into_trainval.py`` to split data into train and validation sets. ```bash -python run_bfcl.py +python split_into_trainval.py --input ./data/multiturn_data_base.jsonl --train ./data/multiturn_data_base_train.jsonl --val ./data/multiturn_data_base_val.jsonl ``` -**Note**: -- `max_workers`: Number of parallel workers (default: `4`) -- `num_runs`: Number of times each task is repeated (default: `1`) -- `model_name`: LLM model name (default: `qwen3-8b`) -- `enable_thinking`: Control the model's thinking mode (default: `False`) -- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_train.jsonl`) -- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`) -- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory - -### 3. Start ReMe Service and Init the task memory pool +### 2. Start ReMe Service After collecting trajectories, Launch the ReMe service (make sure you have installed ReMe environment, if not please follow the steps in the [ReMe Installation Guide](https://github.com/agentscope-ai/ReMe/blob/main/doc/README.md) to install): ```bash -reme \ +reme2 \ backend=http \ http.port=8002 \ - llm.default.model_name=qwen-max-2025-01-25 \ - embedding_model.default.model_name=text-embedding-v4 \ - vector_store.default.backend=local + llms.default.model_name=qwen3-8b \ + embedding_models.default.model_name=text-embedding-v4 \ + vector_stores.default.backend=local \ + vector_stores.default.collection_name=bfcl ``` -and then init the task memory pool: +
+Option: init the task memory pool from scratch -```bash -python init_task_memory_pool.py -``` +- First, collect agent trajectories on training data set without task memory: -**Configuration options in `init_task_memory_pool.py`:** -- `jsonl_file`: Path to the collloaded trajectories -- `service_url`: ReMe service URL (default: `http://localhost:8002`) -- `workspace_id`: Workspace ID for the task memory pool (default: `bfcl_test`) -- `n_threads`: Number of threads for processing (default: `4`) -- `output_file`: Output file to save results (optional) + ```bash + # important: num_runs = 8, use_memory = False, experiment_suffix="wo-memory", data_path="data/multiturn_data_base_train.jsonl" + python run_bfcl.py + ``` -Now you have inited the task memory pool using `local` backend (start on `http://localhost:8002`). Then, use `local_file_to_library.py` script to convert the local file to the memory library or run the following `curl` command: -```bash -curl -X POST "http://0.0.0.0:8002/vector_store" \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "bfcl_test", - "action": "dump", - "path": "./library" - }' -``` -to dump the memory library (default in `./library/bfcl_test.jsonl`). +- Second, using ReMe to construct the initial task memory pool: + ```bash + python init_task_memory_pool.py --jsonl_file ./exp_result/qwen3-8b/with_think/bfcl-multi-turn-base_wo-memory.jsonl + ``` -Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge: -```bash -curl -X POST "http://0.0.0.0:8002/vector_store" \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_id": "bfcl_test", - "action": "load", - "path": "./library" - }' -``` + > Parameters: + > `jsonl_file`: Path to the collloaded trajectories + > `service_url`: ReMe service URL (default: `http://localhost:8002`) + > `n_threads`: Number of threads for processing + > `output_file`: Output file to save results (optional) + Now you have inited the task memory pool using `local` backend. Then, run the following `curl` command to dump the memory library: + ```bash + curl -X POST "http://0.0.0.0:8002/dump_memory" \ + -H "Content-Type: application/json" \ + -d '{ + "dump_file_path": "./library/bfcl.jsonl", + }' + ``` -### 4. Run Experiments on Validation Set +- Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge: + ```bash + curl -X POST "http://0.0.0.0:8002/load_memory" \ + -H "Content-Type: application/json" \ + -d '{ + "load_file_path": "./library/bfcl.jsonl", + "clear_existing": true + }' + ``` +
+ +### 3. Run Experiments on Validation Set Run you can compare agent performance on the validation set with task memory (`use_memory=True`) and without task memory: @@ -108,6 +107,15 @@ Run you can compare agent performance on the validation set with task memory (`u python run_bfcl.py ``` +**Note**: +- `max_workers`: Number of parallel workers +- `num_runs`: Number of times each task is repeated +- `model_name`: LLM model name +- `enable_thinking`: Control the model's thinking mode +- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_val.jsonl`) +- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`) +- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory + After running experiments, analyze the statistical results: ```bash @@ -116,6 +124,6 @@ python run_exp_statistic.py **What this script does:** - Processes all result files in `./exp_result/` -- Calculates best@k metrics for different k values +- Calculates best@k&pass@k metrics for different k values - Generates a summary table showing performance comparisons - Saves results to `experiment_summary.csv` diff --git a/docs/cookbook/faq.md b/docs/cookbook/faq.md index 85ed37a3..66603e37 100644 --- a/docs/cookbook/faq.md +++ b/docs/cookbook/faq.md @@ -10,17 +10,18 @@ This document provides answers to frequently asked questions about our paper "[R reme2 \ backend=http \ http.port=8002 \ - llm.default.model_name=qwen3-8b \ - embedding_model.default.model_name=text-embedding-v4 \ - vector_store.default.backend=es + llms.default.model_name=qwen3-8b \ + embedding_models.default.model_name=text-embedding-v4 \ + vector_stores.default.backend=es \ + vector_stores.default.hosts=http://xx.yy.zz.mm:nn ``` **Evaluation Code:** [run_appworld.py](https://github.com/agentscope-ai/ReMe/blob/main/benchmark/appworld/run_appworld.py) with the following parameters |Experimental Settings|No Memory |ReMe (fixed) |ReMe (dynamic)| |---|---|---|---| -|max_workers| 16|16|16| +|max_workers|16|16|16| |batch_size|8|8|8| |num_runs|4|4|1| -|num_trials|1 |1|3| +|num_trials|1|1|3| |model_name|"qwen3-8b"|"qwen3-8b"|"qwen3-8b"| |use_memory| False| True|True| |use_memory_addition|False|False|True| diff --git a/reme/config/service.yaml b/reme/config/service.yaml index d736f215..31334099 100644 --- a/reme/config/service.yaml +++ b/reme/config/service.yaml @@ -66,7 +66,7 @@ flows: description: "Whether to enable soft comparison between highest and lowest scoring trajectories (default: true)." enable_similarity_comparison: type: boolean - description: "Whether to enable similarity-based comparison between success and failure trajectories (default: true)." + description: "Whether to enable similarity-based comparison between success and failure trajectories (default: false)." max_similarity_sequences: type: integer description: "Maximum number of sequences to compare for similarity (default: 5)." @@ -155,6 +155,7 @@ flows: description: "The path to the memories file." required: - dump_file_path + test: flow_content: TestOp() description: "test" diff --git a/reme/extension/procedural_memory/summary/comparative_extraction.py b/reme/extension/procedural_memory/summary/comparative_extraction.py index 9ca78784..4e728de3 100644 --- a/reme/extension/procedural_memory/summary/comparative_extraction.py +++ b/reme/extension/procedural_memory/summary/comparative_extraction.py @@ -49,8 +49,8 @@ class ComparativeExtraction(BaseOp): comparative_task_memories.extend(soft_task_memories) # Hard comparison: success vs failure (if similarity search is enabled) - if self.context.get("enable_similarity_comparison", True) and success_trajectories and failure_trajectories: - similar_pairs = self._find_similar_step_sequences(success_trajectories, failure_trajectories) + if self.context.get("enable_similarity_comparison", False) and success_trajectories and failure_trajectories: + similar_pairs = await self._find_similar_step_sequences(success_trajectories, failure_trajectories) logger.info(f"Found {len(similar_pairs)} similar pairs for hard comparison") for success_steps, failure_steps, similarity_score in similar_pairs: @@ -182,7 +182,7 @@ class ComparativeExtraction(BaseOp): else: return trajectory.messages - def _find_similar_step_sequences( + async def _find_similar_step_sequences( self, success_trajectories: List[Trajectory], failure_trajectories: List[Trajectory], @@ -227,8 +227,8 @@ class ComparativeExtraction(BaseOp): "embedding_model", ) ): - success_embeddings = self.vector_store.embedding_model.get_embeddings(success_texts) - failure_embeddings = self.vector_store.embedding_model.get_embeddings(failure_texts) + success_embeddings = await self.vector_store.get_embeddings(success_texts) + failure_embeddings = await self.vector_store.get_embeddings(failure_texts) # Calculate similarity and find most similar pairs similarity_threshold = self.context.get("similarity_threshold", 0.5)