mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
Update: check the code&docs for evaluation on bfcl&appworld (#141)
* fix: df.columns bug * fix: await for asynchronous method * update: docs for bfcl&appworld quickstart * update: benchmark/bfcl for new version quickstart * slightly revise bfcl cookbook * update for pre-commit * handle boolean flags in split_into_trainval.py * fix typo in faq.md
This commit is contained in:
zouyingcao
GitHub
parent
3347506e22
commit
65971bafe3
12 changed files with 352 additions and 151 deletions
206
benchmark/bfcl/default_ids.py
Normal file
206
benchmark/bfcl/default_ids.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
# pylint: disable=C0114
|
||||
DEFAULT_TRAIN_IDS: set[str] = {
|
||||
"multi_turn_base_102",
|
||||
"multi_turn_base_107",
|
||||
"multi_turn_base_110",
|
||||
"multi_turn_base_114",
|
||||
"multi_turn_base_115",
|
||||
"multi_turn_base_118",
|
||||
"multi_turn_base_122",
|
||||
"multi_turn_base_123",
|
||||
"multi_turn_base_128",
|
||||
"multi_turn_base_13",
|
||||
"multi_turn_base_130",
|
||||
"multi_turn_base_132",
|
||||
"multi_turn_base_133",
|
||||
"multi_turn_base_143",
|
||||
"multi_turn_base_144",
|
||||
"multi_turn_base_146",
|
||||
"multi_turn_base_15",
|
||||
"multi_turn_base_158",
|
||||
"multi_turn_base_169",
|
||||
"multi_turn_base_17",
|
||||
"multi_turn_base_172",
|
||||
"multi_turn_base_176",
|
||||
"multi_turn_base_182",
|
||||
"multi_turn_base_187",
|
||||
"multi_turn_base_197",
|
||||
"multi_turn_base_199",
|
||||
"multi_turn_base_22",
|
||||
"multi_turn_base_23",
|
||||
"multi_turn_base_24",
|
||||
"multi_turn_base_36",
|
||||
"multi_turn_base_40",
|
||||
"multi_turn_base_44",
|
||||
"multi_turn_base_47",
|
||||
"multi_turn_base_48",
|
||||
"multi_turn_base_5",
|
||||
"multi_turn_base_51",
|
||||
"multi_turn_base_59",
|
||||
"multi_turn_base_63",
|
||||
"multi_turn_base_65",
|
||||
"multi_turn_base_66",
|
||||
"multi_turn_base_67",
|
||||
"multi_turn_base_68",
|
||||
"multi_turn_base_70",
|
||||
"multi_turn_base_75",
|
||||
"multi_turn_base_77",
|
||||
"multi_turn_base_78",
|
||||
"multi_turn_base_79",
|
||||
"multi_turn_base_81",
|
||||
"multi_turn_base_83",
|
||||
"multi_turn_base_93",
|
||||
}
|
||||
|
||||
DEFAULT_VAL_IDS: set[str] = {
|
||||
"multi_turn_base_0",
|
||||
"multi_turn_base_1",
|
||||
"multi_turn_base_10",
|
||||
"multi_turn_base_100",
|
||||
"multi_turn_base_101",
|
||||
"multi_turn_base_103",
|
||||
"multi_turn_base_104",
|
||||
"multi_turn_base_105",
|
||||
"multi_turn_base_106",
|
||||
"multi_turn_base_108",
|
||||
"multi_turn_base_109",
|
||||
"multi_turn_base_11",
|
||||
"multi_turn_base_111",
|
||||
"multi_turn_base_112",
|
||||
"multi_turn_base_113",
|
||||
"multi_turn_base_116",
|
||||
"multi_turn_base_117",
|
||||
"multi_turn_base_119",
|
||||
"multi_turn_base_12",
|
||||
"multi_turn_base_120",
|
||||
"multi_turn_base_121",
|
||||
"multi_turn_base_124",
|
||||
"multi_turn_base_125",
|
||||
"multi_turn_base_126",
|
||||
"multi_turn_base_127",
|
||||
"multi_turn_base_129",
|
||||
"multi_turn_base_131",
|
||||
"multi_turn_base_134",
|
||||
"multi_turn_base_135",
|
||||
"multi_turn_base_136",
|
||||
"multi_turn_base_137",
|
||||
"multi_turn_base_138",
|
||||
"multi_turn_base_139",
|
||||
"multi_turn_base_14",
|
||||
"multi_turn_base_140",
|
||||
"multi_turn_base_141",
|
||||
"multi_turn_base_142",
|
||||
"multi_turn_base_145",
|
||||
"multi_turn_base_147",
|
||||
"multi_turn_base_148",
|
||||
"multi_turn_base_149",
|
||||
"multi_turn_base_150",
|
||||
"multi_turn_base_151",
|
||||
"multi_turn_base_152",
|
||||
"multi_turn_base_153",
|
||||
"multi_turn_base_154",
|
||||
"multi_turn_base_155",
|
||||
"multi_turn_base_156",
|
||||
"multi_turn_base_157",
|
||||
"multi_turn_base_159",
|
||||
"multi_turn_base_16",
|
||||
"multi_turn_base_160",
|
||||
"multi_turn_base_161",
|
||||
"multi_turn_base_162",
|
||||
"multi_turn_base_163",
|
||||
"multi_turn_base_164",
|
||||
"multi_turn_base_165",
|
||||
"multi_turn_base_166",
|
||||
"multi_turn_base_167",
|
||||
"multi_turn_base_168",
|
||||
"multi_turn_base_170",
|
||||
"multi_turn_base_171",
|
||||
"multi_turn_base_173",
|
||||
"multi_turn_base_174",
|
||||
"multi_turn_base_175",
|
||||
"multi_turn_base_177",
|
||||
"multi_turn_base_178",
|
||||
"multi_turn_base_179",
|
||||
"multi_turn_base_18",
|
||||
"multi_turn_base_180",
|
||||
"multi_turn_base_181",
|
||||
"multi_turn_base_183",
|
||||
"multi_turn_base_184",
|
||||
"multi_turn_base_185",
|
||||
"multi_turn_base_186",
|
||||
"multi_turn_base_188",
|
||||
"multi_turn_base_189",
|
||||
"multi_turn_base_19",
|
||||
"multi_turn_base_190",
|
||||
"multi_turn_base_191",
|
||||
"multi_turn_base_192",
|
||||
"multi_turn_base_193",
|
||||
"multi_turn_base_194",
|
||||
"multi_turn_base_195",
|
||||
"multi_turn_base_196",
|
||||
"multi_turn_base_198",
|
||||
"multi_turn_base_2",
|
||||
"multi_turn_base_20",
|
||||
"multi_turn_base_21",
|
||||
"multi_turn_base_25",
|
||||
"multi_turn_base_26",
|
||||
"multi_turn_base_27",
|
||||
"multi_turn_base_28",
|
||||
"multi_turn_base_29",
|
||||
"multi_turn_base_3",
|
||||
"multi_turn_base_30",
|
||||
"multi_turn_base_31",
|
||||
"multi_turn_base_32",
|
||||
"multi_turn_base_33",
|
||||
"multi_turn_base_34",
|
||||
"multi_turn_base_35",
|
||||
"multi_turn_base_37",
|
||||
"multi_turn_base_38",
|
||||
"multi_turn_base_39",
|
||||
"multi_turn_base_4",
|
||||
"multi_turn_base_41",
|
||||
"multi_turn_base_42",
|
||||
"multi_turn_base_43",
|
||||
"multi_turn_base_45",
|
||||
"multi_turn_base_46",
|
||||
"multi_turn_base_49",
|
||||
"multi_turn_base_50",
|
||||
"multi_turn_base_52",
|
||||
"multi_turn_base_53",
|
||||
"multi_turn_base_54",
|
||||
"multi_turn_base_55",
|
||||
"multi_turn_base_56",
|
||||
"multi_turn_base_57",
|
||||
"multi_turn_base_58",
|
||||
"multi_turn_base_6",
|
||||
"multi_turn_base_60",
|
||||
"multi_turn_base_61",
|
||||
"multi_turn_base_62",
|
||||
"multi_turn_base_64",
|
||||
"multi_turn_base_69",
|
||||
"multi_turn_base_7",
|
||||
"multi_turn_base_71",
|
||||
"multi_turn_base_72",
|
||||
"multi_turn_base_73",
|
||||
"multi_turn_base_74",
|
||||
"multi_turn_base_76",
|
||||
"multi_turn_base_8",
|
||||
"multi_turn_base_80",
|
||||
"multi_turn_base_82",
|
||||
"multi_turn_base_84",
|
||||
"multi_turn_base_85",
|
||||
"multi_turn_base_86",
|
||||
"multi_turn_base_87",
|
||||
"multi_turn_base_88",
|
||||
"multi_turn_base_89",
|
||||
"multi_turn_base_9",
|
||||
"multi_turn_base_90",
|
||||
"multi_turn_base_91",
|
||||
"multi_turn_base_92",
|
||||
"multi_turn_base_94",
|
||||
"multi_turn_base_95",
|
||||
"multi_turn_base_96",
|
||||
"multi_turn_base_97",
|
||||
"multi_turn_base_98",
|
||||
"multi_turn_base_99",
|
||||
}
|
||||
|
|
@ -114,6 +114,9 @@ def post_to_summarizer(trajectories: List[Any], service_url: str) -> Dict[str, A
|
|||
|
||||
request_data = {
|
||||
"trajectories": trajectory_dicts,
|
||||
"success_threshold": 1.0,
|
||||
"enable_soft_comparison": True,
|
||||
"validation_threshold": 0.5,
|
||||
}
|
||||
|
||||
try:
|
||||
|
|
@ -156,6 +159,9 @@ def process_trajectories_with_threads(
|
|||
results.append(result)
|
||||
if "memory_list" in result["metadata"]:
|
||||
print(f'✅ Group {group_index} processed: {result["metadata"].get("memory_list", 0)}')
|
||||
memory_list = result["metadata"].get("memory_list", [])
|
||||
response = requests.post(url=f"{service_url}/add_task_memory", json={"memory_list": memory_list})
|
||||
response.raise_for_status()
|
||||
else:
|
||||
print(f"❌ Group {group_index} processed: error")
|
||||
except Exception as e:
|
||||
|
|
@ -174,7 +180,7 @@ def main():
|
|||
"""Main function to convert JSONL to memories using ReMe service."""
|
||||
parser = argparse.ArgumentParser(description="Convert JSONL to memories using ReMe service")
|
||||
parser.add_argument("--jsonl_file", type=str, required=True, help="Path to the JSONL file")
|
||||
parser.add_argument("--service_url", type=str, default="http://localhost:8001", help="ReMe service URL")
|
||||
parser.add_argument("--service_url", type=str, default="http://localhost:8002", help="ReMe service URL")
|
||||
parser.add_argument("--output_file", type=str, help="Output file to save results (optional)")
|
||||
parser.add_argument("--n_threads", type=int, default=4, help="Number of threads for processing")
|
||||
|
||||
|
|
@ -226,21 +232,4 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
main()
|
||||
else:
|
||||
print("Running in compatibility mode...")
|
||||
with open("exp_result/qwen3-8b/with_think/bfcl-multi-turn-base-train_wo-exp.jsonl", "r") as f:
|
||||
data = [json.loads(line) for line in f]
|
||||
|
||||
grouped_trajectories = group_trajectories_by_task_id(data)
|
||||
print(f"Total groups: {len(grouped_trajectories)}")
|
||||
|
||||
results = process_trajectories_with_threads(
|
||||
grouped_trajectories,
|
||||
"http://localhost:8001",
|
||||
n_threads=4,
|
||||
)
|
||||
print(f"Processed {len(results)} groups")
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
"""Load the library data and convert them to the new format"""
|
||||
|
||||
import json
|
||||
|
||||
with open("../../file_vector_store/bfcl_test.jsonl", "r", encoding="utf-8") as f:
|
||||
bfcl = [json.loads(line) for line in f]
|
||||
|
||||
new_bfcl = []
|
||||
for exp in bfcl:
|
||||
new_exp = {}
|
||||
new_exp["workspace_id"] = exp["workspace_id"]
|
||||
new_exp["memory_id"] = exp["unique_id"]
|
||||
new_exp["memory_type"] = exp["metadata"]["memory_type"]
|
||||
|
||||
new_exp["when_to_use"] = exp["content"]
|
||||
new_exp["content"] = exp["metadata"]["content"]
|
||||
new_exp["score"] = exp["metadata"]["score"]
|
||||
|
||||
new_exp["time_created"] = exp["metadata"]["time_created"]
|
||||
new_exp["time_modified"] = exp["metadata"]["time_modified"]
|
||||
new_exp["author"] = exp["metadata"]["author"]
|
||||
|
||||
new_exp["metadata"] = exp["metadata"]["metadata"]
|
||||
new_exp["metadata"]["utility"] = 0
|
||||
new_exp["metadata"]["freq"] = 0
|
||||
|
||||
new_bfcl.append(new_exp)
|
||||
|
||||
with open("../../library/bfcl_test.jsonl", "w", encoding="utf-8") as f:
|
||||
f.writelines(json.dumps(item, ensure_ascii=False) + "\n" for item in new_bfcl)
|
||||
|
|
@ -2,4 +2,5 @@ jinja2
|
|||
loguru
|
||||
openai
|
||||
ray
|
||||
pandas
|
||||
pandas
|
||||
soundfile
|
||||
|
|
@ -131,7 +131,7 @@ def main():
|
|||
run_agent(
|
||||
max_workers=max_workers,
|
||||
model_name=model_name,
|
||||
dataset_name="bfcl-multi-turn-base",
|
||||
dataset_name="bfcl-multi-turn-base-val",
|
||||
experiment_suffix="w-fixed-memory",
|
||||
data_path="data/multiturn_data_base_val.jsonl",
|
||||
answer_path=Path("data/possible_answer"),
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ def run_exp_statistic():
|
|||
|
||||
# Sort columns by the number in column name (best@8, best@4, best@2, best@1)
|
||||
# best_columns = [col for col in df.columns if col.startswith('best@')]
|
||||
best_columns = df.columns
|
||||
best_columns = list(df.columns)
|
||||
best_columns.sort(key=lambda x: x, reverse=False)
|
||||
df = df[best_columns]
|
||||
|
||||
|
|
|
|||
|
|
@ -4,16 +4,46 @@ import argparse
|
|||
import json
|
||||
import random
|
||||
|
||||
from default_ids import DEFAULT_TRAIN_IDS, DEFAULT_VAL_IDS
|
||||
|
||||
def split_jsonl(input_file, train_file, val_file, ratio=0.8):
|
||||
|
||||
def split_jsonl(
|
||||
input_file: str,
|
||||
train_file: str,
|
||||
val_file: str,
|
||||
ratio: float = 0.75,
|
||||
random_split: bool = False,
|
||||
) -> None:
|
||||
"""Split the JSONL file into train and validation sets."""
|
||||
with open(input_file, "r", encoding="utf-8") as f:
|
||||
data = [json.loads(line) for line in f]
|
||||
random.shuffle(data)
|
||||
|
||||
split_idx = int(len(data) * ratio)
|
||||
train_data = data[:split_idx]
|
||||
val_data = data[split_idx:]
|
||||
if random_split:
|
||||
random.shuffle(data)
|
||||
split_idx = int(len(data) * ratio)
|
||||
train_data = data[:split_idx]
|
||||
val_data = data[split_idx:]
|
||||
else:
|
||||
train_data = []
|
||||
val_data = []
|
||||
unknown_ids: list[str] = []
|
||||
for obj in data:
|
||||
if "id" not in obj:
|
||||
raise ValueError(f"Missing 'id' field in input file: {input_file}")
|
||||
obj_id = str(obj["id"])
|
||||
if obj_id in DEFAULT_TRAIN_IDS:
|
||||
train_data.append(obj)
|
||||
elif obj_id in DEFAULT_VAL_IDS:
|
||||
val_data.append(obj)
|
||||
else:
|
||||
unknown_ids.append(obj_id)
|
||||
|
||||
if len(train_data) + len(val_data) != len(data):
|
||||
missing = len(data) - (len(train_data) + len(val_data))
|
||||
examples = ", ".join(unknown_ids) if unknown_ids else "(none)"
|
||||
raise ValueError(
|
||||
f"{missing} samples in {input_file} not found in train_ref/val_ref id sets. Examples: {examples}",
|
||||
)
|
||||
|
||||
with open(train_file, "w", encoding="utf-8") as f:
|
||||
for item in train_data:
|
||||
|
|
@ -29,6 +59,11 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--train", required=True, help="Path to output train file")
|
||||
parser.add_argument("--val", required=True, help="Path to output validation file")
|
||||
parser.add_argument("--ratio", type=float, default=0.5, help="Train ratio (default: 0.8)")
|
||||
|
||||
parser.add_argument(
|
||||
"--random",
|
||||
action="store_true",
|
||||
help="Whether to randomly split input into train/val. "
|
||||
"If false, split strictly by default train/val id sets (see default_ids.py).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
split_jsonl(args.input, args.train, args.val, args.ratio)
|
||||
split_jsonl(args.input, args.train, args.val, args.ratio, args.random)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ This guide helps you quickly set up and run AppWorld experiments with ReMe integ
|
|||
|
||||
```bash
|
||||
git clone https://github.com/agentscope-ai/ReMe.git
|
||||
cd ReMe/cookbook/appworld
|
||||
cd ReMe/benchmark/appworld
|
||||
```
|
||||
|
||||
### 2. Appworld Environment Setup
|
||||
|
|
@ -56,26 +56,16 @@ pip install .
|
|||
Launch the ReMe service to enable memory library functionality:
|
||||
|
||||
```bash
|
||||
reme \
|
||||
reme2 \
|
||||
backend=http \
|
||||
http.port=8002 \
|
||||
llm.default.model_name=qwen-max-latest \
|
||||
embedding_model.default.model_name=text-embedding-v4 \
|
||||
vector_store.default.backend=elasticsearch
|
||||
llms.default.model_name=qwen3-8b \
|
||||
embedding_models.default.model_name=text-embedding-v4 \
|
||||
vector_stores.default.backend=es \
|
||||
vector_stores.default.collection_name=appworld \
|
||||
vector_stores.default.hosts=http://xx.yy.zz.mm:nn
|
||||
```
|
||||
|
||||
add memories for appworld:
|
||||
```bash
|
||||
curl -X POST "http://0.0.0.0:8002/vector_store" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "appworld",
|
||||
"action": "load",
|
||||
"path": "./docs/library"
|
||||
}'
|
||||
```
|
||||
Now you have loaded the ReMe memory library to enable memory-based agent!
|
||||
|
||||
### 4. Common Issues
|
||||
|
||||
**AppWorld data not found**: Ensure `appworld download data` completed successfully
|
||||
|
|
@ -95,21 +85,21 @@ python run_appworld.py
|
|||
```
|
||||
|
||||
**What this does:**
|
||||
- Runs AppWorld tasks on the development dataset
|
||||
- Runs AppWorld tasks on the test-normal set
|
||||
- Compares agent performance with ReMe memory (`use_memory=True`) vs without memory
|
||||
- Uses multiple workers for parallel processing
|
||||
- Runs each task multiple times for statistical significance
|
||||
- Results are automatically saved to `./exp_result/` directory
|
||||
|
||||
**Configuration options in `run_appworld.py`:**
|
||||
- `max_workers`: Number of parallel workers (default: 8)
|
||||
- `num_runs`: Number of times each task is repeated (default: 1)
|
||||
- `max_workers`: Number of parallel workers (default: 16)
|
||||
- `num_runs`: Number of times each task is repeated (default: 4)
|
||||
- `batch_size`: Number of concurrent tasks per batch (default: 8)
|
||||
- `num_trials`: Maximum number of self-reflections, failure-aware reflection mechanism is triggered when num_trials>1 (default: 1)
|
||||
- `model_name`: Task execution model
|
||||
- `use_memory`: Whether to use ReMe memory library
|
||||
- `use_memory_addition`: Whether to enable selective addition
|
||||
- `use_memory_deletion`: Whether to enable utility-based deletion
|
||||
- `model_name`: Task execution model (default: "qwen3-8b")
|
||||
- `use_memory`: Whether to use ReMe memory library (default: True)
|
||||
- `use_memory_addition`: Whether to enable selective addition (default: False)
|
||||
- `use_memory_deletion`: Whether to enable utility-based deletion (default: False)
|
||||
|
||||
### 2. View Experiment Results
|
||||
|
||||
|
|
|
|||
|
|
@ -7,99 +7,98 @@ This guide helps you quickly set up and run BFCL experiments with ReMe integrati
|
|||
|
||||
### 1. BFCL installation
|
||||
|
||||
#### clone the repository
|
||||
#### Clone the repository
|
||||
```bash
|
||||
cd ReMe/benchmark/bfcl
|
||||
git clone https://github.com/ShishirPatil/gorilla.git
|
||||
cd gorilla
|
||||
git checkout ea13468
|
||||
```
|
||||
|
||||
#### Change directory to the `berkeley-function-call-leaderboard`
|
||||
```bash
|
||||
cd gorilla/berkeley-function-call-leaderboard
|
||||
cd berkeley-function-call-leaderboard
|
||||
```
|
||||
|
||||
#### Install the package in editable mode
|
||||
```bash
|
||||
conda create -n bfcl-env python==3.12
|
||||
conda activate bfcl-env
|
||||
pip install -e .
|
||||
cd ../..
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
#### Move the dataset to the data folder under bfcl
|
||||
```bash
|
||||
cp -r bfcl_eval/data {/path/to/bfcl/data}
|
||||
cp -r gorilla/berkeley-function-call-leaderboard/bfcl_eval/data ./
|
||||
```
|
||||
|
||||
**Note**: The original BFCL data is designed as a benchmark dataset and does not have a train/validation split, you can use ``split_into_trainval.py`` to split JSONL file into train and validation sets.
|
||||
#### Preprocess the data to get the suitable data format
|
||||
```bash
|
||||
python preprocess.py
|
||||
```
|
||||
|
||||
### 2. Collect agent trajectories on training data set
|
||||
|
||||
Run the main experiment script to collect agent trajectories on training data set without task memory(`use_memory=False`):
|
||||
**Note**: The original BFCL data is designed as a benchmark dataset and does not have a train/validation split, you can use ``split_into_trainval.py`` to split data into train and validation sets.
|
||||
|
||||
```bash
|
||||
python run_bfcl.py
|
||||
python split_into_trainval.py --input ./data/multiturn_data_base.jsonl --train ./data/multiturn_data_base_train.jsonl --val ./data/multiturn_data_base_val.jsonl
|
||||
```
|
||||
|
||||
**Note**:
|
||||
- `max_workers`: Number of parallel workers (default: `4`)
|
||||
- `num_runs`: Number of times each task is repeated (default: `1`)
|
||||
- `model_name`: LLM model name (default: `qwen3-8b`)
|
||||
- `enable_thinking`: Control the model's thinking mode (default: `False`)
|
||||
- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_train.jsonl`)
|
||||
- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`)
|
||||
- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory
|
||||
|
||||
### 3. Start ReMe Service and Init the task memory pool
|
||||
### 2. Start ReMe Service
|
||||
|
||||
After collecting trajectories, Launch the ReMe service (make sure you have installed ReMe environment, if not please follow the steps in the [ReMe Installation Guide](https://github.com/agentscope-ai/ReMe/blob/main/doc/README.md) to install):
|
||||
|
||||
```bash
|
||||
reme \
|
||||
reme2 \
|
||||
backend=http \
|
||||
http.port=8002 \
|
||||
llm.default.model_name=qwen-max-2025-01-25 \
|
||||
embedding_model.default.model_name=text-embedding-v4 \
|
||||
vector_store.default.backend=local
|
||||
llms.default.model_name=qwen3-8b \
|
||||
embedding_models.default.model_name=text-embedding-v4 \
|
||||
vector_stores.default.backend=local \
|
||||
vector_stores.default.collection_name=bfcl
|
||||
```
|
||||
|
||||
and then init the task memory pool:
|
||||
<details>
|
||||
<summary>Option: init the task memory pool from scratch</summary>
|
||||
|
||||
```bash
|
||||
python init_task_memory_pool.py
|
||||
```
|
||||
- First, collect agent trajectories on training data set without task memory:
|
||||
|
||||
**Configuration options in `init_task_memory_pool.py`:**
|
||||
- `jsonl_file`: Path to the collloaded trajectories
|
||||
- `service_url`: ReMe service URL (default: `http://localhost:8002`)
|
||||
- `workspace_id`: Workspace ID for the task memory pool (default: `bfcl_test`)
|
||||
- `n_threads`: Number of threads for processing (default: `4`)
|
||||
- `output_file`: Output file to save results (optional)
|
||||
```bash
|
||||
# important: num_runs = 8, use_memory = False, experiment_suffix="wo-memory", data_path="data/multiturn_data_base_train.jsonl"
|
||||
python run_bfcl.py
|
||||
```
|
||||
|
||||
Now you have inited the task memory pool using `local` backend (start on `http://localhost:8002`). Then, use `local_file_to_library.py` script to convert the local file to the memory library or run the following `curl` command:
|
||||
```bash
|
||||
curl -X POST "http://0.0.0.0:8002/vector_store" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "bfcl_test",
|
||||
"action": "dump",
|
||||
"path": "./library"
|
||||
}'
|
||||
```
|
||||
to dump the memory library (default in `./library/bfcl_test.jsonl`).
|
||||
- Second, using ReMe to construct the initial task memory pool:
|
||||
```bash
|
||||
python init_task_memory_pool.py --jsonl_file ./exp_result/qwen3-8b/with_think/bfcl-multi-turn-base_wo-memory.jsonl
|
||||
```
|
||||
|
||||
Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge:
|
||||
```bash
|
||||
curl -X POST "http://0.0.0.0:8002/vector_store" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"workspace_id": "bfcl_test",
|
||||
"action": "load",
|
||||
"path": "./library"
|
||||
}'
|
||||
```
|
||||
> Parameters:
|
||||
> `jsonl_file`: Path to the collloaded trajectories
|
||||
> `service_url`: ReMe service URL (default: `http://localhost:8002`)
|
||||
> `n_threads`: Number of threads for processing
|
||||
> `output_file`: Output file to save results (optional)
|
||||
|
||||
Now you have inited the task memory pool using `local` backend. Then, run the following `curl` command to dump the memory library:
|
||||
```bash
|
||||
curl -X POST "http://0.0.0.0:8002/dump_memory" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"dump_file_path": "./library/bfcl.jsonl",
|
||||
}'
|
||||
```
|
||||
|
||||
### 4. Run Experiments on Validation Set
|
||||
- Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge:
|
||||
```bash
|
||||
curl -X POST "http://0.0.0.0:8002/load_memory" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"load_file_path": "./library/bfcl.jsonl",
|
||||
"clear_existing": true
|
||||
}'
|
||||
```
|
||||
</details>
|
||||
|
||||
### 3. Run Experiments on Validation Set
|
||||
|
||||
Run you can compare agent performance on the validation set with task memory (`use_memory=True`) and without task memory:
|
||||
|
||||
|
|
@ -108,6 +107,15 @@ Run you can compare agent performance on the validation set with task memory (`u
|
|||
python run_bfcl.py
|
||||
```
|
||||
|
||||
**Note**:
|
||||
- `max_workers`: Number of parallel workers
|
||||
- `num_runs`: Number of times each task is repeated
|
||||
- `model_name`: LLM model name
|
||||
- `enable_thinking`: Control the model's thinking mode
|
||||
- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_val.jsonl`)
|
||||
- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`)
|
||||
- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory
|
||||
|
||||
After running experiments, analyze the statistical results:
|
||||
|
||||
```bash
|
||||
|
|
@ -116,6 +124,6 @@ python run_exp_statistic.py
|
|||
|
||||
**What this script does:**
|
||||
- Processes all result files in `./exp_result/`
|
||||
- Calculates best@k metrics for different k values
|
||||
- Calculates best@k&pass@k metrics for different k values
|
||||
- Generates a summary table showing performance comparisons
|
||||
- Saves results to `experiment_summary.csv`
|
||||
|
|
|
|||
|
|
@ -10,17 +10,18 @@ This document provides answers to frequently asked questions about our paper "[R
|
|||
reme2 \
|
||||
backend=http \
|
||||
http.port=8002 \
|
||||
llm.default.model_name=qwen3-8b \
|
||||
embedding_model.default.model_name=text-embedding-v4 \
|
||||
vector_store.default.backend=es
|
||||
llms.default.model_name=qwen3-8b \
|
||||
embedding_models.default.model_name=text-embedding-v4 \
|
||||
vector_stores.default.backend=es \
|
||||
vector_stores.default.hosts=http://xx.yy.zz.mm:nn
|
||||
```
|
||||
**Evaluation Code:** [run_appworld.py](https://github.com/agentscope-ai/ReMe/blob/main/benchmark/appworld/run_appworld.py) with the following parameters
|
||||
|Experimental Settings|No Memory |ReMe (fixed) |ReMe (dynamic)|
|
||||
|---|---|---|---|
|
||||
|max_workers| 16|16|16|
|
||||
|max_workers|16|16|16|
|
||||
|batch_size|8|8|8|
|
||||
|num_runs|4|4|1|
|
||||
|num_trials|1 |1|3|
|
||||
|num_trials|1|1|3|
|
||||
|model_name|"qwen3-8b"|"qwen3-8b"|"qwen3-8b"|
|
||||
|use_memory| False| True|True|
|
||||
|use_memory_addition|False|False|True|
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ flows:
|
|||
description: "Whether to enable soft comparison between highest and lowest scoring trajectories (default: true)."
|
||||
enable_similarity_comparison:
|
||||
type: boolean
|
||||
description: "Whether to enable similarity-based comparison between success and failure trajectories (default: true)."
|
||||
description: "Whether to enable similarity-based comparison between success and failure trajectories (default: false)."
|
||||
max_similarity_sequences:
|
||||
type: integer
|
||||
description: "Maximum number of sequences to compare for similarity (default: 5)."
|
||||
|
|
@ -155,6 +155,7 @@ flows:
|
|||
description: "The path to the memories file."
|
||||
required:
|
||||
- dump_file_path
|
||||
|
||||
test:
|
||||
flow_content: TestOp()
|
||||
description: "test"
|
||||
|
|
|
|||
|
|
@ -49,8 +49,8 @@ class ComparativeExtraction(BaseOp):
|
|||
comparative_task_memories.extend(soft_task_memories)
|
||||
|
||||
# Hard comparison: success vs failure (if similarity search is enabled)
|
||||
if self.context.get("enable_similarity_comparison", True) and success_trajectories and failure_trajectories:
|
||||
similar_pairs = self._find_similar_step_sequences(success_trajectories, failure_trajectories)
|
||||
if self.context.get("enable_similarity_comparison", False) and success_trajectories and failure_trajectories:
|
||||
similar_pairs = await self._find_similar_step_sequences(success_trajectories, failure_trajectories)
|
||||
logger.info(f"Found {len(similar_pairs)} similar pairs for hard comparison")
|
||||
|
||||
for success_steps, failure_steps, similarity_score in similar_pairs:
|
||||
|
|
@ -182,7 +182,7 @@ class ComparativeExtraction(BaseOp):
|
|||
else:
|
||||
return trajectory.messages
|
||||
|
||||
def _find_similar_step_sequences(
|
||||
async def _find_similar_step_sequences(
|
||||
self,
|
||||
success_trajectories: List[Trajectory],
|
||||
failure_trajectories: List[Trajectory],
|
||||
|
|
@ -227,8 +227,8 @@ class ComparativeExtraction(BaseOp):
|
|||
"embedding_model",
|
||||
)
|
||||
):
|
||||
success_embeddings = self.vector_store.embedding_model.get_embeddings(success_texts)
|
||||
failure_embeddings = self.vector_store.embedding_model.get_embeddings(failure_texts)
|
||||
success_embeddings = await self.vector_store.get_embeddings(success_texts)
|
||||
failure_embeddings = await self.vector_store.get_embeddings(failure_texts)
|
||||
|
||||
# Calculate similarity and find most similar pairs
|
||||
similarity_threshold = self.context.get("similarity_threshold", 0.5)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue