diff --git a/cookbook/bfcl/bfcl_agent.py b/cookbook/bfcl/bfcl_agent.py
index cd0c3200..db179df9 100644
--- a/cookbook/bfcl/bfcl_agent.py
+++ b/cookbook/bfcl/bfcl_agent.py
@@ -63,14 +63,14 @@ class BFCLAgent:
max_response_size: int = 2000,
num_runs: int = 1,
enable_thinking: bool = False,
- use_experience: bool = False,
- use_fixed_experience: bool = True,
- use_experience_deletion: bool = False,
+ use_memory: bool = False,
+ use_memory_addition: bool = False,
+ use_memory_deletion: bool = False,
delete_freq: int = 10,
freq_threshold: int = 5,
utility_threshold: float = 0.5,
- experience_base_url: str = "http://0.0.0.0:8001/",
- experience_workspace_id: str = "bfcl_8b_0725"):
+ memory_base_url: str = "http://0.0.0.0:8001/",
+ memory_workspace_id: str = "bfcl_8b_0725"):
self.index: int = index
self.task_ids: List[str] = task_ids
@@ -84,17 +84,17 @@ class BFCLAgent:
self.max_response_size: int = max_response_size
self.num_runs: int = num_runs
self.enable_thinking: bool = enable_thinking
- self.use_experience: bool = use_experience
- self.use_fixed_experience: bool = use_fixed_experience if use_experience else True
- self.use_experience_deletion: bool = use_experience_deletion if not use_fixed_experience else False
+ self.use_memory: bool = use_memory
+ self.use_memory_addition: bool = use_memory_addition if use_memory else False
+ self.use_memory_deletion: bool = use_memory_deletion if use_memory else False
self.delete_freq: int = delete_freq
self.freq_threshold: int = freq_threshold
self.utility_threshold: float = utility_threshold
- self.experience_base_url: str = experience_base_url
- self.experience_workspace_id: str = experience_workspace_id
+ self.memory_base_url: str = memory_base_url
+ self.memory_workspace_id: str = memory_workspace_id
self.history: List[List[List[dict]]] = [[] for _ in range(num_runs)]
- self.retrieved_experience_ids: List[List[List[str]]] = [[] for _ in range(num_runs)]
+ self.retrieved_memory_list: List[List[List[Any]]] = [[] for _ in range(num_runs)]
self.test_entry: List[List[Dict[str, Any]]] = [[] for _ in range(num_runs)]
self.original_test_entry: List[List[Dict[str, Any]]] = [[] for _ in range(num_runs)]
self.tool_schema: List[List[List[dict]]] = [[] for _ in range(num_runs)]
@@ -112,74 +112,71 @@ class BFCLAgent:
# 初始历史
msg = self.test_entry[run_id][i].get("messages", [])[0]
- if self.use_experience:
+ if self.use_memory:
query = msg["content"]
- response = self.get_experience(query)
- if len(response["experience_list"]):
- self.retrieved_experience_ids[run_id].append([e["experience_id"] for e in response["experience_list"]])
- exp: str = response["experience_merged"]
- print(f"experience_merged={exp}")
- self.history[run_id].append([self.get_query_with_experience(query, exp)])
- self.update_experience_freq(self.retrieved_experience_ids[run_id][i])
+ response = self.get_memory(query)
+
+ if len(response["metadata"]["memory_list"]):
+ self.retrieved_memory_list[run_id].append(response["metadata"]["memory_list"])
+ exp: str = response["answer"]
+ # print(f"memory_merged={exp}")
+ self.history[run_id].append([self.get_query_with_memory(query, exp)])
else:
- self.retrieved_experience_ids[run_id].append([])
+ self.retrieved_memory_list[run_id].append([])
self.history[run_id].append([msg])
else:
self.history[run_id].append([msg])
self.current_turn[run_id][i] = 1
- def get_query_with_experience(self, query: str, experience: str):
+ def get_query_with_memory(self, query: str, memory: str):
return {
"role": "user",
- "content": "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + experience
+ "content": "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + memory
}
-
- def get_experience(self, query: str):
- response = requests.post(url=self.experience_base_url + "retriever", json={
- "workspace_id": self.experience_workspace_id,
+
+ def get_traj_from_task_history(self, task_id: str, task_history: list, reward: float):
+ return {
+ "task_id": task_id,
+ "messages": task_history,
+ "score": reward
+ }
+
+ def get_memory(self, query: str):
+ response = requests.post(url=self.memory_base_url + "retrieve_task_memory", json={
+ "workspace_id": self.memory_workspace_id,
"query": query,
"top_k": 5
})
- logger.info(f"query:{query}")
if response.status_code != 200:
logger.info(response.text)
return ""
response = response.json()
- logger.info(response)
+ logger.info(f"query: {query}, response: {response}")
return response
- def add_experience(self, trajectories):
- response = requests.post(url=self.experience_base_url + "summarizer", json={
- "workspace_id": self.experience_workspace_id,
- "traj_list": trajectories,
+ def add_memory(self, trajectories):
+ response = requests.post(url=self.memory_base_url + "summary_task_memory", json={
+ "workspace_id": self.memory_workspace_id,
+ "trajectories": trajectories,
})
response.raise_for_status()
response = response.json()
- logger.info(f"add new experiences: {response["experience_list"]}")
+ logger.info(f"add new memorys: {response["metadata"]["memory_list"]}")
- def update_experience_freq(self, experience_ids):
- response = requests.post(url=self.experience_base_url + "vector_store", json={
- "workspace_id": self.experience_workspace_id,
- "action": "update_freq",
- "experience_ids": experience_ids,
+ def update_memory_information(self, memory_list, update_utility: bool=False):
+ response = requests.post(url=self.memory_base_url + "record_task_memory", json={
+ "workspace_id": self.memory_workspace_id,
+ "memory_dicts": memory_list,
+ "update_utility": update_utility
})
response.raise_for_status()
logger.info(response.json())
- def update_experience_utility(self, experience_ids):
- response = requests.post(url=self.experience_base_url + "vector_store", json={
- "workspace_id": self.experience_workspace_id,
- "action": "update_utility",
- "experience_ids": experience_ids,
- })
- response.raise_for_status()
-
- def delete_experience(self):
- response = requests.post(url=self.experience_base_url + "vector_store", json={
- "workspace_id": self.experience_workspace_id,
- "action": "utility_based_delete",
+ def delete_memory(self):
+ response = requests.post(url=self.memory_base_url + "delete_task_memory", json={
+ "workspace_id": self.memory_workspace_id,
"freq_threshold": self.freq_threshold,
"utility_threshold": self.utility_threshold
})
@@ -563,21 +560,18 @@ class BFCLAgent:
break
reward = self.get_reward(run_id, task_index)
- if reward == 1 and self.use_experience and not self.use_fixed_experience:
- # selectively add experiences when succeed
- self.add_experience([{
- "task_id":task_id,
- "messages":self.history[run_id][task_index],
- "score":reward
- }])
+ if self.use_memory:
+ if reward == 1 and self.use_memory_addition: # selectively add memories when succeed
+ new_traj_list = [self.get_traj_from_task_history(task_id, self.history[run_id][task_index], reward)]
+ self.add_memory(new_traj_list)
- if len(self.retrieved_experience_ids[run_id][task_index]):
- # update the utility-related attributes of retrieved experiences
- self.update_experience_utility(self.retrieved_experience_ids[run_id][task_index])
+ # update the freq & utility attributes of retrieved memories
+ update_utility: bool = (reward == 1)
+ self.update_memory_information(self.retrieved_memory_list[run_id][task_index], update_utility)
counter += 1
- if self.use_experience_deletion and counter % self.delete_freq == 0:
- self.delete_experience()
+ if self.use_memory_deletion and counter % self.delete_freq == 0:
+ self.delete_memory()
t_result = {
"run_id": run_id,
diff --git a/cookbook/bfcl/init_task_memory_pool.py b/cookbook/bfcl/init_task_memory_pool.py
new file mode 100644
index 00000000..5f4d9021
--- /dev/null
+++ b/cookbook/bfcl/init_task_memory_pool.py
@@ -0,0 +1,223 @@
+import json
+import requests
+import argparse
+from pathlib import Path
+from typing import List, Dict, Any
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+
+def load_task_case(data_path: str, task_id: str | None) -> Dict[str, Any]:
+ """
+ load training cases by id
+ """
+ if not Path(data_path).exists():
+ raise FileNotFoundError(f"BFCL data file '{data_path}' not found")
+
+ if task_id is None:
+ raise ValueError("task_id is required")
+
+ with open(data_path, "r", encoding="utf-8") as f:
+ if str(task_id).isdigit():
+ idx = int(task_id)
+ for line_no, line in enumerate(f):
+ if line_no == idx:
+ return json.loads(line)
+ raise ValueError(f"Task case index {idx} not found in {data_path}")
+ else:
+ for line in f:
+ data = json.loads(line)
+ if data.get("id") == task_id:
+ return data
+ raise ValueError(f"Task case id '{task_id}' not found in {data_path}")
+
+
+def get_tool_prompt(tools):
+ tool_prompt = "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n"
+ for tool in tools:
+ tool_prompt += "\n" + json.dumps(tool)
+ tool_prompt += "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n"
+ return tool_prompt
+
+
+def group_trajectories_by_task_id(jsonl_entries: List[Dict[str, Any]]) -> List[List[Any]]:
+ """
+ group trajectories by task_id
+
+ Args:
+ jsonl_entries: JSONL entry list
+
+ Returns:
+ List[List[Any]]: trajectory list grouped by task_id
+ """
+ grouped = defaultdict(list)
+
+ for entry in jsonl_entries:
+ task_id = entry.get("task_id", "")
+ taks_case = load_task_case("data/multiturn_data_base.jsonl", task_id)
+ tools = taks_case.get("tools", [{}])
+ from bfcl_utils import extract_tool_schema
+ tool_schema = extract_tool_schema(tools)
+ entry["task_history"][0]["content"] += get_tool_prompt(tool_schema)
+ grouped[task_id].append(entry)
+
+ # retain only the two with the highest and lowest rewards
+ filtered_groups = []
+ for key, trajectories in grouped.items():
+ if len(trajectories) == 1:
+ # when only one trajectory, retain it
+ filtered_groups.append(trajectories)
+ elif len(trajectories) == 2:
+ # when there are two trajectories, retain them
+ filtered_groups.append(trajectories)
+ else:
+ # when there are more than two trajectories, choose the two with the highest and lowest rewards
+ trajectories.sort(key=lambda t: t["reward"])
+ min_reward_traj = trajectories[0] # highest reward
+ max_reward_traj = trajectories[-1] # lowest reward
+ filtered_groups.append([min_reward_traj, max_reward_traj])
+
+ return filtered_groups
+
+
+def post_to_summarizer(trajectories: List[Any], service_url: str, workspace_id: str) -> Dict[str, Any]:
+ trajectory_dicts = [{
+ "task_id": traj["task_id"],
+ "messages": traj["task_history"],
+ "score": traj["reward"]
+ } for traj in trajectories]
+
+ request_data = {
+ "trajectories": trajectory_dicts,
+ "workspace_id": workspace_id
+ }
+
+ try:
+ response = requests.post(f"{service_url}/summary_task_memory", json=request_data)
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ return {"error": str(e), "trajectories_count": len(trajectories)}
+
+
+def process_trajectories_with_threads(grouped_trajectories: List[List[Any]],
+ service_url: str,
+ workspace_id: str,
+ n_threads: int = 4) -> List[Dict[str, Any]]:
+ """
+ use threads to process trajectories
+
+ Args:
+ grouped_trajectories: group trajectory list by task_id
+ service_url: memory summarizer service URL
+ workspace_id: workspace ID
+ n_threads: number of threads
+
+ Returns:
+ all results
+ """
+ results = []
+
+ with ThreadPoolExecutor(max_workers=n_threads) as executor:
+ future_to_group = {
+ executor.submit(post_to_summarizer, group, service_url, workspace_id): i
+ for i, group in enumerate(grouped_trajectories)
+ }
+
+ for future in as_completed(future_to_group):
+ group_index = future_to_group[future]
+ try:
+ result = future.result()
+ result["group_index"] = group_index
+ result["group_size"] = len(grouped_trajectories[group_index])
+ results.append(result)
+ print(f"✅ Group {group_index} processed: {result["metadata"].get('memory_list', 0) if 'memory_list' in result["metadata"] else 'error'}")
+ except Exception as e:
+ error_result = {
+ "group_index": group_index,
+ "group_size": len(grouped_trajectories[group_index]),
+ "error": str(e)
+ }
+ results.append(error_result)
+ print(f"❌ Group {group_index} failed: {e}")
+
+ return results
+
+
+def main():
+ parser = argparse.ArgumentParser(description='Convert JSONL to memories using ReMe service')
+ parser.add_argument('--jsonl_file', type=str, required=True, help='Path to the JSONL file')
+ parser.add_argument('--service_url', type=str, default='http://localhost:8001', help='Reme service URL')
+ parser.add_argument('--workspace_id', type=str, required=True, help='Workspace ID for the task memory pool')
+ parser.add_argument('--output_file', type=str, help='Output file to save results (optional)')
+ parser.add_argument('--n_threads', type=int, default=4, help='Number of threads for processing')
+
+ args = parser.parse_args()
+
+ print(f"Processing JSONL file: {args.jsonl_file}")
+ print(f"Service URL: {args.service_url}")
+ print(f"Workspace ID: {args.workspace_id}")
+ print(f"Threads: {args.n_threads}")
+
+ with open(args.jsonl_file, "r") as f:
+ data = [json.loads(line) for line in f]
+ print(f"Loaded {len(data)} entries from JSONL file")
+
+ grouped_trajectories = group_trajectories_by_task_id(data)
+ print(f"Total groups: {len(grouped_trajectories)}")
+
+ results = process_trajectories_with_threads(
+ grouped_trajectories,
+ args.service_url,
+ args.workspace_id,
+ n_threads=args.n_threads
+ )
+
+ print(f"Processed {len(results)} groups")
+
+ success_count = sum(1 for r in results if 'error' not in r)
+ error_count = len(results) - success_count
+ total_memories = sum(len(r["metadata"].get('memory_list', [])) for r in results if 'memory_list' in r["metadata"])
+
+ print(f"✅ Success: {success_count}")
+ print(f"❌ Errors: {error_count}")
+ print(f"📊 Total task memories created: {total_memories}")
+
+ if args.output_file:
+ try:
+ summary = {
+ "workspace_id": args.workspace_id,
+ "jsonl_file": args.jsonl_file,
+ "total_groups": len(grouped_trajectories),
+ "success_count": success_count,
+ "error_count": error_count,
+ "total_task_memories": total_memories,
+ "results": results
+ }
+
+ with open(args.output_file, 'w') as f:
+ json.dump(summary, f, indent=2)
+ print(f"Results saved to: {args.output_file}")
+ except Exception as e:
+ print(f"Error saving results: {e}")
+
+
+if __name__ == "__main__":
+ import sys
+ if len(sys.argv) > 1:
+ main()
+ else:
+ print("Running in compatibility mode...")
+ with open("exp_result/qwen3-14b/no_think/bfcl-multi-turn-base_wo-exp.jsonl", "r") as f:
+ data = [json.loads(line) for line in f]
+
+ grouped_trajectories = group_trajectories_by_task_id(data)
+ print(f"Total groups: {len(grouped_trajectories)}")
+
+ results = process_trajectories_with_threads(
+ grouped_trajectories,
+ "http://localhost:8001",
+ "bfcl_test",
+ n_threads=4
+ )
+ print(f"Processed {len(results)} groups")
\ No newline at end of file
diff --git a/cookbook/bfcl/local_file_to_library.py b/cookbook/bfcl/local_file_to_library.py
index 28eccc08..afee8374 100644
--- a/cookbook/bfcl/local_file_to_library.py
+++ b/cookbook/bfcl/local_file_to_library.py
@@ -1,21 +1,27 @@
import json
-with open("../../file_vector_store/bfcl_train50_extract_compare_validate.jsonl", 'r') as f:
- appworld = [json.loads(line) for line in f]
+with open("../../file_vector_store/bfcl_test.jsonl", 'r') as f:
+ bfcl = [json.loads(line) for line in f]
-new_appworld = []
-for exp in appworld:
+new_bfcl = []
+for exp in bfcl:
new_exp = {}
new_exp["workspace_id"] = exp["workspace_id"]
- new_exp["experience_id"] = exp["unique_id"]
- new_exp["experience_type"] = "text"
+ new_exp["memory_id"] = exp["unique_id"]
+ new_exp["memory_type"] = exp["metadata"]["memory_type"]
+
new_exp["when_to_use"] = exp["content"]
- new_exp["content"] = exp["metadata"]["experience_content"]
+ new_exp["content"] = exp["metadata"]["content"]
new_exp["score"] = exp["metadata"]["score"]
+
+ new_exp["time_created"] = exp["metadata"]["time_created"]
+ new_exp["time_modified"] = exp["metadata"]["time_modified"]
+ new_exp["author"] = exp["metadata"]["author"]
+
new_exp["metadata"]= exp["metadata"]["metadata"]
- new_appworld.append(new_exp)
+ new_bfcl.append(new_exp)
-with open('../../library/bfcl_train50_extract_compare_validate.jsonl', 'w', encoding='utf-8') as f:
- f.writelines(json.dumps(item, ensure_ascii=False) + '\n' for item in new_appworld)
\ No newline at end of file
+with open('../../library/bfcl_test.jsonl', 'w', encoding='utf-8') as f:
+ f.writelines(json.dumps(item, ensure_ascii=False) + '\n' for item in new_bfcl)
\ No newline at end of file
diff --git a/cookbook/bfcl/quickstart.md b/cookbook/bfcl/quickstart.md
index 5f9e7918..7569646a 100644
--- a/cookbook/bfcl/quickstart.md
+++ b/cookbook/bfcl/quickstart.md
@@ -1,6 +1,6 @@
# BFCL Experiment Quick Start Guide
-This guide helps you quickly set up and run BFCL experiments with ExperienceMaker integration.
+This guide helps you quickly set up and run BFCL experiments with ReMe integration.
## Env Setup
@@ -32,7 +32,7 @@ cp -r bfcl_eval/data {/path/to/bfcl/data}
### 2. Collect agent trajectories on training data set
-Run the main experiment script to collect agent trajectories on training data set without experience(`use_experience=False`):
+Run the main experiment script to collect agent trajectories on training data set without task memory(`use_memory=False`):
```bash
python run_bfcl.py
@@ -40,29 +40,30 @@ python run_bfcl.py
**Note**:
- `max_workers`: Number of parallel workers (default: `4`)
-- `num_runs`: Number of times each task is repeated (default: `4`)
+- `num_runs`: Number of times each task is repeated (default: `1`)
- `model_name`: LLM model name (default: `qwen3-8b`)
- `enable_thinking`: Control the model's thinking mode (default: `False`)
- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_train.jsonl`)
- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`)
- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory
-### 3. Start ExperienceMaker Service and Init the Experience Pool
+### 3. Start ReMe Service and Init the task memory pool
-After collecting trajectories, Launch the ExperienceMaker service to enable experience library functionality:
+After collecting trajectories, Launch the ReMe service to enable memory library functionality:
```bash
# Go back to the project root
cd ../..
-experiencemaker \
- http_service.port=8001 \
+reme \
+ backend=http \
+ http.port=8001 \
llm.default.model_name=qwen-max-2025-01-25 \
embedding_model.default.model_name=text-embedding-v4 \
- vector_store.default.backend=local_file
+ vector_store.default.backend=local
```
-and then init the experience pool:
+and then init the task memory pool:
```bash
python init_exp_pool.py
@@ -70,29 +71,29 @@ python init_exp_pool.py
**Configuration options in `init_exp_pool.py`:**
- `jsonl_file`: Path to the collloaded trajectories
-- `service_url`: Experience maker service URL (default: `http://localhost:8001`)
-- `workspace_id`: Workspace ID for the experience (default: `bfcl_v1`)
+- `service_url`: ReMe service URL (default: `http://localhost:8001`)
+- `workspace_id`: Workspace ID for the task memory pool (default: `bfcl_test`)
- `n_threads`: Number of threads for processing (default: `4`)
- `output_file`: Output file to save results (optional)
-Now you have inited the experience pool using `local_file` backend (start on `http://localhost:8001`). The `local_file_to_library.py` script or use the following `curl` command:
+Now you have inited the task memory pool using `local` backend (start on `http://localhost:8001`). The `local_file_to_library.py` script or use the following `curl` command:
```bash
curl -X POST "http://0.0.0.0:8001/vector_store" \
-H "Content-Type: application/json" \
-d '{
- "workspace_id": "bfcl_v1",
+ "workspace_id": "bfcl_test",
"action": "dump",
"path": "./library"
}'
```
-can convert the local file to the experience library (default in `./library/bfcl_v1.jsonl`).
+can convert the local file to the memory library (default in `./library/bfcl_test.jsonl`).
-Next time, you can import this previously exported experience data to populate the new started workspace with existing knowledge:
+Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge:
```bash
curl -X POST "http://0.0.0.0:8001/vector_store" \
-H "Content-Type: application/json" \
-d '{
- "workspace_id": "bfcl_v1",
+ "workspace_id": "bfcl_test",
"action": "load",
"path": "./library"
}'
@@ -101,7 +102,7 @@ curl -X POST "http://0.0.0.0:8001/vector_store" \
### 4. Run Experiments on Validation Set
-Run you can compare agent performance on the validation set with experience (`use_experience=True`) and without experience:
+Run you can compare agent performance on the validation set with task memory (`use_memory=True`) and without task memory:
```bash
# remember to change the configuration options, e.g., `data_path=./data/multiturn_data_base_val.jsonl`
diff --git a/cookbook/bfcl/run_bfcl.py b/cookbook/bfcl/run_bfcl.py
index c116188a..4eb1570b 100644
--- a/cookbook/bfcl/run_bfcl.py
+++ b/cookbook/bfcl/run_bfcl.py
@@ -1,7 +1,8 @@
import os
import time
import ray
-from ray import logger
+# from ray import logger
+from loguru import logger
from dotenv import load_dotenv
@@ -21,15 +22,15 @@ def run_agent(dataset_name: str,
model_name: str = "qwen3-8b",
data_path: str = "data/multiturn_data_base_val.jsonl",
answer_path: Path = Path("data/possible_answer"),
- use_experience: bool = False,
- use_fixed_experience: bool = True,
- use_experience_deletion: bool = False,
+ use_memory: bool = False,
+ use_memory_addition: bool = True,
+ use_memory_deletion: bool = False,
delete_freq: int = 10,
freq_threshold: int = 5,
utility_threshold: float = 0.5,
enable_thinking: bool = False,
- experience_base_url: str = "http://0.0.0.0:8001/",
- experience_workspace_id: str = "bfcl_8b_0725"):
+ memory_base_url: str = "http://0.0.0.0:8001/",
+ memory_workspace_id: str = "bfcl_test"):
experiment_name = dataset_name + "_" + experiment_suffix
path: Path = Path(f"./exp_result/{model_name}/with_think" if enable_thinking else f"./exp_result/{model_name}/no_think")
path.mkdir(parents=True, exist_ok=True)
@@ -54,15 +55,15 @@ def run_agent(dataset_name: str,
answer_path=answer_path,
model_name=model_name,
num_runs=num_runs,
- use_experience=use_experience,
- use_fixed_experience=use_fixed_experience,
- use_experience_deletion=use_experience_deletion,
+ use_memory=use_memory,
+ use_memory_addition=use_memory_addition,
+ use_memory_deletion=use_memory_deletion,
delete_freq=delete_freq,
freq_threshold=freq_threshold,
utility_threshold=utility_threshold,
enable_thinking=enable_thinking,
- experience_base_url=experience_base_url,
- experience_workspace_id=experience_workspace_id
+ memory_base_url=memory_base_url,
+ memory_workspace_id=memory_workspace_id
)
future = actor.execute.remote()
future_list.append(future)
@@ -83,32 +84,32 @@ def run_agent(dataset_name: str,
def main():
max_workers = 4
- num_runs = 8
- use_experience = False
- use_fixed_experience = False
- use_experience_deletion = True
- experience_base_url = "http://0.0.0.0:8003/"
- experience_workspace_id = "bfcl_train50_extract_compare_validate_add_delete_2"
+ num_runs = 1
+ use_memory = False
+ use_memory_addition = False
+ use_memory_deletion = False
+ memory_base_url = "http://0.0.0.0:8001/"
+ memory_workspace_id = "bfcl_test"
if max_workers > 1:
ray.init(num_cpus=max_workers)
for run_id in range(num_runs):
run_agent(
dataset_name="bfcl-multi-turn-base",
experiment_suffix=f"wo-exp",
- model_name="qwen3-14b",
+ model_name="qwen3-8b",
max_workers=max_workers,
num_runs=1,
- data_path="data/multiturn_data_base_train.jsonl",
+ data_path="data/multiturn_data_base_val.jsonl",
answer_path=Path("data/possible_answer"),
- enable_thinking=True,
- use_experience=use_experience,
- use_fixed_experience=use_fixed_experience,
- use_experience_deletion=use_experience_deletion,
+ enable_thinking=False,
+ use_memory=use_memory,
+ use_memory_addition=use_memory_addition,
+ use_memory_deletion=use_memory_deletion,
delete_freq=5,
freq_threshold=5,
utility_threshold=0.5,
- experience_base_url=experience_base_url,
- experience_workspace_id=experience_workspace_id,
+ memory_base_url=memory_base_url,
+ memory_workspace_id=memory_workspace_id,
)
diff --git a/cookbook/bfcl/run_exp_statistic.py b/cookbook/bfcl/run_exp_statistic.py
index d577bc33..8d689599 100644
--- a/cookbook/bfcl/run_exp_statistic.py
+++ b/cookbook/bfcl/run_exp_statistic.py
@@ -61,7 +61,7 @@ def get_possible_k_values(total_runs: int) -> list:
def run_exp_statistic():
- path: Path = Path(f"./exp_result/qwen-max-latest/no_think")
+ path: Path = Path(f"./exp_result/qwen3-8b/no_think")
# Store results for all experiments
all_results = {}