mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-07 08:26:06 +00:00
update for pre-commit check
This commit is contained in:
caozouying.czy
parent
4ad977408e
commit
3760c22d45
19 changed files with 325 additions and 214 deletions
|
|
@ -1,16 +1,19 @@
|
|||
# flake8: noqa: E402, E501
|
||||
# pylint: disable=E0611
|
||||
"""A minimal ReAct Agent for AppWorld tasks."""
|
||||
import os
|
||||
import re
|
||||
import ray
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
import datetime
|
||||
from typing import List, Any
|
||||
|
||||
|
||||
import ray
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
from loguru import logger
|
||||
from openai import OpenAI
|
||||
from typing import List, Any
|
||||
from jinja2 import Template
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
|
@ -59,7 +62,6 @@ class AppworldReactAgent:
|
|||
self.delete_freq: int = delete_freq
|
||||
self.freq_threshold: int = freq_threshold
|
||||
self.utility_threshold: float = utility_threshold
|
||||
|
||||
|
||||
self.llm_client = OpenAI()
|
||||
self.memory_base_url: str = memory_base_url
|
||||
|
|
@ -73,6 +75,7 @@ class AppworldReactAgent:
|
|||
self.history[run_id].append([])
|
||||
|
||||
def call_llm(self, messages: list) -> str:
|
||||
"""Call the LLM to generate a response to the messages."""
|
||||
for i in range(100):
|
||||
try:
|
||||
response = self.llm_client.chat.completions.create(
|
||||
|
|
@ -92,11 +95,9 @@ class AppworldReactAgent:
|
|||
return "call llm error"
|
||||
|
||||
def prompt_messages(self, run_id, task_index, previous_memories: None, world: AppWorld):
|
||||
"""Prompt the messages to the LLM."""
|
||||
app_descriptions = json.dumps(
|
||||
[
|
||||
{"name": k, "description": v}
|
||||
for (k, v) in world.task.app_descriptions.items()
|
||||
],
|
||||
[{"name": k, "description": v} for (k, v) in world.task.app_descriptions.items()],
|
||||
indent=1,
|
||||
)
|
||||
dictionary = {"supervisor": world.task.supervisor, "app_descriptions": app_descriptions}
|
||||
|
|
@ -107,9 +108,14 @@ class AppworldReactAgent:
|
|||
response = self.get_memory(world.task.instruction)
|
||||
if response and "memory_list" in response["metadata"]:
|
||||
self.retrieved_memory_list[run_id][task_index] = response["metadata"]["memory_list"]
|
||||
task_memory = re.sub(r'\bMemory\s*(\d+)\s*[:]', r'Experience \1:', response["answer"])
|
||||
task_memory = re.sub(r"\bMemory\s*(\d+)\s*[:]", r"Experience \1:", response["answer"])
|
||||
logger.info(f"loaded task_memory: {task_memory}")
|
||||
query = "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + task_memory
|
||||
query = (
|
||||
"Task:\n"
|
||||
+ query
|
||||
+ "\n\nSome Related Experience to help you to complete the task:\n"
|
||||
+ task_memory
|
||||
)
|
||||
else:
|
||||
formatted_memories = []
|
||||
for i, memory in enumerate(previous_memories, 1):
|
||||
|
|
@ -117,24 +123,32 @@ class AppworldReactAgent:
|
|||
memory_content = memory["content"]
|
||||
memory_text = f"Experience {i}:\n When to use: {condition}\n Content: {memory_content}\n"
|
||||
formatted_memories.append(memory_text)
|
||||
query = "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + "\n".join(formatted_memories)
|
||||
query = (
|
||||
"Task:\n"
|
||||
+ query
|
||||
+ "\n\nSome Related Experience to help you to complete the task:\n"
|
||||
+ "\n".join(formatted_memories)
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": sys_prompt},
|
||||
{"role": "user", "content": query}
|
||||
{"role": "user", "content": query},
|
||||
]
|
||||
self.history[run_id][task_index] = messages
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_reward(world) -> float:
|
||||
"""Get the reward for the Appworld world."""
|
||||
tracker = world.evaluate()
|
||||
num_passes = len(tracker.passes)
|
||||
num_failures = len(tracker.failures)
|
||||
return num_passes / (num_passes + num_failures)
|
||||
|
||||
def extract_code_and_fix_content(
|
||||
self, text: str, ignore_multiple_calls=True
|
||||
self,
|
||||
text: str,
|
||||
ignore_multiple_calls=True,
|
||||
) -> tuple[str, str]:
|
||||
"""Extract the code and fix the content."""
|
||||
full_code_regex = r"```python\n(.*?)```"
|
||||
partial_code_regex = r".*```python\n(.*)"
|
||||
|
||||
|
|
@ -151,7 +165,9 @@ class AppworldReactAgent:
|
|||
match_end = re_match.end()
|
||||
# check for partial code match at end (no terminating ```) following the last match
|
||||
partial_match = re.match(
|
||||
partial_code_regex, original_text[match_end:], flags=re.DOTALL
|
||||
partial_code_regex,
|
||||
original_text[match_end:],
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
if partial_match:
|
||||
output_code += partial_match.group(1).strip()
|
||||
|
|
@ -165,6 +181,7 @@ class AppworldReactAgent:
|
|||
return output_code, text
|
||||
|
||||
def execute(self):
|
||||
"""Execute the Appworld tasks."""
|
||||
result = []
|
||||
counter = 0
|
||||
for task_index, task_id in enumerate(tqdm(self.task_ids, desc=f"run_index={self.index}")):
|
||||
|
|
@ -177,16 +194,23 @@ class AppworldReactAgent:
|
|||
before_score = self.get_reward(world)
|
||||
for i in range(self.max_interactions):
|
||||
if i == 0:
|
||||
self.prompt_messages(run_id=run_id, task_index=task_index, previous_memories=previous_memories, world=world)
|
||||
self.prompt_messages(
|
||||
run_id=run_id,
|
||||
task_index=task_index,
|
||||
previous_memories=previous_memories,
|
||||
world=world,
|
||||
)
|
||||
code_msg = self.call_llm(self.history[run_id][task_index])
|
||||
code, text = self.extract_code_and_fix_content(code_msg)
|
||||
code, _ = self.extract_code_and_fix_content(code_msg)
|
||||
self.history[run_id][task_index].append({"role": "assistant", "content": code})
|
||||
|
||||
output = world.execute(code)
|
||||
# if len(output) > self.max_response_size:
|
||||
# # logger.warning(f"output exceed max size={len(output)}")
|
||||
# output = output[: self.max_response_size]
|
||||
self.history[run_id][task_index].append({"role": "user", "content": "Output:\n```\n" + output + "```\n\n"})
|
||||
self.history[run_id][task_index].append(
|
||||
{"role": "user", "content": "Output:\n```\n" + output + "```\n\n"},
|
||||
)
|
||||
|
||||
if world.task_completed():
|
||||
break
|
||||
|
|
@ -196,7 +220,9 @@ class AppworldReactAgent:
|
|||
|
||||
if self.use_memory:
|
||||
if self.use_memory_addition:
|
||||
new_traj_list = [self.get_traj_from_task_history(task_id, self.history[run_id][task_index], after_score)]
|
||||
new_traj_list = [
|
||||
self.get_traj_from_task_history(task_id, self.history[run_id][task_index], after_score),
|
||||
]
|
||||
previous_memories = self.summary_memory(new_traj_list)
|
||||
if after_score == 1:
|
||||
self.add_memory(previous_memories)
|
||||
|
|
@ -206,7 +232,7 @@ class AppworldReactAgent:
|
|||
self.update_memory_information(self.retrieved_memory_list[run_id][task_index], update_utility)
|
||||
|
||||
counter += 1
|
||||
if self.use_memory_deletion: # and counter % self.delete_freq == 0:
|
||||
if self.use_memory_deletion: # and counter % self.delete_freq == 0:
|
||||
self.delete_memory()
|
||||
|
||||
t_result = {
|
||||
|
|
@ -241,10 +267,10 @@ class AppworldReactAgent:
|
|||
url=f"{self.memory_base_url}retrieve_task_memory",
|
||||
json={
|
||||
"query": query,
|
||||
"enable_llm_rerank":False,
|
||||
"enable_score_filter":False,
|
||||
"top_k":5,
|
||||
"enable_llm_rewrite":False,
|
||||
"enable_llm_rerank": False,
|
||||
"enable_score_filter": False,
|
||||
"top_k": 5,
|
||||
"enable_llm_rewrite": False,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -256,12 +282,13 @@ class AppworldReactAgent:
|
|||
return result
|
||||
|
||||
def get_traj_from_task_history(self, task_id: str, task_history: list, reward: float):
|
||||
"""Get the trajectory from the task history."""
|
||||
pattern = r"\n\nSome Related Experience to help you to complete the task:.*"
|
||||
task_history[1]["content"] = re.sub(pattern, "", task_history[1]["content"], flags=re.DOTALL)
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"messages": task_history,
|
||||
"score": reward
|
||||
"score": reward,
|
||||
}
|
||||
|
||||
def summary_memory(self, trajectories):
|
||||
|
|
@ -287,15 +314,17 @@ class AppworldReactAgent:
|
|||
return memory_list
|
||||
|
||||
def add_memory(self, memory_list):
|
||||
"""Add the memory to the memory pool."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}add_task_memory",
|
||||
json={
|
||||
"memory_list": memory_list
|
||||
}
|
||||
"memory_list": memory_list,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
def update_memory_information(self, memory_list, update_utility: bool = False):
|
||||
"""Update the memory information."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}record_task_memory",
|
||||
json={
|
||||
|
|
@ -307,6 +336,7 @@ class AppworldReactAgent:
|
|||
logger.info(response.json())
|
||||
|
||||
def delete_memory(self):
|
||||
"""Delete the memory from the memory pool."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}delete_task_memory",
|
||||
json={
|
||||
|
|
@ -316,7 +346,9 @@ class AppworldReactAgent:
|
|||
)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to run the Appworld React Agent."""
|
||||
dataset_name = "train"
|
||||
task_ids = load_task_ids(dataset_name)
|
||||
agent = AppworldReactAgent(index=0, task_ids=task_ids[0:1], experiment_name=dataset_name, num_trials=1)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# flake8: noqa: E402, E501
|
||||
# pylint: disable=C0114,C0301
|
||||
# This is a basic prompt template containing all the necessary onboarding information to solve AppWorld tasks. It explains the role of the agent and the supervisor, how to explore the API documentation, how to operate the interactive coding environment and call APIs via a simple task, and provides key instructions and disclaimers.
|
||||
|
||||
# You can adapt it as needed by your agent. You can also choose to bypass API docs app and build your own API retrieval, e.g., for FullCodeRefl, IPFunCall, etc, we asked an LLM to predict relevant APIs separately and put its documentation directly in the prompt.
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
# pylint: disable=E0611
|
||||
"""Run the Appworld React Agent."""
|
||||
|
||||
import os
|
||||
import ray
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import ray
|
||||
import requests
|
||||
from loguru import logger
|
||||
from dotenv import load_dotenv
|
||||
from appworld import load_task_ids
|
||||
|
|
@ -29,18 +32,19 @@ def run_agent(
|
|||
delete_freq: int = 10,
|
||||
freq_threshold: int = 5,
|
||||
utility_threshold: float = 0.5,
|
||||
batch_size: int = 4
|
||||
batch_size: int = 4,
|
||||
):
|
||||
"""Run the Appworld React Agent."""
|
||||
experiment_name = dataset_name + "_" + experiment_suffix
|
||||
path: Path = Path(f"./exp_result/{model_name}")
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
task_ids = ["b9c5c9a_3"] # load_task_ids(dataset_name)
|
||||
task_ids = load_task_ids(dataset_name)
|
||||
|
||||
result: list = []
|
||||
|
||||
|
||||
def dump_file():
|
||||
with open(path / f"{experiment_name}.jsonl", "a") as f:
|
||||
with open(path / f"{experiment_name}.jsonl", "a", encoding="utf-8") as f:
|
||||
for x in result:
|
||||
f.write(json.dumps(x) + "\n")
|
||||
|
||||
|
|
@ -48,20 +52,20 @@ def run_agent(
|
|||
# Process tasks in batches
|
||||
total_tasks = len(task_ids)
|
||||
num_batches = (total_tasks + batch_size - 1) // batch_size # Ceiling division
|
||||
|
||||
|
||||
logger.info(f"Total tasks: {total_tasks}, Batch size: {batch_size}, Number of batches: {num_batches}")
|
||||
|
||||
|
||||
for batch_idx in range(num_batches):
|
||||
# Initialize Ray for this batch
|
||||
start_idx = batch_idx * batch_size
|
||||
end_idx = min(start_idx + batch_size, total_tasks)
|
||||
batch_task_ids = task_ids[start_idx:end_idx]
|
||||
|
||||
|
||||
logger.info(f"Starting batch {batch_idx + 1}/{num_batches} with {len(batch_task_ids)} tasks")
|
||||
|
||||
|
||||
# Initialize Ray with the number of CPUs needed for this batch
|
||||
ray.init(num_cpus=len(batch_task_ids))
|
||||
|
||||
|
||||
future_list: list = []
|
||||
for i, task_id in enumerate(batch_task_ids):
|
||||
actor = AppworldReactAgent.remote(
|
||||
|
|
@ -81,7 +85,7 @@ def run_agent(
|
|||
future = actor.execute.remote()
|
||||
future_list.append(future)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
logger.info(f"Batch {batch_idx + 1} submit complete, waiting for results...")
|
||||
|
||||
# Collect results from this batch
|
||||
|
|
@ -93,19 +97,19 @@ def run_agent(
|
|||
result.extend(t_result)
|
||||
else:
|
||||
result.append(t_result)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
logger.exception(f"run ray error with task_id={task_id}")
|
||||
|
||||
|
||||
logger.info(f"Batch {batch_idx + 1}: task {i + 1}/{len(batch_task_ids)} complete")
|
||||
|
||||
|
||||
# Shutdown Ray to free resources before next batch
|
||||
ray.shutdown()
|
||||
logger.info(f"Batch {batch_idx + 1}/{num_batches} complete, Ray resources released")
|
||||
|
||||
|
||||
# Optional: small delay between batches
|
||||
if batch_idx < num_batches - 1:
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
dump_file()
|
||||
|
||||
else:
|
||||
|
|
@ -127,6 +131,7 @@ def run_agent(
|
|||
|
||||
dump_file()
|
||||
|
||||
|
||||
def handle_api_response(response: requests.Response):
|
||||
"""Handle API response with proper error checking"""
|
||||
if response.status_code != 200:
|
||||
|
|
@ -153,19 +158,20 @@ def load_memory(path: str = "docs/library", api_url: str = "http://0.0.0.0:8002/
|
|||
|
||||
|
||||
def main():
|
||||
"""Main function to run the Appworld React Agent."""
|
||||
max_workers = 16
|
||||
batch_size = 8
|
||||
|
||||
|
||||
num_runs = 4 # Number of runs
|
||||
num_trials = 1 # for self-reflection
|
||||
num_trials = 1 # for self-reflection
|
||||
model_name = "qwen3-8b"
|
||||
use_memory = True
|
||||
use_memory_addition = False
|
||||
use_memory_deletion = False
|
||||
memory_base_url = "http://0.0.0.0:8002/"
|
||||
|
||||
|
||||
if use_memory:
|
||||
load_file_path = f"docs/library/paper_data/task/appworld_qwen3_8b.jsonl"
|
||||
load_file_path = "docs/library/paper_data/task/appworld_qwen3_8b.jsonl"
|
||||
load_memory(load_file_path, memory_base_url)
|
||||
|
||||
for i in range(num_runs):
|
||||
|
|
@ -174,7 +180,7 @@ def main():
|
|||
max_workers=max_workers,
|
||||
model_name=model_name,
|
||||
dataset_name="test_normal",
|
||||
experiment_suffix=f"with-fixed-memory",
|
||||
experiment_suffix="with-fixed-memory",
|
||||
num_trials=num_trials,
|
||||
use_memory=use_memory,
|
||||
memory_base_url=memory_base_url,
|
||||
|
|
@ -183,9 +189,9 @@ def main():
|
|||
delete_freq=5,
|
||||
freq_threshold=5,
|
||||
utility_threshold=0.5,
|
||||
batch_size=batch_size
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
"""Run the experiment statistic."""
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
|
@ -31,6 +33,7 @@ def calculate_best_at_k(scores: list, k: int) -> float:
|
|||
|
||||
|
||||
def calculate_pass_at_k(scores: list, k: int) -> float:
|
||||
"""Calculate pass@k."""
|
||||
if len(scores) % k != 0:
|
||||
raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
|
||||
|
||||
|
|
@ -61,16 +64,17 @@ def get_possible_k_values(total_runs: int) -> list:
|
|||
|
||||
|
||||
def run_exp_statistic():
|
||||
"""Run the experiment statistic."""
|
||||
path: Path = Path("./exp_result/qwen3-8b")
|
||||
|
||||
# Store results for all experiments
|
||||
all_results = {}
|
||||
|
||||
for file in [f for f in path.glob("*.jsonl")]:# if not f.stem[-1].isdigit()
|
||||
for file in path.glob("*.jsonl"): # [f for f in path.glob("*.jsonl") if not f.stem[-1].isdigit()]
|
||||
# Group results by task_id
|
||||
task_results = defaultdict(list)
|
||||
|
||||
with open(file, "r") as f:
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -1,26 +1,23 @@
|
|||
# flake8: noqa: E402
|
||||
import os
|
||||
|
||||
os.environ["BFCL_DATA_PATH"] = "data/multiturn_data_base_val.jsonl"
|
||||
os.environ["BFCL_ANSWER_PATH"] = "data/possible_answer"
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv("../../.env")
|
||||
# pylint: disable=too-many-return-statements
|
||||
"""A minimal ReAct Agent for BFCL-v3(multi-turn) tasks."""
|
||||
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import ray
|
||||
import warnings
|
||||
import tempfile
|
||||
import requests
|
||||
import datetime
|
||||
|
||||
from tqdm import tqdm
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
import ray
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
from loguru import logger
|
||||
from openai import OpenAI
|
||||
from typing import Dict, List, Any
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from bfcl_utils import (
|
||||
load_test_case,
|
||||
|
|
@ -48,6 +45,10 @@ from bfcl_eval.utils import (
|
|||
load_file,
|
||||
)
|
||||
|
||||
os.environ["BFCL_DATA_PATH"] = "data/multiturn_data_base_val.jsonl"
|
||||
os.environ["BFCL_ANSWER_PATH"] = "data/possible_answer"
|
||||
load_dotenv("../../.env")
|
||||
|
||||
|
||||
@ray.remote
|
||||
class BFCLAgent:
|
||||
|
|
@ -109,6 +110,7 @@ class BFCLAgent:
|
|||
self.init_state(run_id, task_index)
|
||||
|
||||
def init_state(self, run_id, i) -> Dict[str, Any]:
|
||||
"""Initialize the state of the agent."""
|
||||
self.test_entry[run_id].append(load_test_case(self.data_path, self.task_ids[i]))
|
||||
self.original_test_entry[run_id].append(self.test_entry[run_id][i].get("extra", {}))
|
||||
self.tool_schema[run_id].append(extract_tool_schema(self.test_entry[run_id][i].get("tools", [{}])))
|
||||
|
|
@ -119,12 +121,13 @@ class BFCLAgent:
|
|||
self.current_turn[run_id][i] = 1
|
||||
|
||||
def update_task_history_with_memory(self, run_id, task_index, previous_memories: None):
|
||||
"""Update the task history with memory."""
|
||||
query = self.history[run_id][task_index][0]["content"]
|
||||
if len(previous_memories) == 0:
|
||||
response = self.get_memory(query)
|
||||
if response and "memory_list" in response["metadata"]:
|
||||
self.retrieved_memory_list[run_id][task_index] = response["metadata"]["memory_list"]
|
||||
task_memory = re.sub(r'\bMemory\s*(\d+)\s*[:]', r'Experience \1 :', response["answer"])
|
||||
task_memory = re.sub(r"\bMemory\s*(\d+)\s*[:]", r"Experience \1 :", response["answer"])
|
||||
logger.info(f"loaded task_memory: {task_memory}")
|
||||
self.history[run_id][task_index][0] = self.get_query_with_memory(query, task_memory)
|
||||
else:
|
||||
|
|
@ -137,17 +140,20 @@ class BFCLAgent:
|
|||
self.history[run_id][task_index][0] = self.get_query_with_memory(query, "\n".join(formatted_memories))
|
||||
|
||||
def get_query_with_memory(self, query: str, memory: str):
|
||||
"""Get the query with memory."""
|
||||
return {
|
||||
"role": "user",
|
||||
"content": "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + memory,
|
||||
}
|
||||
|
||||
def get_query_without_experience(self, query: str):
|
||||
"""Get the query without experience."""
|
||||
if "\n\nSome Related Experience" in query:
|
||||
query = query.split("\n\nSome Related Experience")[0].split("Task:\n")[-1]
|
||||
return query
|
||||
|
||||
def get_traj_from_task_history(self, task_id: str, task_history: list, reward: float):
|
||||
"""Get the trajectory from the task history."""
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"messages": task_history,
|
||||
|
|
@ -201,19 +207,21 @@ class BFCLAgent:
|
|||
|
||||
# Extract memory list from response
|
||||
memory_list = result.get("metadata", {}).get("memory_list", [])
|
||||
logger.info(f'add new memories: {memory_list}')
|
||||
logger.info(f"add new memories: {memory_list}")
|
||||
return memory_list
|
||||
|
||||
def add_memory(self, memory_list):
|
||||
"""Add the memory to the memory pool."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}add_task_memory",
|
||||
json={
|
||||
"memory_list": memory_list
|
||||
}
|
||||
"memory_list": memory_list,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
def update_memory_information(self, memory_list, update_utility: bool = False):
|
||||
"""Update the memory information."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}record_task_memory",
|
||||
json={
|
||||
|
|
@ -225,6 +233,7 @@ class BFCLAgent:
|
|||
logger.info(response.json())
|
||||
|
||||
def delete_memory(self):
|
||||
"""Delete the memory from the memory pool."""
|
||||
response = requests.post(
|
||||
url=f"{self.memory_base_url}delete_task_memory",
|
||||
json={
|
||||
|
|
@ -235,6 +244,7 @@ class BFCLAgent:
|
|||
response.raise_for_status()
|
||||
|
||||
def call_llm(self, messages: list, tool_schemas: list[dict]) -> str:
|
||||
"""Call the LLM."""
|
||||
for i in range(100):
|
||||
try:
|
||||
response = self.llm_client.chat.completions.create(
|
||||
|
|
@ -262,46 +272,45 @@ class BFCLAgent:
|
|||
if not chunk.choices:
|
||||
# Handle usage information
|
||||
continue
|
||||
|
||||
delta = chunk.choices[0].delta
|
||||
# Handle AI's thought process (chain reasoning)
|
||||
if hasattr(delta, "reasoning_content") and delta.reasoning_content is not None:
|
||||
reasoning_content += delta.reasoning_content
|
||||
# Handle final response content
|
||||
else:
|
||||
delta = chunk.choices[0].delta
|
||||
# Handle AI's thought process (chain reasoning)
|
||||
if hasattr(delta, "reasoning_content") and delta.reasoning_content is not None:
|
||||
reasoning_content += delta.reasoning_content
|
||||
if not is_answering: # Print title when entering the response phase for the first time
|
||||
is_answering = True
|
||||
if delta.content is not None:
|
||||
answer_content += delta.content
|
||||
|
||||
# Handle final response content
|
||||
else:
|
||||
if not is_answering: # Print title when entering the response phase for the first time
|
||||
is_answering = True
|
||||
if delta.content is not None:
|
||||
answer_content += delta.content
|
||||
# Handle tool invocation information (support parallel tool calls)
|
||||
if delta.tool_calls is not None:
|
||||
for tool_call in delta.tool_calls:
|
||||
index = tool_call.index # Tool call index, used for parallel calls
|
||||
|
||||
# Handle tool invocation information (support parallel tool calls)
|
||||
if delta.tool_calls is not None:
|
||||
for tool_call in delta.tool_calls:
|
||||
index = tool_call.index # Tool call index, used for parallel calls
|
||||
# Dynamically expand tool information storage list
|
||||
while len(tool_info) <= index:
|
||||
tool_info.append(
|
||||
{
|
||||
"id": "",
|
||||
"type": "function",
|
||||
"index": index,
|
||||
"function": {"name": "", "arguments": ""},
|
||||
},
|
||||
)
|
||||
|
||||
# Dynamically expand tool information storage list
|
||||
while len(tool_info) <= index:
|
||||
tool_info.append(
|
||||
{
|
||||
"id": "",
|
||||
"type": "function",
|
||||
"index": index,
|
||||
"function": {"name": "", "arguments": ""},
|
||||
},
|
||||
)
|
||||
# Collect tool call ID (used for subsequent function calls)
|
||||
if tool_call.id:
|
||||
tool_info[index]["id"] += tool_call.id
|
||||
|
||||
# Collect tool call ID (used for subsequent function calls)
|
||||
if tool_call.id:
|
||||
tool_info[index]["id"] += tool_call.id
|
||||
# Collect function name (used for subsequent routing to specific functions)
|
||||
if tool_call.function and tool_call.function.name:
|
||||
tool_info[index]["function"]["name"] += tool_call.function.name
|
||||
|
||||
# Collect function name (used for subsequent routing to specific functions)
|
||||
if tool_call.function and tool_call.function.name:
|
||||
tool_info[index]["function"]["name"] += tool_call.function.name
|
||||
|
||||
# Collect function parameters (in JSON string format, need subsequent parsing)
|
||||
if tool_call.function and tool_call.function.arguments:
|
||||
tool_info[index]["function"]["arguments"] += tool_call.function.arguments
|
||||
# Collect function parameters (in JSON string format, need subsequent parsing)
|
||||
if tool_call.function and tool_call.function.arguments:
|
||||
tool_info[index]["function"]["arguments"] += tool_call.function.arguments
|
||||
msg = {
|
||||
"role": "assistant",
|
||||
"content": answer_content,
|
||||
|
|
@ -398,12 +407,13 @@ class BFCLAgent:
|
|||
args_str = ", ".join([f"{k}={repr(v)}" for k, v in args_dict.items()])
|
||||
execution_list.append(f"{function_name}({args_str})")
|
||||
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
execution_list.append(f"{function_name}()")
|
||||
|
||||
return execution_list
|
||||
|
||||
def get_reward(self, run_id, index) -> float:
|
||||
"""Get the reward."""
|
||||
try:
|
||||
if not self.history[run_id][index] or not self.original_test_entry[run_id][index]:
|
||||
return 0.0
|
||||
|
|
@ -461,11 +471,14 @@ class BFCLAgent:
|
|||
self.categories[index],
|
||||
)
|
||||
print(f"model_result_data: {model_result_data}")
|
||||
print(f"possible_answer: {possible_answer}") if possible_answer else None
|
||||
if possible_answer:
|
||||
print(f"possible_answer: {possible_answer}")
|
||||
else:
|
||||
print("possible_answer: None")
|
||||
|
||||
return accuracy
|
||||
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
|
|
@ -589,6 +602,7 @@ class BFCLAgent:
|
|||
return accuracy, total_count
|
||||
|
||||
def execute(self):
|
||||
"""Execute the agent."""
|
||||
result = []
|
||||
counter = 0
|
||||
for task_index, task_id in enumerate(tqdm(self.task_ids, desc=f"ray_index={self.index}")):
|
||||
|
|
@ -608,10 +622,15 @@ class BFCLAgent:
|
|||
|
||||
env_output = self.env_step(run_id, task_index, self.history[run_id][task_index])
|
||||
# Possible env_output returns after environment interaction:
|
||||
# 1. Triggers a query with available tools list: {"messages": [{"role": "user", "content": user_query}], "tools": tools}
|
||||
# 2. Returns tool invocation result: {"messages": [{"role": "tool", "content": {<execution_results>}, 'tool_call_id': 'chatcmpl-tool-xxx'}]}
|
||||
# <execution_results>: when success, returns result dicts, e.g., {"travel_cost_list": [1140.0]}, when error, returns error message, e.g., {"error": "cd: temporary: No such directory. You cannot use path to change directory."}
|
||||
# 3. Conversation completion: {"messages": [{"role": "env", "content": "[CONVERSATION_COMPLETED]"}]}
|
||||
# 1. Triggers a query with available tools list:
|
||||
# {"messages": [{"role": "user", "content": user_query}], "tools": tools}
|
||||
# 2. Returns tool invocation result: {"messages":
|
||||
# [{"role": "tool", "content": {<exec_results>}, 'tool_call_id': 'chatcmpl-tool-xxx'}]}
|
||||
# <exec_results>: when success, returns result dicts, e.g., {"travel_cost_list": [x]},
|
||||
# when error, returns error message,
|
||||
# e.g., {"error": "cd: temporary: No such directory. You cannot use path ..."}
|
||||
# 3. Conversation completion:
|
||||
# {"messages": [{"role": "env", "content": "[CONVERSATION_COMPLETED]"}]}
|
||||
# 4. Program error: {"messages": [{"role": "env", "content": f"[ERROR] {error_message}"}]}
|
||||
|
||||
# tool_list update
|
||||
|
|
@ -647,7 +666,9 @@ class BFCLAgent:
|
|||
reward = self.get_reward(run_id, task_index)
|
||||
if self.use_memory:
|
||||
if self.use_memory_addition:
|
||||
new_traj_list = [self.get_traj_from_task_history(task_id, self.history[run_id][task_index], reward)]
|
||||
new_traj_list = [
|
||||
self.get_traj_from_task_history(task_id, self.history[run_id][task_index], reward),
|
||||
]
|
||||
previous_memories = self.summary_memory(new_traj_list)
|
||||
if reward == 1:
|
||||
self.add_memory(previous_memories)
|
||||
|
|
@ -688,12 +709,13 @@ class BFCLAgent:
|
|||
|
||||
|
||||
def main():
|
||||
"""Main function to run the BFCLAgent."""
|
||||
with open(os.getenv("BFCL_DATA_PATH"), "r", encoding="utf-8") as f:
|
||||
task_ids = [json.loads(l)["id"] for l in f]
|
||||
dataset_name = "dev"
|
||||
agent = BFCLAgent(
|
||||
index=0,
|
||||
task_id=task_ids[0],
|
||||
task_ids=[task_ids[0]],
|
||||
experiment_name=f"qwen3_8b_{dataset_name}",
|
||||
)
|
||||
result = agent.execute()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
"""Utils for evaluation on BFCL tasks"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
|
@ -18,6 +20,9 @@ from bfcl_eval.model_handler.utils import (
|
|||
|
||||
|
||||
def load_test_case(data_path: str, test_id: str | None) -> Dict[str, Any]:
|
||||
"""
|
||||
load test cases by id
|
||||
"""
|
||||
if not Path(data_path).exists():
|
||||
raise FileNotFoundError(f"BFCL data file '{data_path}' not found")
|
||||
|
||||
|
|
@ -25,7 +30,7 @@ def load_test_case(data_path: str, test_id: str | None) -> Dict[str, Any]:
|
|||
raise ValueError("task_id is required")
|
||||
|
||||
with open(data_path, "r", encoding="utf-8") as f:
|
||||
if str(test_id).isdigit():
|
||||
if str(test_id).isdigit(): # pylint: disable=R1720
|
||||
idx = int(test_id)
|
||||
for line_no, line in enumerate(f):
|
||||
if line_no == idx:
|
||||
|
|
@ -82,7 +87,7 @@ def handle_user_turn(
|
|||
return create_error_response(f"Failed to process user message: {str(e)}")
|
||||
|
||||
|
||||
def handle_tool_calls(
|
||||
def handle_tool_calls( # pylint: disable=W0613
|
||||
tool_calls: List[Dict[str, Any]],
|
||||
decoded_calls: list[str],
|
||||
test_entry: Dict[str, Any],
|
||||
|
|
@ -358,8 +363,6 @@ def capture_and_print_score_files(
|
|||
or content.strip().startswith("[")
|
||||
):
|
||||
try:
|
||||
import json
|
||||
|
||||
lines = content.strip().split("\n")
|
||||
formatted_lines = []
|
||||
for line in lines:
|
||||
|
|
@ -390,6 +393,7 @@ def capture_and_print_score_files(
|
|||
|
||||
|
||||
def extract_tool_schema(tools):
|
||||
for i in range(len(tools)):
|
||||
"""Reformat tool schema"""
|
||||
for i in range(len(tools)): # pylint: disable=C0200
|
||||
tools[i]["function"].pop("response")
|
||||
return tools
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# pylint: disable=W0621,W1514
|
||||
"""Init task memory pool"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
|
|
@ -19,7 +21,7 @@ def load_task_case(data_path: str, task_id: str | None) -> Dict[str, Any]:
|
|||
raise ValueError("task_id is required")
|
||||
|
||||
with open(data_path, "r", encoding="utf-8") as f:
|
||||
if str(task_id).isdigit():
|
||||
if str(task_id).isdigit(): # pylint: disable=R1720
|
||||
idx = int(task_id)
|
||||
for line_no, line in enumerate(f):
|
||||
if line_no == idx:
|
||||
|
|
@ -34,10 +36,18 @@ def load_task_case(data_path: str, task_id: str | None) -> Dict[str, Any]:
|
|||
|
||||
|
||||
def get_tool_prompt(tools):
|
||||
tool_prompt = "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>"
|
||||
"""Construct prompt with provided tools"""
|
||||
tool_prompt = (
|
||||
"\n\n# Tools\n\nYou may call one or more functions to assist with the user query."
|
||||
"\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>"
|
||||
)
|
||||
for tool in tools:
|
||||
tool_prompt += "\n" + json.dumps(tool)
|
||||
tool_prompt += '\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>'
|
||||
tool_prompt += (
|
||||
"\n</tools>\n\nFor each function call, return a json object with function name"
|
||||
" and arguments within <tool_call></tool_call> XML tags:"
|
||||
'\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>'
|
||||
)
|
||||
return tool_prompt
|
||||
|
||||
|
||||
|
|
@ -65,7 +75,7 @@ def group_trajectories_by_task_id(jsonl_entries: List[Dict[str, Any]]) -> List[L
|
|||
|
||||
# retain only the two with the highest and lowest rewards
|
||||
filtered_groups = []
|
||||
for key, trajectories in grouped.items():
|
||||
for _, trajectories in grouped.items():
|
||||
if len(trajectories) == 1:
|
||||
# when only one trajectory, retain it
|
||||
filtered_groups.append(trajectories)
|
||||
|
|
@ -83,6 +93,16 @@ def group_trajectories_by_task_id(jsonl_entries: List[Dict[str, Any]]) -> List[L
|
|||
|
||||
|
||||
def post_to_summarizer(trajectories: List[Any], service_url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
post trajectories to summarizer service
|
||||
|
||||
Args:
|
||||
trajectories: trajectory list
|
||||
service_url: summarizer service URL
|
||||
|
||||
Returns:
|
||||
response json
|
||||
"""
|
||||
trajectory_dicts = [
|
||||
{
|
||||
"task_id": traj["task_id"],
|
||||
|
|
@ -124,8 +144,7 @@ def process_trajectories_with_threads(
|
|||
|
||||
with ThreadPoolExecutor(max_workers=n_threads) as executor:
|
||||
future_to_group = {
|
||||
executor.submit(post_to_summarizer, group, service_url): i
|
||||
for i, group in enumerate(grouped_trajectories)
|
||||
executor.submit(post_to_summarizer, group, service_url): i for i, group in enumerate(grouped_trajectories)
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_group):
|
||||
|
|
@ -135,9 +154,10 @@ def process_trajectories_with_threads(
|
|||
result["group_index"] = group_index
|
||||
result["group_size"] = len(grouped_trajectories[group_index])
|
||||
results.append(result)
|
||||
print(
|
||||
f'✅ Group {group_index} processed: {result["metadata"].get("memory_list", 0) if "memory_list" in result["metadata"] else "error"}',
|
||||
)
|
||||
if "memory_list" in result["metadata"]:
|
||||
print(f'✅ Group {group_index} processed: {result["metadata"].get("memory_list", 0)}')
|
||||
else:
|
||||
print(f"❌ Group {group_index} processed: error")
|
||||
except Exception as e:
|
||||
error_result = {
|
||||
"group_index": group_index,
|
||||
|
|
@ -151,6 +171,7 @@ def process_trajectories_with_threads(
|
|||
|
||||
|
||||
def main():
|
||||
"""Main function to convert JSONL to memories using ReMe service."""
|
||||
parser = argparse.ArgumentParser(description="Convert JSONL to memories using ReMe service")
|
||||
parser.add_argument("--jsonl_file", type=str, required=True, help="Path to the JSONL file")
|
||||
parser.add_argument("--service_url", type=str, default="http://localhost:8001", help="ReMe service URL")
|
||||
|
|
@ -220,7 +241,6 @@ if __name__ == "__main__":
|
|||
results = process_trajectories_with_threads(
|
||||
grouped_trajectories,
|
||||
"http://localhost:8001",
|
||||
"bfcl_v3",
|
||||
n_threads=4,
|
||||
)
|
||||
print(f"Processed {len(results)} groups")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
"""Load the library data and convert them to the new format"""
|
||||
|
||||
import json
|
||||
|
||||
with open("../../file_vector_store/bfcl_test.jsonl", "r") as f:
|
||||
with open("../../file_vector_store/bfcl_test.jsonl", "r", encoding="utf-8") as f:
|
||||
bfcl = [json.loads(line) for line in f]
|
||||
|
||||
new_bfcl = []
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
# pylint: disable=W0621
|
||||
"""Preprocess multi-turn test cases"""
|
||||
|
||||
import json
|
||||
|
||||
|
||||
|
|
@ -18,13 +21,12 @@ def process_multi_turn_test_case(file_path, output_path):
|
|||
Multi-turn test cases don't have the function doc in the prompt. We need to add them here.
|
||||
"""
|
||||
test_cases = []
|
||||
with open(output_path, "w") as outf:
|
||||
|
||||
with open(file_path) as f:
|
||||
with open(output_path, "w", encoding="utf-8") as outf:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
file = f.readlines()
|
||||
for line in file:
|
||||
entry = json.loads(line)
|
||||
if not "multi_turn" in entry["id"]:
|
||||
if "multi_turn" not in entry["id"]:
|
||||
continue
|
||||
test_category: str = entry["id"].rsplit("_", 1)[0]
|
||||
involved_classes = entry["involved_classes"]
|
||||
|
|
@ -32,7 +34,7 @@ def process_multi_turn_test_case(file_path, output_path):
|
|||
for func_collection in involved_classes:
|
||||
# func_doc is a list of dict
|
||||
func_doc = load_file(
|
||||
MULTI_TURN_FUNC_DOC_PATH / MULTI_TURN_FUNC_DOC_FILE_MAPPING[func_collection]
|
||||
MULTI_TURN_FUNC_DOC_PATH / MULTI_TURN_FUNC_DOC_FILE_MAPPING[func_collection],
|
||||
)
|
||||
entry["function"].extend(func_doc)
|
||||
|
||||
|
|
@ -48,21 +50,24 @@ def process_multi_turn_test_case(file_path, output_path):
|
|||
# Remove it from the function list
|
||||
entry["function"].pop(i)
|
||||
break
|
||||
|
||||
|
||||
functions = func_doc_language_specific_pre_processing(entry["function"], test_category)
|
||||
tools = convert_to_tool(functions, GORILLA_TO_OPENAPI, ModelStyle.OpenAI_Completions)
|
||||
|
||||
test_cases.append({
|
||||
"id": entry["id"],
|
||||
"messages": entry["question"][0],
|
||||
"tools": tools,
|
||||
"extra": entry,
|
||||
})
|
||||
test_cases.append(
|
||||
{
|
||||
"id": entry["id"],
|
||||
"messages": entry["question"][0],
|
||||
"tools": tools,
|
||||
"extra": entry,
|
||||
},
|
||||
)
|
||||
outf.write(json.dumps(test_cases[-1], ensure_ascii=False) + "\n")
|
||||
|
||||
return test_cases
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = Path("./gorilla/berkeley-function-call-leaderboard/bfcl_eval/data/BFCL_v3_multi_turn_base.json")
|
||||
output_path = "data/multiturn_data_base.jsonl"
|
||||
preprocessed_test_cases = process_multi_turn_test_case(file_path, output_path)
|
||||
preprocessed_test_cases = process_multi_turn_test_case(file_path, output_path)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import ray
|
||||
"""Run evaluation on BFCL-V3-Multi-Turn-Base dataset."""
|
||||
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import ray
|
||||
import requests
|
||||
from loguru import logger
|
||||
from dotenv import load_dotenv
|
||||
from bfcl_agent import BFCLAgent
|
||||
|
|
@ -26,8 +28,9 @@ def run_agent(
|
|||
use_memory_deletion: bool = False,
|
||||
delete_freq: int = 10,
|
||||
freq_threshold: int = 5,
|
||||
utility_threshold: float = 0.5
|
||||
utility_threshold: float = 0.5,
|
||||
):
|
||||
"""Run the agent"""
|
||||
experiment_name = dataset_name + "_" + experiment_suffix
|
||||
path: Path = Path(
|
||||
f"./exp_result/{model_name}/with_think" if enable_thinking else f"./exp_result/{model_name}/no_think",
|
||||
|
|
@ -35,12 +38,12 @@ def run_agent(
|
|||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(data_path, "r", encoding="utf-8") as f:
|
||||
task_ids = [json.loads(l)["id"] for l in f]
|
||||
task_ids = [json.loads(line)["id"] for line in f]
|
||||
|
||||
result: list = []
|
||||
|
||||
def dump_file():
|
||||
with open(path / f"{experiment_name}.jsonl", "a") as f:
|
||||
with open(path / f"{experiment_name}.jsonl", "a", encoding="utf-8") as f:
|
||||
for x in result:
|
||||
f.write(json.dumps(x) + "\n")
|
||||
|
||||
|
|
@ -61,7 +64,7 @@ def run_agent(
|
|||
delete_freq=delete_freq,
|
||||
freq_threshold=freq_threshold,
|
||||
utility_threshold=utility_threshold,
|
||||
enable_thinking=enable_thinking
|
||||
enable_thinking=enable_thinking,
|
||||
)
|
||||
future = actor.execute.remote()
|
||||
future_list.append(future)
|
||||
|
|
@ -79,6 +82,7 @@ def run_agent(
|
|||
logger.info(f"{i + 1}/{len(task_ids)} complete")
|
||||
dump_file()
|
||||
|
||||
|
||||
def handle_api_response(response: requests.Response):
|
||||
"""Handle API response with proper error checking"""
|
||||
if response.status_code != 200:
|
||||
|
|
@ -105,10 +109,11 @@ def load_memory(path: str = "docs/library", api_url: str = "http://0.0.0.0:8002/
|
|||
|
||||
|
||||
def main():
|
||||
"""Main function"""
|
||||
max_workers = 4
|
||||
if max_workers > 1:
|
||||
ray.init(num_cpus=max_workers)
|
||||
|
||||
|
||||
num_runs = 4
|
||||
num_trials = 1
|
||||
model_name = "qwen3-8b"
|
||||
|
|
@ -119,15 +124,15 @@ def main():
|
|||
memory_base_url = "http://0.0.0.0:8003/"
|
||||
|
||||
if use_memory:
|
||||
load_file_path = f"docs/library/paper_data/task/bfcl_qwen3_8b.jsonl"
|
||||
load_file_path = "docs/library/paper_data/task/bfcl_qwen3_8b.jsonl"
|
||||
load_memory(load_file_path, memory_base_url)
|
||||
|
||||
for run_id in range(num_runs):
|
||||
|
||||
for _ in range(num_runs):
|
||||
run_agent(
|
||||
max_workers=max_workers,
|
||||
model_name=model_name,
|
||||
dataset_name="bfcl-multi-turn-base",
|
||||
experiment_suffix=f"w-fixed-memory",
|
||||
experiment_suffix="w-fixed-memory",
|
||||
data_path="data/multiturn_data_base_val.jsonl",
|
||||
answer_path=Path("data/possible_answer"),
|
||||
enable_thinking=enable_thinking,
|
||||
|
|
@ -138,7 +143,7 @@ def main():
|
|||
use_memory_deletion=use_memory_deletion,
|
||||
delete_freq=5,
|
||||
freq_threshold=5,
|
||||
utility_threshold=0.5
|
||||
utility_threshold=0.5,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
"""Run the experiment statistic."""
|
||||
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
|
@ -31,6 +33,7 @@ def calculate_best_at_k(scores: list, k: int) -> float:
|
|||
|
||||
|
||||
def calculate_pass_at_k(scores: list, k: int) -> float:
|
||||
"""Calculate pass@k."""
|
||||
if len(scores) % k != 0:
|
||||
raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
|
||||
|
||||
|
|
@ -61,15 +64,16 @@ def get_possible_k_values(total_runs: int) -> list:
|
|||
|
||||
|
||||
def run_exp_statistic():
|
||||
path: Path = Path(f"./exp_result/qwen3-8b/with_think")
|
||||
"""Run the experiment statistic."""
|
||||
path: Path = Path("./exp_result/qwen3-8b/with_think")
|
||||
|
||||
# Store results for all experiments
|
||||
all_results = {}
|
||||
for file in [f for f in path.glob("*.jsonl")]:
|
||||
for file in path.glob("*.jsonl"):
|
||||
# Group results by task_id
|
||||
task_results = defaultdict(list)
|
||||
print(file)
|
||||
with open(file, "r") as f:
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
|
@ -137,7 +141,7 @@ def run_exp_statistic():
|
|||
|
||||
# Sort columns by the number in column name (best@8, best@4, best@2, best@1)
|
||||
# best_columns = [col for col in df.columns if col.startswith('best@')]
|
||||
best_columns = [col for col in df.columns]
|
||||
best_columns = df.columns
|
||||
best_columns.sort(key=lambda x: x, reverse=False)
|
||||
df = df[best_columns]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
"""Split the JSONL file into train and validation sets."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
|
||||
|
||||
def split_jsonl(input_file, train_file, val_file, ratio=0.8):
|
||||
"""Split the JSONL file into train and validation sets."""
|
||||
with open(input_file, "r", encoding="utf-8") as f:
|
||||
data = [json.loads(line) for line in f]
|
||||
random.shuffle(data)
|
||||
|
|
|
|||
|
|
@ -17,49 +17,6 @@ flows:
|
|||
flow_content: TestOp()
|
||||
description: "test"
|
||||
|
||||
llms:
|
||||
default:
|
||||
backend: openai
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
# model_name: qwen3-30b-a3b-thinking-2507
|
||||
request_interval: 1
|
||||
# temperature: 0.0001
|
||||
|
||||
qwen3_max_instruct:
|
||||
backend: openai
|
||||
model_name: qwen3-max
|
||||
request_interval: 2
|
||||
|
||||
qwen-plus-thinking:
|
||||
backend: openai
|
||||
model_name: qwen-plus
|
||||
request_interval: 1
|
||||
extra_body:
|
||||
enable_thinking: True
|
||||
|
||||
embedding_models:
|
||||
default:
|
||||
backend: openai
|
||||
model_name: text-embedding-v4
|
||||
dimensions: 1024
|
||||
|
||||
vector_stores:
|
||||
default:
|
||||
backend: chroma
|
||||
# backend: local
|
||||
embedding_model: default
|
||||
collection_name: reme
|
||||
|
||||
token_counters:
|
||||
default:
|
||||
backend: base
|
||||
|
||||
hf:
|
||||
backend: hf
|
||||
model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
use_mirror: true
|
||||
|
||||
flows:
|
||||
retrieve_task_memory:
|
||||
flow_content: BuildQuery() >> MemoryRetrieval() >> RerankMemory() >> RewriteMemory()
|
||||
description: "Retrieves the most relevant top-k memory experiences from historical data based on the current query to enhance task-solving capabilities"
|
||||
|
|
@ -128,7 +85,7 @@ flows:
|
|||
max_existing_task_memories:
|
||||
type: integer
|
||||
description: "Maximum number of existing task memories to check for deduplication (default: 1000)."
|
||||
required:
|
||||
required:
|
||||
- trajectories
|
||||
|
||||
add_task_memory:
|
||||
|
|
@ -174,7 +131,7 @@ flows:
|
|||
required:
|
||||
- memory_list
|
||||
- update_utility
|
||||
|
||||
|
||||
load_memory:
|
||||
flow_content: LoadMemory()
|
||||
description: "Load memories from disk into the vector store"
|
||||
|
|
@ -189,7 +146,7 @@ flows:
|
|||
description: "If True, clears existing memories before loading (default: False)."
|
||||
required:
|
||||
- load_file_path
|
||||
|
||||
|
||||
dump_memory:
|
||||
flow_content: DumpMemory()
|
||||
description: "Dump the vector store memories to disk"
|
||||
|
|
@ -201,3 +158,45 @@ flows:
|
|||
description: "The path to the memories file."
|
||||
required:
|
||||
- dump_file_path
|
||||
|
||||
llms:
|
||||
default:
|
||||
backend: openai
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
# model_name: qwen3-30b-a3b-thinking-2507
|
||||
request_interval: 1
|
||||
# temperature: 0.0001
|
||||
|
||||
qwen3_max_instruct:
|
||||
backend: openai
|
||||
model_name: qwen3-max
|
||||
request_interval: 2
|
||||
|
||||
qwen-plus-thinking:
|
||||
backend: openai
|
||||
model_name: qwen-plus
|
||||
request_interval: 1
|
||||
extra_body:
|
||||
enable_thinking: True
|
||||
|
||||
embedding_models:
|
||||
default:
|
||||
backend: openai
|
||||
model_name: text-embedding-v4
|
||||
dimensions: 1024
|
||||
|
||||
vector_stores:
|
||||
default:
|
||||
backend: chroma
|
||||
# backend: local
|
||||
embedding_model: default
|
||||
collection_name: reme
|
||||
|
||||
token_counters:
|
||||
default:
|
||||
backend: base
|
||||
|
||||
hf:
|
||||
backend: hf
|
||||
model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct
|
||||
use_mirror: true
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
"""Procedural memory workflow."""
|
||||
|
||||
from ...core import R
|
||||
|
||||
from .dump_memory import DumpMemory
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ class LoadMemory(BaseOp):
|
|||
if not file_path.exists():
|
||||
logger.error(f"File not found: {load_file_path}")
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
# Attempt to retrieve the event loop associated with the current thread
|
||||
loop = asyncio.get_running_loop()
|
||||
|
|
@ -53,7 +53,7 @@ class LoadMemory(BaseOp):
|
|||
except RuntimeError:
|
||||
# Start a new event loop to run the coroutine to completion
|
||||
print("No running event loop found, starting a new one")
|
||||
|
||||
|
||||
clear_existing: bool = self.context.get("clear_existing", False)
|
||||
if clear_existing:
|
||||
await self.vector_store.delete_all()
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ class MemoryAddition(BaseOp):
|
|||
4. Inserts them into the vector store
|
||||
"""
|
||||
raw_memory_list = self.context.memory_list
|
||||
insert_memory_list: List[MemoryNode] = [
|
||||
MemoryNode(**x) if isinstance(x, dict) else x for x in raw_memory_list
|
||||
]
|
||||
insert_memory_list: List[MemoryNode] = [MemoryNode(**x) if isinstance(x, dict) else x for x in raw_memory_list]
|
||||
if insert_memory_list:
|
||||
insert_nodes: List[VectorNode] = [x.to_vector_node() for x in insert_memory_list]
|
||||
await self.vector_store.insert(nodes=insert_nodes)
|
||||
|
|
|
|||
|
|
@ -68,7 +68,11 @@ class MemoryDeduplication(BaseOp):
|
|||
continue
|
||||
|
||||
# Check similarity with current batch task memories
|
||||
if await self._is_similar_to_current_task_memories(current_embedding, unique_task_memories, similarity_threshold):
|
||||
if await self._is_similar_to_current_task_memories(
|
||||
current_embedding,
|
||||
unique_task_memories,
|
||||
similarity_threshold,
|
||||
):
|
||||
logger.debug(f"Skipping duplicate in current batch: {str(task_memory.when_to_use)[:50]}...")
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class MemoryValidation(BaseOp):
|
|||
parsed = json.loads(raw_json)
|
||||
except json.JSONDecodeError as json_err:
|
||||
logger.warning(
|
||||
f"JSONDecodeError in task_memory_validation, fallback to regex parse: {json_err}"
|
||||
f"JSONDecodeError in task_memory_validation, fallback to regex parse: {json_err}",
|
||||
)
|
||||
is_valid_match = re.search(r'"is_valid"\s*:\s*(true|false)', raw_json, re.IGNORECASE)
|
||||
score_match = re.search(r'"score"\s*:\s*([0-9]+(?:\.[0-9]+)?)', raw_json)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from loguru import logger
|
|||
from ...core.enumeration import Role
|
||||
from ...core.schema.message import Message, Trajectory
|
||||
|
||||
|
||||
def merge_messages_content(messages: list[Message | dict]) -> str:
|
||||
"""Merge messages content into a formatted string representation.
|
||||
|
||||
|
|
@ -139,4 +140,3 @@ def get_trajectory_context(trajectory: Trajectory, step_sequence: list[Message])
|
|||
except Exception as e:
|
||||
logger.error(f"Error getting trajectory context: {e}")
|
||||
return f"Query: {trajectory.metadata.get('query', 'N/A')}"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue