From f2e55402b5c600f0fac753c09ae051febd6e57d1 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Tue, 22 Jul 2025 13:15:40 +0800 Subject: [PATCH] update doc --- cookbook/appworld/__init__.py | 0 cookbook/react/__init__.py | 0 cookbook/react/agent.py | 329 ------------------ cookbook/react/quick_start.md | 86 ----- cookbook/react/simple_agent.py | 96 ----- cookbook/react/simple_agent_prompt.yaml | 23 -- cookbook/react/zhaoan.py | 205 ----------- cookbook/vector_store/elasticsearch.md | 31 -- {cookbook/material => doc}/framework.png | Bin doc/global_params.md | 124 +++++++ {cookbook/material => doc}/logo.png | Bin {cookbook/material => doc}/logo_v2.png | Bin doc/operations.md | 27 ++ .../vector_store_quick_start.md | 0 14 files changed, 151 insertions(+), 770 deletions(-) delete mode 100644 cookbook/appworld/__init__.py delete mode 100644 cookbook/react/__init__.py delete mode 100644 cookbook/react/agent.py delete mode 100644 cookbook/react/quick_start.md delete mode 100644 cookbook/react/simple_agent.py delete mode 100644 cookbook/react/simple_agent_prompt.yaml delete mode 100644 cookbook/react/zhaoan.py delete mode 100644 cookbook/vector_store/elasticsearch.md rename {cookbook/material => doc}/framework.png (100%) create mode 100644 doc/global_params.md rename {cookbook/material => doc}/logo.png (100%) rename {cookbook/material => doc}/logo_v2.png (100%) create mode 100644 doc/operations.md rename {cookbook/material => doc}/vector_store_quick_start.md (100%) diff --git a/cookbook/appworld/__init__.py b/cookbook/appworld/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cookbook/react/__init__.py b/cookbook/react/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cookbook/react/agent.py b/cookbook/react/agent.py deleted file mode 100644 index fe970f84..00000000 --- a/cookbook/react/agent.py +++ /dev/null @@ -1,329 +0,0 @@ -import json -import re -import time -from concurrent.futures import ProcessPoolExecutor -from pathlib import Path - -import numpy as np -from appworld import AppWorld, load_task_ids, evaluate_task -from appworld.apps.model_lib import CachedDBHandler -from appworld.task import Task -from jinja2 import Template -from loguru import logger -from openai import OpenAI -from tqdm import tqdm - -from experiencemaker.utils.util_function import load_env_keys - -load_env_keys("../../.env") - -# This is a basic prompt template containing all the necessary onboarding information to solve AppWorld tasks. It explains the role of the agent and the supervisor, how to explore the API documentation, how to operate the interactive coding environment and call APIs via a simple task, and provides key instructions and disclaimers. - -# You can adapt it as needed by your agent. You can also choose to bypass API docs app and build your own API retrieval, e.g., for FullCodeRefl, IPFunCall, etc, we asked an LLM to predict relevant APIs separately and put its documentation directly in the prompt. -PROMPT_TEMPLATE = """ -USER: -I am your supervisor and you are a super intelligent AI Assistant whose job is to achieve my day-to-day tasks completely autonomously. - -To do this, you will need to interact with app/s (e.g., spotify, venmo, etc) using their associated APIs on my behalf. For this you will undertake a *multi-step conversation* using a python REPL environment. That is, you will write the python code and the environment will execute it and show you the result, based on which, you will write python code for the next step and so on, until you've achieved the goal. This environment will let you interact with app/s using their associated APIs on my behalf. - -Here are three key APIs that you need to know to get more information - -# To get a list of apps that are available to you. -print(apis.api_docs.show_app_descriptions()) - -# To get the list of apis under any app listed above, e.g. supervisor -print(apis.api_docs.show_api_descriptions(app_name='supervisor')) - -# To get the specification of a particular api, e.g. supervisor app's show_account_passwords -print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords')) - -Each code execution will produce an output that you can use in subsequent calls. Using these APIs, you can now generate code, that the environment will execute, to solve the task. - -For example, consider the task: - -My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}. - -Task: - -What is the password for my Spotify account? - -ASSISTANT: -# Okay. Lets first find which apps are available to get the password by looking at the app descriptions. -print(apis.api_docs.show_app_descriptions()) - -USER: -[ - { - "name": "api_docs", - "description": "An app to search and explore API documentation." - }, - { - "name": "supervisor", - "description": "An app to access supervisor's personal information, account credentials, addresses, payment cards, and manage the assigned task." - }, - ... - { - "name": "spotify", - "description": "A music streaming app to stream songs and manage song, album and playlist libraries." - }, - { - "name": "venmo", - "description": "A social payment app to send, receive and request money to and from others." - }, - ... -] - - -ASSISTANT: -# Looks like the supervisor app could help me with that. Lets see what apis are available under this app. -print(apis.api_docs.show_api_descriptions(app_name='supervisor')) - - -USER: -[ - ... - "show_account_passwords : Show your supervisor's account passwords." - ... -] - - -ASSISTANT: -# I can use `show_account_passwords` to get the passwords. Let me see its detailed specification to understand its arguments and output structure. -print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords')) - -USER: -{ - 'app_name': 'supervisor', - 'api_name': 'show_account_passwords', - 'path': '/account_passwords', - 'method': 'GET', - 'description': "Show your supervisor's app account passwords.", - 'parameters': [], - 'response_schemas': { - 'success': [{'account_name': 'string', 'password': 'string'}], - 'failure': {'message': 'string'} - } -} - -ASSISTANT: -# Okay, it requires no arguments. So I can just call it directly. -print(apis.supervisor.show_account_passwords()) - -USER: -[ - { - "account_name": "spotify", - "password": "dummy_spotify_pass" - }, - { - "account_name": "file_system", - "password": "dummy_fs_pass" - }, - ... -] - - -ASSISTANT: -# So the Spotify password is an entry in the `passwords` list with the account_name=spotify. -spotify_password = [account_password["account_name"] == "spotify" for account_password in passwords][0]["password"] -print(spotify_password) - - -USER: -dummy_spotify_pass - -ASSISTANT: -# When the task is completed, I need to call apis.supervisor.complete_task(). If there is an answer, I need to pass it as an argument `answer`. I will pass the spotify_password as an answer. -apis.supervisor.complete_task(answer=spotify_password) - - -USER: -Marked the active task complete. - - ----------------------------------------------- - -USER: -**Key instructions and disclaimers**: - -1. The email addresses, access tokens and variables (e.g. spotify_password) in the example above were only for demonstration. Obtain the correct information by calling relevant APIs yourself. -2. Only generate valid code blocks, i.e., do not put them in ```...``` or add any extra formatting. Any thoughts should be put as code comments. -3. You can use the variables from the previous code blocks in the subsequent code blocks. -4. Write small chunks of code and only one chunk of code in every step. Make sure everything is working correctly before making any irreversible change. -5. The provided Python environment has access to its standard library. But modules and functions that have a risk of affecting the underlying OS, file system or process are disabled. You will get an error if do call them. -6. Any reference to a file system in the task instructions means the file system *app*, operable via given APIs, and not the actual file system the code is running on. So do not write code making calls to os-level modules and functions. -7. To interact with apps, only use the provided APIs, and not the corresponding Python packages. E.g., do NOT use `spotipy` for Spotify. Remember, the environment only has the standard library. -8. The provided API documentation has both the input arguments and the output JSON schemas. All calls to APIs and parsing its outputs must be as per this documentation. -9. For APIs that return results in "pages", make sure to consider all pages. -10. To obtain current date or time, use Python functions like `datetime.now()` or obtain it from the phone app. Do not rely on your existing knowledge of what the current date or time is. -11. For all temporal requests, use proper time boundaries, e.g., if I ask for something that happened yesterday, make sure to consider the time between 00:00:00 and 23:59:59. All requests are concerning a single, default (no) time zone. -12. Any reference to my friends, family or any other person or relation refers to the people in my phone's contacts list. -13. All my personal information, and information about my app account credentials, physical addresses and owned payment cards are stored in the "supervisor" app. You can access them via the APIs provided by the supervisor app. -14. Once you have completed the task, call `apis.supervisor.complete_task()`. If the task asks for some information, return it as the answer argument, i.e. call `apis.supervisor.complete_task(answer=)`. For tasks that do not require an answer, just skip the answer argument or pass it as None. -15. The answers, when given, should be just entity or number, not full sentences, e.g., `answer=10` for "How many songs are in the Spotify queue?". When an answer is a number, it should be in numbers, not in words, e.g., "10" and not "ten". -16. You can also pass `status="fail"` in the complete_task API if you are sure you cannot solve it and want to exit. -17. You must make all decisions completely autonomously and not ask for any clarifications or confirmations from me or anyone else. - -USER: -Using these APIs, now generate code to solve the actual task: - -My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}. - -Task: - -{{ instruction }} -""" - - -class MinimalReactAgent: - """A minimal ReAct Agent for AppWorld tasks.""" - - def __init__(self, task: Task): - self.task = task - self.history: list[dict] = self.prompt_messages() - - @staticmethod - def call_llm(messages: list[dict]) -> str: - for i in range(100): - try: - client = OpenAI() - # Change this function to modify the base llm - response = client.chat.completions.create( - model="qwen-max-2025-01-25", messages=messages, temperature=0.6, max_tokens=400, seed=123 - ) - return response.choices[0].message.content - except Exception as e: - logger.exception("") - time.sleep(1 + i * 10) - - return "call llm error" - - def prompt_messages(self) -> list[dict]: - dictionary = {"supervisor": self.task.supervisor, "instruction": self.task.instruction} - prompt = Template(PROMPT_TEMPLATE.lstrip()).render(dictionary) - # Extract and return the OpenAI JSON formatted messages from the prompt - messages: list[dict] = [] - last_start = 0 - for match in re.finditer("(USER|ASSISTANT|SYSTEM):\n", prompt): - last_end = match.span()[0] - if len(messages) == 0: - if last_end != 0: - raise ValueError( - f"Start of the prompt has no assigned role: {prompt[:last_end]}" - ) - else: - messages[-1]["content"] = prompt[last_start:last_end] - mesg_type = match.group(1).lower() - messages.append({"role": mesg_type, "content": None}) - last_start = match.span()[1] - messages[-1]["content"] = prompt[last_start:] - return messages - - def next_code_block(self, last_execution_output: str | None = None) -> str: - if last_execution_output is not None: - self.history.append({"role": "user", "content": last_execution_output}) - code = self.call_llm(self.history) - self.history.append({"role": "assistant", "content": code}) - return code - - -def run_one_agent(task_index: int, task_id: str, experiment_name: str, max_interactions: int = 50): - with AppWorld(task_id=task_id, experiment_name=experiment_name) as world: - print("instruction: " + world.task.instruction) - agent = MinimalReactAgent(world.task) - output: str | None = None - messages: list = [{"supervisor": world.task.supervisor, "instruction": world.task.instruction}] - path: Path = Path(f"./exp_result/{experiment_name}") - - for i in range(max_interactions): - code = agent.next_code_block(output) - messages.append({"role": "assistant", "content": code}) - output = world.execute(code) - if len(output) > 2000: - output = output[:2000] - messages.append({"role": "user", "content": output, "actual_size": len(output)}) - else: - messages.append({"role": "user", "content": output}) - logger.info(f"task_index={task_index} task_id={task_id} steps={i}") - - with open(path / f"{task_index}_{task_id}.jsonl", "w") as f: - json.dump(messages, f, indent=2) - - eval_result = world.evaluate().to_dict() - logger.info(f"===== {i} {json.dumps(eval_result)}=====") - - if world.task_completed(): - messages.append({"role": "task_completed", "content": "task_completed"}) - logger.info(f"task_index={task_index} task_id={task_id} complete.") - break - - with open(path / f"{task_index}_{task_id}.jsonl", "w") as f: - json.dump(messages, f, indent=2) - - return messages - - -def run_agent(dataset_name: str, max_workers: int = 1): - experiment_name = "agent_" + dataset_name - path: Path = Path(f"./exp_result/{experiment_name}") - path.mkdir(parents=True, exist_ok=True) - - task_ids = load_task_ids(dataset_name) - with ProcessPoolExecutor(max_workers=max_workers) as executor: - task_list: list = [] - for index, task_id in enumerate(task_ids): - task = executor.submit(run_one_agent, task_index=index, task_id=task_id, experiment_name=experiment_name) - task_list.append((task_id, task)) - time.sleep(1) - - for i, (task_id, task) in enumerate(task_list): - task.result() - - -def eval_agent(dataset_name: str): - experiment_name = "agent_" + dataset_name - path: Path = Path(f"./exp_result/{experiment_name}") - ratio_list = [] - success_list = [] - if not CachedDBHandler.is_empty(): - raise Exception( - "The cached DB handler is not empty. You likely have an open AppWorld somewhere. " - "Consider calling world.close() on the open one or AppWorld.close_all() to force " - "close all." - ) - - CachedDBHandler.reset() - for file in tqdm(path.iterdir(), desc=experiment_name): - if file.is_file() and file.suffix == ".jsonl": - task_index, task_id = file.stem.split("_", 1) - tracker = evaluate_task( - task_id=task_id, - experiment_name=experiment_name, - suppress_errors=True, - save_report=False) - num_passes = len(tracker.passes) - num_failures = len(tracker.failures) - ratio: float = num_passes / (num_passes + num_failures) - success: float = float(num_failures == 0) - # logger.info(f"task_index={task_index} task_id={task_id} ratio={ratio} success={success}") - - ratio_list.append(ratio) - success_list.append(success) - - CachedDBHandler.reset() - - logger.info(f"experiment_name={experiment_name} size={len(ratio_list)} " - f"ratio={np.mean(ratio_list)} success={np.mean(success_list)}") - - -if __name__ == "__main__": - # pydantic 1.10.22, - run_agent(dataset_name="train") - run_agent(dataset_name="dev") - run_agent(dataset_name="test_normal") - run_agent(dataset_name="test_challenge") - - # eval_agent(dataset_name="train") - # eval_agent(dataset_name="dev") - # eval_agent(dataset_name="test_normal") - # eval_agent(dataset_name="test_challenge") diff --git a/cookbook/react/quick_start.md b/cookbook/react/quick_start.md deleted file mode 100644 index 239b9561..00000000 --- a/cookbook/react/quick_start.md +++ /dev/null @@ -1,86 +0,0 @@ -# Quick Start - -## Hello Experience Maker -Here is a simple user guide for ExperienceMaker. - -### Step0: Preparation Work - -#### Prepare LLM & EMBEDDING_MODEL -We need to prepare the API services for the LLM and the Embedding model. -Since we are using an OpenAI-compatible service, we only need to write the `OPENAI_API_KEY` and `OPENAI_BASE_URL` into the environment. -```shell -export OPENAI_API_KEY="sk-xxx" -export OPENAI_BASE_URL="xxx" -``` - -#### Prepare Vector Store -If you want to use vector store, you need to set up a vector database. Don't forget to set up the `ES_HOSTS`. -- Elasticsearch [quick start](../vector_store/elasticsearch.md) - -#### Prepare Your Own Agent -Assume you have a runnable agent. -Here, we use a basic LLM combined with a simple react framework including three tools(code, web_search, terminate) as an example. -```python -class YourOwnAgent(...): - ... - def think(self, **kwargs) -> bool: - ... - - def act(self, **kwargs): - ... - - def run(self, query: str): - ... -``` - -Here is an [example code](./your_own_agent.py) with [prompt](./your_own_agent_prompt.yaml). To use this simple agent, you will need to set up `DASHSCOPE_API_KEY`. -```shell -export DASHSCOPE_API_KEY="sk-xxx" -``` - -### Step1: Start ExperienceMaker Http Service -- Install dependencies. -```shell -pip install . -``` - -- We start our context and summary services in `simple` mode. Next, you can use standard HTTP interfaces to call the services. -```shell -python -m experiencemaker.em_service \ - --port=8001 \ - --llm='{"backend": "openai_compatible", "model_name": "qwen3-32b", "temperature": 0.6}' \ - --embedding_model='{"backend": "openai_compatible", "model_name": "text-embedding-v4", "dimensions": 1024}' \ - --vector_store='{"backend": "elasticsearch"}' \ - --context_generator='{"backend": "simple"}' \ - --summarizer='{"backend": "simple"}' -``` - -### Step2: Enhance Your Own Agent - -Call the capabilities of ContextGenerator and Summarizer through `EMClient`. - -```python -from experiencemaker.em_client import EMClient - -em_client = EMClient(base_url="http://0.0.0.0:8001") -``` - -Assume you have a list of messages generated by an agent; you can use the summarizer to generate experiences from them. - -```python -request = SummarizerRequest(trajectories=[Trajectory(query=query, steps=messages, answer=messages[-1].content, done=True)], workspace_id="w_1234") -response = em_client.call_summarizer(request) -for experience in response.experiences: - print(experience.model_dump_json()) -``` - -Assume you have a query; you can use the context generator to generate a new query from the query and the related -experiences. - -```python -request = ContextGeneratorRequest(trajectory=Trajectory(query=query), retrieve_top_k=1, workspace_id="w_1234") -response = em_client.call_context_generator(request) -new_query = f"{response.context_msg.content}\n\nUser Question\n{query}" -``` - -The complete code can be found in the implementation of [YourOwnAgentEnhanced](./your_own_agent_enhanced.py). diff --git a/cookbook/react/simple_agent.py b/cookbook/react/simple_agent.py deleted file mode 100644 index dde6afed..00000000 --- a/cookbook/react/simple_agent.py +++ /dev/null @@ -1,96 +0,0 @@ -import datetime -from pathlib import Path -from typing import List - -from loguru import logger -from pydantic import Field, BaseModel - -from experiencemaker.model.base_llm import BaseLLM -from experiencemaker.module.prompt.prompt_mixin import PromptMixin -from experiencemaker.schema.trajectory import Message, ActionMessage, ToolCall, StateMessage -from experiencemaker.tool import CodeTool, DashscopeSearchTool, TerminateTool -from experiencemaker.tool.base_tool import BaseTool - - -class AgentContext(BaseModel): - current_step: int = Field(default=-1) - query: str = Field(default="") - messages: List[Message] = Field(default_factory=list) - metadata: dict = Field(default_factory=dict) - has_terminate_tool: bool = Field(default=False) - - -class SimpleAgent(PromptMixin): - llm: BaseLLM | None = Field(default=None) - max_steps: int = Field(default=10) - tools: List[BaseTool] = [CodeTool(), DashscopeSearchTool(), TerminateTool()] - prompt_file_path: Path = Path(__file__).parent / "simple_agent_prompt.yaml" - - def think(self, context: AgentContext): - now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - tool_names = [x.name for x in self.tools] - - if context.current_step == 0: - user_prompt = self.prompt_format(prompt_name="role_prompt", - time=now_time, - tools=", ".join(tool_names), - query=context.query) - - elif context.has_terminate_tool: - user_prompt = self.prompt_format(prompt_name="final_prompt", query=context.query) - - else: - user_prompt = self.prompt_format(prompt_name="next_prompt", query=context.query) - - context.messages.append(Message(content=user_prompt)) - logger.info(f"step.{context.current_step} user_prompt={user_prompt}") - - if context.has_terminate_tool: - action_msg: ActionMessage = self.llm.chat(context.messages) - - else: - action_msg: ActionMessage = self.llm.chat(context.messages, tools=self.tools) - for tool in action_msg.tool_calls: - if tool.name == "terminate": - context.has_terminate_tool = True - break - - context.messages.append(action_msg) - action_msg_context: str = action_msg.content + "\n\n" + action_msg.reasoning_content - logger.info(f"step.{context.current_step} action_msg_context={action_msg_context} " - f"tool_calls={action_msg.tool_calls}") - return True if action_msg.tool_calls else False - - def act(self, context: AgentContext): - action_msg = context.messages[-1] - assert isinstance(action_msg, ActionMessage) - - tool_dict = {tool.name: tool for tool in self.tools} - - new_tool_calls: List[ToolCall] = [] - for tool_call in action_msg.tool_calls: - if tool_call.name not in tool_dict: - continue - - new_tool_call = tool_call.model_copy(deep=True) - tool = tool_dict[tool_call.name] - new_tool_call.result = tool.execute(**tool_call.argument_dict) - new_tool_calls.append(new_tool_call) - - state_msg = StateMessage(tool_calls=new_tool_calls) - state_msg.tool_result_to_content() - context.messages.append(state_msg) - logger.info(f"step.{context.current_step} state_msg_context={state_msg.content}") - - def run(self, query: str) -> List[Message]: - context: AgentContext = AgentContext(query=query) - - for i in range(self.max_steps): - context.current_step = i - - should_act: bool = self.think(context) - if should_act: - self.act(context) - else: - break - return context.messages diff --git a/cookbook/react/simple_agent_prompt.yaml b/cookbook/react/simple_agent_prompt.yaml deleted file mode 100644 index eaa1cc81..00000000 --- a/cookbook/react/simple_agent_prompt.yaml +++ /dev/null @@ -1,23 +0,0 @@ -role_prompt: | - You are a helpful assistant named BeyondAgent. - The current time is {time}. - - Please proactively choose the most suitable tool or combination of tools based on the user's question, including {tools} etc. - For complex tasks, you can break down the problem step by step and use different tools to solve it incrementally. - Please determine the response language based on the language of the user's question. - - {query} - -next_prompt: | - User's question - {query} - - Plan the most suitable tool or combination of tools based on the context and the user's question. - For complex tasks, break down the problem and use different tools step by step to solve it, don't give up easily. - Try calling the same tool multiple times with different parameters to obtain information from various perspectives. - If the task is completed and the user's question can now be answered, use the **terminate** tool. - -final_prompt: | - Please integrate the context and provide a complete answer to the user's question: - {query} - diff --git a/cookbook/react/zhaoan.py b/cookbook/react/zhaoan.py deleted file mode 100644 index 1ea2a351..00000000 --- a/cookbook/react/zhaoan.py +++ /dev/null @@ -1,205 +0,0 @@ -# ========== Standard and Third-party Imports ========== -import os -import time -import json -import psutil -import statistics -import openai -from loguru import logger -from rich.progress import Progress -from Config.config import TaskConfig -from collections import defaultdict -from llm_client.llm_client import LLMClient -from World_client.env_client import EnvClient -from EM_client.em_client import EMClient -from Summarizer.summarizer import summarize_experience, generate_context -from concurrent.futures import ThreadPoolExecutor, as_completed -from tasks.utils import extract_task, insert_context_before_task, get_task_difficulty - -exp_name = "W_0103" -w_id = "w_0102" - -llm_client = LLMClient(api_key="sk-wE5x9PGlWJn3lwlllprnobZqsWhsfxuc47dobxXYTb0LZM0D", base_url="http://8.130.177.212:3000/v1") - -def run_environment_interaction(client, agent, instance_id, max_interactions) -> int: - - output = None - for i in range(max_interactions): - code = agent.next_code_block(output) - action = {"role": "assistant", "content": code} - result = client.step(instance_id, action) - output = result["state"].get('content', '') - - # Terminate early if the environment signals completion - if result.get('is_terminated', False): - print(f"Terminated after {i + 1} turns") - break - return client.evaluate(instance_id) - - -class ReactAgent: - """ - Agent for proposing the next code block, maintaining - conversation history and interacting with the LLM API. - """ - def __init__(self, history, llm_client): - self.history: list[dict] = history # Initial conversation context - self.llm_client = llm_client - - def next_code_block(self, last_execution_output: str | None = None) -> str: - - if last_execution_output is not None: - self.history.append({"role": "user", "content": last_execution_output}) - code = None - max_tries = 3 - sleep_sec = 3 - for attempt in range(max_tries): - try: - time.sleep(sleep_sec) - code = llm_client.call_llm(self.history) - break - except openai.OpenAIError as e: - logger.error(f"Rate limit error in LLM call on attempt: {attempt + 1}/{max_tries}:{str(e)}") - - #Wait 5 seconds before retrying again - if attempt < max_tries - 1: - logger.info(f"Rate limit exceeded, retrying in {sleep_sec * 2} seconds") - time.sleep(sleep_sec * 2) - except Exception as e: - logger.exception(f"Unexpected error in LLM call: {str(e)}") - - if code is None: - logger.error(f"Failed to generate code after all retries.") - self.history.append({"role": "assistant", "content": code}) - return code - -def evaluate_task(task_id: str, count: int, config: TaskConfig) -> dict: - """ - Executes a single task (multiple runs if best_at > 1), records results, returns highest result. - """ - em_client = EMClient(base_url="http://0.0.0.0:8003") - app_client = EnvClient(base_url="http://localhost:9000") - task_difficulty = get_task_difficulty(task_id) - runs = [] - - try: - for i in range(config.best_at): - print("\n\n" + "*" * 20 + f" Task: {count} | {config.sample_size} " + "*" * 20) - # --- New env & agent every run - init_response = app_client.create_instance(config.env_type, task_id) - instance_id = init_response["info"]["instance_id"] - init_content = init_response["state"]["content"] - - if config.run_with_experience: - task_instruction = extract_task(init_content) - enhanced_content = generate_context(em_client, task_instruction, w_id) - prompt = insert_context_before_task(init_content, enhanced_content) - history = [{"role": "user", "content": prompt}] - else: - history = [{"role": "user", "content": init_content}] - - agent = ReactAgent(history, llm_client) - - # Run or evaluate task as per configuration - score = run_environment_interaction(app_client, agent, instance_id, config.max_interactions) - runs.append((score, agent.history.copy(), init_response)) - print(f"Task id: {task_id} | Run #{i + 1} | Score: {score}") - - try: - success = app_client.release_instance(instance_id) - print(f"Instance released: {success}") - except Exception as e: - logger.exception(f"Failed to release {instance_id}: {str(e)}") - - #Find best run - max_score, max_score_history, max_score_init_response = max(runs, key=lambda x: x[0]) - - # Output task run summary to terminal - print(f"task_id: {task_id} \n" - f"difficulty: {task_difficulty} \n" - f"Score: {max_score} out of {[r[0] for r in runs]} \n") - - result = { - 'task_id': task_id, - 'difficulty': task_difficulty, - 'score': max_score - } - - history_dir = f"/Users/seanlu/PycharmProjects/Simple_ReAct/experiments/{exp_name}" - output_filename = os.path.join(history_dir, f"history_{config.experiment_name}_{task_id}.json") - - - os.makedirs(os.path.dirname(output_filename), exist_ok=True) - with open(output_filename, "w") as f: - json.dump(max_score_history, f, indent=2) - - # Save summarized experience for later training or review - if config.create_exp: - experience_dir = f"/Users/seanlu/PycharmProjects/Simple_ReAct/experiments/Experiences/{exp_name}" - experience_filename = os.path.join(experience_dir, f"{task_id}.json") - os.makedirs(os.path.dirname(experience_filename), exist_ok=True) - instruction = extract_task(max_score_init_response["state"]["content"]) - summarize_experience(em_client, instruction, max_score_history , experience_filename, w_id) - return result - - except Exception as e: - logger.exception(f"Exception in evaluate_task for task_id: {task_id}: {str(e)}") - # Return a failure result for error tracking/statistics - return { - "task_id": task_id, - "difficulty": task_difficulty, - "score": 0, - "error": str(e) - } - # Always attempt to release any used environment instance to prevent resource leaks - - -# ========== Parallel Experiment Pipeline ========== -def main(): - # ---- Load experiment configuration, tasks, and dataset ---- - main_app_client = EnvClient(base_url="http://localhost:9000") - env_type = "appworld" - task_ids = main_app_client.get_task_ids(env_type) - sample_size = 57 - experiment_name = exp_name - max_interactions = 35 - all_results = [] - - config = TaskConfig( - experiment_name=experiment_name, max_interactions=max_interactions, sample_size=sample_size, - env_type=env_type, run_with_experience=False, create_exp=False, best_at = 2 - ) - - # ---- Launch N parallel workers for multiprocessing ---- - with ThreadPoolExecutor(max_workers=20) as executor: - futures = [] - for idx, task_id in enumerate(task_ids[:sample_size]): - futures.append(executor.submit(evaluate_task, task_id, idx + 1, config)) - - with Progress() as progress: - task = progress.add_task("[green]Running experiments...", total=sample_size) - for idx, future in enumerate(as_completed(futures)): - result = future.result() - all_results.append(result) - # Log memory usage in progress bar - mem_mb = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) - progress.update( - task, - advance=1, - description=f"Mem: {mem_mb:.1f} MB | {len(all_results)}/{sample_size} complete" - ) - - difficulty_scores = defaultdict(list) - for res in all_results: - if 'difficulty' in res and 'score' in res: - difficulty_scores[res['difficulty']].append(res['score']) - - for diff, scores in sorted(difficulty_scores.items()): - avg = statistics.mean(scores) if scores else 0 - print(f"Difficulty {diff}: {len(scores)} tasks, Average Score: {avg}") - - print(f"Overall Average: {statistics.mean([r['score'] for r in all_results])}") - print(f"Best of: {config.best_at}") - -if __name__ == "__main__": - main() diff --git a/cookbook/vector_store/elasticsearch.md b/cookbook/vector_store/elasticsearch.md deleted file mode 100644 index fcff9186..00000000 --- a/cookbook/vector_store/elasticsearch.md +++ /dev/null @@ -1,31 +0,0 @@ -## Elasticsearch Vector Store -If a vector database is involved, you will need an Elasticsearch environment. You can refer to the following steps. - -### Install Docker Desktop -If you don’t have Docker installed, download and install [Docker Desktop](https://www.docker.com/products/docker-desktop) for your operating system. - -### Set up Elasticsearch -You can choose one of the following three options. - -#### All in One Script -To set up [Elasticsearch](https://www.elastic.co/docs/solutions/search/run-elasticsearch-locally) and Kibana locally, run the start-local script in the command line: -```shell -curl -fsSL https://elastic.co/start-local | sh -``` - -#### Docker Run Image with Http Host -```shell -docker pull docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.0 -docker run -p 9200:9200 \ - -e "discovery.type=single-node" \ - -e "xpack.security.enabled=false" \ - -e "xpack.license.self_generated.type=trial" \ - -e "http.host=0.0.0.0" \ - docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.0 -``` - -### Inject Environment Variables -Inject variables of the `ES_HOSTS` into the environment where you use Elasticsearch. -```shell -export ES_HOSTS=http://localhost:9200 -``` \ No newline at end of file diff --git a/cookbook/material/framework.png b/doc/framework.png similarity index 100% rename from cookbook/material/framework.png rename to doc/framework.png diff --git a/doc/global_params.md b/doc/global_params.md new file mode 100644 index 00000000..fc9f87ad --- /dev/null +++ b/doc/global_params.md @@ -0,0 +1,124 @@ +# Global Params Documentation + +This document describes all available command-line parameters for ExperienceMaker. The application +uses [OmegaConf](https://omegaconf.readthedocs.io/) for configuration management, supporting both YAML files and +command-line overrides. + +## Basic Usage + +```bash +experiencemaker [parameter1=value1] [parameter2=value2] ... +``` + +## Configuration Loading Priority + +1. Default values from `AppConfig` dataclass +2. Pre-defined YAML configuration file (default: `demo_config.yaml`) +3. Custom YAML file (if `config_path` is specified) +4. Command-line overrides + +## Basic Configuration Parameters + +| Parameter | Type | Default Value | Description | Example | +|----------------------|--------|-----------------|----------------------------------------------------------------------|-------------------------------------------| +| `pre_defined_config` | string | `"demo_config"` | Name of the pre-defined configuration file (without .yaml extension) | `pre_defined_config=full_pipeline_config` | +| `config_path` | string | `""` | Path to custom configuration YAML file | `config_path=/path/to/config.yaml` | + +## HTTP Service Configuration + +| Parameter | Type | Default Value | Description | Example | +|-----------------------------------|---------|---------------|-----------------------------------|---------------------------------------| +| `http_service.host` | string | `"0.0.0.0"` | Host address for the HTTP service | `http_service.host=127.0.0.1` | +| `http_service.port` | integer | `8001` | Port number for the HTTP service | `http_service.port=8080` | +| `http_service.timeout_keep_alive` | integer | `600` | Keep-alive timeout in seconds | `http_service.timeout_keep_alive=600` | +| `http_service.limit_concurrency` | integer | `64` | Maximum concurrent connections | `http_service.limit_concurrency=128` | + +## Thread Pool Configuration + +| Parameter | Type | Default Value | Description | Example | +|---------------------------|---------|---------------|----------------------------------|------------------------------| +| `thread_pool.max_workers` | integer | `10` | Maximum number of worker threads | `thread_pool.max_workers=20` | + +## API Pipeline Configuration + +| Parameter | Type | Default Value | Description | Example | +|--------------------|--------|---------------|------------------------------------------|--------------------------------------------------------------| +| `api.retriever` | string | `""` | Pipeline definition for retriever API | `api.retriever="build_query_op->recall_vector_store_op"` | +| `api.summarizer` | string | `""` | Pipeline definition for summarizer API | `api.summarizer="simple_summary_op->update_vector_store_op"` | +| `api.vector_store` | string | `""` | Pipeline definition for vector store API | `api.vector_store="vector_store_action_op"` | +| `api.agent` | string | `""` | Pipeline definition for agent API | `api.agent="react_op"` | + +## Operation Configuration + +Operations are configured using the pattern `op.{operation_name}.{parameter}`. Each operation can have the following +parameters: + +| Parameter | Type | Default Value | Description | Example | +|------------------------------|--------|---------------|--------------------------------------------|------------------------------------------------------------| +| `op.{name}.backend` | string | `""` | Backend implementation class name | `op.build_query_op.backend=build_query_op` | +| `op.{name}.prompt_file_path` | string | `""` | Path to prompt template file | `op.react_op.prompt_file_path=/path/to/prompt.yaml` | +| `op.{name}.prompt_dict` | dict | `{}` | Direct prompt configuration dictionary | `op.react_op.prompt_dict.system="You are an AI assistant"` | +| `op.{name}.llm` | string | `""` | Reference to LLM configuration | `op.react_op.llm=default` | +| `op.{name}.embedding_model` | string | `""` | Reference to embedding model configuration | `op.recall_op.embedding_model=default` | +| `op.{name}.vector_store` | string | `""` | Reference to vector store configuration | `op.recall_op.vector_store=default` | +| `op.{name}.params.{param}` | any | `{}` | Operation-specific parameters | `op.build_query_op.params.enable_llm_build=false` | + +## LLM Configuration + +| Parameter | Type | Default Value | Description | Example | +|-----------------------------|--------|---------------|----------------------------|-----------------------------------------| +| `llm.{name}.backend` | string | `""` | LLM backend implementation | `llm.default.backend=openai_compatible` | +| `llm.{name}.model_name` | string | `""` | Model name identifier | `llm.default.model_name=qwen3-32b` | +| `llm.{name}.params.{param}` | any | `{}` | LLM-specific parameters | `llm.default.params.temperature=0.6` | + +## Embedding Model Configuration + +| Parameter | Type | Default Value | Description | Example | +|-----------------------------------------|--------|---------------|----------------------------------------|--------------------------------------------------------| +| `embedding_model.{name}.backend` | string | `""` | Embedding model backend implementation | `embedding_model.default.backend=openai_compatible` | +| `embedding_model.{name}.model_name` | string | `""` | Embedding model name identifier | `embedding_model.default.model_name=text-embedding-v4` | +| `embedding_model.{name}.params.{param}` | any | `{}` | Model-specific parameters | `embedding_model.default.params.dimensions=1024` | + +## Vector Store Configuration + +| Parameter | Type | Default Value | Description | Example | +|---------------------------------------|--------|---------------|--------------------------------------------|-----------------------------------------------------------| +| `vector_store.{name}.backend` | string | `""` | Vector store backend implementation | `vector_store.default.backend=elasticsearch` | +| `vector_store.{name}.embedding_model` | string | `""` | Reference to embedding model configuration | `vector_store.default.embedding_model=default` | +| `vector_store.{name}.params.{param}` | any | `{}` | Vector store-specific parameters | `vector_store.default.params.store_dir=file_vector_store` | + +## Complete Example + +Here's a complete example showing how to configure the entire system: + +```bash +experiencemaker \ + http_service.port=8080 \ + thread_pool.max_workers=20 \ + llm.default.backend=openai_compatible \ + llm.default.model_name=qwen3-32b \ + llm.default.params.temperature=0.6 \ + embedding_model.default.backend=openai_compatible \ + embedding_model.default.model_name=text-embedding-v4 \ + embedding_model.default.params.dimensions=1024 \ + vector_store.default.backend=elasticsearch \ + vector_store.default.embedding_model=default \ +``` + +## Configuration File vs Command Line + +You can also create a YAML configuration file and override specific parameters: + +1. Create a custom configuration file (`my_config.yaml`) +2. Use it with command-line overrides: + +```bash +experiencemaker config_path=my_config.yaml llm.default.model_name=qwen3-32b http_service.port=8080 +``` + +## Parameter Validation + +- All parameters are validated according to their types +- Referenced configurations (like `llm`, `embedding_model`, `vector_store`) must exist +- Backend implementations must be registered in their respective registries +- Nested parameters use dot notation for access \ No newline at end of file diff --git a/cookbook/material/logo.png b/doc/logo.png similarity index 100% rename from cookbook/material/logo.png rename to doc/logo.png diff --git a/cookbook/material/logo_v2.png b/doc/logo_v2.png similarity index 100% rename from cookbook/material/logo_v2.png rename to doc/logo_v2.png diff --git a/doc/operations.md b/doc/operations.md new file mode 100644 index 00000000..418780c5 --- /dev/null +++ b/doc/operations.md @@ -0,0 +1,27 @@ +# Operations Documentation + +This document provides an overview of all operations in the ExperienceMaker framework. + +## Operations Overview + +| Op Name | Class | Description | Parameters | +|--------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Build Query | `BuildQueryOp` | Builds retrieval query from user request. Extracts query from request.query or constructs it from messages using LLM if enabled. | `op.build_query_op.params.enable_llm_build = true/false` - Enable LLM-based query construction from messages | +| Recall Experience | `RecallExperienceOp` | Recalls relevant experiences from vector store based on the built query. Performs semantic search and retrieves top-k candidates. | `op.recall_experience_op.params.retrieve_top_k = 15` - Number of experiences to retrieve
`op.recall_experience_op.params.query_enhancement = false` - Enable query enhancement with message context | +| Rerank Experience | `RerankExperienceOp` | Reranks and filters recalled experiences using LLM evaluation and score-based filtering to improve relevance. | `op.rerank_experience_op.params.enable_llm_rerank = true` - Enable LLM-based reranking
`op.rerank_experience_op.params.enable_score_filter = false` - Enable score-based filtering
`op.rerank_experience_op.params.min_score_threshold = 0.3` - Minimum score threshold for filtering
`op.rerank_experience_op.params.top_k = 5` - Number of top experiences to return | +| Rewrite Experience | `RewriteExperienceOp` | Generates and rewrites context messages from reranked experiences to make them more relevant and actionable for the current task. | `op.rewrite_experience_op.params.enable_llm_rewrite = true` - Enable LLM-based context rewriting | +| Merge Experience | `MergeExperienceOp` | Merges the list of experiences into a single formatted message that can be used as context in downstream operations. | - | +| Trajectory Preprocess | `TrajectoryPreprocessOp` | Preprocesses trajectories by validating their structure and classifying them into success/failure categories based on score thresholds. Sets up context for downstream operations. | `op.trajectory_preprocess_op.params.success_threshold = 1.0` - Score threshold to classify trajectories as successful | +| Trajectory Segmentation | `TrajectorySegmentationOp` | Segments trajectories into meaningful step sequences using LLM-based analysis. Identifies natural breakpoints based on logical completion, context switches, tool boundaries, and reasoning phases. | `op.trajectory_segmentation_op.params.segment_target = "all"` - Which trajectories to segment ("all", "success", "failure") | +| Experience Validation | `ExperienceValidationOp` | Validates the quality and usefulness of extracted experiences using LLM-based assessment. Evaluates actionability, accuracy, relevance, clarity, and uniqueness of experiences. | `op.experience_validation_op.params.validation_threshold = 0.3` - Minimum score threshold for experience validation | +| Experience Storage | `ExperienceStorageOp` | Stores validated and deduplicated experiences to the vector database. Converts experiences to vector nodes and handles workspace-specific storage operations. | `op.experience_storage_op.params.default_workspace_id = "default"` - Default workspace ID when not specified in request | +| Experience Deduplication | `ExperienceDeduplicationOp` | Removes duplicate experiences by comparing embeddings of experience content. Performs similarity analysis against both existing stored experiences and current batch experiences. | `op.experience_deduplication_op.params.similarity_threshold = 0.5` - Cosine similarity threshold for duplicate detection
`op.experience_deduplication_op.params.max_existing_experiences = 1000` - Maximum number of existing experiences to compare against | +| Comparative Extraction | `ComparativeExtractionOp` | Extracts comparative experiences by analyzing differences between high/low scoring trajectories (soft comparison) and success/failure patterns (hard comparison). Uses similarity analysis to find comparable step sequences. | `op.comparative_extraction_op.params.enable_soft_comparison = true` - Enable highest vs lowest score comparison
`op.comparative_extraction_op.params.enable_similarity_comparison = false` - Enable success vs failure similarity comparison
`op.comparative_extraction_op.params.max_similarity_sequences = 5` - Maximum sequences to compare for similarity
`op.comparative_extraction_op.params.similarity_threshold = 0.3` - Similarity threshold for step sequence matching
`op.comparative_extraction_op.params.max_similarity_pairs = 3` - Maximum similar pairs to extract experiences from | +| Simple Summary | `SimpleSummaryOp` | Generates simple text summaries from individual trajectories by analyzing execution process and results. Creates basic experiences from trajectory completion status. | `op.simple_summary_op.params.success_score_threshold = 0.9` - Score threshold to classify trajectory as successful | +| Success Extraction | `SuccessExtractionOp` | Extracts actionable experiences specifically from successful trajectories. Processes both segmented step sequences and entire trajectories to identify successful patterns and strategies. | - | +| Failure Extraction | `FailureExtractionOp` | Extracts learning experiences from failed trajectories to identify common pitfalls and failure patterns. Processes both segmented sequences and complete trajectories for failure analysis. | - | +| Update Vector Store | `UpdateVectorStoreOp` | Updates vector store by inserting new vector nodes or deleting existing ones. Handles batch operations for experience storage and management in workspace-specific vector databases. | - | +| Recall Vector Store | `RecallVectorStoreOp` | Retrieves relevant experiences from vector store using semantic search. Filters results by score threshold and removes duplicates based on content similarity. | `op.recall_vector_store_op.params.threshold_score = ` - Minimum similarity score threshold for filtering results (optional) | +| Vector Store Action | `VectorStoreActionOp` | Performs administrative actions on vector store workspaces including copy, delete, dump, and load operations. Manages workspace lifecycle and data migration between environments. | Action-specific parameters passed through request context (workspace IDs, file paths, etc.) | +| React V1 | `ReactV1Op` | Implements ReAct (Reasoning and Acting) agent framework for interactive task execution. Manages tool usage, reasoning steps, and multi-step problem solving with configurable tools and step limits. | `op.react_v1_op.params.max_steps = 10` - Maximum number of reasoning/action steps
`op.react_v1_op.params.tool_names = "code_tool,dashscope_search_tool,terminate_tool"` - Comma-separated list of available tools | + diff --git a/cookbook/material/vector_store_quick_start.md b/doc/vector_store_quick_start.md similarity index 100% rename from cookbook/material/vector_store_quick_start.md rename to doc/vector_store_quick_start.md