diff --git a/cookbook/appworld/appworld_react_agent.py b/cookbook/appworld/appworld_react_agent.py index c10b9d03..0f3d0a38 100644 --- a/cookbook/appworld/appworld_react_agent.py +++ b/cookbook/appworld/appworld_react_agent.py @@ -34,10 +34,12 @@ class AppworldReactAgent: model_name: str = "qwen3-8b", temperature: float = 0.9, max_interactions: int = 30, - max_response_size: int = 2000, + max_response_size: int = 4096, num_runs: int = 1, use_experience: bool = False, - make_experience: bool = False): + make_experience: bool = False, + base_url: str = "http://0.0.0.0:8001/", + workspace_id: str="appworld_8b_0725"): self.index: int = index self.task_ids: List[str] = task_ids @@ -49,6 +51,8 @@ class AppworldReactAgent: self.num_runs: int = num_runs self.use_experience: bool = use_experience self.make_experience: bool = make_experience + self.base_url = base_url + self.workspace_id = workspace_id self.llm_client = OpenAI() @@ -71,7 +75,6 @@ class AppworldReactAgent: return "call llm error" def prompt_messages(self,world: AppWorld) -> list[dict]: - logger.info(f"use experience: {self.use_experience}") if self.use_experience: experience = self.get_experience(world.task.instruction) logger.info(f"loaded experience: {experience}") @@ -113,8 +116,6 @@ class AppworldReactAgent: with AppWorld(task_id=task_id, experiment_name=f"{self.experiment_name}_run_{run_id}") as world: history = self.prompt_messages(world=world) before_score = self.get_reward(world) - logger.info(f"ray_id={self.index} task_index={task_index} run_id={run_id} " - f"instruction={world.task.instruction} before_score={before_score:.4f}") for i in range(self.max_interactions): code = self.call_llm(history) @@ -122,12 +123,10 @@ class AppworldReactAgent: output = world.execute(code) if len(output) > self.max_response_size: - logger.warning(f"output exceed max size={len(output)}") + # logger.warning(f"output exceed max size={len(output)}") output = output[:self.max_response_size] history.append({"role": "user", "content": output}) - logger.info(f"ray_id={self.index} task_index={task_index} run_id={run_id} step={i} complete~") - if world.task_completed(): break @@ -151,14 +150,11 @@ class AppworldReactAgent: return result def get_experience(self, query: str): - base_url = "http://0.0.0.0:8001/" - workspace_id = "appworld_8b_0724" - response = requests.post(url=base_url + "retriever", json={ - "workspace_id": workspace_id, + response = requests.post(url=self.base_url + "retriever", json={ + "workspace_id": self.workspace_id, "query": query, "top_k": 5 }) - logger.info(f"query:{query}") if response.status_code != 200: print(response.text) diff --git a/cookbook/appworld/quickstart.md b/cookbook/appworld/quickstart.md new file mode 100644 index 00000000..49f49ec7 --- /dev/null +++ b/cookbook/appworld/quickstart.md @@ -0,0 +1,140 @@ +# AppWorld Experiment Quick Start Guide + +This guide helps you quickly set up and run AppWorld experiments with ExperienceMaker integration. + +## Env Setup + +### 1. Clone the Repository + +```bash +git clone https://github.com/modelscope/ExperienceMaker.git +cd ExperienceMaker/cookbook/appworld +``` + +### 2. Appworld Environment Setup + +Create a new conda environment with Python 3.12: + +```bash +conda create -p ./appworld-env python==3.12 +conda activate ./appworld-env +``` + +Install required Python packages: + +```bash +pip install -r requirements.txt +``` + +Install AppWorld and download the dataset: + +```bash +pip install appworld +appworld install +appworld download data +``` + +**Note**: The AppWorld data will be saved in the current directory. + +### 3. Start ExperienceMaker Service + +Install ExperienceMaker (if not already installed) +If you haven't installed the ExperienceMaker environment yet, follow these steps: +```bash +# Go back to the project root +cd ../.. + +# Create ExperienceMaker environment +conda create -p ./em-env python==3.12 +conda activate ./em-env + +# Install ExperienceMaker +pip install . +``` + +Launch the ExperienceMaker service to enable experience library functionality: + +```bash +experiencemaker \ + http_service.port=8001 \ + llm.default.model_name=qwen-max-latest \ + embedding_model.default.model_name=text-embedding-v4 \ + vector_store.default.backend=local_file +``` + +add experiences for appworld: +```bash +curl -X POST "http://0.0.0.0:8001/vector_store" \ + -H "Content-Type: application/json" \ + -d '{ + "workspace_id": "appworld", + "action": "dump", + "path": "./experience_library" + }' +``` +Now you have loaded the ExperienceMaker experience library to enable experience-based agent! + +### 4. Common Issues + +**AppWorld data not found**: Ensure `appworld download data` completed successfully + +**pydantic version issue**: AppWorld depends on an older version of pydantic, which is why a separate environment is needed. If you encounter issues running the experiments, try `pip install appworld` to override the dependencies. + + + +## Run Experiments + +### 1. Test: With Experience vs Without Experience + +Run the main experiment script to compare performance with and without experience: + +```bash +python run_appworld.py +``` + +**What this does:** +- Runs AppWorld tasks on the development dataset +- Compares agent performance with experience (`use_experience=True`) vs without experience +- Uses multiple workers for parallel processing +- Runs each task multiple times for statistical significance +- Results are automatically saved to `./exp_result/` directory + +**Configuration options in `run_appworld.py`:** +- `max_workers`: Number of parallel workers (default: 6) +- `num_runs`: Number of times each task is repeated (default: 4) +- `use_experience`: Whether to use ExperienceMaker experience library + +### 2. View Experiment Results + +After running experiments, analyze the statistical results: + +```bash +python run_exp_statistic.py +``` + +**What this script does:** +- Processes all result files in `./exp_result/` +- Calculates best@k metrics for different k values +- Generates a summary table showing performance comparisons +- Saves results to `experiment_summary.csv` + +**Metrics explained:** +- `best@k`: Takes groups of k runs per task, finds the maximum score in each group, then averages these maximums +- Higher k values show potential performance, lower k values show consistency + +**Output Files** + +- `./exp_result/*.jsonl`: Raw experiment results for each configuration +- `./exp_result/experiment_summary.csv`: Statistical summary table +- Console output: Real-time progress and summary statistics + +## Understanding Results + +The experiment compares: +1. **Baseline**: Agent without experience library +2. **With Experience**: Agent enhanced with ExperienceMaker experience library + +Key metrics to look for: +- **best@1**: Average performance across all single runs +- **best@k**: Performance when taking the best of k attempts +- Improvement percentage when using experience vs baseline \ No newline at end of file diff --git a/cookbook/appworld/requirements.txt b/cookbook/appworld/requirements.txt index 680712ca..86ebcb1f 100644 --- a/cookbook/appworld/requirements.txt +++ b/cookbook/appworld/requirements.txt @@ -1,4 +1,5 @@ jinja2 loguru openai -ray \ No newline at end of file +ray +pandas \ No newline at end of file diff --git a/cookbook/appworld/run_appworld.py b/cookbook/appworld/run_appworld.py index afd4021f..eff60921 100644 --- a/cookbook/appworld/run_appworld.py +++ b/cookbook/appworld/run_appworld.py @@ -17,7 +17,7 @@ from appworld import load_task_ids from appworld_react_agent import AppworldReactAgent -def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_runs: int = 1, use_experience: bool = False): +def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_runs: int = 1, use_experience: bool = False, workspace_id: str="appworld", exp_url: str = "http://0.0.0.0:8001/") : experiment_name = dataset_name + "_" + experiment_suffix path: Path = Path(f"./exp_result") path.mkdir(parents=True, exist_ok=True) @@ -26,7 +26,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r result: list = [] def dump_file(): - with open(path / f"{experiment_name}.jsonl", "w") as f: + with open(path / f"{experiment_name}.jsonl", "a") as f: for x in result: f.write(json.dumps(x) + "\n") @@ -39,7 +39,9 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r task_ids=worker_task_ids, experiment_name=experiment_name, num_runs=num_runs, - use_experience=use_experience) + use_experience=use_experience, + workspace_id=workspace_id, + exp_url=exp_url) future = actor.execute.remote() future_list.append(future) time.sleep(1) @@ -54,7 +56,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r result.append(t_result) logger.info(f"worker {i + 1}/{max_workers} complete") - dump_file() + dump_file() else: for index, task_id in enumerate(task_ids): @@ -68,16 +70,25 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r result.extend(task_results) else: result.append(task_results) - dump_file() + dump_file() def main(): - max_workers = 4 + max_workers = 6 num_runs = 4 # Run each task 4 times if max_workers > 1: - ray.init(num_cpus=4) + ray.init(num_cpus=6) # run_agent(dataset_name="train", experiment_suffix="v2", max_workers=max_workers, num_runs=num_runs) - run_agent(dataset_name="dev", experiment_suffix="add-exp", max_workers=max_workers, num_runs=num_runs, use_experience=True) + + logger.info("Start running experiments without experience") + for i in range(num_runs): + run_agent(dataset_name="dev", experiment_suffix=f"no-exp", max_workers=max_workers, num_runs=1, + use_experience=False, workspace_id="appworld_8b_0725") + + logger.info("Start running experiments with experience") + for i in range(num_runs): + run_agent(dataset_name="dev", experiment_suffix=f"add-exp", max_workers=max_workers, num_runs=1, use_experience=True,workspace_id="appworld_8b_0725") + if __name__ == "__main__":