add quickstart

cookbook/appworld/appworld_react_agent.py

@@ -34,10 +34,12 @@ class AppworldReactAgent:
                  model_name: str = "qwen3-8b",
                  temperature: float = 0.9,
                  max_interactions: int = 30,
-                 max_response_size: int = 2000,
+                 max_response_size: int = 4096,
                  num_runs: int = 1,
                  use_experience: bool = False,
-                 make_experience: bool = False):
+                 make_experience: bool = False,
+                 base_url: str = "http://0.0.0.0:8001/",
+                 workspace_id: str="appworld_8b_0725"):
 
         self.index: int = index
         self.task_ids: List[str] = task_ids
@@ -49,6 +51,8 @@ class AppworldReactAgent:
         self.num_runs: int = num_runs
         self.use_experience: bool = use_experience
         self.make_experience: bool = make_experience
+        self.base_url = base_url
+        self.workspace_id = workspace_id
 
         self.llm_client = OpenAI()
 
@@ -71,7 +75,6 @@ class AppworldReactAgent:
         return "call llm error"
 
     def prompt_messages(self,world: AppWorld) -> list[dict]:
-        logger.info(f"use experience: {self.use_experience}")
         if self.use_experience:
             experience = self.get_experience(world.task.instruction)
             logger.info(f"loaded experience: {experience}")
@@ -113,8 +116,6 @@ class AppworldReactAgent:
                 with AppWorld(task_id=task_id, experiment_name=f"{self.experiment_name}_run_{run_id}") as world:
                     history = self.prompt_messages(world=world)
                     before_score = self.get_reward(world)
-                    logger.info(f"ray_id={self.index} task_index={task_index} run_id={run_id} "
-                                f"instruction={world.task.instruction} before_score={before_score:.4f}")
 
                     for i in range(self.max_interactions):
                         code = self.call_llm(history)
@@ -122,12 +123,10 @@ class AppworldReactAgent:
 
                         output = world.execute(code)
                         if len(output) > self.max_response_size:
-                            logger.warning(f"output exceed max size={len(output)}")
+                            # logger.warning(f"output exceed max size={len(output)}")
                             output = output[:self.max_response_size]
                         history.append({"role": "user", "content": output})
 
-                        logger.info(f"ray_id={self.index} task_index={task_index} run_id={run_id} step={i} complete~")
-
                         if world.task_completed():
                             break
 
@@ -151,14 +150,11 @@ class AppworldReactAgent:
         return result
 
     def get_experience(self, query: str):
-        base_url = "http://0.0.0.0:8001/"
-        workspace_id = "appworld_8b_0724"
-        response = requests.post(url=base_url + "retriever", json={
-            "workspace_id": workspace_id,
+        response = requests.post(url=self.base_url + "retriever", json={
+            "workspace_id": self.workspace_id,
             "query": query,
             "top_k": 5
         })
-        logger.info(f"query:{query}")
 
         if response.status_code != 200:
             print(response.text)

cookbook/appworld/quickstart.md

@@ -0,0 +1,140 @@
+# AppWorld Experiment Quick Start Guide
+
+This guide helps you quickly set up and run AppWorld experiments with ExperienceMaker integration.
+
+## Env Setup
+
+### 1. Clone the Repository
+
+```bash
+git clone https://github.com/modelscope/ExperienceMaker.git
+cd ExperienceMaker/cookbook/appworld
+```
+
+### 2. Appworld Environment Setup
+
+Create a new conda environment with Python 3.12:
+
+```bash
+conda create -p ./appworld-env python==3.12
+conda activate ./appworld-env
+```
+
+Install required Python packages:
+
+```bash
+pip install -r requirements.txt
+```
+
+Install AppWorld and download the dataset:
+
+```bash
+pip install appworld
+appworld install
+appworld download data
+```
+
+**Note**: The AppWorld data will be saved in the current directory.
+
+### 3. Start ExperienceMaker Service
+
+Install ExperienceMaker (if not already installed)
+If you haven't installed the ExperienceMaker environment yet, follow these steps:
+```bash
+# Go back to the project root
+cd ../..
+
+# Create ExperienceMaker environment
+conda create -p ./em-env python==3.12
+conda activate ./em-env
+
+# Install ExperienceMaker
+pip install .
+```
+
+Launch the ExperienceMaker service to enable experience library functionality:
+
+```bash
+experiencemaker \
+  http_service.port=8001 \
+  llm.default.model_name=qwen-max-latest \
+  embedding_model.default.model_name=text-embedding-v4 \
+  vector_store.default.backend=local_file
+```
+
+add experiences for appworld:
+```bash
+curl -X POST "http://0.0.0.0:8001/vector_store" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "appworld",
+    "action": "dump",
+    "path": "./experience_library"
+  }'
+```
+Now you have loaded the ExperienceMaker experience library to enable experience-based agent!
+
+### 4. Common Issues
+
+**AppWorld data not found**: Ensure `appworld download data` completed successfully
+
+**pydantic version issue**:  AppWorld depends on an older version of pydantic, which is why a separate environment is needed. If you encounter issues running the experiments, try `pip install appworld` to override the dependencies.
+
+
+
+## Run Experiments
+
+### 1. Test: With Experience vs Without Experience
+
+Run the main experiment script to compare performance with and without experience:
+
+```bash
+python run_appworld.py
+```
+
+**What this does:**
+- Runs AppWorld tasks on the development dataset
+- Compares agent performance with experience (`use_experience=True`) vs without experience
+- Uses multiple workers for parallel processing
+- Runs each task multiple times for statistical significance
+- Results are automatically saved to `./exp_result/` directory
+
+**Configuration options in `run_appworld.py`:**
+- `max_workers`: Number of parallel workers (default: 6)
+- `num_runs`: Number of times each task is repeated (default: 4)
+- `use_experience`: Whether to use ExperienceMaker experience library
+
+### 2. View Experiment Results
+
+After running experiments, analyze the statistical results:
+
+```bash
+python run_exp_statistic.py
+```
+
+**What this script does:**
+- Processes all result files in `./exp_result/`
+- Calculates best@k metrics for different k values
+- Generates a summary table showing performance comparisons
+- Saves results to `experiment_summary.csv`
+
+**Metrics explained:**
+- `best@k`: Takes groups of k runs per task, finds the maximum score in each group, then averages these maximums
+- Higher k values show potential performance, lower k values show consistency
+
+**Output Files**
+
+- `./exp_result/*.jsonl`: Raw experiment results for each configuration
+- `./exp_result/experiment_summary.csv`: Statistical summary table
+- Console output: Real-time progress and summary statistics
+
+## Understanding Results
+
+The experiment compares:
+1. **Baseline**: Agent without experience library
+2. **With Experience**: Agent enhanced with ExperienceMaker experience library
+
+Key metrics to look for:
+- **best@1**: Average performance across all single runs
+- **best@k**: Performance when taking the best of k attempts
+- Improvement percentage when using experience vs baseline

cookbook/appworld/requirements.txt

@@ -1,4 +1,5 @@
 jinja2
 loguru
 openai
-ray
+ray
+pandas

cookbook/appworld/run_appworld.py

@@ -17,7 +17,7 @@ from appworld import load_task_ids
 from appworld_react_agent import AppworldReactAgent
 
 
-def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_runs: int = 1, use_experience: bool = False):
+def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_runs: int = 1, use_experience: bool = False, workspace_id: str="appworld", exp_url: str = "http://0.0.0.0:8001/") :
     experiment_name = dataset_name + "_" + experiment_suffix
     path: Path = Path(f"./exp_result")
     path.mkdir(parents=True, exist_ok=True)
@@ -26,7 +26,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r
     result: list = []
 
     def dump_file():
-        with open(path / f"{experiment_name}.jsonl", "w") as f:
+        with open(path / f"{experiment_name}.jsonl", "a") as f:
             for x in result:
                 f.write(json.dumps(x) + "\n")
 
@@ -39,7 +39,9 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r
                                               task_ids=worker_task_ids,
                                               experiment_name=experiment_name,
                                               num_runs=num_runs,
-                                              use_experience=use_experience)
+                                              use_experience=use_experience,
+                                              workspace_id=workspace_id,
+                                              exp_url=exp_url)
             future = actor.execute.remote()
             future_list.append(future)
             time.sleep(1)
@@ -54,7 +56,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r
                     result.append(t_result)
 
             logger.info(f"worker {i + 1}/{max_workers} complete")
-            dump_file()
+        dump_file()
 
     else:
         for index, task_id in enumerate(task_ids):
@@ -68,16 +70,25 @@ def run_agent(dataset_name: str, experiment_suffix: str, max_workers: int, num_r
                 result.extend(task_results)
             else:
                 result.append(task_results)
-            dump_file()
+        dump_file()
 
 
 def main():
-    max_workers = 4
+    max_workers = 6
     num_runs = 4  # Run each task 4 times
     if max_workers > 1:
-        ray.init(num_cpus=4)
+        ray.init(num_cpus=6)
     # run_agent(dataset_name="train", experiment_suffix="v2", max_workers=max_workers, num_runs=num_runs)
-    run_agent(dataset_name="dev", experiment_suffix="add-exp", max_workers=max_workers, num_runs=num_runs, use_experience=True)
+
+    logger.info("Start running experiments without experience")
+    for i in range(num_runs):
+        run_agent(dataset_name="dev", experiment_suffix=f"no-exp", max_workers=max_workers, num_runs=1,
+                  use_experience=False, workspace_id="appworld_8b_0725")
+
+    logger.info("Start running experiments with experience")
+    for i in range(num_runs):
+        run_agent(dataset_name="dev", experiment_suffix=f"add-exp", max_workers=max_workers, num_runs=1, use_experience=True,workspace_id="appworld_8b_0725")
+
 
 
 if __name__ == "__main__":