update readme & roadmap

README.md

@@ -15,20 +15,22 @@
   <strong>A comprehensive framework for AI agent experience generation and reuse</strong><br>
   <em>Empowering agents to learn from the past and excel in the future</em>
 </p>
-
 ---
 
 ## 📰 What's New
 - **[2025-08]** 🎉 ExperienceMaker v0.1.0 is now available on [PyPI](https://pypi.org/project/experiencemaker/)!
 - **[2025-07]** 📚 Complete documentation and quick start guides released
 - **[2025-07]** 🚀 Multi-backend vector store support (Elasticsearch & ChromaDB)
+---
 
 ## 📰 What's Next
-- **Pre-built Experience Libraries**: Domain repositories (finance/coding/education/research) + community marketplace
+- **Pre-built Experience Libraries**: Domain repositories (Finance/Coding/Education/Research) + community marketplace
 - **Rich Experience Formats**: Executable code/tool configs/pipeline templates/workflows
 - **Experience Validation**: Quality analysis + cross-task effectiveness + auto-refinement
 - **Universal Trajectory Extraction**: Raw logs/multimodal data/execution traces → experiences
 
+Exciting features and improvements are on the horizon! Check out our detailed [Future Roadmap](./doc/future_roadmap.md) for upcoming enhancements.
+
 ---
 
 ## 🌟 What is ExperienceMaker?
@@ -251,13 +253,7 @@ We test ExperienceMaker on Appworld with qwen3-8b:
 
 ### 🔧 Experiment on BFCL-V3
 
-Detailed benchmarking results and performance analysis coming soon.
-
----
-
-## 🛣️ Future Roadmap
-
-Exciting features and improvements are on the horizon! Check out our detailed [Future Roadmap](./doc/future_roadmap.md) for upcoming enhancements.
+Coming Soon! Stay tuned for comprehensive evaluation results.
 
 ---
 
@@ -298,8 +294,6 @@ We warmly welcome contributions from the community! Here's how you can help make
 - Best practices guides and design patterns
 - Translation and localization efforts
 
-**Getting Started**: Fork the repository, create a feature branch, and submit a pull request. Please follow our coding standards and include comprehensive tests for new functionality.
-
 ---
 ## 📄 Citation
 If you use ExperienceMaker in your research or projects, please cite:

cookbook/simple_demo/test_appworld/appworld_react_agent.py

@@ -8,6 +8,7 @@ load_dotenv("../../../.env")
 import re
 import time
 import json
+import ray
 
 from appworld import AppWorld, load_task_ids
 from jinja2 import Template
@@ -17,7 +18,7 @@ from openai import OpenAI
 from prompt import PROMPT_TEMPLATE
 
 
-# @ray.remote
+@ray.remote
 class AppworldReactAgent:
     """A minimal ReAct Agent for AppWorld tasks."""
 
@@ -25,7 +26,7 @@ class AppworldReactAgent:
                  index: int,
                  task_id: str,
                  experiment_name: str,
-                 model_name: str = "qwen3-8b",
+                 model_name: str = "qwen3-32b",
                  temperature: float = 0.9,
                  max_interactions: int = 30,
                  max_response_size: int = 2000):
@@ -111,9 +112,7 @@ class AppworldReactAgent:
                     output = self.next_step(code)
                     self.history.append({"role": "user", "content": output})
 
-                    logger.info(f"index={self.index} task_id={self.task_id} iteration={i} ")
-                    # f"code=\n{code}\n output=\n{output}\n "
-                    # f"score={self.get_reward():.4f}")
+                    logger.info(f"index={self.index} task_id={self.task_id} iteration={i}")
 
                     if self.world.task_completed():
                         break

cookbook/simple_demo/test_appworld/run_appworld.py

@@ -2,7 +2,6 @@ import os
 
 import ray
 from ray import logger
-from tqdm import tqdm
 
 os.environ["APPWORLD_ROOT"] = "."
 from dotenv import load_dotenv
@@ -17,7 +16,7 @@ from appworld import load_task_ids
 from appworld_react_agent import AppworldReactAgent
 
 
-def run_agent(dataset_name: str, experiment_suffix: str, multi_thread: bool = False):
+def run_agent(dataset_name: str, experiment_suffix: str, multi_process: bool = True):
     experiment_name = dataset_name + "_" + experiment_suffix
     path: Path = Path(f"./exp_result")
     path.mkdir(parents=True, exist_ok=True)
@@ -30,7 +29,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, multi_thread: bool = Fa
             for x in result:
                 f.write(json.dumps(x) + "\n")
 
-    if multi_thread:
+    if multi_process:
         future_list: list = []
         for index, task_id in enumerate(task_ids):
             actor = AppworldReactAgent.remote(index=index, task_id=task_id, experiment_name=experiment_name)
@@ -49,8 +48,7 @@ def run_agent(dataset_name: str, experiment_suffix: str, multi_thread: bool = Fa
             dump_file()
 
 
-
 if __name__ == "__main__":
-    # ray.init(num_cpus=8)
+    ray.init(num_cpus=4)
     # run_agent(dataset_name="train", experiment_suffix="v2")
     run_agent(dataset_name="dev", experiment_suffix="v2")

cookbook/simple_demo/test_appworld/run_exp_statistic.py

@@ -0,0 +1,36 @@
+import json
+from pathlib import Path
+
+from loguru import logger
+
+
+def run_exp_statistic():
+    path: Path = Path(f"./exp_result")
+
+    for file in path.glob("*.jsonl"):
+        with open(file, "r") as f:
+            task_completed_list = []
+            before_score_list = []
+            after_score_list = []
+            task_success_list = []
+            for line in f:
+                if not line.strip():
+                    continue
+                data = json.loads(line)
+                task_completed_list.append(1 if data["task_completed"] is True else 0)
+                before_score_list.append(data["before_score"])
+                after_score_list.append(data["after_score"])
+                task_success_list.append(data["after_score"] > 0.9)
+
+            task_completed_ratio = sum(task_completed_list) / len(task_completed_list)
+            before_score_ratio = sum(before_score_list) / len(before_score_list)
+            after_score_ratio = sum(after_score_list) / len(after_score_list)
+            task_success_ratio = sum(task_success_list) / len(task_success_list)
+            logger.info(f"task_completed_ratio={task_completed_ratio:.2f} "
+                        f"before_score_ratio={before_score_ratio:.2f} "
+                        f"after_score_ratio={after_score_ratio:.2f} "
+                        f"task_success_ratio={task_success_ratio:.2f}")
+
+
+if __name__ == "__main__":
+    run_exp_statistic()

doc/future_roadmap.md

@@ -7,7 +7,13 @@ We aim to build curated experience libraries for complex scenarios, providing ba
 Just as financial analysts develop analytical frameworks, senior engineers establish coding standards, and education experts create teaching methodologies, AI agents can build professional experience repositories. Start your AI projects standing on the shoulders of giants.
 
 **Core Features:**
-- [ ] Pre-built experience libraries for key domains (finance, programming, education, research, etc.)
+
+- [ ] Pre-built experience libraries for key domains
+    - [ ] Finance
+    - [ ] Coding
+    - [ ] Education
+    - [ ] Research
+    - etc
 - [ ] Experience marketplace: community-driven experience sharing and exchange
 
 ## P0 - Support for Rich Experience Formats
@@ -34,4 +40,16 @@ Transform valuable experience data from daily work into usable insights:
 - Optimization insights from code commits
 - Decision-making processes from meeting recordings
 
-Enable AI to naturally become stronger through everyday work, rather than wasting real-world experience data due to format limitations.
+Enable AI to naturally become stronger through everyday work, rather than wasting real-world experience data due to format limitations.
+
+## Current TODO
+
+- [x] op dev & test ready
+- [ ] integrate into beyond-agent @jinli
+- [ ] cook_book-appworld code & readme @jiaji
+- [ ] cook_book-bfcl-v3 op @zouyin delay 0730
+- [ ] fix multi-process bug @jinli
+- [ ] logo optimize @jiaji
+- [ ] Ready-made Experience Store @jinli, add appworld/bfcl-v3 default experience store @jiaji
+- [ ] op config make up @jiaji
+- [ ] config make up, easy to understand @jinli

example.env

@@ -0,0 +1,11 @@
+OPENAI_API_KEY=sk-xxxx
+OPENAI_BASE_URL=https://xxxx/v1
+
+EMBEDDING_API_KEY=sk-xxxx
+EMBEDDING_BASE_URL=https://xxxx/v1
+
+LLM_API_KEY=sk-xxxx
+LLM_BASE_URL=https://xxxx/v1
+
+ES_HOSTS=http://0.0.0.0:9200
+DASHSCOPE_API_KEY=sk-xxxx