From 15da7cc73049e6f3ebd0d86ce4b600730905d72a Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Mon, 9 Jun 2025 19:44:27 +0800 Subject: [PATCH] add simple summarizer --- experiencemaker/model/base_embedding_model.py | 5 +- experiencemaker/model/base_llm.py | 4 +- .../agent_wrapper/agent_wrapper_mixin.py | 1 + .../base_context_generator.py | 5 ++ .../module/summarizer/base_summarizer.py | 29 ++++--- .../module/summarizer/simple_summarizer.py | 68 +++++++++++++++ .../summarizer/simple_summarizer_prompt.yaml | 22 +++++ experiencemaker/schema/experience.py | 83 +++++++++++++++++++ experiencemaker/schema/trajectory.py | 8 -- experiencemaker/storage/__init__.py | 3 - experiencemaker/storage/base_vector_store.py | 4 + experiencemaker/storage/es_vector_store.py | 7 +- experiencemaker/storage/file_vector_store.py | 3 +- 13 files changed, 212 insertions(+), 30 deletions(-) create mode 100644 experiencemaker/module/summarizer/simple_summarizer.py create mode 100644 experiencemaker/module/summarizer/simple_summarizer_prompt.yaml create mode 100644 experiencemaker/schema/experience.py diff --git a/experiencemaker/model/base_embedding_model.py b/experiencemaker/model/base_embedding_model.py index 0391f5d8..ef09a74e 100644 --- a/experiencemaker/model/base_embedding_model.py +++ b/experiencemaker/model/base_embedding_model.py @@ -7,8 +7,6 @@ from pydantic import BaseModel, Field from experiencemaker.schema.vector_store_node import VectorStoreNode from experiencemaker.utils.registry import Registry -EMBEDDING_MODEL_REGISTRY = Registry("embedding_model") - class BaseEmbeddingModel(BaseModel, ABC): model_name: str = Field(default=..., description="model name") @@ -87,3 +85,6 @@ class BaseEmbeddingModel(BaseModel, ABC): else: raise RuntimeError(f"unsupported type={type(nodes)}") + + +EMBEDDING_MODEL_REGISTRY = Registry[BaseEmbeddingModel]("embedding_model") diff --git a/experiencemaker/model/base_llm.py b/experiencemaker/model/base_llm.py index f5632aa0..a055e6f9 100644 --- a/experiencemaker/model/base_llm.py +++ b/experiencemaker/model/base_llm.py @@ -8,7 +8,6 @@ from experiencemaker.schema.trajectory import Message, ActionMessage from experiencemaker.tool.base_tool import BaseTool from experiencemaker.utils.registry import Registry -LLM_REGISTRY = Registry("llm") class BaseLLM(BaseModel, ABC): model_name: str = Field(...) @@ -110,3 +109,6 @@ class BaseLLM(BaseModel, ABC): raise e return None + + +LLM_REGISTRY = Registry[BaseLLM]("llm") diff --git a/experiencemaker/module/agent_wrapper/agent_wrapper_mixin.py b/experiencemaker/module/agent_wrapper/agent_wrapper_mixin.py index 5bfc2ae4..bcaf4450 100644 --- a/experiencemaker/module/agent_wrapper/agent_wrapper_mixin.py +++ b/experiencemaker/module/agent_wrapper/agent_wrapper_mixin.py @@ -9,6 +9,7 @@ from experiencemaker.utils.registry import Registry class AgentWrapperMixin(BaseModel, ABC): context_generator: BaseContextGenerator | None = Field(default=None) + workspace_id: str = Field(default="") def execute(self, query: str, **kwargs) -> Trajectory: raise NotImplementedError diff --git a/experiencemaker/module/context_generator/base_context_generator.py b/experiencemaker/module/context_generator/base_context_generator.py index 4934e372..ebe61af7 100644 --- a/experiencemaker/module/context_generator/base_context_generator.py +++ b/experiencemaker/module/context_generator/base_context_generator.py @@ -3,6 +3,8 @@ from typing import List from pydantic import Field, BaseModel +from experiencemaker.model.base_embedding_model import BaseEmbeddingModel +from experiencemaker.model.base_llm import BaseLLM from experiencemaker.schema.trajectory import Trajectory, ContextMessage from experiencemaker.schema.vector_store_node import VectorStoreNode from experiencemaker.storage.base_vector_store import BaseVectorStore @@ -11,6 +13,9 @@ from experiencemaker.utils.registry import Registry class BaseContextGenerator(BaseModel, ABC): vector_store: BaseVectorStore | None = Field(default=None) + llm: BaseLLM | None = Field(default=None) + embedding_model: BaseEmbeddingModel | None = Field(default=None) + workspace_id: str = Field(default="") def _build_retrieve_query(self, trajectory: Trajectory, **kwargs) -> str: raise NotImplementedError diff --git a/experiencemaker/module/summarizer/base_summarizer.py b/experiencemaker/module/summarizer/base_summarizer.py index b06889d2..d5f93c40 100644 --- a/experiencemaker/module/summarizer/base_summarizer.py +++ b/experiencemaker/module/summarizer/base_summarizer.py @@ -3,24 +3,33 @@ from typing import List from pydantic import Field, BaseModel -from experiencemaker.schema.trajectory import Trajectory, Sample +from experiencemaker.model.base_embedding_model import BaseEmbeddingModel +from experiencemaker.model.base_llm import BaseLLM +from experiencemaker.schema.experience import Experience +from experiencemaker.schema.trajectory import Trajectory +from experiencemaker.schema.vector_store_node import VectorStoreNode from experiencemaker.storage.base_vector_store import BaseVectorStore +from experiencemaker.utils.registry import Registry class BaseSummarizer(BaseModel, ABC): vector_store: BaseVectorStore | None = Field(default=None) + llm: BaseLLM | None = Field(default=None) + embedding_model: BaseEmbeddingModel | None = Field(default=None) + workspace_id: str = Field(default="") - def _extract_samples(self, trajectories: List[Trajectory], **kwargs) -> List[Sample]: + def _extract_experiences(self, trajectories: List[Trajectory], **kwargs) -> List[Experience]: raise NotImplementedError - def _insert_into_database(self, samples: List[Sample], **kwargs): - raise NotImplementedError + def execute(self, trajectories: List[Trajectory], return_experience: bool = True, **kwargs) -> List[Experience]: + experiences: List[Experience] = self._extract_experiences(trajectories, **kwargs) - def execute(self, trajectories: List[Trajectory], return_samples: bool = True, **kwargs) -> List[Sample]: - samples: List[Sample] = self._extract_samples(trajectories, **kwargs) - self._insert_into_database(samples, **kwargs) + nodes: List[VectorStoreNode] = [x.to_vector_store_node() for x in experiences] + self.vector_store.insert(nodes, **kwargs) - if return_samples: - return samples + if return_experience: + return experiences + return [] - return [] \ No newline at end of file + +SUMMARIZER_REGISTRY = Registry[BaseSummarizer]("summarizer") diff --git a/experiencemaker/module/summarizer/simple_summarizer.py b/experiencemaker/module/summarizer/simple_summarizer.py new file mode 100644 index 00000000..3bfb820f --- /dev/null +++ b/experiencemaker/module/summarizer/simple_summarizer.py @@ -0,0 +1,68 @@ +from pathlib import Path +from typing import List + +from loguru import logger +from pydantic import Field + +from experiencemaker.enumeration.role import Role +from experiencemaker.module.prompt.prompt_mixin import PromptMixin +from experiencemaker.module.summarizer.base_summarizer import BaseSummarizer, SUMMARIZER_REGISTRY +from experiencemaker.schema.experience import Experience +from experiencemaker.schema.trajectory import Trajectory, Message, ActionMessage +from experiencemaker.utils.util_function import get_html_match_content + + +class SimpleSummarizer(BaseSummarizer, PromptMixin): + max_retries: int = Field(default=5, description="max retries") + prompt_file_path: Path = Path(__file__).parent / "simple_summarizer_prompt.yaml" + + def _extract_trajectory_experience(self, trajectory: Trajectory) -> Experience | None: + step_content_collector: List[str] = [] + + for step in trajectory.steps: + step_index = len(step_content_collector) + + if step.role is Role.ASSISTANT: + line = f"### step.{step_index} role={step.role.value} content=\n{step.content}\n{step.reasoning_content}\n" + if step.tool_calls: + for tool_call in step.tool_calls: + line += f" - tool call={tool_call.name}\n params={tool_call.arguments}\n" + step_content_collector.append(line) + + elif step.role is Role.USER: + line = f"### step.{step_index} role={step.role.value} content=\n{step.content}\n" + step_content_collector.append(line) + + elif step.role is Role.TOOL: + line = f"### step.{step_index} role={step.role.value} tool call result=\n{step.content}\n" + step_content_collector.append(line) + + prompt = self.prompt_format(prompt_name="summary_prompt", + query=trajectory.query, + execution_process="\n".join(step_content_collector).strip(), + answer=trajectory.answer) + + for i in range(self.max_retries): + action_message: ActionMessage = self.llm.chat(messages=[Message(content=prompt)]) + experience_str = get_html_match_content(action_message.content, key="experience") + condition_str = get_html_match_content(action_message.content, key="condition") + if experience_str and condition_str: + return Experience(experience_workspace_id=self.workspace_id, + experience_role=self.llm.model_name, + experience_desc=condition_str, + experience_content=experience_str) + else: + logger.warning(f"action_message.content={action_message.content} re.search failed.") + + return None + + def _extract_experiences(self, trajectories: List[Trajectory], **kwargs) -> List[Experience]: + experiences: List[Experience] = [] + for trajectory in trajectories: + experience: Experience = self._extract_trajectory_experience(trajectory) + if experience: + experiences.append(experience) + return experiences + + +SUMMARIZER_REGISTRY.register(SimpleSummarizer, "simple") diff --git a/experiencemaker/module/summarizer/simple_summarizer_prompt.yaml b/experiencemaker/module/summarizer/simple_summarizer_prompt.yaml new file mode 100644 index 00000000..3a0f7283 --- /dev/null +++ b/experiencemaker/module/summarizer/simple_summarizer_prompt.yaml @@ -0,0 +1,22 @@ +summary_prompt: | + # Role + You are a helpful assistant named BeyondAgent. + + # User Question + {query} + + # Execution Process + {execution_process} + + # Answer + {answer} + + # Task + Reflect on the strengths and weaknesses of the **Execution Process** and the **Answer** based on the **User Question**. + Finally, summarize generalized experience from handling such problems to accumulate experience for future similar tasks. + The experience should be broadly applicable, such as how to use tools effectively or approaches to solving certain types of problems. + Also, specify the conditions or scenarios in which these experience are applicable. + + # Output Format + Output the scenarios or conditions in which applying this experience would be particularly effective... + Output generalized experience, concise content is required... \ No newline at end of file diff --git a/experiencemaker/schema/experience.py b/experiencemaker/schema/experience.py new file mode 100644 index 00000000..017a6fb1 --- /dev/null +++ b/experiencemaker/schema/experience.py @@ -0,0 +1,83 @@ +import datetime +from typing import List +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from experiencemaker.schema.vector_store_node import VectorStoreNode + + +class ExperienceFunctionArg(BaseModel): + arg_name: str = Field(default=..., description="argument name") + arg_type: str = Field(default=..., description="argument type, like: 'str', 'int', 'bool'") + required: bool = Field(default=True, description="whether the argument is required") + + +class ExperienceFunction(BaseModel): + func_code: str = Field(default=..., description="function code") + func_name: str = Field(default=..., description="function name") + func_args: List[ExperienceFunctionArg] = Field(default_factory=list, description="function arguments") + + +class Experience(BaseModel): + experience_id: str = Field(default_factory=lambda: uuid4().hex, description="experience unique id") + experience_workspace_id: str = Field(default="", description="unique workspace id") + experience_role: str = Field(default="", description="experience role") + experience_desc: str = Field(default="", description="use condition/purpose. It will be used in vector matching") + experience_content: str | bytes = Field(default="", description="content of the experience") + experience_function: ExperienceFunction | None = Field(default=None, description="experience function(optional)") + experience_score: float = Field(default=0.0, description="score of the experience") + experience_created_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + experience_modified_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + metadata: dict = Field(default_factory=dict, description="additional metadata") + + def to_vector_store_node(self) -> VectorStoreNode: + metadata: dict = { + "experience_role": self.experience_role, + "experience_content": self.experience_content, + "experience_function": self.experience_function.model_dump(), + "experience_score": self.experience_score, + "experience_created_time": self.experience_created_time, + "experience_modified_time": self.experience_modified_time, + "metadata": self.metadata, + } + return VectorStoreNode( + unique_id=self.experience_id, + workspace_id=self.experience_workspace_id, + content=self.experience_desc, + metadata=metadata) + + @classmethod + def from_vector_store_node(cls, node: VectorStoreNode) -> "Experience": + return cls( + experience_id=node.unique_id, + experience_workspace_id=node.workspace_id, + experience_role=node.metadata.get("experience_role", ""), + experience_desc=node.content, + experience_content=node.metadata.get("experience_content", ""), + experience_function=node.metadata.get("experience_function", None), + experience_score=node.metadata.get("experience_score", 0.0), + experience_created_time=node.metadata.get("experience_created_time", ""), + experience_modified_time=node.metadata.get("experience_modified_time", ""), + metadata=node.metadata.get("metadata", {})) + + +if __name__ == "__main__": + e1 = Experience( + experience_workspace_id="w_1024", + experience_role="qwen3", + experience_desc="test desc", + experience_content="test content", + experience_function=ExperienceFunction( + func_code="def a():\n return", + func_name="a", + func_args=[ExperienceFunctionArg(arg_name="x", arg_type="str", required=True)] + ), + experience_score=0.99, + metadata={"haha": 1} + ) + print(e1.model_dump_json(indent=2)) + v1 = e1.to_vector_store_node() + print(v1.model_dump_json(indent=2)) + e2 = Experience.from_vector_store_node(v1) + print(e2.model_dump_json(indent=2)) diff --git a/experiencemaker/schema/trajectory.py b/experiencemaker/schema/trajectory.py index 64dd5b09..14018b56 100644 --- a/experiencemaker/schema/trajectory.py +++ b/experiencemaker/schema/trajectory.py @@ -118,11 +118,3 @@ class Trajectory(BaseModel): self.query = "" self.answer = "" self.metadata.clear() - - -class Experience(BaseModel): - experience_id: str = Field(default_factory=lambda: uuid4().hex, description="experience unique id") - experience_desc: str = Field(default="", description="use condition or use purpose for vector matching") - experience_content: str | bytes = Field(default="", description="content of the experience") - experience_score: float = Field(default=0.0, description="score of the experience") - metadata: dict = Field(default_factory=dict, description="additional metadata") diff --git a/experiencemaker/storage/__init__.py b/experiencemaker/storage/__init__.py index 5df9193b..e69de29b 100644 --- a/experiencemaker/storage/__init__.py +++ b/experiencemaker/storage/__init__.py @@ -1,3 +0,0 @@ -from experiencemaker.utils.registry import Registry - -VECTOR_STORE_REGISTRY = Registry("vector_store") diff --git a/experiencemaker/storage/base_vector_store.py b/experiencemaker/storage/base_vector_store.py index 9674c6cb..e4644aff 100644 --- a/experiencemaker/storage/base_vector_store.py +++ b/experiencemaker/storage/base_vector_store.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, Field from experiencemaker.model.base_embedding_model import BaseEmbeddingModel from experiencemaker.schema.vector_store_node import VectorStoreNode +from experiencemaker.utils.registry import Registry class BaseVectorStore(BaseModel, ABC): @@ -24,3 +25,6 @@ class BaseVectorStore(BaseModel, ABC): def retrieve_by_query(self, query: str, top_k: int = 3, **kwargs) -> List[VectorStoreNode]: raise NotImplementedError + + +VECTOR_STORE_REGISTRY = Registry[BaseVectorStore]("vector_store") diff --git a/experiencemaker/storage/es_vector_store.py b/experiencemaker/storage/es_vector_store.py index b0de2d85..d3865a86 100644 --- a/experiencemaker/storage/es_vector_store.py +++ b/experiencemaker/storage/es_vector_store.py @@ -7,8 +7,7 @@ from loguru import logger from pydantic import Field, PrivateAttr, model_validator from experiencemaker.schema.vector_store_node import VectorStoreNode -from experiencemaker.storage import VECTOR_STORE_REGISTRY -from experiencemaker.storage.base_vector_store import BaseVectorStore +from experiencemaker.storage.base_vector_store import BaseVectorStore, VECTOR_STORE_REGISTRY class EsVectorStore(BaseVectorStore): @@ -104,7 +103,7 @@ class EsVectorStore(BaseVectorStore): self.retrieve_filters.clear() return self - def insert(self, nodes: VectorStoreNode | List[VectorStoreNode], refresh_index: bool = False, **kwargs): + def insert(self, nodes: VectorStoreNode | List[VectorStoreNode], refresh_index: bool = True, **kwargs): if isinstance(nodes, VectorStoreNode): nodes = [nodes] @@ -119,7 +118,7 @@ class EsVectorStore(BaseVectorStore): if refresh_index: self.refresh_index() - def update(self, nodes: VectorStoreNode | List[VectorStoreNode], refresh_index: bool = False, **kwargs): + def update(self, nodes: VectorStoreNode | List[VectorStoreNode], refresh_index: bool = True, **kwargs): if isinstance(nodes, VectorStoreNode): nodes = [nodes] diff --git a/experiencemaker/storage/file_vector_store.py b/experiencemaker/storage/file_vector_store.py index 8d8bf772..3fb3c8e9 100644 --- a/experiencemaker/storage/file_vector_store.py +++ b/experiencemaker/storage/file_vector_store.py @@ -8,8 +8,7 @@ from loguru import logger from pydantic import Field, model_validator, PrivateAttr from experiencemaker.schema.vector_store_node import VectorStoreNode -from experiencemaker.storage import VECTOR_STORE_REGISTRY -from experiencemaker.storage.base_vector_store import BaseVectorStore +from experiencemaker.storage.base_vector_store import BaseVectorStore, VECTOR_STORE_REGISTRY class FileVectorStore(BaseVectorStore):