add simple_comparative_summary_op

cookbook/finance/web_search.py

@@ -0,0 +1,33 @@
+import json
+import os
+from pathlib import Path
+
+from tavily import TavilyClient
+
+cache_path: Path = Path("./web_search_cache")
+cache_path.mkdir(parents=True, exist_ok=True)
+
+
+def web_search(query: str, enable_print: bool = True, enable_cache: bool = True):
+    if enable_cache:
+        ...
+
+    client = TavilyClient(api_key=os.environ["TVLY_API_KEY"])
+    response = client.search(
+        include_answer=True,
+        include_raw_content=True,
+        query=query)
+
+    # response = client.get_search_context(query=query)
+    # response = json.loads(response)
+    if enable_print:
+        print(json.dumps(response, indent=2, ensure_ascii=False))
+
+    return response
+
+
+if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv("../../.env")
+    web_search("恒生医药为什么一直涨")

experiencemaker/op/__init__.py

@@ -15,6 +15,7 @@ from experiencemaker.op.summarizer.experience_validation_op import ExperienceVal
 from experiencemaker.op.summarizer.experience_deduplication_op import ExperienceDeduplicationOp
 from experiencemaker.op.summarizer.experience_validation_op import ExperienceValidationOp
 from experiencemaker.op.summarizer.trajectory_segmentation_op import TrajectorySegmentationOp
+from experiencemaker.op.summarizer.simple_comparative_summary_op import SimpleComparativeSummaryOp
 
 from experiencemaker.op.retriever.rerank_experience_op import RerankExperienceOp
 from experiencemaker.op.retriever.rewrite_experience_op import RewriteExperienceOp

experiencemaker/op/summarizer/simple_comparative_summary_op.py

@@ -0,0 +1,71 @@
+import json
+from typing import List, Dict
+
+from loguru import logger
+
+from experiencemaker.op import OP_REGISTRY
+from experiencemaker.op.base_op import BaseOp
+from experiencemaker.schema.experience import TextExperience, ExperienceMeta, BaseExperience
+from experiencemaker.schema.message import Message, Trajectory
+from experiencemaker.schema.request import SummarizerRequest
+from experiencemaker.schema.response import SummarizerResponse
+from experiencemaker.utils.op_utils import merge_messages_content
+
+
+@OP_REGISTRY.register()
+class SimpleComparativeSummaryOp(BaseOp):
+    current_path: str = __file__
+
+    def compare_summary_trajectory(self, trajectory_a: Trajectory, trajectory_b: Trajectory) -> List[BaseExperience]:
+        summary_prompt = self.prompt_format(prompt_name="summary_prompt",
+                                            execution_process_a=merge_messages_content(trajectory_a.messages),
+                                            execution_process_b=merge_messages_content(trajectory_b.messages),
+                                            summary_example=self.get_prompt("summary_example"))
+
+        def parse_content(message: Message):
+            content = message.content
+            experience_list = []
+            try:
+                content = content.split("```")[1].strip()
+                if content.startswith("json"):
+                    content = content.strip("json")
+
+                for exp_dict in json.loads(content):
+                    when_to_use = exp_dict.get("when_to_use", "").strip()
+                    experience = exp_dict.get("experience", "").strip()
+                    if when_to_use and experience:
+                        experience_list.append(TextExperience(workspace_id=self.context.request.workspace_id,
+                                                              when_to_use=when_to_use,
+                                                              content=experience,
+                                                              metadata=ExperienceMeta(author=self.llm.model_name)))
+
+                return experience_list
+
+            except Exception as e:
+                logger.exception(f"parse content failed!\n{content}")
+                raise e
+
+        return self.llm.chat(messages=[Message(content=summary_prompt)], callback_fn=parse_content)
+
+    def execute(self):
+        request: SummarizerRequest = self.context.request
+        response: SummarizerResponse = self.context.response
+
+        task_id_dict: Dict[str, List[Trajectory]] = {}
+        for trajectory in request.traj_list:
+            if trajectory.task_id not in task_id_dict:
+                task_id_dict[trajectory.task_id] = []
+            task_id_dict[trajectory.task_id].append(trajectory)
+
+        for task_id, trajectories in task_id_dict.items():
+            trajectories: List[Trajectory] = sorted(trajectories, key=lambda x: x.score, reverse=True)
+            if len(trajectories) < 2:
+                continue
+
+            if trajectories[0].score > trajectories[-1].score:
+                self.submit_task(self.compare_summary_trajectory, trajectory_a=trajectories[0],
+                                 trajectory_b=trajectories[-1])
+
+        response.experience_list = self.join_task()
+        for e in response.experience_list:
+            logger.info(f"add experience when_to_use={e.when_to_use}\ncontent={e.content}")

experiencemaker/op/summarizer/simple_comparative_summary_prompt.yaml

@@ -0,0 +1,32 @@
+summary_prompt: |
+  # Execution Process A
+  {execution_process_a}
+  
+  # Execution Process B
+  {execution_process_b}
+
+  # Task
+  **Execution Process A** and **Execution Process B** represent two distinct execution trajectories of two agents, with Execution Process A being superior to Execution Process B.
+  Analyze these two trajectories and identify areas where Execution Process B can be improved relative to Execution Process A.
+  The insights derived should be generalizable, offering guidance for solving similar problems in the future.
+  These insights can include positive recommendations or point out pitfalls to avoid.
+  The format of the insights can be plain text or a snippet of code addressing a specific issue.
+  If no insights can be derived, return an empty list [].
+  For each insight, first specify the context in which it applies (when to use), followed by the insight itself. Provide up to two insights.
+
+  # Output Format
+  {summary_example}
+
+summary_example: |
+  ```json
+  [
+    {
+      "when_to_use": "...",
+      "experience": "..."
+    },
+    {
+      "when_to_use": "...",
+      "experience": "..."
+    }
+  ]
+  ```

experiencemaker/schema/message.py

@@ -62,6 +62,7 @@ class Message(BaseModel):
 
 
 class Trajectory(BaseModel):
+    task_id: str = Field(default="")
     messages: List[Message] = Field(default_factory=list)
     score: float = Field(default=0.0)
     metadata: dict = Field(default_factory=dict)