diff --git a/cookbook/appworld/agent.py b/cookbook/appworld/agent.py index 2b6f9939..fe970f84 100644 --- a/cookbook/appworld/agent.py +++ b/cookbook/appworld/agent.py @@ -317,12 +317,13 @@ def eval_agent(dataset_name: str): if __name__ == "__main__": - # run_agent(dataset_name="train") - # run_agent(dataset_name="dev") - # run_agent(dataset_name="test_normal") - # run_agent(dataset_name="test_challenge") + # pydantic 1.10.22, + run_agent(dataset_name="train") + run_agent(dataset_name="dev") + run_agent(dataset_name="test_normal") + run_agent(dataset_name="test_challenge") - eval_agent(dataset_name="train") - eval_agent(dataset_name="dev") - eval_agent(dataset_name="test_normal") - eval_agent(dataset_name="test_challenge") + # eval_agent(dataset_name="train") + # eval_agent(dataset_name="dev") + # eval_agent(dataset_name="test_normal") + # eval_agent(dataset_name="test_challenge") diff --git a/cookbook/step_agent/demo_log.txt b/cookbook/step_agent/demo_log.txt deleted file mode 100644 index e17eee3d..00000000 --- a/cookbook/step_agent/demo_log.txt +++ /dev/null @@ -1,169 +0,0 @@ -2025-06-18 14:30:26.301 | WARNING | __main__::270 - skip key=origin_config info.annotation= -2025-06-18 14:30:26.301 | INFO | __main__::278 - service.kwargs={ - "host": "0.0.0.0", - "port": 8001, - "timeout_keep_alive": 600000, - "limit_concurrency": 32, - "llm": { - "backend": "openai_compatible", - "model_name": "qwen-max-2025-01-25", - "temperature": 0.6 - }, - "embedding_model": { - "backend": "openai_compatible", - "model_name": "text-embedding-v4", - "dimensions": 1024 - }, - "vector_store": { - "backend": "local_file", - "store_dir": "./step_experiences/" - }, - "context_generator": { - "backend": "step", - "enable_llm_rerank": true, - "enable_context_rewrite": true, - "enable_score_filter": false, - "vector_retrieve_top_k": 15, - "final_top_k": 5, - "min_score_threshold": 0.3 - }, - "summarizer": { - "backend": "step", - "enable_step_segmentation": false, - "enable_similar_comparison": false, - "enable_experience_validation": true, - "max_retries": 3, - "max_workers": 16 - } -} -2025-06-18 14:30:26.335 | INFO | __main__:init_llm:51 - llm is inited with backend=openai_compatible params={'model_name': 'qwen-max-2025-01-25', 'temperature': 0.6} -2025-06-18 14:30:26.348 | INFO | __main__:init_embedding_model:70 - embedding_model is inited with backend=openai_compatible params={'model_name': 'text-embedding-v4', 'dimensions': 1024} -2025-06-18 14:30:26.349 | INFO | __main__:init_vector_store:91 - vector_store is inited with backend=local_file params={'store_dir': './step_experiences/'} -2025-06-18 14:30:26.350 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=context_rewrite_prompt -2025-06-18 14:30:26.350 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=experience_rerank_prompt -2025-06-18 14:30:26.350 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=context_generation_prompt -2025-06-18 14:30:26.350 | INFO | __main__:init_context_generator:117 - context_generator is inited with backend=step params={'enable_llm_rerank': True, 'enable_context_rewrite': True, 'enable_score_filter': False, 'vector_retrieve_top_k': 15, 'final_top_k': 5, 'min_score_threshold': 0.3} -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=success_step_experience_prompt -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=failure_step_experience_prompt -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=comparative_step_experience_prompt -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=general_step_experience_prompt -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=step_segmentation_prompt -2025-06-18 14:30:26.352 | INFO | experiencemaker.module.prompt.prompt_mixin:init_prompt:30 - add prompt_dict key=experience_validation_prompt -2025-06-18 14:30:26.352 | INFO | __main__:init_summarizer:132 - summarizer is inited with backend=step params={'enable_step_segmentation': False, 'enable_similar_comparison': False, 'enable_experience_validation': True, 'max_retries': 3, 'max_workers': 16} -INFO: Started server process [94237] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:8001 (Press CTRL+C to quit) -2025-06-18 14:35:52.470 | INFO | __main__:call_summarizer:225 - workspace_id=w_agent_enhanced metadata={} trajectories= -{ - "id": "6a7e7204439d45a593416c50f563116e", - "steps": [ - { - "role": "user", - "content": "What is the capital of France?", - "reasoning_content": "", - "tool_calls": [], - "timestamp": "2025-06-18 14:35:52.464309", - "add_reasoning_content_when_content_is_empty": false, - "metadata": {} - }, - { - "role": "assistant", - "content": "Paris", - "reasoning_content": "", - "tool_calls": [], - "timestamp": "2025-06-18 14:35:52.464361", - "add_reasoning_content_when_content_is_empty": false, - "metadata": {} - } - ], - "is_terminated": false, - "reward": { - "outcome": 1.0, - "description": "Outcome 1 denotes success, and 0 denotes failure.", - "metadata": {} - }, - "query": "What is the capital of France?", - "answer": "", - "metadata": {} -} -2025-06-18 14:35:52.470 | INFO | experiencemaker.module.summarizer.step_summarizer:_extract_experiences:70 - Starting step-level experience extraction pipeline for 1 trajectories -2025-06-18 14:35:52.470 | INFO | experiencemaker.module.summarizer.step_summarizer:_async_extract_step_experiences_from_success:122 - Extracting step experiences from 1 successful trajectories -2025-06-18 14:35:57.993 | INFO | experiencemaker.module.summarizer.step_summarizer:_async_validate_experiences:331 - Validating 1 extracted experiences -2025-06-18 14:36:13.472 | INFO | experiencemaker.module.summarizer.step_summarizer:_async_validate_experiences:348 - Validated 1 out of 1 experiences -2025-06-18 14:36:13.472 | INFO | experiencemaker.module.summarizer.step_summarizer:_async_extract_experiences:116 - Extracted 1 validated step experiences -2025-06-18 14:36:13.687 | INFO | experiencemaker.storage.file_vector_store:update:111 - update w_agent_enhanced nodes.size=1 all.size=1 update_cnt=0 -2025-06-18 14:36:13.687 | INFO | __main__:call_summarizer:232 - workspace_id=w_agent_enhanced experiences_content= -{ - "experience_id": "72b620febc7e4b30b94d200e3aa21f3b", - "experience_workspace_id": "w_agent_enhanced", - "experience_role": "", - "experience_desc": "When the query is a simple, fact-based question requiring a direct and concise answer.", - "experience_content": "The agent immediately provided the correct answer without unnecessary elaboration or additional steps. This worked well because the query was straightforward, and the response matched the user's expected level of detail, ensuring efficiency and clarity.", - "experience_function": null, - "experience_score": 0.0, - "experience_created_time": "2025-06-18 14:35:57", - "experience_modified_time": "2025-06-18 14:35:57", - "metadata": { - "when_to_use": "When the query is a simple, fact-based question requiring a direct and concise answer.", - "experience": "The agent immediately provided the correct answer without unnecessary elaboration or additional steps. This worked well because the query was straightforward, and the response matched the user's expected level of detail, ensuring efficiency and clarity.", - "tags": [ - "fact-based", - "direct-answer", - "efficiency" - ], - "confidence": 0.95, - "step_type": "action", - "tools_used": [ - "knowledge-base" - ] - } -} -INFO: 127.0.0.1:55361 - "POST /summarizer HTTP/1.1" 200 OK -2025-06-18 14:36:13.694 | INFO | __main__:call_context_generator:206 - workspace_id=w_agent_enhanced metadata={} trajectory= -{ - "id": "5d56ba6e010e4c6ea496ca006ccaf33d", - "steps": [ - { - "role": "user", - "content": "What is the capital of France?", - "reasoning_content": "", - "tool_calls": [], - "timestamp": "2025-06-18 14:36:13.694224", - "add_reasoning_content_when_content_is_empty": false, - "metadata": {} - }, - { - "role": "assistant", - "content": "Paris", - "reasoning_content": "", - "tool_calls": [], - "timestamp": "2025-06-18 14:36:13.694256", - "add_reasoning_content_when_content_is_empty": false, - "metadata": {} - } - ], - "is_terminated": false, - "reward": { - "outcome": 1.0, - "description": "Outcome 1 denotes success, and 0 denotes failure.", - "metadata": {} - }, - "query": "What is the capital of France?", - "answer": "", - "metadata": {} -} -2025-06-18 14:36:13.694 | INFO | experiencemaker.module.context_generator.step_context_generator:_hybrid_retrieve:105 - Starting hybrid retrieval for query: 'What is the capital of France?' -2025-06-18 14:36:13.885 | INFO | experiencemaker.module.context_generator.step_context_generator:_vector_retrieve:138 - Vector retrieval found 1 candidates -2025-06-18 14:36:20.455 | INFO | experiencemaker.module.context_generator.step_context_generator:_hybrid_retrieve:124 - Hybrid retrieval completed: 1 experiences selected -2025-06-18 14:36:39.916 | INFO | experiencemaker.module.context_generator.step_context_generator:_rewrite_context:202 - Context successfully rewritten for current task -2025-06-18 14:36:39.916 | INFO | __main__:call_context_generator:211 - workspace_id=w_agent_enhanced context_msg={ - "role": "context_assistant", - "content": "For the current task, which involves answering a straightforward, fact-based question about the capital of France, the guidance is to provide an accurate and concise response. The user seeks immediate clarity without additional elaboration or unrelated information. Drawing from past experiences with similar queries, prioritize delivering the correct answer—'Paris'—in a clear and direct manner. This approach ensures that the response is both useful and aligned with the user's expectations for simplicity and precision.", - "reasoning_content": "", - "tool_calls": [], - "timestamp": "2025-06-18 14:36:39.916579", - "add_reasoning_content_when_content_is_empty": false, - "metadata": {} -} -INFO: 127.0.0.1:55377 - "POST /context_generator HTTP/1.1" 200 OK diff --git a/cookbook/step_agent/examples.py b/cookbook/step_agent/examples.py deleted file mode 100644 index 75932fd6..00000000 --- a/cookbook/step_agent/examples.py +++ /dev/null @@ -1,261 +0,0 @@ -""" -Step Experience Service Usage Examples - -This file demonstrates how to use the step-level experience extraction and context generation service. -""" - -import json -import requests -from typing import List, Dict, Any - -from experiencemaker.enumeration.role import Role -from experiencemaker.schema.trajectory import Trajectory, Message, ToolCall - -# Service configuration -SERVICE_URL = "http://localhost:8001" -WORKSPACE_ID = "test_workspace" - - -def create_sample_trajectory(query: str, steps: List[Message], done: bool = True) -> Trajectory: - """Create a sample trajectory for testing""" - trajectory = Trajectory( - query=query, - steps=steps, - is_terminated=done, - metadata={ - "domain": "coding", - "task_type": "problem_solving" - } - ) - return trajectory - - -def example_1_extract_step_experiences(): - """Example 1: Extract step-level experiences from trajectories""" - print("=== Example 1: Extract Step-Level Experiences ===") - - # Create sample successful trajectory - successful_trajectory = create_sample_trajectory( - query="How to implement a binary search algorithm?", - steps=[ - Message( - role=Role.USER, - content="How to implement a binary search algorithm?" - ), - Message( - role=Role.ASSISTANT, - content="I'll help you implement a binary search algorithm. Let me start by explaining the concept.", - reasoning_content="Need to first explain the concept before implementation" - ), - Message( - role=Role.ASSISTANT, - content="Here's the implementation:\n\ndef binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1", - tool_calls=[ - ToolCall( - index=0, - id="call_1", - name="code_execution", - arguments='{"code": "def binary_search(arr, target): ..."}', - result="Code executed successfully" - ) - ] - ), - Message( - role=Role.TOOL, - content="Code executed successfully. Binary search implementation is correct." - ) - ], - done=True - ) - - # Create sample failed trajectory - failed_trajectory = create_sample_trajectory( - query="How to implement a binary search algorithm?", - steps=[ - Message( - role=Role.USER, - content="How to implement a binary search algorithm?" - ), - Message( - role=Role.ASSISTANT, - content="Here's a binary search:\n\ndef binary_search(arr, target):\n for i in range(len(arr)):\n if arr[i] == target:\n return i\n return -1", - reasoning_content="Implementing binary search as linear search by mistake" - ), - Message( - role=Role.TOOL, - content="Error: This is actually a linear search, not binary search. Binary search requires sorted array and divide-and-conquer approach." - ) - ], - done=False - ) - - # Extract experiences using summarizer - summarizer_request = { - "trajectories": [successful_trajectory.model_dump(), failed_trajectory.model_dump()], - "workspace_id": WORKSPACE_ID - } - - try: - response = requests.post(f"{SERVICE_URL}/summarizer", json=summarizer_request) - response.raise_for_status() - - result = response.json() - print(f"✅ Extracted {len(result['experiences'])} step-level experiences") - - for i, experience in enumerate(result['experiences']): - print(f"\nExperience {i + 1}:") - print(f" Condition: {experience['experience_desc']}") - print(f" Content: {experience['experience_content'][:100]}...") - print(f" Role: {experience['experience_role']}") - - except requests.exceptions.RequestException as e: - print(f"❌ Error extracting experiences: {e}") - - -def example_2_generate_step_context(): - """Example 2: Generate context from step-level experiences""" - print("\n=== Example 2: Generate Step-Level Context ===") - - # Create a new trajectory that needs context - current_trajectory = create_sample_trajectory( - query="How to implement a quick sort algorithm?", - steps=[ - Message( - role=Role.USER, - content="How to implement a quick sort algorithm?" - ), - Message( - role=Role.ASSISTANT, - content="I need to implement a quick sort algorithm. Let me think about the approach.", - reasoning_content="Quick sort is a divide-and-conquer algorithm" - ) - ], - done=False - ) - - # Generate context using context generator - context_request = { - "trajectory": current_trajectory.model_dump(), - "workspace_id": WORKSPACE_ID - } - - try: - response = requests.post(f"{SERVICE_URL}/context_generator", json=context_request) - response.raise_for_status() - - result = response.json() - context_message = result['context_msg'] - - print("✅ Generated step-level context:") - print(f"Content: {context_message['content']}") - - if 'metadata' in context_message: - print(f"Metadata: {context_message['metadata']}") - - except requests.exceptions.RequestException as e: - print(f"❌ Error generating context: {e}") - - -def example_3_full_agent_execution(): - """Example 3: Full agent execution with step-level experience""" - print("\n=== Example 3: Full Agent Execution with Step Experience ===") - - # Execute agent with step-level context - agent_request = { - "query": "Implement a merge sort algorithm with proper error handling", - "workspace_id": WORKSPACE_ID - } - - try: - response = requests.post(f"{SERVICE_URL}/agent_wrapper", json=agent_request) - response.raise_for_status() - - result = response.json() - trajectory = result['trajectory'] - - print("✅ Agent execution completed:") - print(f"Query: {trajectory['query']}") - print(f"Steps: {len(trajectory['steps'])}") - print(f"Done: {trajectory['done']}") - print(f"Answer: {trajectory.get('answer', 'No answer')[:200]}...") - - except requests.exceptions.RequestException as e: - print(f"❌ Error in agent execution: {e}") - - -def example_4_custom_configuration(): - """Example 4: Custom configuration for different use cases""" - print("\n=== Example 4: Custom Configuration Examples ===") - - # Configuration for research-intensive tasks - research_config = { - "context_generator": { - "backend": "step", - "enable_llm_rerank": True, - "enable_context_rewrite": True, - "enable_score_filter": True, - "vector_retrieve_top_k": 20, - "final_top_k": 8, - "min_score_threshold": 0.2 - }, - "summarizer": { - "backend": "step", - "enable_step_segmentation": True, - "enable_similar_comparison": True, - "enable_experience_validation": True, - "max_retries": 5 - } - } - - # Configuration for quick tasks - quick_config = { - "context_generator": { - "backend": "step", - "enable_llm_rerank": False, - "enable_context_rewrite": False, - "enable_score_filter": False, - "vector_retrieve_top_k": 5, - "final_top_k": 2, - "min_score_threshold": 0.5 - }, - "summarizer": { - "backend": "step", - "enable_step_segmentation": False, - "enable_similar_comparison": False, - "enable_experience_validation": False, - "max_retries": 1 - } - } - - print("📋 Research-intensive configuration:") - print(json.dumps(research_config, indent=2)) - - print("\n📋 Quick task configuration:") - print(json.dumps(quick_config, indent=2)) - - -def main(): - """Run all examples""" - print("🚀 Step Experience Service Usage Examples") - print("=" * 50) - - # Wait for service to be ready - try: - response = requests.get(f"{SERVICE_URL}/docs") - print("✅ Service is running and ready") - except requests.exceptions.RequestException: - print("❌ Service is not running. Please start the service first:") - print(" run.sh") - return - - # Run examples - example_1_extract_step_experiences() - example_2_generate_step_context() - example_3_full_agent_execution() - example_4_custom_configuration() - - print("\n🎉 All examples completed!") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/cookbook/step_agent/pipeline.png b/cookbook/step_agent/pipeline.png deleted file mode 100644 index 9ef2be7e..00000000 Binary files a/cookbook/step_agent/pipeline.png and /dev/null differ diff --git a/cookbook/step_agent/readme.md b/cookbook/step_agent/readme.md deleted file mode 100644 index ff4f8d7e..00000000 --- a/cookbook/step_agent/readme.md +++ /dev/null @@ -1,98 +0,0 @@ -# Step Summarizer & Context Generator - -A step-level experience extraction and context generation module for the ExperienceMaker framework. - -## Overview -![pipeline.png](pipeline.png) -This module provides advanced step-level experience extraction and context generation capabilities: - -- **StepSummarizer**: Extracts reusable experiences from individual steps or step sequences in trajectories -- **StepContextGenerator**: Retrieves and utilizes step-level experiences to provide relevant context for agent execution - -## Key Features - -### StepSummarizer Features -- **Trajectory segmentation** into meaningful step sequences -- **Step-level experience extraction** from successful and failed trajectories -- **Similarity-based comparison** for finding related experiences and compare between success and failure cases -- **Experience validation** for quality assurance - -### StepContextGenerator Features -- **Score-based filtering** for quality control -- **Llm reranking** for reranking after recalling -- **Context rewriting** to make experiences more relevant to current tasks -- **Hybrid retrieval** combining vector search with LLM reranking - -## Configuration Options - -### StepSummarizer Parameters -- `enable_step_segmentation`: Enable trajectory segmentation (default: False) -- `enable_similar_comparison`: Enable similarity search for comparison (default: False) -- `enable_experience_validation`: Enable experience validation (default: True) -- `max_retries`: Maximum retries for LLM calls (default: 3) - -### StepContextGenerator Parameters -- `enable_llm_rerank`: Enable LLM-based reranking (default: True) -- `enable_context_rewrite`: Enable context rewriting (default: True) -- `enable_score_filter`: Enable score-based filtering (default: False) -- `vector_retrieve_top_k`: Number of candidates to retrieve (default: 15) -- `final_top_k`: Final number of experiences to return (default: 5) -- `min_score_threshold`: Minimum score threshold for filtering (default: 0.3) - -## Quick Start - -### 1. Basic Setup -```bash -# Start the service with default configuration -bash run.sh - - -### 3. Run Examples -```python -# Test the system with example trajectories -python examples.py -``` - -## Usage Examples - -### Extract Step Experiences -```python -# Extract experiences from trajectories -summarizer_request = { - "trajectories": [successful_trajectory, failed_trajectory], - "workspace_id": "my_workspace", - # "metadata": { - # "em_config": { - # "summarizer": { - # "backend": "step", - # "enable_step_segmentation": True, - # "enable_similar_comparison": True, - # "enable_experience_validation": True - # } - # } - # } -} - -response = requests.post(f"{SERVICE_URL}/summarizer", json=summarizer_request) -``` - -### Generate Context -```python -# Generate context for current task -context_request = { - "trajectory": current_trajectory, - "workspace_id": "my_workspace", - # "metadata": { - # "em_config": { - # "context_generator": { - # "backend": "step", - # "enable_llm_rerank": True, - # "enable_context_rewrite": True, - # "final_top_k": 5 - # } - # } - # } -} - -response = requests.post(f"{SERVICE_URL}/context_generator", json=context_request) -``` \ No newline at end of file diff --git a/cookbook/step_agent/run.sh b/cookbook/step_agent/run.sh deleted file mode 100644 index 8ba2c40d..00000000 --- a/cookbook/step_agent/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Step Experience Service Startup Script -# This script launches the experiencemaker service with step-level experience extraction and context generation - -# StepSummarizer Parameters -# enable_step_segmentation: Segment trajectories into meaningful step sequences -# enable_similar_comparison: Compare similar sequences between success/failure -# enable_experience_validation: Validate experience quality before storage - -# StepContextGenerator Parameters -# enable_llm_rerank: Rerank retrieved experiences by relevance using LLM -# enable_context_rewrite: Rewrite context to be more task-specific -# enable_score_filter: Filter experiences by quality scores - -echo "Starting Step Experience Service..." - -python -m experiencemaker.em_service \ - --port=8001 \ - --llm='{"backend": "openai_compatible", "model_name": "qwen-max-2025-01-25", "temperature": 0.6}' \ - --embedding_model='{"backend": "openai_compatible", "model_name": "text-embedding-v4", "dimensions": 1024}' \ - --vector_store='{"backend": "local_file", "store_dir": "./step_experiences/"}' \ - --context_generator='{"backend": "step", "enable_llm_rerank": true, "enable_context_rewrite": true, "enable_score_filter": false, "vector_retrieve_top_k": 15, "final_top_k": 5, "min_score_threshold": 0.3}' \ - --summarizer='{"backend": "step", "enable_step_segmentation": false, "enable_similar_comparison": false, "enable_experience_validation": true, "max_retries": 3, "max_workers": 32}' - -echo "Step Experience Service started on port 8001" \ No newline at end of file diff --git a/v1/__init__.py b/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/config.yaml b/v1/config.yaml new file mode 100644 index 00000000..24d8aa5f --- /dev/null +++ b/v1/config.yaml @@ -0,0 +1,67 @@ +# demo config.yaml + +#python -m experiencemaker.em_service \ +# --port=8001 \ +# --llm='{"backend": "openai_compatible", "model_name": "qwen3-32b", "temperature": 0.6}' \ +# --embedding_model='{"backend": "openai_compatible", "model_name": "text-embedding-v4", "dimensions": 1024}' \ +# --vector_store='{"backend": "elasticsearch"}' \ + +http_service: + host: "0.0.0.0" + port: 8001 + timeout_keep_alive: 600 + limit_concurrency: 64 + +# -http_service.port=8001 + +thread_pool: + max_workers: 20 + + +api: + step_retriever: mock1_op->mock2_op->mock3_op + step_summarizer: mock1_op->[mock4_op->mock2_op|mock5_op]->mock3_op + vector_store: mock6_op + +# -api.step_retriever=mock1_op->[mock4_op->mock2_op|mock5_op]->mock3_op + + +# -op.mock1_op.a=1 + +op: + mock1_op: + backend: mock1_op + a: 1 + b: 2 + llm: default + vector_store: default + mock2_op: + backend: mock2_op + a: 1 + mock3_op: + backend: mock3_op + mock4_op: + backend: mock4_op + mock5_op: + backend: mock5_op + mock6_op: + backend: mock6_op + +llm: + default: + backend: openai_compatible + model_name: qwen3-32b + temperature: 0.6 + +embedding_model: + default: + backend: openai_compatible + model_name: text-embedding-v4 + dimensions: 1024 + +vector_store: + default: + backend: elasticsearch + embedding_model: default + hosts: "http://localhost:9200" + diff --git a/v1/config/__init__.py b/v1/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/config_handler.py b/v1/config_handler.py new file mode 100644 index 00000000..852e868b --- /dev/null +++ b/v1/config_handler.py @@ -0,0 +1,4 @@ +class ConfigHandler(object): + + def __init__(self): + pass diff --git a/v1/enumeration/__init__.py b/v1/enumeration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/enumeration/chunk_enum.py b/v1/enumeration/chunk_enum.py new file mode 100644 index 00000000..975f0b1f --- /dev/null +++ b/v1/enumeration/chunk_enum.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class ChunkEnum(str, Enum): + THINK = "think" + ANSWER = "answer" + TOOL = "tool" + USAGE = "usage" + ERROR = "error" diff --git a/v1/enumeration/http_enum.py b/v1/enumeration/http_enum.py new file mode 100644 index 00000000..f5ebe79f --- /dev/null +++ b/v1/enumeration/http_enum.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class HttpEnum(str, Enum): + GET = "get" + POST = "post" + HEAD = "head" + PUT = "put" + DELETE = "delete" diff --git a/v1/enumeration/role.py b/v1/enumeration/role.py new file mode 100644 index 00000000..a9b8efdb --- /dev/null +++ b/v1/enumeration/role.py @@ -0,0 +1,8 @@ +from enum import Enum + + +class Role(str, Enum): + SYSTEM = "system" + USER = "user" + ASSISTANT = "assistant" + TOOL = "tool" diff --git a/v1/faiss_vector_store.py b/v1/faiss_vector_store.py new file mode 100644 index 00000000..8f5fe893 --- /dev/null +++ b/v1/faiss_vector_store.py @@ -0,0 +1,70 @@ +import faiss +import numpy as np +from typing import List +from pydantic import Field, model_validator, PrivateAttr +from experiencemaker.model.base_embedding_model import BaseEmbeddingModel +from experiencemaker.schema.vector_store_node import VectorStoreNode +from experiencemaker.storage.base_vector_store import BaseVectorStore, VECTOR_STORE_REGISTRY + +@VECTOR_STORE_REGISTRY.register("faiss") +class FaissVectorStore(BaseVectorStore): + dim: int = Field(default=768) + _index: faiss.IndexFlatIP = PrivateAttr() + _id_map: dict = PrivateAttr(default_factory=dict) + _vectors: list = PrivateAttr(default_factory=list) + + @model_validator(mode="after") + def init_faiss(self): + self._index = faiss.IndexFlatIP(self.dim) + self._id_map = {} + self._vectors = [] + return self + + def exist_index(self, index_name: str | None = None) -> bool: + return hasattr(self, '_index') + + def create_index(self, index_name: str | None = None): + self._index = faiss.IndexFlatIP(self.dim) + self._id_map = {} + self._vectors = [] + + def delete_index(self, index_name: str | None = None): + self._index = faiss.IndexFlatIP(self.dim) + self._id_map = {} + self._vectors = [] + + def exist_id(self, unique_id: str, index_name: str | None = None): + return unique_id in self._id_map + + def insert(self, nodes: VectorStoreNode | List[VectorStoreNode], index_name: str | None = None, **kwargs): + if isinstance(nodes, VectorStoreNode): + nodes = [nodes] + nodes = self.embedding_model.get_node_embeddings(nodes) + for node in nodes: + vec = np.array(node.vector, dtype=np.float32) + self._index.add(vec.reshape(1, -1)) + self._id_map[len(self._vectors)] = node.unique_id + self._vectors.append(node) + + def update(self, nodes: VectorStoreNode | List[VectorStoreNode], index_name: str | None = None, **kwargs): + self.delete_index() + all_nodes = self._vectors + (nodes if isinstance(nodes, list) else [nodes]) + self.insert(all_nodes) + + def delete_by_id(self, unique_id: str, index_name: str | None = None, **kwargs): + idx_to_remove = [i for i, node in enumerate(self._vectors) if node.unique_id == unique_id] + if idx_to_remove: + self._vectors = [node for node in self._vectors if node.unique_id != unique_id] + self.create_index() + self.insert(self._vectors) + + def retrieve_by_id(self, unique_id: str, index_name: str | None = None, **kwargs) -> VectorStoreNode | None: + for node in self._vectors: + if node.unique_id == unique_id: + return node + return None + + def retrieve_by_query(self, query: str, top_k: int = 1, index_name: str | None = None, **kwargs) -> List[VectorStoreNode]: + query_vec = np.array(self.embedding_model.get_embeddings(query), dtype=np.float32).reshape(1, -1) + D, I = self._index.search(query_vec, top_k) + return [self._vectors[i] for i in I[0] if i >= 0] \ No newline at end of file diff --git a/v1/main.py b/v1/main.py new file mode 100644 index 00000000..161a752d --- /dev/null +++ b/v1/main.py @@ -0,0 +1,29 @@ +# main.py +# config.py +import json +from dataclasses import dataclass, field, asdict +from typing import List + +from omegaconf import OmegaConf + + +main() + +if __name__ == "__main__": + default_cfg = OmegaConf.structured(AppConfig) + + # 2. 从 YAML 文件加载 + yaml_cfg = OmegaConf.load("config.yaml") + + # 3. 合并（YAML 覆盖默认） + cfg = OmegaConf.merge(default_cfg, yaml_cfg) + + # 4. 再合并命令行（命令行优先级最高） + cli_cfg = OmegaConf.from_cli() + cfg = OmegaConf.merge(cfg, cli_cfg) + + # 5. 转为 dataclass 实例 + cfg_obj: AppConfig = OmegaConf.to_object(cfg) # 递归转为 dataclass + + print(json.dumps(asdict(cfg_obj), indent=2)) + # print(cfg_obj) # 如果你需要 dataclass 实例 diff --git a/v1/op/__init__.py b/v1/op/__init__.py new file mode 100644 index 00000000..a09751c6 --- /dev/null +++ b/v1/op/__init__.py @@ -0,0 +1,3 @@ +from v1.utils.registry import Registry + +OPERATION_REGISTRY = Registry() diff --git a/v1/op/base_op.py b/v1/op/base_op.py new file mode 100644 index 00000000..d54854d7 --- /dev/null +++ b/v1/op/base_op.py @@ -0,0 +1,25 @@ +from abc import abstractmethod, ABC + +from loguru import logger + +from v1.pipeline.pipeline_context import PipelineContext +from v1.utils.timer import Timer + + +class BaseOp(ABC): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.timer = Timer(name=self.__class__.__name__) + + @abstractmethod + def execute(self, context: PipelineContext): + ... + + def execute_wrap(self, context: PipelineContext): + try: + with self.timer: + return self.execute(context) + + except Exception as e: + logger.exception(f"OP.{self.__class__.__name__} execute failed, error={e.args}") diff --git a/v1/op/mock_op.py b/v1/op/mock_op.py new file mode 100644 index 00000000..ea2f4a83 --- /dev/null +++ b/v1/op/mock_op.py @@ -0,0 +1,45 @@ +import time + +from loguru import logger + +from v1.op import OPERATION_REGISTRY +from v1.op.base_op import BaseOp +from v1.pipeline.pipeline_context import PipelineContext + + +@OPERATION_REGISTRY.register("mock1") +class MockOp1(BaseOp): + + def __init__(self, a: int, b: str, **kwargs): + super().__init__(**kwargs) + self.a = a + self.b = b + + def execute(self, context: PipelineContext): + time.sleep(3) + logger.info(f"enter class={self.__class__.__name__}. a={self.a} b={self.b}") + + +@OPERATION_REGISTRY.register("mock2") +class MockOp2(MockOp1): + ... + + +@OPERATION_REGISTRY.register("mock3") +class MockOp3(MockOp1): + ... + + +@OPERATION_REGISTRY.register("mock4") +class MockOp4(MockOp1): + ... + + +@OPERATION_REGISTRY.register("mock5") +class MockOp5(MockOp1): + ... + + +@OPERATION_REGISTRY.register("mock6") +class MockOp6(MockOp1): + ... diff --git a/v1/pipeline/__init__.py b/v1/pipeline/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/pipeline/pipeline.py b/v1/pipeline/pipeline.py new file mode 100644 index 00000000..f984055b --- /dev/null +++ b/v1/pipeline/pipeline.py @@ -0,0 +1,102 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed +from itertools import zip_longest +from typing import Dict, List + +from loguru import logger + +from v1.op.base_op import BaseOp +from v1.pipeline.pipeline_context import PipelineContext +from v1.utils.timer import Timer, timer + + +class Pipeline(object): + seq_symbol: str = "->" + parallel_symbol: str = "|" + + def __init__(self, + name: str, + pipeline: str, + op_config_dict: Dict[str, dict], + op_registry: Dict[str, type[BaseOp]], + context: PipelineContext, + thread_pool: ThreadPoolExecutor): + + self.name: str = name + self.pipeline_list: List[str | List[str]] = self._parse_pipline(pipeline) + self.op_config_dict: Dict[str, dict] = op_config_dict + self.op_registry: Dict[str, type[BaseOp]] = op_registry + self.context: PipelineContext = context + self.thread_pool: ThreadPoolExecutor = thread_pool + + def _parse_pipline(self, pipeline: str) -> List[str | List[str]]: + pipeline_list: List[str | List[str]] = [] + + for pipeline_split1 in pipeline.split("["): + for sub_pipeline in pipeline_split1.split("]"): + sub_pipeline = sub_pipeline.strip().strip(self.seq_symbol) + if not sub_pipeline: + continue + + if self.parallel_symbol in sub_pipeline: + self.pipeline_list.append(sub_pipeline.split(self.parallel_symbol)) + else: + self.pipeline_list.append(sub_pipeline) + + return pipeline_list + + def execute_sub_pipeline(self, pipeline: str, context: PipelineContext): + for op in pipeline.split(self.seq_symbol): + op = op.strip() + if not op: + continue + + assert op in self.op_config_dict, f"op({op}).config is missing!" + backend = self.op_config_dict.pop("backend", "") + assert backend in self.op_registry, f"op({op}).backend({backend}) is not registered!" + + op_cls = self.op_registry[backend] + op_obj: BaseOp = op_cls(**self.op_config_dict[op]) + op_obj.execute_wrap(context) + + def parse_sub_pipeline(self, pipeline: str): + for op in pipeline.split(self.seq_symbol): + op = op.strip() + if not op: + continue + + yield op + + @timer() + def print_pipeline(self): + i: int = 0 + for pipeline in self.pipeline_list: + if isinstance(pipeline, str): + for op in self.parse_sub_pipeline(pipeline): + i += 1 + logger.info(f"stage_{i}: {op}") + + elif isinstance(pipeline, list): + for op_list in zip_longest(*[self.parse_sub_pipeline(x) for x in pipeline], fillvalue="-"): + i += 1 + logger.info(f"stage{i}: {' | '.join(op_list)}") + + else: + raise ValueError(f"unknown pipeline.type={type(pipeline)}") + + @timer() + def execute_pipeline(self): + for i, pipeline in enumerate(self.pipeline_list): + with Timer(f"step_{i}"): + if isinstance(pipeline, str): + self.execute_sub_pipeline(pipeline, self.context) + + else: + future_list = [] + for sub_pipeline in pipeline: + future = self.thread_pool.submit(self.execute_sub_pipeline, + pipeline=sub_pipeline, + context=self.context) + future_list.append(future) + + for future in as_completed(future_list): + future.result() diff --git a/v1/pipeline/pipeline_context.py b/v1/pipeline/pipeline_context.py new file mode 100644 index 00000000..c13937b2 --- /dev/null +++ b/v1/pipeline/pipeline_context.py @@ -0,0 +1,7 @@ +from concurrent.futures import ThreadPoolExecutor + + +class PipelineContext(object): + + def __init__(self): + self.thread_pool: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=10) \ No newline at end of file diff --git a/v1/schema/__init__.py b/v1/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/schema/app_config.py b/v1/schema/app_config.py new file mode 100644 index 00000000..7d1a2bed --- /dev/null +++ b/v1/schema/app_config.py @@ -0,0 +1,63 @@ +from dataclasses import dataclass, field +from typing import Dict + + +@dataclass +class HttpServiceConfig: + host: str = field(default="0.0.0.0") + port: int = field(default=8001) + timeout_keep_alive: int = field(default=600) + limit_concurrency: int = field(default=64) + + +@dataclass +class ThreadPoolConfig: + max_workers: int = field(default=10) + + +@dataclass +class APIConfig: + retriever_pipeline: str = field(default="") + summarizer_pipeline: str = field(default="") + vector_store_pipeline: str = field(default="") + agent_pipeline: str = field(default="") + + +@dataclass +class OpConfig: + backend: str = field(default="") + llm: str = field(default="") + embedding_model: str = field(default="") + vector_store: str = field(default="") + params: dict = field(default_factory=dict) + + +@dataclass +class LLMConfig: + backend: str = field(default="") + model: str = field(default="") + params: dict = field(default_factory=dict) + + +@dataclass +class EmbeddingModelConfig: + backend: str = field(default="") + model: str = field(default="") + params: dict = field(default_factory=dict) + + +@dataclass +class VectorStoreConfig: + backend: str = field(default="") + embedding_model: str = field(default="") + params: dict = field(default_factory=dict) + + +class AppConfig: + http_service: HttpServiceConfig = field(default_factory=HttpServiceConfig) + thread_pool: ThreadPoolConfig = field(default_factory=ThreadPoolConfig) + api: APIConfig = field(default_factory=APIConfig) + op: Dict[str, OpConfig] = field(default_factory=dict) + llm: Dict[str, LLMConfig] = field(default_factory=dict) + embedding_model: Dict[str, EmbeddingModelConfig] = field(default_factory=dict) + vector_store: Dict[str, VectorStoreConfig] = field(default_factory=dict) diff --git a/v1/schema/experience.py b/v1/schema/experience.py new file mode 100644 index 00000000..0fc1ae64 --- /dev/null +++ b/v1/schema/experience.py @@ -0,0 +1,76 @@ +import datetime +from abc import ABC +from typing import List +from uuid import uuid4 + +from pydantic import BaseModel, Field + +from experiencemaker.schema.vector_store_node import VectorStoreNode + + +class ExperienceMeta(BaseModel): + author: str = Field(default="") + created_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + modified_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + extra_info: dict | None = Field(default=None) + + +class BaseExperienceNode(BaseModel, ABC): + experience_id: str = Field(default_factory=lambda: uuid4().hex) + experience_type: str = Field(default="text") + workspace_id: str = Field(default="") + experience_meta: ExperienceMeta | None = Field(default=None) + + def to_vector_store_node(self) -> VectorStoreNode: + ... + + @classmethod + def from_vector_store_node(cls, node: VectorStoreNode) -> "TextExperienceNode": + ... + + +class TextExperienceNode(BaseExperienceNode): + experience_type: str = Field(default="text") + when_to_use: str = Field(default="") + content: str | bytes = Field(default="") + score: float | None = Field(default=None) + + +class FunctionArg(BaseModel): + arg_name: str = Field(default=...) + arg_type: str = Field(default=...) + required: bool = Field(default=True) + + +class Function(BaseModel): + func_code: str = Field(default=..., description="function code") + func_name: str = Field(default=..., description="function name") + func_args: List[FunctionArg] = Field(default_factory=list) + + +class FuncExperienceNode(BaseExperienceNode): + """ + TODO + """ + experience_type: str = Field(default="func") + functions: List[Function] = Field(default_factory=list) + + +class PersonalExperienceNode(BaseExperienceNode): + """ + TODO: memory node from MemoryScope + """ + experience_type: str = Field(default="personal") + person: str = Field(default="") + topic: str = Field(default="") + content: str | bytes = Field(default="") + + +class KnowledgeExperienceNode(BaseExperienceNode): + """ + TODO + """ + experience_type: str = Field(default="knowledge") + topic: str = Field(default="") + content: str | bytes = Field(default="") + score: float | None = Field(default=None) diff --git a/v1/schema/message.py b/v1/schema/message.py new file mode 100644 index 00000000..9f431cc6 --- /dev/null +++ b/v1/schema/message.py @@ -0,0 +1,66 @@ +import json +from typing import List + +from pydantic import BaseModel, Field + +from v1.enumeration.role import Role + + +class ToolCall(BaseModel): + index: int = Field(default=0) + id: str = Field(default="") + name: str = Field(default="") + arguments: str = Field(default="") + type: str = Field(default="function") + + @model_validator(mode="before") # noqa + @classmethod + def init_tool_call(cls, data: dict): + tool_type = data.get("type", "") + tool_type_dict = data.get(tool_type, {}) + + for key in ["name", "arguments"]: + if key not in data: + data[key] = tool_type_dict.get(key, "") + return data + + @property + def argument_dict(self) -> dict: + return json.loads(self.arguments) + + def simple_dump(self) -> dict: + return { + "id": self.id, + self.type: { + "arguments": self.arguments, + "name": self.name + }, + "type": self.type, + "index": self.index, + } + + +class Message(BaseModel): + role: Role = Field(default=Role.USER) + content: str | bytes = Field(default="") + reasoning_content: str = Field(default="") + tool_calls: List[ToolCall] = Field(default_factory=list) + metadata: dict = Field(default_factory=dict) + + def simple_dump(self, add_reason_when_empty: bool = True) -> dict: + if self.content: + result = {"role": self.role.value, "content": self.content} + elif add_reason_when_empty and self.reasoning_content: + result = {"role": self.role.value, "content": self.reasoning_content} + else: + result = {"role": self.role.value, "content": ""} + + if self.tool_calls: + result["tool_calls"] = [x.simple_dump() for x in self.tool_calls] + return result + + +class Trajectory(BaseModel): + messages: List[Message] = Field(default_factory=list) + score: float = Field(default=0.0) + metadata: dict = Field(default_factory=dict) diff --git a/v1/schema/request.py b/v1/schema/request.py new file mode 100644 index 00000000..b927628b --- /dev/null +++ b/v1/schema/request.py @@ -0,0 +1,30 @@ +from abc import ABC +from typing import List + +from pydantic import BaseModel, Field + +from v1.schema.message import Message, Trajectory + + +class BaseRequest(BaseModel, ABC): + metadata: dict = Field(default_factory=dict) + workspace_id: str = Field(default="") + + +class RetrieverRequest(BaseRequest): + query: str = Field(default="") + messages: List[Message] = Field(default_factory=list) + top_k: int = Field(default=3) + + +class SummarizerRequest(BaseRequest): + traj_list: List[Trajectory] = Field(default_factory=list) + + +class VectorStoreRequest(BaseModel): + action: str = Field(default="") + params: dict = Field(default_factory=dict) + + +class AgentRequest(BaseModel): + query: str = Field(default="") diff --git a/v1/schema/response.py b/v1/schema/response.py new file mode 100644 index 00000000..738930e7 --- /dev/null +++ b/v1/schema/response.py @@ -0,0 +1,30 @@ +from abc import ABC +from typing import List + +from pydantic import BaseModel, Field + +from experiencemaker.schema.experience import Experience +from experiencemaker.schema.trajectory import Trajectory, ContextMessage + + +class BaseResponse(BaseModel, ABC): + success: bool = Field(default=True) + metadata: dict = Field(default_factory=dict) + +# +# class AgentWrapperResponse(BaseResponse): +# trajectory: Trajectory = Field(default_factory=Trajectory) + + +class ContextGeneratorResponse(BaseResponse): + experience: list[dict] = Field(default_factory=list) + + merge_experience: str = Field(default="") + + # when to use, experience, response + + + + +class SummarizerResponse(BaseResponse): + experiences: List[dict] = Field(default_factory=list) diff --git a/v1/schema/vector_store_node.py b/v1/schema/vector_store_node.py new file mode 100644 index 00000000..49ce532d --- /dev/null +++ b/v1/schema/vector_store_node.py @@ -0,0 +1,11 @@ +from typing import List +from uuid import uuid4 +from pydantic import BaseModel, Field + + +class VectorStoreNode(BaseModel): + unique_id: str = Field(default_factory=lambda: uuid4().hex) + workspace_id: str = Field(default="") + content: str = Field(default="") + vector: List[float] | None = Field(default=None) + metadata: dict = Field(default_factory=dict) diff --git a/v1/service/__init__.py b/v1/service/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/service/service.py b/v1/service/service.py new file mode 100644 index 00000000..d6a78903 --- /dev/null +++ b/v1/service/service.py @@ -0,0 +1,2 @@ + +class Service \ No newline at end of file diff --git a/v1/singleton.py b/v1/singleton.py new file mode 100644 index 00000000..df7a1b33 --- /dev/null +++ b/v1/singleton.py @@ -0,0 +1,9 @@ +def singleton(cls): + _instance = {} + + def _singleton(*args, **kwargs): + if cls not in _instance: + _instance[cls] = cls(*args, **kwargs) + return _instance[cls] + + return _singleton diff --git a/v1/sqlite_vector_store.py b/v1/sqlite_vector_store.py new file mode 100644 index 00000000..f447b1f7 --- /dev/null +++ b/v1/sqlite_vector_store.py @@ -0,0 +1,113 @@ +import sqlite3 +import json +from typing import List +from pydantic import Field, model_validator, PrivateAttr +from experiencemaker.model.base_embedding_model import BaseEmbeddingModel +from experiencemaker.schema.vector_store_node import VectorStoreNode +from experiencemaker.storage.base_vector_store import BaseVectorStore, VECTOR_STORE_REGISTRY + +@VECTOR_STORE_REGISTRY.register("sqlite") +class SQLiteVectorStore(BaseVectorStore): + db_path: str = Field(default="./vector_store.db") + _conn: sqlite3.Connection = PrivateAttr() + + @model_validator(mode="after") + def init_db(self): + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) + self._conn.execute(f""" + CREATE TABLE IF NOT EXISTS {self.index_name} ( + unique_id TEXT PRIMARY KEY, + workspace_id TEXT, + content TEXT, + metadata TEXT, + vector TEXT + ) + """) + self._conn.commit() + return self + + def exist_index(self, index_name: str | None = None) -> bool: + index = index_name or self.index_name + cursor = self._conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?", (index,)) + return cursor.fetchone() is not None + + def create_index(self, index_name: str | None = None): + index = index_name or self.index_name + self._conn.execute(f""" + CREATE TABLE IF NOT EXISTS {index} ( + unique_id TEXT PRIMARY KEY, + workspace_id TEXT, + content TEXT, + metadata TEXT, + vector TEXT + ) + """) + self._conn.commit() + + def delete_index(self, index_name: str | None = None): + index = index_name or self.index_name + self._conn.execute(f"DROP TABLE IF EXISTS {index}") + self._conn.commit() + + def exist_id(self, unique_id: str, index_name: str | None = None): + index = index_name or self.index_name + cursor = self._conn.execute(f"SELECT 1 FROM {index} WHERE unique_id=?", (unique_id,)) + return cursor.fetchone() is not None + + def insert(self, nodes: VectorStoreNode | List[VectorStoreNode], index_name: str | None = None, **kwargs): + index = index_name or self.index_name + if isinstance(nodes, VectorStoreNode): + nodes = [nodes] + nodes = self.embedding_model.get_node_embeddings(nodes) + for node in nodes: + self._conn.execute(f"REPLACE INTO {index} (unique_id, workspace_id, content, metadata, vector) VALUES (?, ?, ?, ?, ?)", + (node.unique_id, node.workspace_id, node.content, json.dumps(node.metadata), json.dumps(node.vector))) + self._conn.commit() + + def update(self, nodes: VectorStoreNode | List[VectorStoreNode], index_name: str | None = None, **kwargs): + self.insert(nodes, index_name=index_name) + + def delete_by_id(self, unique_id: str, index_name: str | None = None, **kwargs): + index = index_name or self.index_name + self._conn.execute(f"DELETE FROM {index} WHERE unique_id=?", (unique_id,)) + self._conn.commit() + + def retrieve_by_id(self, unique_id: str, index_name: str | None = None, **kwargs) -> VectorStoreNode | None: + index = index_name or self.index_name + cursor = self._conn.execute(f"SELECT unique_id, workspace_id, content, metadata, vector FROM {index} WHERE unique_id=?", (unique_id,)) + row = cursor.fetchone() + if row: + return VectorStoreNode( + unique_id=row[0], + workspace_id=row[1], + content=row[2], + metadata=json.loads(row[3]), + vector=json.loads(row[4]) + ) + return None + + def retrieve_by_query(self, query: str, top_k: int = 1, index_name: str | None = None, **kwargs) -> List[VectorStoreNode]: + index = index_name or self.index_name + query_vec = self.embedding_model.get_embeddings(query) + cursor = self._conn.execute(f"SELECT unique_id, workspace_id, content, metadata, vector FROM {index}") + results = [] + for row in cursor: + node = VectorStoreNode( + unique_id=row[0], + workspace_id=row[1], + content=row[2], + metadata=json.loads(row[3]), + vector=json.loads(row[4]) + ) + node.metadata["score"] = self._cosine_similarity(query_vec, node.vector) + results.append(node) + results.sort(key=lambda x: x.metadata["score"], reverse=True) + return results[:top_k] + + @staticmethod + def _cosine_similarity(vec1, vec2): + import math + dot = sum(x * y for x, y in zip(vec1, vec2)) + norm1 = math.sqrt(sum(x * x for x in vec1)) + norm2 = math.sqrt(sum(y * y for y in vec2)) + return dot / (norm1 * norm2) if norm1 and norm2 else 0.0 \ No newline at end of file diff --git a/v1/tt.py b/v1/tt.py new file mode 100644 index 00000000..8485b126 --- /dev/null +++ b/v1/tt.py @@ -0,0 +1,15 @@ +class Pipe(): + + def __init__(self): + pass + + + +if __name__ == "__main__": + line = "a->[c->d->d|e->f]->[f->g]->h" + ops = [] + for sub_line in line.split("["): + for sub_line2 in sub_line.split("]"): + ops.append(sub_line2) + + print(ops) \ No newline at end of file diff --git a/v1/utils/__init__.py b/v1/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/v1/utils/registry.py b/v1/utils/registry.py new file mode 100644 index 00000000..9ee86e3c --- /dev/null +++ b/v1/utils/registry.py @@ -0,0 +1,31 @@ +from typing import List + +from loguru import logger + + +class Registry(object): + def __init__(self): + self._registry = {} + + def register(self, name: str = None): + + def decorator(cls): + class_name = name if name is not None else cls.__name__ + if class_name in self._registry: + logger.warning(f"name={class_name} is already registered, will be overwritten.") + self._registry[class_name] = cls + return cls + + return decorator + + def __getitem__(self, name: str): + if name not in self._registry: + raise KeyError(f"name={name} is not registered!") + return self._registry[name] + + def __contains__(self, name: str): + return name in self._registry + + @property + def registered_names(self) -> List[str]: + return sorted(self._registry.keys()) diff --git a/v1/utils/timer.py b/v1/utils/timer.py new file mode 100644 index 00000000..318b59cf --- /dev/null +++ b/v1/utils/timer.py @@ -0,0 +1,54 @@ +import time +from typing import Optional + +from loguru import logger + + +class Timer(object): + def __init__(self, name: str, use_ms: bool = False, stack_level: int = 2): + self.name: str = name + self.use_ms: bool = use_ms + self.stack_level: int = stack_level + + self.time_start: float = 0 + self.time_end: float = 0 + self.time_cost: float = 0 + + def __enter__(self, *args, **kwargs): + self.time_start = time.time() + logger.info(f"========== timer={self.name} start ==========", stacklevel=self.stack_level) + return self + + def __exit__(self, *args): + self.time_end = time.time() + self.time_cost = self.time_end - self.time_start + if self.use_ms: + time_str = f"{self.time_cost * 1000:.2f}ms" + else: + time_str = f"{self.time_cost:.3f}s" + + logger.info(f"========== timer={self.name} end, time_cost={time_str} ==========", stacklevel=self.stack_level) + + +def timer(name: Optional[str] = None, use_ms: bool = False, stack_level: int = 2): + def decorator(func): + def wrapper(*args, **kwargs): + with Timer(name=name or func.__name__, use_ms=use_ms, stack_level=stack_level + 1): + return func(*args, **kwargs) + + return wrapper + + return decorator + + +if __name__ == "__main__": + import random + + + @timer("run_func_final", use_ms=True) + def run_func(): + time.sleep(random.uniform(0.05, 0.15)) + print("done") + + + run_func()