diff --git a/experiencemaker/cookbook/__init__.py b/cookbook/__init__.py similarity index 100% rename from experiencemaker/cookbook/__init__.py rename to cookbook/__init__.py diff --git a/experiencemaker/cookbook/appworld/__init__.py b/cookbook/appworld/__init__.py similarity index 100% rename from experiencemaker/cookbook/appworld/__init__.py rename to cookbook/appworld/__init__.py diff --git a/experiencemaker/cookbook/appworld/appworld_react_agent.py b/cookbook/appworld/appworld_react_agent.py similarity index 100% rename from experiencemaker/cookbook/appworld/appworld_react_agent.py rename to cookbook/appworld/appworld_react_agent.py diff --git a/experiencemaker/cookbook/appworld/prompt.py b/cookbook/appworld/prompt.py similarity index 100% rename from experiencemaker/cookbook/appworld/prompt.py rename to cookbook/appworld/prompt.py diff --git a/experiencemaker/cookbook/appworld/quickstart.md b/cookbook/appworld/quickstart.md similarity index 100% rename from experiencemaker/cookbook/appworld/quickstart.md rename to cookbook/appworld/quickstart.md diff --git a/experiencemaker/cookbook/appworld/requirements.txt b/cookbook/appworld/requirements.txt similarity index 100% rename from experiencemaker/cookbook/appworld/requirements.txt rename to cookbook/appworld/requirements.txt diff --git a/experiencemaker/cookbook/appworld/run_appworld.py b/cookbook/appworld/run_appworld.py similarity index 100% rename from experiencemaker/cookbook/appworld/run_appworld.py rename to cookbook/appworld/run_appworld.py diff --git a/experiencemaker/cookbook/appworld/run_exp_statistic.py b/cookbook/appworld/run_exp_statistic.py similarity index 100% rename from experiencemaker/cookbook/appworld/run_exp_statistic.py rename to cookbook/appworld/run_exp_statistic.py diff --git a/experiencemaker/cookbook/bfcl/__init__.py b/cookbook/bfcl/__init__.py similarity index 100% rename from experiencemaker/cookbook/bfcl/__init__.py rename to cookbook/bfcl/__init__.py diff --git a/experiencemaker/cookbook/bfcl/bfcl_agent.py b/cookbook/bfcl/bfcl_agent.py similarity index 100% rename from experiencemaker/cookbook/bfcl/bfcl_agent.py rename to cookbook/bfcl/bfcl_agent.py diff --git a/experiencemaker/cookbook/bfcl/bfcl_utils.py b/cookbook/bfcl/bfcl_utils.py similarity index 100% rename from experiencemaker/cookbook/bfcl/bfcl_utils.py rename to cookbook/bfcl/bfcl_utils.py diff --git a/experiencemaker/cookbook/bfcl/evaluation.ipynb b/cookbook/bfcl/evaluation.ipynb similarity index 100% rename from experiencemaker/cookbook/bfcl/evaluation.ipynb rename to cookbook/bfcl/evaluation.ipynb diff --git a/experiencemaker/cookbook/bfcl/init_exp_pool.py b/cookbook/bfcl/init_exp_pool.py similarity index 100% rename from experiencemaker/cookbook/bfcl/init_exp_pool.py rename to cookbook/bfcl/init_exp_pool.py diff --git a/experiencemaker/cookbook/bfcl/local_file_to_library.py b/cookbook/bfcl/local_file_to_library.py similarity index 100% rename from experiencemaker/cookbook/bfcl/local_file_to_library.py rename to cookbook/bfcl/local_file_to_library.py diff --git a/experiencemaker/cookbook/bfcl/quickstart.md b/cookbook/bfcl/quickstart.md similarity index 100% rename from experiencemaker/cookbook/bfcl/quickstart.md rename to cookbook/bfcl/quickstart.md diff --git a/experiencemaker/cookbook/bfcl/run_bfcl.py b/cookbook/bfcl/run_bfcl.py similarity index 100% rename from experiencemaker/cookbook/bfcl/run_bfcl.py rename to cookbook/bfcl/run_bfcl.py diff --git a/experiencemaker/cookbook/bfcl/run_exp_statistic.py b/cookbook/bfcl/run_exp_statistic.py similarity index 100% rename from experiencemaker/cookbook/bfcl/run_exp_statistic.py rename to cookbook/bfcl/run_exp_statistic.py diff --git a/experiencemaker/cookbook/bfcl/split_into_trainval.py b/cookbook/bfcl/split_into_trainval.py similarity index 100% rename from experiencemaker/cookbook/bfcl/split_into_trainval.py rename to cookbook/bfcl/split_into_trainval.py diff --git a/experiencemaker/cookbook/frozenlake/__init__.py b/cookbook/frozenlake/__init__.py similarity index 100% rename from experiencemaker/cookbook/frozenlake/__init__.py rename to cookbook/frozenlake/__init__.py diff --git a/experiencemaker/cookbook/frozenlake/frozenlake_prompts.yaml b/cookbook/frozenlake/frozenlake_prompts.yaml similarity index 100% rename from experiencemaker/cookbook/frozenlake/frozenlake_prompts.yaml rename to cookbook/frozenlake/frozenlake_prompts.yaml diff --git a/experiencemaker/cookbook/frozenlake/frozenlake_react_agent.py b/cookbook/frozenlake/frozenlake_react_agent.py similarity index 100% rename from experiencemaker/cookbook/frozenlake/frozenlake_react_agent.py rename to cookbook/frozenlake/frozenlake_react_agent.py diff --git a/experiencemaker/cookbook/frozenlake/map_manager.py b/cookbook/frozenlake/map_manager.py similarity index 100% rename from experiencemaker/cookbook/frozenlake/map_manager.py rename to cookbook/frozenlake/map_manager.py diff --git a/experiencemaker/cookbook/frozenlake/quickstart.md b/cookbook/frozenlake/quickstart.md similarity index 100% rename from experiencemaker/cookbook/frozenlake/quickstart.md rename to cookbook/frozenlake/quickstart.md diff --git a/experiencemaker/cookbook/frozenlake/run_exp_statistic.py b/cookbook/frozenlake/run_exp_statistic.py similarity index 100% rename from experiencemaker/cookbook/frozenlake/run_exp_statistic.py rename to cookbook/frozenlake/run_exp_statistic.py diff --git a/experiencemaker/cookbook/frozenlake/run_frozenlake.py b/cookbook/frozenlake/run_frozenlake.py similarity index 100% rename from experiencemaker/cookbook/frozenlake/run_frozenlake.py rename to cookbook/frozenlake/run_frozenlake.py diff --git a/experiencemaker/cookbook/simple_demo/__init__.py b/cookbook/simple_demo/__init__.py similarity index 100% rename from experiencemaker/cookbook/simple_demo/__init__.py rename to cookbook/simple_demo/__init__.py diff --git a/experiencemaker/cookbook/simple_demo/experience.jsonl b/cookbook/simple_demo/experience.jsonl similarity index 100% rename from experiencemaker/cookbook/simple_demo/experience.jsonl rename to cookbook/simple_demo/experience.jsonl diff --git a/experiencemaker/cookbook/simple_demo/messages.jsonl b/cookbook/simple_demo/messages.jsonl similarity index 100% rename from experiencemaker/cookbook/simple_demo/messages.jsonl rename to cookbook/simple_demo/messages.jsonl diff --git a/experiencemaker/cookbook/simple_demo/quick_start.md b/cookbook/simple_demo/quick_start.md similarity index 100% rename from experiencemaker/cookbook/simple_demo/quick_start.md rename to cookbook/simple_demo/quick_start.md diff --git a/experiencemaker/cookbook/simple_demo/simple_demo.py b/cookbook/simple_demo/simple_demo.py similarity index 100% rename from experiencemaker/cookbook/simple_demo/simple_demo.py rename to cookbook/simple_demo/simple_demo.py diff --git a/experiencemaker/README.md b/doc/README.md similarity index 100% rename from experiencemaker/README.md rename to doc/README.md diff --git a/experiencemaker/doc/configuration_guide.md b/doc/configuration_guide.md similarity index 100% rename from experiencemaker/doc/configuration_guide.md rename to doc/configuration_guide.md diff --git a/experiencemaker/doc/figure/framework.png b/doc/figure/framework.png similarity index 100% rename from experiencemaker/doc/figure/framework.png rename to doc/figure/framework.png diff --git a/experiencemaker/doc/figure/frozenlake_failure.gif b/doc/figure/frozenlake_failure.gif similarity index 100% rename from experiencemaker/doc/figure/frozenlake_failure.gif rename to doc/figure/frozenlake_failure.gif diff --git a/experiencemaker/doc/figure/frozenlake_success.gif b/doc/figure/frozenlake_success.gif similarity index 100% rename from experiencemaker/doc/figure/frozenlake_success.gif rename to doc/figure/frozenlake_success.gif diff --git a/experiencemaker/doc/figure/logo.jpg b/doc/figure/logo.jpg similarity index 100% rename from experiencemaker/doc/figure/logo.jpg rename to doc/figure/logo.jpg diff --git a/experiencemaker/doc/figure/logo.png b/doc/figure/logo.png similarity index 100% rename from experiencemaker/doc/figure/logo.png rename to doc/figure/logo.png diff --git a/experiencemaker/doc/figure/logo_v2.png b/doc/figure/logo_v2.png similarity index 100% rename from experiencemaker/doc/figure/logo_v2.png rename to doc/figure/logo_v2.png diff --git a/experiencemaker/doc/figure/logo_v3.jpg b/doc/figure/logo_v3.jpg similarity index 100% rename from experiencemaker/doc/figure/logo_v3.jpg rename to doc/figure/logo_v3.jpg diff --git a/experiencemaker/doc/future_roadmap.md b/doc/future_roadmap.md similarity index 100% rename from experiencemaker/doc/future_roadmap.md rename to doc/future_roadmap.md diff --git a/experiencemaker/doc/mcp_quick_start.md b/doc/mcp_quick_start.md similarity index 100% rename from experiencemaker/doc/mcp_quick_start.md rename to doc/mcp_quick_start.md diff --git a/experiencemaker/doc/operations_documentation.md b/doc/operations_documentation.md similarity index 100% rename from experiencemaker/doc/operations_documentation.md rename to doc/operations_documentation.md diff --git a/experiencemaker/doc/vector_store_setup.md b/doc/vector_store_setup.md similarity index 100% rename from experiencemaker/doc/vector_store_setup.md rename to doc/vector_store_setup.md diff --git a/experiencemaker/example.env b/experiencemaker/example.env deleted file mode 100644 index ce62c9ce..00000000 --- a/experiencemaker/example.env +++ /dev/null @@ -1,11 +0,0 @@ -OPENAI_API_KEY=sk-xxxx -OPENAI_BASE_URL=https://xxxx/v1 - -EMBEDDING_API_KEY=sk-xxxx -EMBEDDING_BASE_URL=https://xxxx/v1 - -LLM_API_KEY=sk-xxxx -LLM_BASE_URL=https://xxxx/v1 - -ES_HOSTS=http://0.0.0.0:9200 -DASHSCOPE_API_KEY=sk-xxxx diff --git a/experiencemaker/experiencemaker/__init__.py b/experiencemaker/experiencemaker/__init__.py deleted file mode 100644 index 9ce0fe86..00000000 --- a/experiencemaker/experiencemaker/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# from .app import main - -__version__ = "0.1.1" - -__all__ = ["main"] - - -# python -m build -# twine upload dist/* \ No newline at end of file diff --git a/experiencemaker/experiencemaker/app.py b/experiencemaker/experiencemaker/app.py deleted file mode 100644 index f50f31e6..00000000 --- a/experiencemaker/experiencemaker/app.py +++ /dev/null @@ -1,52 +0,0 @@ -import sys - -import uvicorn -from dotenv import load_dotenv -from fastapi import FastAPI - -from experiencemaker.schema.request import RetrieverRequest, SummarizerRequest, VectorStoreRequest, AgentRequest -from experiencemaker.schema.response import RetrieverResponse, SummarizerResponse, VectorStoreResponse, AgentResponse -from experiencemaker.service.experience_maker_service import ExperienceMakerService - -load_dotenv() - -app = FastAPI() -service = ExperienceMakerService(sys.argv[1:]) - -@app.post('/retriever', response_model=RetrieverResponse) -def call_retriever(request: RetrieverRequest): - return service(api="retriever", request=request) - - -@app.post('/summarizer', response_model=SummarizerResponse) -def call_summarizer(request: SummarizerRequest): - return service(api="summarizer", request=request) - - -@app.post('/vector_store', response_model=VectorStoreResponse) -def call_vector_store(request: VectorStoreRequest): - return service(api="vector_store", request=request) - - -@app.post('/agent', response_model=AgentResponse) -def call_agent(request: AgentRequest): - return service(api="agent", request=request) - - -def main(): - uvicorn.run(app=app, - host=service.http_service_config.host, - port=service.http_service_config.port, - timeout_keep_alive=service.http_service_config.timeout_keep_alive, - limit_concurrency=service.http_service_config.limit_concurrency) - - -if __name__ == "__main__": - main() - -# start with: -# experiencemaker \ -# http_service.port=8001 \ -# llm.default.model_name=qwen3-32b \ -# embedding_model.default.model_name=text-embedding-v4 \ -# vector_store.default.backend=local_file \ No newline at end of file diff --git a/experiencemaker/experiencemaker/config/__init__.py b/experiencemaker/experiencemaker/config/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/config/config_parser.py b/experiencemaker/experiencemaker/config/config_parser.py deleted file mode 100644 index 432b81d3..00000000 --- a/experiencemaker/experiencemaker/config/config_parser.py +++ /dev/null @@ -1,80 +0,0 @@ -import json -from pathlib import Path - -from loguru import logger -from omegaconf import OmegaConf, DictConfig - -from experiencemaker.schema.app_config import AppConfig - - -class ConfigParser: - """ - Configuration parser that handles loading and merging configurations from multiple sources. - - The configuration loading priority (from lowest to highest): - 1. Default configuration from AppConfig schema - 2. YAML configuration file - 3. Command line arguments - 4. Runtime keyword arguments - """ - - def __init__(self, args: list): - """ - Initialize the configuration parser with command line arguments. - - Args: - args: List of command line arguments in dotlist format (e.g., ['key=value']) - """ - # Step 1: Initialize with default configuration from AppConfig schema - self.app_config: DictConfig = OmegaConf.structured(AppConfig) - - # Step 2: Load configuration from YAML file - # First, parse CLI arguments to check if custom config path is specified - cli_config: DictConfig = OmegaConf.from_dotlist(args) - temp_config: AppConfig = OmegaConf.to_object(OmegaConf.merge(self.app_config, cli_config)) - - # Determine config file path: either from CLI args or use predefined config - if temp_config.config_path: - # Use custom config path if provided - config_path = Path(temp_config.config_path) - else: - # Use predefined config name from the config directory - pre_defined_config = temp_config.pre_defined_config - if not pre_defined_config.endswith(".yaml"): - pre_defined_config += ".yaml" - config_path = Path(__file__).parent / pre_defined_config - - logger.info(f"load config from path={config_path}") - yaml_config = OmegaConf.load(config_path) - # Merge YAML config with default config - self.app_config = OmegaConf.merge(self.app_config, yaml_config) - - # Step 3: Merge CLI arguments (highest priority) - self.app_config = OmegaConf.merge(self.app_config, cli_config) - - # Log the final merged configuration - app_config_dict = OmegaConf.to_container(self.app_config, resolve=True) - logger.info(f"app_config=\n{json.dumps(app_config_dict, indent=2, ensure_ascii=False)}") - - def get_app_config(self, **kwargs) -> AppConfig: - """ - Get the application configuration with optional runtime overrides. - - Args: - **kwargs: Additional configuration parameters to override at runtime - - Returns: - AppConfig: The final application configuration object - """ - # Create a copy of the current configuration - app_config = self.app_config.copy() - - # Apply runtime overrides if provided - if kwargs: - # Convert kwargs to dotlist format for OmegaConf - kwargs_list = [f"{k}={v}" for k, v in kwargs.items()] - update_config = OmegaConf.from_dotlist(kwargs_list) - app_config = OmegaConf.merge(app_config, update_config) - - # Convert OmegaConf DictConfig to structured AppConfig object - return OmegaConf.to_object(app_config) diff --git a/experiencemaker/experiencemaker/config/mock_config.yaml b/experiencemaker/experiencemaker/config/mock_config.yaml deleted file mode 100644 index 9fcc4555..00000000 --- a/experiencemaker/experiencemaker/config/mock_config.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# demo config.yaml - -http_service: - host: "0.0.0.0" - port: 8001 - timeout_keep_alive: 600 - limit_concurrency: 64 - -thread_pool: - max_workers: 10 - -api: - retriever: mock1_op->[mock4_op->mock2_op|mock5_op]->[mock3_op|mock6_op] - summarizer: mock1_op->[mock4_op->mock2_op|mock5_op]->mock3_op - vector_store: mock6_op - -op: - mock1_op: - backend: mock1_op - llm: default - vector_store: default - params: - a: 1 - b: 2 - mock2_op: - backend: mock2_op - params: - a: 1 - mock3_op: - backend: mock3_op - mock4_op: - backend: mock4_op - mock5_op: - backend: mock5_op - mock6_op: - backend: mock6_op - -llm: - default: - backend: openai_compatible - model_name: qwen3-32b - params: - temperature: 0.6 - -embedding_model: - default: - backend: openai_compatible - model_name: text-embedding-v4 - params: - dimensions: 1024 - -vector_store: - default: - backend: elasticsearch - embedding_model: default - params: - hosts: "http://localhost:9200" - diff --git a/experiencemaker/experiencemaker/embedding_model/__init__.py b/experiencemaker/experiencemaker/embedding_model/__init__.py deleted file mode 100644 index 2adb1018..00000000 --- a/experiencemaker/experiencemaker/embedding_model/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from experiencemaker.utils.registry import Registry - -EMBEDDING_MODEL_REGISTRY = Registry() - -from experiencemaker.embedding_model.openai_compatible_embedding_model import OpenAICompatibleEmbeddingModel diff --git a/experiencemaker/experiencemaker/embedding_model/base_embedding_model.py b/experiencemaker/experiencemaker/embedding_model/base_embedding_model.py deleted file mode 100644 index b9a59a00..00000000 --- a/experiencemaker/experiencemaker/embedding_model/base_embedding_model.py +++ /dev/null @@ -1,103 +0,0 @@ -from abc import ABC -from typing import List - -from loguru import logger -from pydantic import BaseModel, Field - -from experiencemaker.schema.vector_node import VectorNode - - -class BaseEmbeddingModel(BaseModel, ABC): - """ - Abstract base class for embedding models. - - This class provides a common interface for various embedding model implementations, - including retry logic, error handling, and batch processing capabilities. - """ - # Model configuration fields - model_name: str = Field(default=..., description="Name of the embedding model") - dimensions: int = Field(default=..., description="Dimensionality of the embedding vectors") - max_retries: int = Field(default=3, description="Maximum number of retry attempts on failure") - raise_exception: bool = Field(default=True, description="Whether to raise exceptions after max retries") - max_batch_size: int = Field(default=10, description="Maximum batch size for processing (text-embedding-v4 should not exceed 10)") - - def _get_embeddings(self, input_text: str | List[str]): - """ - Abstract method to get embeddings from the model. - - This method must be implemented by concrete subclasses to provide - the actual embedding functionality. - - Args: - input_text: Single text string or list of text strings to embed - - Returns: - Embedding vector(s) corresponding to the input text(s) - """ - raise NotImplementedError - - def get_embeddings(self, input_text: str | List[str]): - """ - Get embeddings with retry logic and error handling. - - This method wraps the _get_embeddings method with automatic retry - functionality in case of failures. - - Args: - input_text: Single text string or list of text strings to embed - - Returns: - Embedding vector(s) or None if all retries failed and raise_exception is False - """ - # Retry loop with exponential backoff potential - for i in range(self.max_retries): - try: - return self._get_embeddings(input_text) - - except Exception as e: - logger.exception(f"embedding model name={self.model_name} encounter error with e={e.args}") - # If this is the last retry and raise_exception is True, re-raise the exception - if i == self.max_retries - 1 and self.raise_exception: - raise e - - # Return None if all retries failed and raise_exception is False - return None - - def get_node_embeddings(self, nodes: VectorNode | List[VectorNode]): - """ - Generate embeddings for VectorNode objects and update their vector fields. - - This method handles both single nodes and lists of nodes, with automatic - batching for efficient processing of large node lists. - - Args: - nodes: Single VectorNode or list of VectorNode objects to embed - - Returns: - The same node(s) with updated vector fields containing embeddings - - Raises: - RuntimeError: If unsupported node type is provided - """ - # Handle single VectorNode - if isinstance(nodes, VectorNode): - nodes.vector = self.get_embeddings(nodes.content) - return nodes - - # Handle list of VectorNodes with batch processing - elif isinstance(nodes, list): - # Process nodes in batches to respect max_batch_size limits - embeddings = [emb for i in range(0, len(nodes), self.max_batch_size) for emb in - self.get_embeddings(input_text=[node.content for node in nodes[i:i + self.max_batch_size]])] - - # Validate that we got the expected number of embeddings - if len(embeddings) != len(nodes): - logger.warning(f"embeddings.size={len(embeddings)} <> nodes.size={len(nodes)}") - else: - # Assign embeddings to corresponding nodes - for node, embedding in zip(nodes, embeddings): - node.vector = embedding - return nodes - - else: - raise RuntimeError(f"unsupported type={type(nodes)}") diff --git a/experiencemaker/experiencemaker/embedding_model/openai_compatible_embedding_model.py b/experiencemaker/experiencemaker/embedding_model/openai_compatible_embedding_model.py deleted file mode 100644 index bf420608..00000000 --- a/experiencemaker/experiencemaker/embedding_model/openai_compatible_embedding_model.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -from typing import Literal, List - -from dotenv import load_dotenv -from openai import OpenAI -from pydantic import Field, PrivateAttr, model_validator - -from experiencemaker.embedding_model import EMBEDDING_MODEL_REGISTRY -from experiencemaker.embedding_model.base_embedding_model import BaseEmbeddingModel - - -@EMBEDDING_MODEL_REGISTRY.register("openai_compatible") -class OpenAICompatibleEmbeddingModel(BaseEmbeddingModel): - """ - OpenAI-compatible embedding model implementation. - - This class provides an implementation of BaseEmbeddingModel that works with - OpenAI-compatible embedding APIs, including OpenAI's official API and - other services that follow the same interface. - """ - # API configuration fields - api_key: str = Field(default_factory=lambda: os.getenv("EMBEDDING_API_KEY"), description="API key for authentication") - base_url: str = Field(default_factory=lambda: os.getenv("EMBEDDING_BASE_URL"), description="Base URL for the API endpoint") - model_name: str = Field(default="", description="Name of the embedding model to use") - dimensions: int = Field(default=1024, description="Dimensionality of the embedding vectors") - encoding_format: Literal["float", "base64"] = Field(default="float", description="Encoding format for embeddings") - - # Private OpenAI client instance - _client: OpenAI = PrivateAttr() - - @model_validator(mode="after") - def init_client(self): - """ - Initialize the OpenAI client after model validation. - - This method is called automatically after Pydantic model validation - to set up the OpenAI client with the provided API key and base URL. - - Returns: - self: The model instance for method chaining - """ - self._client = OpenAI(api_key=self.api_key, base_url=self.base_url) - return self - - def _get_embeddings(self, input_text: str | List[str]): - """ - Get embeddings from the OpenAI-compatible API. - - This method implements the abstract _get_embeddings method from BaseEmbeddingModel - by calling the OpenAI-compatible embeddings API. - - Args: - input_text: Single text string or list of text strings to embed - - Returns: - Embedding vector(s) corresponding to the input text(s) - - Raises: - RuntimeError: If unsupported input type is provided - """ - completion = self._client.embeddings.create( - model=self.model_name, - input=input_text, - dimensions=self.dimensions, - encoding_format=self.encoding_format - ) - - if isinstance(input_text, str): - return completion.data[0].embedding - - elif isinstance(input_text, list): - result_emb = [[] for _ in range(len(input_text))] - for emb in completion.data: - result_emb[emb.index] = emb.embedding - return result_emb - - else: - raise RuntimeError(f"unsupported type={type(input_text)}") - - -def main(): - load_dotenv() - model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - res1 = model.get_embeddings( - "The clothes are of good quality and look good, definitely worth the wait. I love them.") - res2 = model.get_embeddings(["aa", "bb"]) - print(res1) - print(res2) - - -if __name__ == "__main__": - main() - # launch with: python -m experiencemaker.model.openai_compatible_embedding_model diff --git a/experiencemaker/experiencemaker/enumeration/__init__.py b/experiencemaker/experiencemaker/enumeration/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/enumeration/agent_state.py b/experiencemaker/experiencemaker/enumeration/agent_state.py deleted file mode 100644 index ae7269a0..00000000 --- a/experiencemaker/experiencemaker/enumeration/agent_state.py +++ /dev/null @@ -1,8 +0,0 @@ -from enum import Enum - - -class AgentState(str, Enum): - IDLE = "idle" - RUNNING = "running" - COMPLETE = "complete" - FAILED = "failed" diff --git a/experiencemaker/experiencemaker/enumeration/chunk_enum.py b/experiencemaker/experiencemaker/enumeration/chunk_enum.py deleted file mode 100644 index 975f0b1f..00000000 --- a/experiencemaker/experiencemaker/enumeration/chunk_enum.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import Enum - - -class ChunkEnum(str, Enum): - THINK = "think" - ANSWER = "answer" - TOOL = "tool" - USAGE = "usage" - ERROR = "error" diff --git a/experiencemaker/experiencemaker/enumeration/http_enum.py b/experiencemaker/experiencemaker/enumeration/http_enum.py deleted file mode 100644 index f5ebe79f..00000000 --- a/experiencemaker/experiencemaker/enumeration/http_enum.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import Enum - - -class HttpEnum(str, Enum): - GET = "get" - POST = "post" - HEAD = "head" - PUT = "put" - DELETE = "delete" diff --git a/experiencemaker/experiencemaker/enumeration/role.py b/experiencemaker/experiencemaker/enumeration/role.py deleted file mode 100644 index a9b8efdb..00000000 --- a/experiencemaker/experiencemaker/enumeration/role.py +++ /dev/null @@ -1,8 +0,0 @@ -from enum import Enum - - -class Role(str, Enum): - SYSTEM = "system" - USER = "user" - ASSISTANT = "assistant" - TOOL = "tool" diff --git a/experiencemaker/experiencemaker/llm/__init__.py b/experiencemaker/experiencemaker/llm/__init__.py deleted file mode 100644 index 6706fd7a..00000000 --- a/experiencemaker/experiencemaker/llm/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from experiencemaker.utils.registry import Registry - -LLM_REGISTRY = Registry() - -from experiencemaker.llm.openai_compatible_llm import OpenAICompatibleBaseLLM diff --git a/experiencemaker/experiencemaker/llm/base_llm.py b/experiencemaker/experiencemaker/llm/base_llm.py deleted file mode 100644 index f34031a4..00000000 --- a/experiencemaker/experiencemaker/llm/base_llm.py +++ /dev/null @@ -1,138 +0,0 @@ -import time -from abc import ABC -from typing import List, Literal, Callable - -from loguru import logger -from pydantic import Field, BaseModel - -from experiencemaker.schema.message import Message -from experiencemaker.tool.base_tool import BaseTool - - -class BaseLLM(BaseModel, ABC): - """ - Abstract base class for Large Language Model (LLM) implementations. - - This class defines the common interface and configuration parameters - that all LLM implementations should support. It provides a standardized - way to interact with different LLM providers while handling common - concerns like retries, error handling, and streaming. - """ - # Core model configuration - model_name: str = Field(..., description="Name of the LLM model to use") - - # Generation parameters - seed: int = Field(default=42, description="Random seed for reproducible outputs") - top_p: float | None = Field(default=None, description="Top-p (nucleus) sampling parameter") - # stream: bool = Field(default=True) # Commented out - streaming is handled per request - stream_options: dict = Field(default={"include_usage": True}, description="Options for streaming responses") - temperature: float = Field(default=0.0000001, description="Sampling temperature (low for deterministic outputs)") - presence_penalty: float | None = Field(default=None, description="Presence penalty to reduce repetition") - - # Model-specific features - enable_thinking: bool = Field(default=True, description="Enable reasoning/thinking mode for supported models") - - # Tool usage configuration - tool_choice: Literal["none", "auto", "required"] = Field(default="auto", description="Strategy for tool selection") - parallel_tool_calls: bool = Field(default=True, description="Allow multiple tool calls in parallel") - - # Error handling and reliability - max_retries: int = Field(default=5, description="Maximum number of retry attempts on failure") - raise_exception: bool = Field(default=False, description="Whether to raise exceptions or return default values") - - def stream_chat(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs): - """ - Stream chat completions from the LLM. - - This method should yield chunks of the response as they become available, - allowing for real-time display of the model's output. - - Args: - messages: List of conversation messages - tools: Optional list of tools the model can use - **kwargs: Additional model-specific parameters - - Yields: - Chunks of the streaming response with their types - """ - raise NotImplementedError - - def stream_print(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs): - """ - Stream chat completions and print them to console in real-time. - - This is a convenience method for debugging and interactive use, - combining streaming with formatted console output. - - Args: - messages: List of conversation messages - tools: Optional list of tools the model can use - **kwargs: Additional model-specific parameters - """ - raise NotImplementedError - - def _chat(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs) -> Message: - """ - Internal method to perform a single chat completion. - - This method should be implemented by subclasses to handle the actual - communication with the LLM provider. It's called by the public chat() - method which adds retry logic and error handling. - - Args: - messages: List of conversation messages - tools: Optional list of tools the model can use - **kwargs: Additional model-specific parameters - - Returns: - The complete response message from the LLM - """ - raise NotImplementedError - - def chat(self, messages: List[Message], tools: List[BaseTool] = None, callback_fn: Callable = None, - default_value=None, **kwargs): - """ - Perform a chat completion with retry logic and error handling. - - This is the main public interface for chat completions. It wraps the - internal _chat() method with robust error handling, exponential backoff, - and optional callback processing. - - Args: - messages: List of conversation messages - tools: Optional list of tools the model can use - callback_fn: Optional callback to process the response message - default_value: Value to return if all retries fail (when raise_exception=False) - **kwargs: Additional model-specific parameters - - Returns: - The response message (possibly processed by callback_fn) or default_value - - Raises: - Exception: If raise_exception=True and all retries fail - """ - for i in range(self.max_retries): - try: - # Attempt to get response from the model - message: Message = self._chat(messages, tools, **kwargs) - - # Apply callback function if provided - if callback_fn: - return callback_fn(message) - else: - return message - - except Exception as e: - logger.exception(f"chat with model={self.model_name} encounter error with e={e.args}") - - # Exponential backoff: wait longer after each failure - time.sleep(1 + i) - - # Handle final retry failure - if i == self.max_retries - 1: - if self.raise_exception: - raise e - else: - return default_value - - return None diff --git a/experiencemaker/experiencemaker/llm/openai_compatible_llm.py b/experiencemaker/experiencemaker/llm/openai_compatible_llm.py deleted file mode 100644 index a004ca78..00000000 --- a/experiencemaker/experiencemaker/llm/openai_compatible_llm.py +++ /dev/null @@ -1,282 +0,0 @@ -import os -from typing import List - -from dotenv import load_dotenv -from loguru import logger -from openai import OpenAI -from openai.types import CompletionUsage -from pydantic import Field, PrivateAttr, model_validator - -from experiencemaker.enumeration.chunk_enum import ChunkEnum -from experiencemaker.enumeration.role import Role -from experiencemaker.llm import LLM_REGISTRY -from experiencemaker.llm.base_llm import BaseLLM -from experiencemaker.schema.message import Message, ToolCall -from experiencemaker.tool.base_tool import BaseTool - - -@LLM_REGISTRY.register("openai_compatible") -class OpenAICompatibleBaseLLM(BaseLLM): - """ - OpenAI-compatible LLM implementation supporting streaming and tool calls. - - This class implements the BaseLLM interface for OpenAI-compatible APIs, - including support for: - - Streaming responses with different chunk types (thinking, answer, tools) - - Tool calling with parallel execution - - Reasoning/thinking content from supported models - - Robust error handling and retries - """ - - # API configuration - api_key: str = Field(default_factory=lambda: os.getenv("LLM_API_KEY"), description="API key for authentication") - base_url: str = Field(default_factory=lambda: os.getenv("LLM_BASE_URL"), description="Base URL for the API endpoint") - _client: OpenAI = PrivateAttr() - - @model_validator(mode="after") - def init_client(self): - """ - Initialize the OpenAI client after model validation. - - This validator runs after all field validation is complete, - ensuring we have valid API credentials before creating the client. - - Returns: - Self for method chaining - """ - self._client = OpenAI(api_key=self.api_key, base_url=self.base_url) - return self - - def stream_chat(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs): - """ - Stream chat completions from OpenAI-compatible API. - - This method handles streaming responses and categorizes chunks into different types: - - THINK: Reasoning/thinking content from the model - - ANSWER: Regular response content - - TOOL: Tool calls that need to be executed - - USAGE: Token usage statistics - - ERROR: Error information - - Args: - messages: List of conversation messages - tools: Optional list of tools available to the model - **kwargs: Additional parameters - - Yields: - Tuple of (chunk_content, ChunkEnum) for each streaming piece - """ - for i in range(self.max_retries): - try: - # Create streaming completion request - completion = self._client.chat.completions.create( - model=self.model_name, - messages=[x.simple_dump() for x in messages], - seed=self.seed, - top_p=self.top_p, - stream=True, - stream_options=self.stream_options, - temperature=self.temperature, - extra_body={"enable_thinking": self.enable_thinking}, # Enable reasoning mode - tools=[x.simple_dump() for x in tools] if tools else None, - tool_choice=self.tool_choice, - parallel_tool_calls=self.parallel_tool_calls) - - # Initialize tool call tracking - ret_tools = [] # Accumulate tool calls across chunks - is_answering = False # Track when model starts answering - - # Process each chunk in the streaming response - for chunk in completion: - # Handle chunks without choices (usually usage info) - if not chunk.choices: - yield chunk.usage, ChunkEnum.USAGE - - else: - delta = chunk.choices[0].delta - - # Handle reasoning/thinking content (model's internal thoughts) - if hasattr(delta, 'reasoning_content') and delta.reasoning_content is not None: - yield delta.reasoning_content, ChunkEnum.THINK - - else: - # Mark transition from thinking to answering - if not is_answering: - is_answering = True - - # Handle regular response content - if delta.content is not None: - yield delta.content, ChunkEnum.ANSWER - - # Handle tool calls (function calling) - if delta.tool_calls is not None: - for tool_call in delta.tool_calls: - index = tool_call.index - - # Ensure we have enough tool call slots - while len(ret_tools) <= index: - ret_tools.append(ToolCall(index=index)) - - # Accumulate tool call information across chunks - if tool_call.id: - ret_tools[index].id += tool_call.id - - if tool_call.function and tool_call.function.name: - ret_tools[index].name += tool_call.function.name - - if tool_call.function and tool_call.function.arguments: - ret_tools[index].arguments += tool_call.function.arguments - - # Yield completed tool calls after streaming finishes - if ret_tools: - tool_dict = {x.name: x for x in tools} if tools else {} - for tool in ret_tools: - # Only yield tool calls that correspond to available tools - if tool.name not in tool_dict: - continue - - yield tool, ChunkEnum.TOOL - - return # Success - exit retry loop - - except Exception as e: - logger.exception(f"stream chat with model={self.model_name} encounter error with e={e.args}") - - # Handle retry logic - if i == self.max_retries - 1 and self.raise_exception: - raise e - else: - yield e.args, ChunkEnum.ERROR - - def _chat(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs) -> Message: - """ - Perform a complete chat completion by aggregating streaming chunks. - - This method consumes the entire streaming response and combines all - chunks into a single Message object. It separates reasoning content, - regular answer content, and tool calls. - - Args: - messages: List of conversation messages - tools: Optional list of tools available to the model - **kwargs: Additional parameters - - Returns: - Complete Message with all content aggregated - """ - # Initialize content accumulators - reasoning_content = "" # Model's internal reasoning - answer_content = "" # Final response content - tool_calls = [] # List of tool calls to execute - - # Consume streaming response and aggregate chunks by type - for chunk, chunk_enum in self.stream_chat(messages, tools, **kwargs): - if chunk_enum is ChunkEnum.THINK: - reasoning_content += chunk - - elif chunk_enum is ChunkEnum.ANSWER: - answer_content += chunk - - elif chunk_enum is ChunkEnum.TOOL: - tool_calls.append(chunk) - - # Note: USAGE and ERROR chunks are ignored in non-streaming mode - - # Construct complete response message - return Message(role=Role.ASSISTANT, - reasoning_content=reasoning_content, - content=answer_content, - tool_calls=tool_calls) - - def stream_print(self, messages: List[Message], tools: List[BaseTool] = None, **kwargs): - """ - Stream chat completions with formatted console output. - - This method provides a real-time view of the model's response, - with different formatting for different types of content: - - Thinking content is wrapped in tags - - Answer content is printed directly - - Tool calls are formatted as JSON - - Usage statistics and errors are clearly marked - - Args: - messages: List of conversation messages - tools: Optional list of tools available to the model - **kwargs: Additional parameters - """ - # Track which sections we've entered for proper formatting - enter_think = False # Whether we've started printing thinking content - enter_answer = False # Whether we've started printing answer content - - # Process each streaming chunk with appropriate formatting - for chunk, chunk_enum in self.stream_chat(messages, tools, **kwargs): - if chunk_enum is ChunkEnum.USAGE: - # Display token usage statistics - if isinstance(chunk, CompletionUsage): - print(f"\n{chunk.model_dump_json(indent=2)}") - else: - print(f"\n{chunk}") - - elif chunk_enum is ChunkEnum.THINK: - # Format thinking/reasoning content - if not enter_think: - enter_think = True - print("\n", end="") - print(chunk, end="") - - elif chunk_enum is ChunkEnum.ANSWER: - # Format regular answer content - if not enter_answer: - enter_answer = True - # Close thinking section if we were in it - if enter_think: - print("\n") - print(chunk, end="") - - elif chunk_enum is ChunkEnum.TOOL: - # Format tool calls as structured JSON - assert isinstance(chunk, ToolCall) - print(f"\n{chunk.model_dump_json(indent=2)}", end="") - - elif chunk_enum is ChunkEnum.ERROR: - # Display error information - print(f"\n{chunk}", end="") - - -def main(): - """ - Demo function to test the OpenAI-compatible LLM implementation. - - This function demonstrates: - 1. Basic chat without tools - 2. Chat with tool usage (search and code tools) - 3. Real-time streaming output formatting - """ - from experiencemaker.tool.dashscope_search_tool import DashscopeSearchTool - from experiencemaker.tool.code_tool import CodeTool - from experiencemaker.enumeration.role import Role - - # Load environment variables for API credentials - load_dotenv() - - # Initialize the LLM with a specific model - model_name = "qwen-max-2025-01-25" - llm = OpenAICompatibleBaseLLM(model_name=model_name) - - # Set up available tools - tools: List[BaseTool] = [DashscopeSearchTool(), CodeTool()] - - # Test 1: Simple greeting without tools - print("=== Test 1: Simple Chat ===") - llm.stream_print([Message(role=Role.USER, content="hello")], []) - - print("\n" + "=" * 20) - - # Test 2: Complex query that might use tools - print("\n=== Test 2: Chat with Tools ===") - llm.stream_print([Message(role=Role.USER, content="What's the weather like in Beijing today?")], tools) - - -if __name__ == "__main__": - main() - # Launch with: python -m experiencemaker.llm.openai_compatible_llm diff --git a/experiencemaker/experiencemaker/mcp_server.py b/experiencemaker/experiencemaker/mcp_server.py deleted file mode 100644 index 33a33b5e..00000000 --- a/experiencemaker/experiencemaker/mcp_server.py +++ /dev/null @@ -1,111 +0,0 @@ -import sys -from typing import List - -from dotenv import load_dotenv -from fastmcp import FastMCP - -from experiencemaker.service.experience_maker_service import ExperienceMakerService - -load_dotenv() - -mcp = FastMCP("ExperienceMaker") -service = ExperienceMakerService(sys.argv[1:]) - - -@mcp.tool -def retriever(query: str, - messages: List[dict] = None, - top_k: int = 1, - workspace_id: str = "default", - config: dict = None) -> dict: - """ - Retrieve experiences from the workspace based on a query. - - Args: - query: Query string - messages: List of messages - top_k: Number of top experiences to retrieve - workspace_id: Workspace identifier - config: Additional configuration parameters - - Returns: - Dictionary containing retrieved experiences - """ - return service(api="retriever", request={ - "query": query, - "messages": messages if messages else [], - "top_k": top_k, - "workspace_id": workspace_id, - "config": config if config else {}, - }).model_dump() - - -@mcp.tool -def summarizer(traj_list: List[dict], workspace_id: str = "default", config: dict = None) -> dict: - """ - Summarize trajectories into experiences. - - Args: - traj_list: List of trajectories - workspace_id: Workspace identifier - config: Additional configuration parameters - - Returns: - experiences - """ - return service(api="summarizer", request={ - "traj_list": traj_list, - "workspace_id": workspace_id, - "config": config if config else {}, - }).model_dump() - - -@mcp.tool -def vector_store(action: str, - src_workspace_id: str = "", - workspace_id: str = "", - path: str = "./", - config: dict = None) -> dict: - """ - Perform vector store operations. - - Args: - action: Action to perform (e.g., "copy", "delete", "dump", "load") - src_workspace_id: Source workspace identifier - workspace_id: Workspace identifier - path: Path to the vector store - config: Additional configuration parameters - - Returns: - Dictionary containing the result of the vector store operation - """ - return service(api="vector_store", request={ - "action": action, - "src_workspace_id": src_workspace_id, - "workspace_id": workspace_id, - "path": path, - "config": config if config else {}, - }).model_dump() - - -def main(): - mcp_transport: str = service.init_app_config.mcp_transport - if mcp_transport == "sse": - mcp.run(transport="sse", host=service.http_service_config.host, port=service.http_service_config.port) - elif mcp_transport == "stdio": - mcp.run(transport="stdio") - else: - raise ValueError(f"Unsupported mcp transport: {mcp_transport}") - - -if __name__ == "__main__": - main() - - -# start with: -# experiencemaker_mcp \ -# mcp_transport=stdio \ -# http_service.port=8001 \ -# llm.default.model_name=qwen3-32b \ -# embedding_model.default.model_name=text-embedding-v4 \ -# vector_store.default.backend=local_file \ No newline at end of file diff --git a/experiencemaker/experiencemaker/op/__init__.py b/experiencemaker/experiencemaker/op/__init__.py deleted file mode 100644 index 6f72c4d5..00000000 --- a/experiencemaker/experiencemaker/op/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -from experiencemaker.utils.registry import Registry - -OP_REGISTRY = Registry() - -from experiencemaker.op.mock_op import Mock1Op, Mock2Op, Mock3Op, Mock4Op, Mock5Op, Mock6Op -from experiencemaker.op.retriever.build_query_op import BuildQueryOp -from experiencemaker.op.retriever.merge_experience_op import MergeExperienceOp -from experiencemaker.op.summarizer.simple_summary_op import SimpleSummaryOp - -from experiencemaker.op.summarizer.trajectory_preprocess_op import TrajectoryPreprocessOp -from experiencemaker.op.summarizer.comparative_extraction_op import ComparativeExtractionOp -from experiencemaker.op.summarizer.success_extraction_op import SuccessExtractionOp -from experiencemaker.op.summarizer.failure_extraction_op import FailureExtractionOp -from experiencemaker.op.summarizer.experience_validation_op import ExperienceValidationOp -from experiencemaker.op.summarizer.experience_deduplication_op import ExperienceDeduplicationOp -from experiencemaker.op.summarizer.experience_validation_op import ExperienceValidationOp -from experiencemaker.op.summarizer.trajectory_segmentation_op import TrajectorySegmentationOp -from experiencemaker.op.summarizer.simple_comparative_summary_op import SimpleComparativeSummaryOp - -from experiencemaker.op.retriever.rerank_experience_op import RerankExperienceOp -from experiencemaker.op.retriever.rewrite_experience_op import RewriteExperienceOp - -from experiencemaker.op.vector_store.update_vector_store_op import UpdateVectorStoreOp -from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp -from experiencemaker.op.vector_store.vector_store_action_op import VectorStoreActionOp -from experiencemaker.op.react.react_v1_op import ReactV1Op diff --git a/experiencemaker/experiencemaker/op/base_op.py b/experiencemaker/experiencemaker/op/base_op.py deleted file mode 100644 index 9b2c075e..00000000 --- a/experiencemaker/experiencemaker/op/base_op.py +++ /dev/null @@ -1,125 +0,0 @@ -from abc import abstractmethod, ABC -from concurrent.futures import Future -from pathlib import Path -from typing import List - -from loguru import logger -from tqdm import tqdm - -from experiencemaker.embedding_model import EMBEDDING_MODEL_REGISTRY -from experiencemaker.embedding_model.base_embedding_model import BaseEmbeddingModel -from experiencemaker.llm import LLM_REGISTRY -from experiencemaker.llm.base_llm import BaseLLM -from experiencemaker.op.prompt_mixin import PromptMixin -from experiencemaker.pipeline.pipeline_context import PipelineContext -from experiencemaker.schema.app_config import OpConfig, LLMConfig, EmbeddingModelConfig -from experiencemaker.utils.common_utils import camel_to_snake -from experiencemaker.utils.timer import Timer -from experiencemaker.vector_store.base_vector_store import BaseVectorStore - - -class BaseOp(PromptMixin, ABC): - current_path: str = __file__ - - def __init__(self, context: PipelineContext, op_config: OpConfig): - super().__init__() - self.context: PipelineContext = context - self.op_config: OpConfig = op_config - self.timer = Timer(name=self.simple_name) - - self._prepare_prompt() - - self._llm: BaseLLM | None = None - self._embedding_model: BaseEmbeddingModel | None = None - self._vector_store: BaseVectorStore | None = None - - self.task_list: List[Future] = [] - - def _prepare_prompt(self): - if self.op_config.prompt_file_path: - prompt_file_path = self.op_config.prompt_file_path - else: - prompt_name = self.simple_name.replace("_op", "_prompt.yaml") - prompt_file_path = Path(self.current_path).parent / prompt_name - - # Load custom prompts from prompt file - self.load_prompt_by_file(prompt_file_path=prompt_file_path) - - # Load custom prompts from config - self.load_prompt_dict(prompt_dict=self.op_config.prompt_dict) - - @property - def simple_name(self) -> str: - return camel_to_snake(self.__class__.__name__) - - @property - def op_params(self) -> dict: - return self.op_config.params - - @abstractmethod - def execute(self): - ... - - def execute_wrap(self): - try: - with self.timer: - return self.execute() - - except Exception as e: - logger.exception(f"op={self.simple_name} execute failed, error={e.args}") - - def submit_task(self, fn, *args, **kwargs): - task = self.context.thread_pool.submit(fn, *args, **kwargs) - self.task_list.append(task) - return self - - def join_task(self, task_desc: str = None) -> list: - result = [] - for task in tqdm(self.task_list, desc=task_desc or (self.simple_name + ".join_task")): - t_result = task.result() - if t_result: - if isinstance(t_result, list): - result.extend(t_result) - else: - result.append(t_result) - self.task_list.clear() - return result - - @property - def llm(self) -> BaseLLM: - if self._llm is None: - llm_name: str = self.op_config.llm - assert llm_name in self.context.app_config.llm, f"llm={llm_name} not found in app_config.llm!" - llm_config: LLMConfig = self.context.app_config.llm[llm_name] - - assert llm_config.backend in LLM_REGISTRY, f"llm.backend={llm_config.backend} not found in LLM_REGISTRY!" - llm_cls = LLM_REGISTRY[llm_config.backend] - self._llm = llm_cls(model_name=llm_config.model_name, **llm_config.params) - - return self._llm - - @property - def embedding_model(self): - if self._embedding_model is None: - embedding_model_name: str = self.op_config.embedding_model - assert embedding_model_name in self.context.app_config.embedding_model, \ - f"embedding_model={embedding_model_name} not found in app_config.embedding_model!" - embedding_model_config: EmbeddingModelConfig = self.context.app_config.embedding_model[embedding_model_name] - - assert embedding_model_config.backend in EMBEDDING_MODEL_REGISTRY, \ - f"embedding_model.backend={embedding_model_config.backend} not found in EMBEDDING_MODEL_REGISTRY!" - embedding_model_cls = EMBEDDING_MODEL_REGISTRY[embedding_model_config.backend] - self._embedding_model = embedding_model_cls(model_name=embedding_model_config.model_name, - **embedding_model_config.params) - - return self._embedding_model - - @property - def vector_store(self): - if self._vector_store is None: - vector_store_name: str = self.op_config.vector_store - assert vector_store_name in self.context.vector_store_dict, \ - f"vector_store={vector_store_name} not found in vector_store_dict!" - self._vector_store = self.context.vector_store_dict[vector_store_name] - - return self._vector_store diff --git a/experiencemaker/experiencemaker/op/mock_op.py b/experiencemaker/experiencemaker/op/mock_op.py deleted file mode 100644 index a3a539e3..00000000 --- a/experiencemaker/experiencemaker/op/mock_op.py +++ /dev/null @@ -1,40 +0,0 @@ -import time - -from loguru import logger - -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp - - -@OP_REGISTRY.register() -class Mock1Op(BaseOp): - def execute(self): - time.sleep(1) - a: int = self.op_params["a"] - b: str = self.op_params["b"] - logger.info(f"enter class={self.simple_name}. a={a} b={b}") - - -@OP_REGISTRY.register() -class Mock2Op(Mock1Op): - ... - - -@OP_REGISTRY.register() -class Mock3Op(Mock1Op): - ... - - -@OP_REGISTRY.register() -class Mock4Op(Mock1Op): - ... - - -@OP_REGISTRY.register() -class Mock5Op(Mock1Op): - ... - - -@OP_REGISTRY.register() -class Mock6Op(Mock1Op): - ... diff --git a/experiencemaker/experiencemaker/op/prompt_mixin.py b/experiencemaker/experiencemaker/op/prompt_mixin.py deleted file mode 100644 index b933fc8d..00000000 --- a/experiencemaker/experiencemaker/op/prompt_mixin.py +++ /dev/null @@ -1,74 +0,0 @@ -from pathlib import Path - -import yaml -from loguru import logger - - -class PromptMixin: - - def __init__(self): - self._prompt_dict: dict = {} - - def load_prompt_by_file(self, prompt_file_path: Path | str = None): - if prompt_file_path is None: - return - - if isinstance(prompt_file_path, str): - prompt_file_path = Path(prompt_file_path) - - if not prompt_file_path.exists(): - return - - with prompt_file_path.open() as f: - prompt_dict = yaml.load(f, yaml.FullLoader) - self.load_prompt_dict(prompt_dict) - - def load_prompt_dict(self, prompt_dict: dict = None): - if not prompt_dict: - return - - for key, value in prompt_dict.items(): - if isinstance(value, str): - if key in self._prompt_dict: - self._prompt_dict[key] = value - logger.warning(f"prompt_dict key={key} overwrite!") - - else: - self._prompt_dict[key] = value - logger.info(f"add prompt_dict key={key}") - - def prompt_format(self, prompt_name: str, **kwargs): - prompt = self._prompt_dict[prompt_name] - - flag_kwargs = {k: v for k, v in kwargs.items() if isinstance(v, bool)} - other_kwargs = {k: v for k, v in kwargs.items() if not isinstance(v, bool)} - - if flag_kwargs: - split_prompt = [] - for line in prompt.strip().split("\n"): - hit = False - hit_flag = True - for key, flag in kwargs.items(): - if not line.startswith(f"[{key}]"): - continue - - else: - hit = True - hit_flag = flag - line = line.strip(f"[{key}]") - break - - if not hit: - split_prompt.append(line) - elif hit_flag: - split_prompt.append(line) - - prompt = "\n".join(split_prompt) - - if other_kwargs: - prompt = prompt.format(**other_kwargs) - - return prompt - - def get_prompt(self, key: str): - return self._prompt_dict[key] diff --git a/experiencemaker/experiencemaker/op/retriever/__init__.py b/experiencemaker/experiencemaker/op/retriever/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/op/retriever/build_query_op.py b/experiencemaker/experiencemaker/op/retriever/build_query_op.py deleted file mode 100644 index 97d5e63d..00000000 --- a/experiencemaker/experiencemaker/op/retriever/build_query_op.py +++ /dev/null @@ -1,42 +0,0 @@ -from loguru import logger - -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.request import RetrieverRequest -from experiencemaker.utils.op_utils import merge_messages_content - - -@OP_REGISTRY.register() -class BuildQueryOp(BaseOp): - current_path: str = __file__ - - def execute(self): - request: RetrieverRequest = self.context.request - if request.query: - query = request.query - - elif request.messages: - enable_llm_build: str = str(self.op_params.get("enable_llm_build")) - if enable_llm_build and enable_llm_build.lower() == "true": - execution_process = merge_messages_content(request.messages) - query = self.prompt_format(prompt_name="query_build", execution_process=execution_process) - - else: - context_parts = [] - message_summaries = [] - for message in request.messages[-3:]: # Last 3 messages - content = message.content[:200] + "..." if len(message.content) > 200 else message.content - message_summaries.append(f"- {message.role.value}: {content}") - if message_summaries: - context_parts.append("Recent messages:\n" + "\n".join(message_summaries)) - - query = "\n\n".join(context_parts) - - else: - raise RuntimeError("query or messages is required!") - - logger.info(f"build.query={query}") - - from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp - self.context.set_context(RecallVectorStoreOp.SEARCH_QUERY, query) - self.context.set_context(RecallVectorStoreOp.SEARCH_MESSAGE, request.messages) diff --git a/experiencemaker/experiencemaker/op/retriever/build_query_prompt.yaml b/experiencemaker/experiencemaker/op/retriever/build_query_prompt.yaml deleted file mode 100644 index 3af908af..00000000 --- a/experiencemaker/experiencemaker/op/retriever/build_query_prompt.yaml +++ /dev/null @@ -1,6 +0,0 @@ -query_build: | - # Execution Process - {execution_process} - - Read through the entire execution process to understand which part is currently being executed. - Generate a `query` that reflects the current state, which will later be used to search for similar problems in the database and help resolve the issue at hand. \ No newline at end of file diff --git a/experiencemaker/experiencemaker/op/retriever/merge_experience_op.py b/experiencemaker/experiencemaker/op/retriever/merge_experience_op.py deleted file mode 100644 index 95ab9baa..00000000 --- a/experiencemaker/experiencemaker/op/retriever/merge_experience_op.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import List - -from loguru import logger - -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.experience import BaseExperience -from experiencemaker.schema.response import RetrieverResponse - - -@OP_REGISTRY.register() -class MergeExperienceOp(BaseOp): - - def execute(self): - response: RetrieverResponse = self.context.response - experience_list: List[BaseExperience] = response.experience_list - - if not experience_list: - return - - content_collector = ["Previous Experience"] - for experience in experience_list: - if not experience.content: - continue - - content_collector.append(f"- when_to_use: {experience.when_to_use}\n" - f"content: {experience.content}\n") - content_collector.append("Please consider the helpful parts from these in answering the question, " - "to make the response more comprehensive and substantial.") - response.experience_merged = "\n".join(content_collector) - logger.info(f"experience_merged={response.experience_merged}") diff --git a/experiencemaker/experiencemaker/op/vector_store/__init__.py b/experiencemaker/experiencemaker/op/vector_store/__init__.py deleted file mode 100644 index efa3f21c..00000000 --- a/experiencemaker/experiencemaker/op/vector_store/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -1. retrieve: - search: query(context), workspace_id(request), top_k(request) -2. summary: - insert: nodes(context), workspace_id(request) - delete: ids(context), workspace_id(request) - search: query(context), workspace_id(request), top_k(request.config.op) -3. vector: - dump: workspace_id(request), path(str), max_size(int) - load: workspace_id(request), path(str) - delete: workspace_id(request) - copy: source_id, target_id, max_size(int) -""" diff --git a/experiencemaker/experiencemaker/op/vector_store/recall_vector_store_op.py b/experiencemaker/experiencemaker/op/vector_store/recall_vector_store_op.py deleted file mode 100644 index 07ee355d..00000000 --- a/experiencemaker/experiencemaker/op/vector_store/recall_vector_store_op.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import List - -from loguru import logger - -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.experience import BaseExperience, vector_node_to_experience -from experiencemaker.schema.request import RetrieverRequest -from experiencemaker.schema.response import RetrieverResponse -from experiencemaker.schema.vector_node import VectorNode - - -@OP_REGISTRY.register() -class RecallVectorStoreOp(BaseOp): - SEARCH_QUERY = "search_query" - SEARCH_MESSAGE = "search_message" - - def execute(self): - # get query - query = self.context.get_context(self.SEARCH_QUERY) - assert query, "query should be not empty!" - - # retrieve from vector store - request: RetrieverRequest = self.context.request - nodes: List[VectorNode] = self.vector_store.search(query=query, - workspace_id=request.workspace_id, - top_k=request.top_k) - - # convert to experience, filter duplicate - experience_list: List[BaseExperience] = [] - experience_content_list: List[str] = [] - for node in nodes: - experience: BaseExperience = vector_node_to_experience(node) - if experience.content not in experience_content_list: - experience_list.append(experience) - experience_content_list.append(experience.content) - experience_size = len(experience_list) - logger.info(f"retrieve experience size={experience_size}") - - # filter by score - threshold_score: float | None = self.op_params.get("threshold_score", None) - if threshold_score is not None: - experience_list = [e for e in experience_list if e.score >= threshold_score or e.score is None] - logger.info(f"after filter by threshold_score size={len(experience_list)}") - - # set response - response: RetrieverResponse = self.context.response - response.experience_list = experience_list diff --git a/experiencemaker/experiencemaker/pipeline/__init__.py b/experiencemaker/experiencemaker/pipeline/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/pipeline/pipeline.py b/experiencemaker/experiencemaker/pipeline/pipeline.py deleted file mode 100644 index 67ef8284..00000000 --- a/experiencemaker/experiencemaker/pipeline/pipeline.py +++ /dev/null @@ -1,94 +0,0 @@ -from concurrent.futures import as_completed -from itertools import zip_longest -from typing import List - -from loguru import logger - -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.pipeline.pipeline_context import PipelineContext -from experiencemaker.utils.timer import Timer, timer - - -class Pipeline: - seq_symbol: str = "->" - parallel_symbol: str = "|" - - def __init__(self, pipeline: str, context: PipelineContext): - self.pipeline_list: List[str | List[str]] = self._parse_pipline(pipeline) - self.context: PipelineContext = context - - def _parse_pipline(self, pipeline: str) -> List[str | List[str]]: - pipeline_list: List[str | List[str]] = [] - - for pipeline_split1 in pipeline.split("["): - for sub_pipeline in pipeline_split1.split("]"): - sub_pipeline = sub_pipeline.strip().strip(self.seq_symbol) - if not sub_pipeline: - continue - - if self.parallel_symbol in sub_pipeline: - pipeline_list.append(sub_pipeline.split(self.parallel_symbol)) - else: - pipeline_list.append(sub_pipeline) - logger.info(f"add sub_pipeline={sub_pipeline}") - return pipeline_list - - def _execute_sub_pipeline(self, pipeline: str): - op_config_dict = self.context.app_config.op - for op in pipeline.split(self.seq_symbol): - op = op.strip() - if not op: - continue - - assert op in op_config_dict, f"op={op} config is missing!" - op_config = op_config_dict[op] - - assert op_config.backend in OP_REGISTRY, f"op={op} backend={op_config.backend} is not registered!" - op_cls = OP_REGISTRY[op_config.backend] - - op_obj: BaseOp = op_cls(context=self.context, op_config=op_config) - op_obj.execute_wrap() - - def _parse_sub_pipeline(self, pipeline: str): - for op in pipeline.split(self.seq_symbol): - op = op.strip() - if not op: - continue - - yield op - - def print_pipeline(self): - i: int = 0 - for pipeline in self.pipeline_list: - if isinstance(pipeline, str): - for op in self._parse_sub_pipeline(pipeline): - i += 1 - logger.info(f"stage_{i}: {op}") - - elif isinstance(pipeline, list): - parallel_pipeline = [self._parse_sub_pipeline(x) for x in pipeline] - for op_list in zip_longest(*parallel_pipeline, fillvalue="-"): - i += 1 - logger.info(f"stage{i}: {' | '.join(op_list)}") - else: - raise ValueError(f"unknown pipeline.type={type(pipeline)}") - - @timer(name="pipeline.execute") - def __call__(self, enable_print: bool = True): - if enable_print: - self.print_pipeline() - - for i, pipeline in enumerate(self.pipeline_list): - with Timer(f"step_{i}"): - if isinstance(pipeline, str): - self._execute_sub_pipeline(pipeline) - - else: - future_list = [] - for sub_pipeline in pipeline: - future = self.context.thread_pool.submit(self._execute_sub_pipeline, pipeline=sub_pipeline) - future_list.append(future) - - for future in as_completed(future_list): - future.result() diff --git a/experiencemaker/experiencemaker/pipeline/pipeline_context.py b/experiencemaker/experiencemaker/pipeline/pipeline_context.py deleted file mode 100644 index 29bfa49e..00000000 --- a/experiencemaker/experiencemaker/pipeline/pipeline_context.py +++ /dev/null @@ -1,37 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from typing import Dict - -from experiencemaker.schema.app_config import AppConfig -from experiencemaker.vector_store.base_vector_store import BaseVectorStore - - -class PipelineContext: - - def __init__(self, **kwargs): - self._context: dict = {**kwargs} - - def get_context(self, key: str, default=None): - return self._context.get(key, default) - - def set_context(self, key: str, value): - self._context[key] = value - - @property - def request(self): - return self._context["request"] - - @property - def response(self): - return self._context["response"] - - @property - def app_config(self) -> AppConfig: - return self._context["app_config"] - - @property - def thread_pool(self) -> ThreadPoolExecutor: - return self._context["thread_pool"] - - @property - def vector_store_dict(self) -> Dict[str, BaseVectorStore]: - return self._context["vector_store_dict"] diff --git a/experiencemaker/experiencemaker/schema/__init__.py b/experiencemaker/experiencemaker/schema/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/schema/app_config.py b/experiencemaker/experiencemaker/schema/app_config.py deleted file mode 100644 index 6524487f..00000000 --- a/experiencemaker/experiencemaker/schema/app_config.py +++ /dev/null @@ -1,69 +0,0 @@ -from dataclasses import dataclass, field -from typing import Dict - - -@dataclass -class HttpServiceConfig: - host: str = field(default="0.0.0.0") - port: int = field(default=8001) - timeout_keep_alive: int = field(default=600) - limit_concurrency: int = field(default=64) - - -@dataclass -class ThreadPoolConfig: - max_workers: int = field(default=10) - - -@dataclass -class APIConfig: - retriever: str = field(default="") - summarizer: str = field(default="") - vector_store: str = field(default="") - agent: str = field(default="") - - -@dataclass -class OpConfig: - backend: str = field(default="") - prompt_file_path: str = field(default="") - prompt_dict: dict = field(default_factory=dict) - llm: str = field(default="") - embedding_model: str = field(default="") - vector_store: str = field(default="") - params: dict = field(default_factory=dict) - - -@dataclass -class LLMConfig: - backend: str = field(default="") - model_name: str = field(default="") - params: dict = field(default_factory=dict) - - -@dataclass -class EmbeddingModelConfig: - backend: str = field(default="") - model_name: str = field(default="") - params: dict = field(default_factory=dict) - - -@dataclass -class VectorStoreConfig: - backend: str = field(default="") - embedding_model: str = field(default="") - params: dict = field(default_factory=dict) - - -@dataclass -class AppConfig: - pre_defined_config: str = field(default="default_config") - config_path: str = field(default="") - mcp_transport: str = field(default="sse") - http_service: HttpServiceConfig = field(default_factory=HttpServiceConfig) - thread_pool: ThreadPoolConfig = field(default_factory=ThreadPoolConfig) - api: APIConfig = field(default_factory=APIConfig) - op: Dict[str, OpConfig] = field(default_factory=dict) - llm: Dict[str, LLMConfig] = field(default_factory=dict) - embedding_model: Dict[str, EmbeddingModelConfig] = field(default_factory=dict) - vector_store: Dict[str, VectorStoreConfig] = field(default_factory=dict) diff --git a/experiencemaker/experiencemaker/schema/experience.py b/experiencemaker/experiencemaker/schema/experience.py deleted file mode 100644 index 7d93643a..00000000 --- a/experiencemaker/experiencemaker/schema/experience.py +++ /dev/null @@ -1,144 +0,0 @@ -import datetime -from abc import ABC -from typing import List -from uuid import uuid4 - -from loguru import logger -from pydantic import BaseModel, Field - -from experiencemaker.schema.vector_node import VectorNode - - -class ExperienceMeta(BaseModel): - author: str = Field(default="") - created_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - modified_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - extra_info: dict | None = Field(default=None) - - def update_modified_time(self): - self.modified_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - -class BaseExperience(BaseModel, ABC): - workspace_id: str = Field(default="") - - experience_id: str = Field(default_factory=lambda: uuid4().hex) - experience_type: str = Field(default="") - - when_to_use: str = Field(default="") - content: str | bytes = Field(default="") - score: float | None = Field(default=None) - metadata: ExperienceMeta = Field(default_factory=ExperienceMeta) - - def to_vector_node(self) -> VectorNode: - raise NotImplementedError - - @classmethod - def from_vector_node(cls, node: VectorNode): - raise NotImplementedError - - -class TextExperience(BaseExperience): - experience_type: str = Field(default="text") - - def to_vector_node(self) -> VectorNode: - return VectorNode(unique_id=self.experience_id, - workspace_id=self.workspace_id, - content=self.when_to_use, - metadata={ - "experience_type": self.experience_type, - "experience_content": self.content, - "score": self.score, - "metadata": self.metadata.model_dump(), - }) - - @classmethod - def from_vector_node(cls, node: VectorNode): - return cls(workspace_id=node.workspace_id, - experience_id=node.unique_id, - experience_type=node.metadata.get("experience_type"), - when_to_use=node.content, - content=node.metadata.get("experience_content"), - score=node.metadata.get("score"), - metadata=node.metadata.get("metadata")) - - -class FunctionArg(BaseModel): - arg_name: str = Field(default=...) - arg_type: str = Field(default=...) - required: bool = Field(default=True) - - -class Function(BaseModel): - func_code: str = Field(default=..., description="function code") - func_name: str = Field(default=..., description="function name") - func_args: List[FunctionArg] = Field(default_factory=list) - - -class FuncExperience(BaseExperience): - experience_type: str = Field(default="function") - functions: List[Function] = Field(default_factory=list) - - -class PersonalExperience(BaseExperience): - experience_type: str = Field(default="personal") - person: str = Field(default="") - topic: str = Field(default="") - - -class KnowledgeExperience(BaseExperience): - experience_type: str = Field(default="knowledge") - topic: str = Field(default="") - - -def vector_node_to_experience(node: VectorNode) -> BaseExperience: - experience_type = node.metadata.get("experience_type") - if experience_type == "text": - return TextExperience.from_vector_node(node) - - elif experience_type == "function": - return FuncExperience.from_vector_node(node) - - elif experience_type == "personal": - return PersonalExperience.from_vector_node(node) - - elif experience_type == "knowledge": - return KnowledgeExperience.from_vector_node(node) - - else: - logger.warning(f"experience type {experience_type} not supported") - return TextExperience.from_vector_node(node) - - -def dict_to_experience(experience_dict: dict) -> BaseExperience: - experience_type = experience_dict.get("experience_type", "text") - if experience_type == "text": - return TextExperience(**experience_dict) - - elif experience_type == "function": - return FuncExperience(**experience_dict) - - elif experience_type == "personal": - return PersonalExperience(**experience_dict) - - elif experience_type == "knowledge": - return KnowledgeExperience(**experience_dict) - - else: - logger.warning(f"experience type {experience_type} not supported") - return TextExperience(**experience_dict) - - -if __name__ == "__main__": - e1 = TextExperience( - workspace_id="w_1024", - experience_id="123", - when_to_use="test case use", - content="test content", - score=0.99, - metadata=ExperienceMeta(author="user")) - print(e1.model_dump_json(indent=2)) - v1 = e1.to_vector_node() - print(v1.model_dump_json(indent=2)) - e2 = vector_node_to_experience(v1) - print(e2.model_dump_json(indent=2)) diff --git a/experiencemaker/experiencemaker/schema/message.py b/experiencemaker/experiencemaker/schema/message.py deleted file mode 100644 index e41b3be8..00000000 --- a/experiencemaker/experiencemaker/schema/message.py +++ /dev/null @@ -1,68 +0,0 @@ -import json -from typing import List - -from pydantic import BaseModel, Field, model_validator - -from experiencemaker.enumeration.role import Role - - -class ToolCall(BaseModel): - index: int = Field(default=0) - id: str = Field(default="") - name: str = Field(default="") - arguments: str = Field(default="") - type: str = Field(default="function") - - @model_validator(mode="before") # noqa - @classmethod - def init_tool_call(cls, data: dict): - tool_type = data.get("type", "") - tool_type_dict = data.get(tool_type, {}) - - for key in ["name", "arguments"]: - if key not in data: - data[key] = tool_type_dict.get(key, "") - return data - - @property - def argument_dict(self) -> dict: - return json.loads(self.arguments) - - def simple_dump(self) -> dict: - return { - "id": self.id, - self.type: { - "arguments": self.arguments, - "name": self.name - }, - "type": self.type, - "index": self.index, - } - -class Message(BaseModel): - role: Role = Field(default=Role.USER) - content: str | bytes = Field(default="") - reasoning_content: str = Field(default="") - tool_calls: List[ToolCall] = Field(default_factory=list) - tool_call_id: str = Field(default="") - metadata: dict = Field(default_factory=dict) - - def simple_dump(self, add_reason_when_empty: bool = True) -> dict: - result: dict - if self.content: - result = {"role": self.role.value, "content": self.content} - elif add_reason_when_empty and self.reasoning_content: - result = {"role": self.role.value, "content": self.reasoning_content} - else: - result = {"role": self.role.value, "content": ""} - - if self.tool_calls: - result["tool_calls"] = [x.simple_dump() for x in self.tool_calls] - return result - - -class Trajectory(BaseModel): - task_id: str = Field(default="") - messages: List[Message] = Field(default_factory=list) - score: float = Field(default=0.0) - metadata: dict = Field(default_factory=dict) diff --git a/experiencemaker/experiencemaker/schema/request.py b/experiencemaker/experiencemaker/schema/request.py deleted file mode 100644 index 50468b32..00000000 --- a/experiencemaker/experiencemaker/schema/request.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import List - -from pydantic import BaseModel, Field - -from experiencemaker.schema.message import Message, Trajectory - - -class BaseRequest(BaseModel): - workspace_id: str = Field(default="default") - config: dict = Field(default_factory=dict) - - -class RetrieverRequest(BaseRequest): - query: str = Field(default="") - messages: List[Message] = Field(default_factory=list) - top_k: int = Field(default=1) - - -class SummarizerRequest(BaseRequest): - traj_list: List[Trajectory] = Field(default_factory=list) - - -class VectorStoreRequest(BaseRequest): - action: str = Field(default="") - src_workspace_id: str = Field(default="") - path: str = Field(default="") - experience_ids: List[str] = Field(default_factory=list) - freq_threshold: int = Field(default=5) - utility_threshold: float = Field(default=0.5) - - -class AgentRequest(BaseRequest): - query: str = Field(default="") - messages: List[Message] = Field(default_factory=list) - diff --git a/experiencemaker/experiencemaker/schema/response.py b/experiencemaker/experiencemaker/schema/response.py deleted file mode 100644 index f4ecaf18..00000000 --- a/experiencemaker/experiencemaker/schema/response.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import List - -from pydantic import BaseModel, Field - -from experiencemaker.schema.experience import BaseExperience -from experiencemaker.schema.message import Message - - -class BaseResponse(BaseModel): - success: bool = Field(default=True) - metadata: dict = Field(default_factory=dict) - - -class RetrieverResponse(BaseResponse): - experience_list: List[BaseExperience] = Field(default_factory=list) - experience_merged: str = Field(default="") - - -class SummarizerResponse(BaseResponse): - experience_list: List[BaseExperience] = Field(default_factory=list) - deleted_experience_ids: List[str] = Field(default_factory=list) - - -class VectorStoreResponse(BaseResponse): - ... - -class AgentResponse(BaseResponse): - answer: str = Field(default="") - messages: List[Message] = Field(default_factory=list) diff --git a/experiencemaker/experiencemaker/schema/vector_node.py b/experiencemaker/experiencemaker/schema/vector_node.py deleted file mode 100644 index 7897dbb0..00000000 --- a/experiencemaker/experiencemaker/schema/vector_node.py +++ /dev/null @@ -1,13 +0,0 @@ -from typing import List -from uuid import uuid4 -from pydantic import BaseModel, Field - - -class VectorNode(BaseModel): - unique_id: str = Field(default_factory=lambda: uuid4().hex) - workspace_id: str = Field(default="") - content: str = Field(default="") - vector: List[float] | None = Field(default=None) - freq: int = Field(default=0) - utility: int = Field(default=0) - metadata: dict = Field(default_factory=dict) diff --git a/experiencemaker/experiencemaker/service/__init__.py b/experiencemaker/experiencemaker/service/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/service/experience_maker_client.py b/experiencemaker/experiencemaker/service/experience_maker_client.py deleted file mode 100644 index 1cfd175b..00000000 --- a/experiencemaker/experiencemaker/service/experience_maker_client.py +++ /dev/null @@ -1,37 +0,0 @@ -from pydantic import Field - -from experiencemaker.schema.message import Trajectory, Message -from experiencemaker.schema.request import RetrieverRequest, SummarizerRequest, VectorStoreRequest, AgentRequest -from experiencemaker.schema.response import RetrieverResponse, SummarizerResponse, VectorStoreResponse, AgentResponse -from experiencemaker.utils.http_client import HttpClient - - -class ExperienceMakerClient(HttpClient): - base_url: str = Field(default="http://0.0.0.0:8001") - - def call_retriever(self, request: RetrieverRequest): - self.url = self.base_url + "/retriever" - return RetrieverResponse(**self.request(json_data=request.model_dump())) - - def call_summarizer(self, request: SummarizerRequest): - self.url = self.base_url + "/summarizer" - return SummarizerResponse(**self.request(json_data=request.model_dump())) - - def call_vector_store(self, request: VectorStoreRequest): - self.url = self.base_url + "/vector_store" - return VectorStoreResponse(**self.request(json_data=request.model_dump())) - - def call_agent(self, request: AgentRequest): - self.url = self.base_url + "/agent" - return AgentResponse(**self.request(json_data=request.model_dump())) - - -if __name__ == "__main__": - client = ExperienceMakerClient() - workspace_id = "t123" - response = client.call_summarizer( - SummarizerRequest(workspace_id=workspace_id, - traj_list=[Trajectory(messages=[Message(content="hello world!")])])) - print(response.model_dump()) - response = client.call_retriever(RetrieverRequest(workspace_id=workspace_id, query="hello world")) - print(response.model_dump()) diff --git a/experiencemaker/experiencemaker/service/experience_maker_service.py b/experiencemaker/experiencemaker/service/experience_maker_service.py deleted file mode 100644 index 8e97230b..00000000 --- a/experiencemaker/experiencemaker/service/experience_maker_service.py +++ /dev/null @@ -1,96 +0,0 @@ -from concurrent.futures import ThreadPoolExecutor -from typing import List - -from loguru import logger - -from experiencemaker.config.config_parser import ConfigParser -from experiencemaker.embedding_model import EMBEDDING_MODEL_REGISTRY -from experiencemaker.pipeline.pipeline import Pipeline -from experiencemaker.pipeline.pipeline_context import PipelineContext -from experiencemaker.schema.app_config import AppConfig, HttpServiceConfig, EmbeddingModelConfig -from experiencemaker.schema.request import SummarizerRequest, RetrieverRequest, VectorStoreRequest, AgentRequest, \ - BaseRequest -from experiencemaker.schema.response import SummarizerResponse, RetrieverResponse, VectorStoreResponse, AgentResponse, \ - BaseResponse -from experiencemaker.vector_store import VECTOR_STORE_REGISTRY - - -class ExperienceMakerService: - - def __init__(self, args: List[str]): - self.config_parser = ConfigParser(args) - self.init_app_config: AppConfig = self.config_parser.get_app_config() - self.thread_pool = ThreadPoolExecutor(max_workers=self.init_app_config.thread_pool.max_workers) - - # The vectorstore is initialized at the very beginning and then used directly afterward. - self.vector_store_dict: dict = {} - for name, config in self.init_app_config.vector_store.items(): - assert config.backend in VECTOR_STORE_REGISTRY, f"backend={config.backend} is not existed" - vector_store_cls = VECTOR_STORE_REGISTRY[config.backend] - - assert config.embedding_model in self.init_app_config.embedding_model, \ - f"embedding_model={config.embedding_model} is not existed" - embedding_model_config: EmbeddingModelConfig = self.init_app_config.embedding_model[config.embedding_model] - - assert embedding_model_config.backend in EMBEDDING_MODEL_REGISTRY, \ - f"embedding_model={embedding_model_config.backend} is not existed" - embedding_model_cls = EMBEDDING_MODEL_REGISTRY[embedding_model_config.backend] - embedding_model = embedding_model_cls(model_name=embedding_model_config.model_name, - **embedding_model_config.params) - - self.vector_store_dict[name] = vector_store_cls(embedding_model=embedding_model, **config.params) - - @property - def http_service_config(self) -> HttpServiceConfig: - return self.init_app_config.http_service - - def __call__(self, api: str, request: dict | BaseRequest) -> BaseResponse: - if isinstance(request, dict): - app_config: AppConfig = self.config_parser.get_app_config(**request["config"]) - else: - app_config: AppConfig = self.config_parser.get_app_config(**request.config) - - if api == "retriever": - if isinstance(request, dict): - request = RetrieverRequest(**request) - response = RetrieverResponse() - pipeline = app_config.api.retriever - - elif api == "summarizer": - if isinstance(request, dict): - request = SummarizerRequest(**request) - response = SummarizerResponse() - pipeline = app_config.api.summarizer - - elif api == "vector_store": - if isinstance(request, dict): - request = VectorStoreRequest(**request) - response = VectorStoreResponse() - pipeline = app_config.api.vector_store - - elif api == "agent": - if isinstance(request, dict): - request = AgentRequest(**request) - response = AgentResponse() - pipeline = app_config.api.agent - - else: - raise RuntimeError(f"Invalid service.api={api}") - - logger.info(f"request={request.model_dump_json()}") - - try: - context = PipelineContext(app_config=app_config, - thread_pool=self.thread_pool, - request=request, - response=response, - vector_store_dict=self.vector_store_dict) - pipeline = Pipeline(pipeline=pipeline, context=context) - pipeline() - - except Exception as e: - logger.exception(f"api={api} encounter error={e.args}") - response.success = False - response.metadata["error"] = str(e) - - return response diff --git a/experiencemaker/experiencemaker/service/mcp_client.py b/experiencemaker/experiencemaker/service/mcp_client.py deleted file mode 100644 index 9be64e3a..00000000 --- a/experiencemaker/experiencemaker/service/mcp_client.py +++ /dev/null @@ -1,87 +0,0 @@ -import asyncio -import json -from typing import List - -from fastmcp import Client -from pydantic import BaseModel, Field - -from experiencemaker.schema.request import RetrieverRequest, SummarizerRequest, VectorStoreRequest, AgentRequest -from experiencemaker.schema.response import RetrieverResponse, SummarizerResponse, VectorStoreResponse, AgentResponse - - -class MCPClient(BaseModel): - base_url: str = Field(default="http://0.0.0.0:8001/sse") - enable_sse: bool = Field(default=True) - timeout: int = Field(default=300) - - _client: Client | None = None - - async def __aenter__(self): - if self.enable_sse: - self._client = Client(self.base_url) - else: - self._client = Client("stdio") - - await self._client.__aenter__() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - if self._client: - await self._client.__aexit__(exc_type, exc_val, exc_tb) - - async def list_tools(self) -> List[str]: - tools = await self._client.list_tools() - return [tool.name for tool in tools] - - async def call_retriever(self, request: RetrieverRequest) -> RetrieverResponse: - result = await self._client.call_tool("retriever", request.model_dump()) - return RetrieverResponse(**result.structured_content) - - async def call_summarizer(self, request: SummarizerRequest) -> SummarizerResponse: - result = await self._client.call_tool("summarizer", request.model_dump()) - return SummarizerResponse(**result.structured_content) - - async def call_vector_store(self, request: VectorStoreRequest) -> VectorStoreResponse: - result = await self._client.call_tool("vector_store", request.model_dump()) - return VectorStoreResponse(**result.structured_content) - - async def call_agent(self, request: AgentRequest) -> AgentResponse: - result = await self._client.call_tool("agent", request.model_dump()) - return AgentResponse(**result.structured_content) - - -async def main(): - """Example usage of MCPClient""" - async with MCPClient() as client: - # List available tools - tools = await client.list_tools() - print("Available tools:", json.dumps(tools, ensure_ascii=False, indent=2)) - - # Example retriever call - retriever_request = RetrieverRequest( - workspace_id="test_workspace", - query="hello world", - top_k=5) - - try: - response = await client.call_retriever(retriever_request) - print("Retriever response:", response.model_dump()) - except Exception as e: - print(f"Error calling retriever: {e}") - - # Example summarizer call - from experiencemaker.schema.message import Trajectory, Message - - summarizer_request = SummarizerRequest( - workspace_id="test_workspace", - traj_list=[Trajectory(messages=[Message(content="hello world!")])]) - - try: - response = await client.call_summarizer(summarizer_request) - print("Summarizer response:", response.model_dump()) - except Exception as e: - print(f"Error calling summarizer: {e}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/experiencemaker/experiencemaker/tool/__init__.py b/experiencemaker/experiencemaker/tool/__init__.py deleted file mode 100644 index c10d8c6d..00000000 --- a/experiencemaker/experiencemaker/tool/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from experiencemaker.utils.registry import Registry - -TOOL_REGISTRY = Registry() - -from experiencemaker.tool.code_tool import CodeTool -from experiencemaker.tool.dashscope_search_tool import DashscopeSearchTool -from experiencemaker.tool.tavily_search_tool import TavilySearchTool -from experiencemaker.tool.terminate_tool import TerminateTool -from experiencemaker.tool.mcp_tool import MCPTool diff --git a/experiencemaker/experiencemaker/tool/base_tool.py b/experiencemaker/experiencemaker/tool/base_tool.py deleted file mode 100644 index 0dc09edf..00000000 --- a/experiencemaker/experiencemaker/tool/base_tool.py +++ /dev/null @@ -1,80 +0,0 @@ -from abc import ABC - -from loguru import logger -from pydantic import BaseModel, Field - - -class BaseTool(BaseModel, ABC): - tool_id: str = Field(default="") - name: str = Field(..., description="tool name") - description: str = Field(..., description="tool description") - tool_type: str = Field(default="function") - parameters: dict = Field(default_factory=dict, description="tool parameters") - arguments: dict = Field(default_factory=dict, description="execute arguments") - - enable_cache: bool = Field(default=False, description="whether to cache the tool result") - cached_result: dict = Field(default_factory=dict, description="tool execution result") - - max_retries: int = Field(default=3, description="max retries") - raise_exception: bool = Field(default=True, description="raise exception") - success: bool = Field(default=True, description="whether the tool executed successfully") - - def reset(self): - self.arguments.clear() - self.cached_result.clear() - self.success = True - - def _execute(self, **kwargs): - raise NotImplementedError - - def execute(self, **kwargs): - cache_id = "" - if self.enable_cache: - cache_id = self.get_cache_id(**kwargs) - if cache_id in self.cached_result: - return self.cached_result[cache_id] - - for i in range(self.max_retries): - try: - if self.enable_cache: - self.cached_result[cache_id] = self._execute(**kwargs) - return self.cached_result[cache_id] - - else: - return self._execute(**kwargs) - - except Exception as e: - logger.exception(f"using tool.name={self.name} encounter error with e={e.args}") - if i == self.max_retries - 1 and self.raise_exception: - raise e - - return None - - - def simple_dump(self) -> dict: - """ - It may be in other different tool params formats; different versions are completed here. - """ - return { - "type": self.tool_type, - self.tool_type: { - "name": self.name, - "description": self.description, - "parameters": self.parameters, - }, - } - - @property - def input_schema(self) -> dict: - return self.parameters.get("properties", {}) - - @property - def output_schema(self) -> dict: - raise NotImplementedError - - def refresh(self): - # for mcp - raise NotImplementedError - - def get_cache_id(self, **kwargs) -> str: - raise NotImplementedError diff --git a/experiencemaker/experiencemaker/tool/code_tool.py b/experiencemaker/experiencemaker/tool/code_tool.py deleted file mode 100644 index 21c9b265..00000000 --- a/experiencemaker/experiencemaker/tool/code_tool.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys -from io import StringIO - -from experiencemaker.tool import TOOL_REGISTRY -from experiencemaker.tool.base_tool import BaseTool - - -@TOOL_REGISTRY.register() -class CodeTool(BaseTool): - name: str = "python_execute" - description: str = "Execute python code can be used in scenarios such as analysis or calculation, and the final result can be printed using the `print` function." - parameters: dict = { - "type": "object", - "properties": { - "code": { - "type": "string", - "description": "code to be executed. Please do not execute any matplotlib code here.", - } - }, - "required": ["code"] - } - - def _execute(self, code: str, **kwargs): - old_stdout = sys.stdout - redirected_output = sys.stdout = StringIO() - - try: - exec(code) - result = redirected_output.getvalue() - - except Exception as e: - self.success = False - result = str(e) - - sys.stdout = old_stdout - - return result - - -if __name__ == '__main__': - tool = CodeTool() - print(tool.execute(code="print('Hello World')")) - print(tool.execute(code="print('Hello World!'")) diff --git a/experiencemaker/experiencemaker/tool/dashscope_search_tool.py b/experiencemaker/experiencemaker/tool/dashscope_search_tool.py deleted file mode 100644 index 04a8746f..00000000 --- a/experiencemaker/experiencemaker/tool/dashscope_search_tool.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -from typing import Literal - -import dashscope -from dashscope.api_entities.dashscope_response import Message -from dotenv import load_dotenv -from loguru import logger -from pydantic import Field - -from experiencemaker.tool import TOOL_REGISTRY -from experiencemaker.tool.base_tool import BaseTool - - -@TOOL_REGISTRY.register() -class DashscopeSearchTool(BaseTool): - name: str = "web_search" - description: str = "Use search keywords to retrieve relevant information from the internet. " \ - "If there are multiple search keywords, please use each keyword separately to call this tool." - parameters: dict = { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "search keyword", - } - }, - "required": ["query"] - } - - model_name: Literal["qwen-plus-2025-04-28", "qwq-plus-latest", "qwen-max-2025-01-25"] = \ - Field(default="qwen-plus-2025-04-28") - api_key: str = Field(default_factory=lambda: os.environ["DASHSCOPE_API_KEY"]) - stream_print: bool = Field(default=False) - temperature: float = Field(default=0.0000001) - use_role_prompt: bool = Field(default=True) - role_prompt: str = """ -# user's question -{question} - -# task -Extract the original content related to the user's question directly from the context, maintain accuracy, and avoid excessive processing. """.strip() - return_only_content: bool = Field(default=True) - - def parse_reasoning_response(self, response, result: dict): - is_answering = False - is_first_chunk = True - - for chunk in response: - if is_first_chunk: - result["search_results"] = chunk.output.search_info["search_results"] - - if self.stream_print: - print("=" * 20 + "search result" + "=" * 20) - for web in result["search_results"]: - print(f"[{web['index']}]: [{web['title']}]({web['url']})") - print("=" * 20 + "thinking process" + "=" * 20) - result["reasoning_content"] += chunk.output.choices[0].message.reasoning_content - - if self.stream_print: - print(chunk.output.choices[0].message.reasoning_content, end="", flush=True) - is_first_chunk = False - - else: - if chunk.output.choices[0].message.content == "" \ - and chunk.output.choices[0].message.reasoning_content == "": - pass - - else: - if chunk.output.choices[0].message.reasoning_content != "" and \ - chunk.output.choices[0].message.content == "": - - if self.stream_print: - print(chunk.output.choices[0].message.reasoning_content, end="", flush=True) - result["reasoning_content"] += chunk.output.choices[0].message.reasoning_content - - elif chunk.output.choices[0].message.content != "": - if not is_answering: - if self.stream_print: - print("\n" + "=" * 20 + "complete answer" + "=" * 20) - is_answering = True - - if self.stream_print: - print(chunk.output.choices[0].message.content, end="", flush=True) - result["answer_content"] += chunk.output.choices[0].message.content - - def parse_response(self, response, result: dict): - is_first_chunk = True - - for chunk in response: - if is_first_chunk: - result["search_results"] = chunk.output.search_info["search_results"] - - if self.stream_print: - print("=" * 20 + "search result" + "=" * 20) - for web in result["search_results"]: - print(f"[{web['index']}]: [{web['title']}]({web['url']})") - print("\n" + "=" * 20 + "complete answer" + "=" * 20) - is_first_chunk = False - - else: - if chunk.output.choices[0].message.content == "": - pass - - else: - if chunk.output.choices[0].message.content != "": - if self.stream_print: - print(chunk.output.choices[0].message.content, end="", flush=True) - result["answer_content"] += chunk.output.choices[0].message.content - - def execute(self, query: str = "", **kwargs): - result = { - "search_results": [], - "reasoning_content": "", - "answer_content": "" - } - user_query = self.role_prompt.format(question=query) if self.use_role_prompt else query - messages = [Message(role="user", content=user_query)] - - response = dashscope.Generation.call( - api_key=self.api_key, - model=self.model_name, - messages=messages, - enable_thinking=True, - enable_search=True, - search_options={ - "forced_search": True, - "enable_source": True, - "enable_citation": False, - "search_strategy": "pro" - }, - stream=True, - incremental_output=True, - result_format="message", - ) - - if self.model_name != "qwen-max-2025-01-25": - self.parse_reasoning_response(response, result) - else: - self.parse_response(response, result) - - if self.return_only_content: - return result["answer_content"] - else: - return result - - -def main(): - load_dotenv() - query = "What is artificial intelligence?" - - tool = DashscopeSearchTool(stream_print=True) - logger.info(tool.execute(query=query)) - - tool = DashscopeSearchTool(stream_print=False) - logger.info(tool.execute(query=query)) - - tool = DashscopeSearchTool(stream_print=True, model_name="qwen-max-2025-01-25") - logger.info(tool.execute(query=query)) - - -if __name__ == '__main__': - main() diff --git a/experiencemaker/experiencemaker/tool/mcp_tool.py b/experiencemaker/experiencemaker/tool/mcp_tool.py deleted file mode 100644 index 96c1ba9f..00000000 --- a/experiencemaker/experiencemaker/tool/mcp_tool.py +++ /dev/null @@ -1,77 +0,0 @@ -import asyncio -from typing import List - -from mcp import ClientSession -from mcp.client.sse import sse_client -from pydantic import Field, model_validator - -from experiencemaker.tool import TOOL_REGISTRY -from experiencemaker.tool.base_tool import BaseTool - - -@TOOL_REGISTRY.register() -class MCPTool(BaseTool): - server_url: str = Field(..., description="MCP server URL") - tool_name_list: List[str] = Field(default_factory=list) - cache_tools: dict = Field(default_factory=dict, alias="cache_tools") - - @model_validator(mode="after") - def refresh_tools(self): - self.refresh() - return self - - async def _get_tools(self): - async with sse_client(url=self.server_url) as streams: - async with ClientSession(streams[0], streams[1]) as session: - await session.initialize() - tools = await session.list_tools() - return tools - - def refresh(self): - self.tool_name_list.clear() - self.cache_tools.clear() - - if "sse" in self.server_url: - original_tool_list = asyncio.run(self._get_tools()) - for tool in original_tool_list.tools: - self.cache_tools[tool.name] = tool - self.tool_name_list.append(tool.name) - else: - raise NotImplementedError("Non-SSE refresh not implemented yet") - - @property - def input_schema(self) -> dict: - return {x: self.cache_tools[x].inputSchema for x in self.cache_tools} - - @property - def output_schema(self) -> dict: - raise NotImplementedError("Output schema not implemented yet") - - def get_tool_description(self, tool_name: str, schema: bool = False) -> str: - if tool_name not in self.cache_tools: - raise RuntimeError(f"Tool {tool_name} not found") - - tool = self.cache_tools.get(tool_name) - description = f"tool={tool_name} description={tool.description}\n" - if schema: - description += f"input_schema={self.input_schema[tool_name]}\n" \ - f"output_schema={self.output_schema[tool_name]}\n" - return description.strip() - - async def async_execute(self, tool_name: str, **kwargs): - if "sse" in self.server_url: - async with sse_client(url=self.server_url) as streams: - async with ClientSession(streams[0], streams[1]) as session: - await session.initialize() - results = await session.call_tool(tool_name, kwargs) - return results.content[0].text, results.isError - - else: - raise NotImplementedError("Non-SSE execute not implemented yet") - - def _execute(self, **kwargs): - return asyncio.run(self.async_execute(**kwargs)) - - def get_cache_id(self, **kwargs) -> str: - # Implement a method to generate a unique cache ID based on the input - return f"{kwargs.get('tool_name')}_{hash(frozenset(kwargs.get('args', {}).items()))}" diff --git a/experiencemaker/experiencemaker/tool/tavily_search_tool.py b/experiencemaker/experiencemaker/tool/tavily_search_tool.py deleted file mode 100644 index 9d1c564d..00000000 --- a/experiencemaker/experiencemaker/tool/tavily_search_tool.py +++ /dev/null @@ -1,109 +0,0 @@ -import json -import os -import re -import time -from typing import Literal - -from loguru import logger -from pydantic import Field, model_validator, PrivateAttr -from tavily import TavilyClient - -from experiencemaker.tool import TOOL_REGISTRY -from experiencemaker.tool.base_tool import BaseTool - - -@TOOL_REGISTRY.register() -class TavilySearchTool(BaseTool): - name: str = "web_search" - description: str = "Use query to retrieve relevant information from the internet." - parameters: dict = { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "search query", - } - }, - "required": ["query"] - } - enable_print: bool = Field(default=True) - enable_cache: bool = Field(default=False) - cache_path: str = Field(default="./web_search_cache") - topic: Literal["general", "news", "finance"] = Field(default="general", description="finance, general") - - _client: TavilyClient | None = PrivateAttr() - - @model_validator(mode="after") - def init(self): - if not os.path.exists(self.cache_path): - os.makedirs(self.cache_path) - - self._client = TavilyClient() - return self - - def load_cache(self, cache_name: str = "default") -> dict: - cache_file = os.path.join(self.cache_path, cache_name + ".jsonl") - if not os.path.exists(cache_file): - return {} - - with open(cache_file) as f: - return json.load(f) - - def dump_cache(self, cache_dict: dict, cache_name: str = "default"): - cache_file = os.path.join(self.cache_path, cache_name + ".jsonl") - with open(cache_file, "w") as f: - return json.dump(cache_dict, f, indent=2, ensure_ascii=False) - - @staticmethod - def remove_urls_and_images(text): - pattern = re.compile(r'https?://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]') - result = pattern.sub("", text) - return result - - def post_process(self, response): - if self.enable_print: - logger.info("response=\n" + json.dumps(response, indent=2, ensure_ascii=False)) - - return response - - def execute(self, query: str = "", **kwargs): - assert query, "Query cannot be empty" - - cache_dict = {} - if self.enable_cache: - cache_dict = self.load_cache() - if query in cache_dict: - return self.post_process(cache_dict[query]) - - for i in range(self.max_retries): - try: - response = self._client.search(query=query, topic=self.topic) - url_info_dict = {item["url"]: item for item in response["results"]} - response_extract = self._client.extract(urls=[item["url"] for item in response["results"]], - format="text") - - final_result = {} - for item in response_extract["results"]: - url = item["url"] - final_result[url] = url_info_dict[url] - final_result[url]["raw_content"] = item["raw_content"] - - if self.enable_cache: - cache_dict[query] = final_result - self.dump_cache(cache_dict) - - return self.post_process(final_result) - - except Exception as e: - logger.exception(f"tavily search with query={query} encounter error with e={e.args}") - time.sleep(i + 1) - - return None - - -if __name__ == "__main__": - from dotenv import load_dotenv - - load_dotenv() - tool = TavilySearchTool() - tool.execute(query="A股医药为什么一直涨") diff --git a/experiencemaker/experiencemaker/tool/terminate_tool.py b/experiencemaker/experiencemaker/tool/terminate_tool.py deleted file mode 100644 index fcb777cd..00000000 --- a/experiencemaker/experiencemaker/tool/terminate_tool.py +++ /dev/null @@ -1,23 +0,0 @@ -from experiencemaker.tool import TOOL_REGISTRY -from experiencemaker.tool.base_tool import BaseTool - - -@TOOL_REGISTRY.register() -class TerminateTool(BaseTool): - name: str = "terminate" - description: str = "If you can answer the user's question based on the context, be sure to use the **terminate** tool." - parameters: dict = { - "type": "object", - "properties": { - "status": { - "type": "string", - "description": "Please determine whether the user's question has been completed. (success / failure)", - "enum": ["success", "failure"], - } - }, - "required": ["status"], - } - - def execute(self, status: str): - self.success = status in ["success", "failure"] - return f"The interaction has been completed with status: {status}" diff --git a/experiencemaker/experiencemaker/utils/__init__.py b/experiencemaker/experiencemaker/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/experiencemaker/experiencemaker/utils/common_utils.py b/experiencemaker/experiencemaker/utils/common_utils.py deleted file mode 100644 index b78b8a1a..00000000 --- a/experiencemaker/experiencemaker/utils/common_utils.py +++ /dev/null @@ -1,17 +0,0 @@ -import re - - -def camel_to_snake(content: str) -> str: - """ - BaseWorker -> base_worker - """ - snake_str = re.sub(r'(? str: - """ - base_worker -> BaseWorker - """ - camel_str = "".join(x.capitalize() for x in content.split("_")) - return camel_str diff --git a/experiencemaker/experiencemaker/utils/file_handler.py b/experiencemaker/experiencemaker/utils/file_handler.py deleted file mode 100644 index 3e680907..00000000 --- a/experiencemaker/experiencemaker/utils/file_handler.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -from pathlib import Path - -import yaml - - -class FileHandler: - - def __init__(self, file_path: str | Path): - self.file_path: Path = Path(file_path) - suffix = Path(self.file_path).suffix - if suffix == ".json": - self._obj = json - elif suffix == ".yaml": - self._obj = yaml - else: - raise ValueError(f"unsupported file type={suffix}") - - def dump(self, config, **kwargs): - with open(self.file_path, "w") as f: - self._obj.dump(config, f, **kwargs) - - def load(self, **kwargs): - with open(self.file_path, "r") as f: - return self._obj.load(f, **kwargs) diff --git a/experiencemaker/experiencemaker/utils/http_client.py b/experiencemaker/experiencemaker/utils/http_client.py deleted file mode 100644 index 3f12e308..00000000 --- a/experiencemaker/experiencemaker/utils/http_client.py +++ /dev/null @@ -1,156 +0,0 @@ -import http -import time -from typing import Any - -import requests -from loguru import logger -from pydantic import BaseModel, Field, PrivateAttr, model_validator - -from experiencemaker.enumeration.http_enum import HttpEnum - - -class HttpClient(BaseModel): - url: str = Field(default="") - keep_alive: bool = Field(default=False, description="if true, use session to keep long connection") - timeout: int = Field(default=300, description="request timeout, second") - - return_default_if_error: bool = Field(default=True) - request_start_time: float = Field(default_factory=time.time) - request_time_cost: float = Field(default=0.0, description="request time cost") - - retry_sleep_time: float = Field(default=0.5, description="interval time for retry") - retry_time_multiplier: float = Field(default=2.0, description="retry time multiplier") - retry_max_count: int = Field(default=1, description="maximum number of retries") - - _client: Any = PrivateAttr() - - @model_validator(mode="after") - def init_client(self): - self._client = requests.Session() if self.keep_alive else requests - return self - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - self.request_time_cost: float = time.time() - self.request_start_time - - def close(self): - if isinstance(self._client, requests.Session): - self._client.close() - - def _request(self, - data: str = None, - json_data: dict = None, - headers: dict = None, - stream: bool = False, - http_enum: HttpEnum | str = HttpEnum.POST): - - if isinstance(http_enum, str): - http_enum = HttpEnum(http_enum) - - if http_enum is HttpEnum.POST: - response: requests.Response = self._client.post(url=self.url, - data=data, - json=json_data, - headers=headers, - stream=stream, - timeout=self.timeout) - - elif http_enum is HttpEnum.GET: - response: requests.Response = self._client.get(url=self.url, - data=data, - json=json_data, - headers=headers, - stream=stream, - timeout=self.timeout) - - else: - raise NotImplementedError - - if response.status_code != http.HTTPStatus.OK: - raise RuntimeError(f"request failed! content={response.json()}") - - return response - - def parse_result(self, response: requests.Response | Any = None, **kwargs): - return response.json() - - def return_default(self, **kwargs): - return None - - def request(self, - data: str | Any = None, - json_data: dict = None, - headers: dict = None, - http_enum: HttpEnum | str = HttpEnum.POST, - **kwargs): - - retry_sleep_time = self.retry_sleep_time - for i in range(self.retry_max_count): - try: - response = self._request(data=data, json_data=json_data, headers=headers, http_enum=http_enum) - result = self.parse_result(response=response, - data=data, - json_data=json_data, - headers=headers, - http_enum=http_enum, - **kwargs) - return result - - except Exception as e: - logger.exception(f"{self.__class__.__name__} {i}th request failed with args={e.args}") - - if i == self.retry_max_count - 1: - if self.return_default_if_error: - return self.return_default() - else: - raise e - - retry_sleep_time *= self.retry_time_multiplier - time.sleep(retry_sleep_time) - - return None - - def request_stream(self, - data: str = None, - json_data: dict = None, - headers: dict = None, - http_enum: HttpEnum | str = HttpEnum.POST, - **kwargs): - - retry_sleep_time = self.retry_sleep_time - for i in range(self.retry_max_count): - try: - response = self._request(data=data, - json_data=json_data, - headers=headers, - stream=True, - http_enum=http_enum) - request_context = {} - for iter_idx, line in enumerate(response.iter_lines()): - yield self.parse_result(line=line, - request_context=request_context, - index=iter_idx, - data=data, - json_data=json_data, - headers=headers, - http_enum=http_enum, - **kwargs) - - return None - - except Exception as e: - logger.exception(f"{self.__class__.__name__} {i}th request failed with args={e.args}") - - if i == self.retry_max_count - 1: - if self.return_default_if_error: - return self.return_default() - else: - raise e - - retry_sleep_time *= self.retry_time_multiplier - time.sleep(retry_sleep_time) - - return None diff --git a/experiencemaker/experiencemaker/utils/op_utils.py b/experiencemaker/experiencemaker/utils/op_utils.py deleted file mode 100644 index a6474225..00000000 --- a/experiencemaker/experiencemaker/utils/op_utils.py +++ /dev/null @@ -1,102 +0,0 @@ -from typing import List - -from experiencemaker.enumeration.role import Role -from experiencemaker.schema.message import Message, Trajectory -import json -import re -from loguru import logger - -def merge_messages_content(messages: List[Message | dict]) -> str: - content_collector = [] - for i, message in enumerate(messages): - if isinstance(message, dict): - message = Message(**message) - - if message.role is Role.ASSISTANT: - line = f"### step.{i} role={message.role.value} content=\n{message.reasoning_content}\n\n{message.content}\n" - if message.tool_calls: - for tool_call in message.tool_calls: - line += f" - tool call={tool_call.name}\n params={tool_call.arguments}\n" - content_collector.append(line) - - elif message.role is Role.USER: - line = f"### step.{i} role={message.role.value} content=\n{message.content}\n" - content_collector.append(line) - - elif message.role is Role.TOOL: - line = f"### step.{i} role={message.role.value} tool call result=\n{message.content}\n" - content_collector.append(line) - - return "\n".join(content_collector) - - -def parse_json_experience_response(response: str) -> List[dict]: - """Parse JSON formatted experience response""" - try: - # Extract JSON blocks - json_pattern = r'```json\s*([\s\S]*?)\s*```' - json_blocks = re.findall(json_pattern, response) - - if json_blocks: - parsed = json.loads(json_blocks[0]) - - # Handle array format - if isinstance(parsed, list): - experiences = [] - for exp_data in parsed: - if isinstance(exp_data, dict) and ( - ("when_to_use" in exp_data and "experience" in exp_data) or - ("condition" in exp_data and "experience" in exp_data) - ): - experiences.append(exp_data) - - return experiences - - - # Handle single object - elif isinstance(parsed, dict) and ( - ("when_to_use" in parsed and "experience" in parsed) or - ("condition" in parsed and "experience" in parsed) - ): - return [parsed] - - # Fallback: try to parse entire response - parsed = json.loads(response) - if isinstance(parsed, list): - return parsed - elif isinstance(parsed, dict): - return [parsed] - - except json.JSONDecodeError as e: - logger.warning(f"Failed to parse JSON experience response: {e}") - - return [] - -def get_trajectory_context(trajectory: Trajectory, step_sequence: List[Message]) -> str: - """Get context of step sequence within trajectory""" - try: - # Find position of step sequence in trajectory - start_idx = 0 - for i, step in enumerate(trajectory.messages): - if step == step_sequence[0]: - start_idx = i - break - - # Extract before and after context - context_before = trajectory.messages[max(0, start_idx - 2):start_idx] - context_after = trajectory.messages[start_idx + len(step_sequence):start_idx + len(step_sequence) + 2] - - context = f"Query: {trajectory.metadata.get('query', 'N/A')}\n" - - if context_before: - context += "Previous steps:\n" + "\n".join( - [f"- {step.content[:100]}..." for step in context_before]) + "\n" - - if context_after: - context += "Following steps:\n" + "\n".join([f"- {step.content[:100]}..." for step in context_after]) - - return context - - except Exception as e: - logger.error(f"Error getting trajectory context: {e}") - return f"Query: {trajectory.metadata.get('query', 'N/A')}" \ No newline at end of file diff --git a/experiencemaker/experiencemaker/utils/registry.py b/experiencemaker/experiencemaker/utils/registry.py deleted file mode 100644 index b5f207dc..00000000 --- a/experiencemaker/experiencemaker/utils/registry.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import List - -from loguru import logger - -from experiencemaker.utils.common_utils import camel_to_snake - - -class Registry(object): - def __init__(self): - self._registry = {} - - def register(self, name: str = ""): - - def decorator(cls): - class_name = name if name else camel_to_snake(cls.__name__) - if class_name in self._registry: - logger.warning(f"name={class_name} is already registered, will be overwritten.") - self._registry[class_name] = cls - return cls - - return decorator - - def __getitem__(self, name: str): - if name not in self._registry: - raise KeyError(f"name={name} is not registered!") - return self._registry[name] - - def __contains__(self, name: str): - return name in self._registry - - @property - def registered_names(self) -> List[str]: - return sorted(self._registry.keys()) diff --git a/experiencemaker/experiencemaker/utils/singleton.py b/experiencemaker/experiencemaker/utils/singleton.py deleted file mode 100644 index df7a1b33..00000000 --- a/experiencemaker/experiencemaker/utils/singleton.py +++ /dev/null @@ -1,9 +0,0 @@ -def singleton(cls): - _instance = {} - - def _singleton(*args, **kwargs): - if cls not in _instance: - _instance[cls] = cls(*args, **kwargs) - return _instance[cls] - - return _singleton diff --git a/experiencemaker/experiencemaker/utils/timer.py b/experiencemaker/experiencemaker/utils/timer.py deleted file mode 100644 index 74f8a3f9..00000000 --- a/experiencemaker/experiencemaker/utils/timer.py +++ /dev/null @@ -1,53 +0,0 @@ -import time - -from loguru import logger - - -class Timer(object): - def __init__(self, name: str, use_ms: bool = False, stack_level: int = 2): - self.name: str = name - self.use_ms: bool = use_ms - self.stack_level: int = stack_level - - self.time_start: float = 0 - self.time_end: float = 0 - self.time_cost: float = 0 - - def __enter__(self, *args, **kwargs): - self.time_start = time.time() - logger.info(f"---------- enter {self.name} ----------", stacklevel=self.stack_level) - return self - - def __exit__(self, *args): - self.time_end = time.time() - self.time_cost = self.time_end - self.time_start - if self.use_ms: - time_str = f"{self.time_cost * 1000:.2f}ms" - else: - time_str = f"{self.time_cost:.3f}s" - - logger.info(f"---------- leave {self.name} [{time_str}] ----------", stacklevel=self.stack_level) - - -def timer(name: str = None, use_ms: bool = False, stack_level: int = 2): - def decorator(func): - def wrapper(*args, **kwargs): - with Timer(name=name or func.__name__, use_ms=use_ms, stack_level=stack_level + 1): - return func(*args, **kwargs) - - return wrapper - - return decorator - - -if __name__ == "__main__": - import random - - - @timer("run_func_final", use_ms=True) - def run_func(): - time.sleep(random.uniform(0.05, 0.15)) - print("done") - - - run_func() diff --git a/experiencemaker/experiencemaker/vector_store/__init__.py b/experiencemaker/experiencemaker/vector_store/__init__.py deleted file mode 100644 index 034285dc..00000000 --- a/experiencemaker/experiencemaker/vector_store/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from experiencemaker.utils.registry import Registry - -VECTOR_STORE_REGISTRY = Registry() - -from experiencemaker.vector_store.es_vector_store import EsVectorStore -from experiencemaker.vector_store.chroma_vector_store import ChromaVectorStore -from experiencemaker.vector_store.file_vector_store import FileVectorStore diff --git a/experiencemaker/experiencemaker/vector_store/base_vector_store.py b/experiencemaker/experiencemaker/vector_store/base_vector_store.py deleted file mode 100644 index cdf32126..00000000 --- a/experiencemaker/experiencemaker/vector_store/base_vector_store.py +++ /dev/null @@ -1,144 +0,0 @@ -import fcntl -import json -from abc import ABC -from pathlib import Path -from typing import List, Iterable - -from loguru import logger -from pydantic import BaseModel, Field -from tqdm import tqdm - -from experiencemaker.embedding_model.base_embedding_model import BaseEmbeddingModel -from experiencemaker.schema.vector_node import VectorNode - - -class BaseVectorStore(BaseModel, ABC): - embedding_model: BaseEmbeddingModel | None = Field(default=None) - batch_size: int = Field(default=1024) - - @staticmethod - def _load_from_path(workspace_id: str, path: str | Path, callback_fn=None, **kwargs) -> Iterable[VectorNode]: - workspace_path = Path(path) / f"{workspace_id}.jsonl" - if not workspace_path.exists(): - logger.warning(f"workspace_path={workspace_path} is not exists!") - return - - with workspace_path.open() as f: - fcntl.flock(f, fcntl.LOCK_SH) - try: - for line in tqdm(f, desc="load from path"): - if line.strip(): - node_dict = json.loads(line.strip()) - if callback_fn: - node = callback_fn(node_dict) - else: - node = VectorNode(**node_dict, **kwargs) - node.workspace_id = workspace_id - yield node - - finally: - fcntl.flock(f, fcntl.LOCK_UN) - - @staticmethod - def _dump_to_path(nodes: Iterable[VectorNode], workspace_id: str, path: str | Path = "", callback_fn=None, - ensure_ascii: bool = False, **kwargs): - dump_path: Path = Path(path) - dump_path.mkdir(parents=True, exist_ok=True) - dump_file = dump_path / f"{workspace_id}.jsonl" - - count = 0 - with dump_file.open("w") as f: - fcntl.flock(f, fcntl.LOCK_EX) - try: - for node in tqdm(nodes, desc="dump to path"): - node.workspace_id = workspace_id - if callback_fn: - node_dict = callback_fn(node) - else: - node_dict = node.model_dump() - assert isinstance(node_dict, dict) - f.write(json.dumps(node_dict, ensure_ascii=ensure_ascii, **kwargs)) - f.write("\n") - count += 1 - - return {"size": count} - finally: - fcntl.flock(f, fcntl.LOCK_UN) - - def exist_workspace(self, workspace_id: str, **kwargs) -> bool: - raise NotImplementedError - - def delete_workspace(self, workspace_id: str, **kwargs): - raise NotImplementedError - - def create_workspace(self, workspace_id: str, **kwargs): - raise NotImplementedError - - def _iter_workspace_nodes(self, workspace_id: str, **kwargs) -> Iterable[VectorNode]: - raise NotImplementedError - - def dump_workspace(self, workspace_id: str, path: str | Path = "", callback_fn=None, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id, **kwargs): - logger.warning(f"workspace_id={workspace_id} is not exist!") - return {} - - return self._dump_to_path(nodes=self._iter_workspace_nodes(workspace_id=workspace_id, **kwargs), - workspace_id=workspace_id, - path=path, - callback_fn=callback_fn, - **kwargs) - - def load_workspace(self, workspace_id: str, path: str | Path = "", nodes: List[VectorNode] = None, callback_fn=None, - **kwargs): - if self.exist_workspace(workspace_id, **kwargs): - self.delete_workspace(workspace_id=workspace_id, **kwargs) - logger.info(f"delete workspace_id={workspace_id}") - - self.create_workspace(workspace_id=workspace_id, **kwargs) - - all_nodes: List[VectorNode] = [] - if nodes: - all_nodes.extend(nodes) - for node in self._load_from_path(path=path, workspace_id=workspace_id, callback_fn=callback_fn, **kwargs): - all_nodes.append(node) - self.insert(nodes=all_nodes, workspace_id=workspace_id, **kwargs) - return {"size": len(all_nodes)} - - def copy_workspace(self, src_workspace_id: str, dest_workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=src_workspace_id, **kwargs): - logger.warning(f"src_workspace_id={src_workspace_id} is not exist!") - return {} - - if not self.exist_workspace(dest_workspace_id, **kwargs): - self.create_workspace(workspace_id=dest_workspace_id, **kwargs) - - nodes = [] - node_size = 0 - for node in self._iter_workspace_nodes(workspace_id=src_workspace_id, **kwargs): - nodes.append(node) - node_size += 1 - if len(nodes) >= self.batch_size: - self.insert(nodes=nodes, workspace_id=dest_workspace_id, **kwargs) - nodes.clear() - - if nodes: - self.insert(nodes=nodes, workspace_id=dest_workspace_id, **kwargs) - return {"size": node_size} - - def search(self, query: str, workspace_id: str, top_k: int = 1, **kwargs) -> List[VectorNode]: - raise NotImplementedError - - def insert(self, nodes: VectorNode | List[VectorNode], workspace_id: str, **kwargs): - raise NotImplementedError - - def delete(self, node_ids: str | List[str], workspace_id: str, **kwargs): - raise NotImplementedError - - def update_freq(self, node_ids: str | List[str], workspace_id: str, **kwargs): - raise NotImplementedError - - def update_utility(self, node_ids: str | List[str], workspace_id: str, **kwargs): - raise NotImplementedError - - def utility_based_delete(self, workspace_id: str, freq_threshold: int, utility_threshold: float, **kwargs): - raise NotImplementedError \ No newline at end of file diff --git a/experiencemaker/experiencemaker/vector_store/chroma_vector_store.py b/experiencemaker/experiencemaker/vector_store/chroma_vector_store.py deleted file mode 100644 index cb493451..00000000 --- a/experiencemaker/experiencemaker/vector_store/chroma_vector_store.py +++ /dev/null @@ -1,188 +0,0 @@ -from typing import List, Iterable - -import chromadb -from chromadb import Collection -from chromadb.config import Settings -from loguru import logger -from pydantic import Field, PrivateAttr, model_validator - -from experiencemaker.embedding_model.openai_compatible_embedding_model import OpenAICompatibleEmbeddingModel -from experiencemaker.schema.vector_node import VectorNode -from experiencemaker.vector_store import VECTOR_STORE_REGISTRY -from experiencemaker.vector_store.base_vector_store import BaseVectorStore - - -@VECTOR_STORE_REGISTRY.register("chroma") -class ChromaVectorStore(BaseVectorStore): - store_dir: str = Field(default="./chroma_vector_store") - collections: dict = Field(default_factory=dict) - _client: chromadb.Client = PrivateAttr() - - @model_validator(mode="after") - def init_client(self): - self._client = chromadb.Client(Settings(persist_directory=self.store_dir)) - return self - - def _get_collection(self, workspace_id: str) -> Collection: - if workspace_id not in self.collections: - self.collections[workspace_id] = self._client.get_or_create_collection(workspace_id) - return self.collections[workspace_id] - - def exist_workspace(self, workspace_id: str, **kwargs) -> bool: - return workspace_id in [c.name for c in self._client.list_collections()] - - def delete_workspace(self, workspace_id: str, **kwargs): - self._client.delete_collection(workspace_id) - if workspace_id in self.collections: - del self.collections[workspace_id] - - def create_workspace(self, workspace_id: str, **kwargs): - self.collections[workspace_id] = self._client.get_or_create_collection(workspace_id) - - def _iter_workspace_nodes(self, workspace_id: str, **kwargs) -> Iterable[VectorNode]: - collection: Collection = self._get_collection(workspace_id) - results = collection.get() - for i in range(len(results["ids"])): - node = VectorNode(workspace_id=workspace_id, - unique_id=results["ids"][i], - content=results["documents"][i], - metadata=results["metadatas"][i]) - yield node - - def search(self, query: str, workspace_id: str, top_k: int = 1, **kwargs) -> List[VectorNode]: - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return [] - - collection: Collection = self._get_collection(workspace_id) - query_vector = self.embedding_model.get_embeddings(query) - results = collection.query(query_embeddings=[query_vector], n_results=top_k) - nodes = [] - for i in range(len(results["ids"][0])): - node = VectorNode(workspace_id=workspace_id, - unique_id=results["ids"][0][i], - content=results["documents"][0][i], - metadata=results["metadatas"][0][i]) - nodes.append(node) - return nodes - - def insert(self, nodes: VectorNode | List[VectorNode], workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - self.create_workspace(workspace_id=workspace_id) - - if isinstance(nodes, VectorNode): - nodes = [nodes] - - embedded_nodes = [node for node in nodes if node.vector] - not_embedded_nodes = [node for node in nodes if not node.vector] - now_embedded_nodes = self.embedding_model.get_node_embeddings(not_embedded_nodes) - all_nodes = embedded_nodes + now_embedded_nodes - - collection: Collection = self._get_collection(workspace_id) - collection.add(ids=[n.unique_id for n in all_nodes], - embeddings=[n.vector for n in all_nodes], - documents=[n.content for n in all_nodes], - metadatas=[n.metadata for n in all_nodes]) - - def delete(self, node_ids: str | List[str], workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - collection: Collection = self._get_collection(workspace_id) - collection.delete(ids=node_ids) - - -def main(): - from dotenv import load_dotenv - load_dotenv() - - embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - workspace_id = "chroma_test_index" - - chroma_store = ChromaVectorStore( - embedding_model=embedding_model, - store_dir="./chroma_test_db" - ) - - if chroma_store.exist_workspace(workspace_id): - chroma_store.delete_workspace(workspace_id) - chroma_store.create_workspace(workspace_id) - - sample_nodes = [ - VectorNode( - unique_id="node1", - workspace_id=workspace_id, - content="Artificial intelligence is a technology that simulates human intelligence.", - metadata={ - "node_type": "n1", - "category": "tech" - } - ), - VectorNode( - unique_id="node2", - workspace_id=workspace_id, - content="AI is the future of mankind.", - metadata={ - "node_type": "n1", - "category": "tech" - } - ), - VectorNode( - unique_id="node3", - workspace_id=workspace_id, - content="I want to eat fish!", - metadata={ - "node_type": "n2", - "category": "food" - } - ), - VectorNode( - unique_id="node4", - workspace_id=workspace_id, - content="The bigger the storm, the more expensive the fish.", - metadata={ - "node_type": "n1", - "category": "food" - } - ), - ] - - chroma_store.insert(sample_nodes, workspace_id=workspace_id) - - logger.info("=" * 20) - results = chroma_store.search("What is AI?", top_k=5, workspace_id=workspace_id) - for r in results: - logger.info(r.model_dump(exclude={"vector"})) - logger.info("=" * 20) - - node2_update = VectorNode( - unique_id="node2", - workspace_id=workspace_id, - content="AI is the future of humanity and technology.", - metadata={ - "node_type": "n1", - "category": "tech", - "updated": True - } - ) - chroma_store.delete(node2_update.unique_id, workspace_id=workspace_id) - chroma_store.insert(node2_update, workspace_id=workspace_id) - - logger.info("Updated Result:") - results = chroma_store.search("fish?", top_k=10, workspace_id=workspace_id) - for r in results: - logger.info(r.model_dump(exclude={"vector"})) - logger.info("=" * 20) - - chroma_store.dump_workspace(workspace_id=workspace_id) - - chroma_store.delete_workspace(workspace_id=workspace_id) - - -if __name__ == "__main__": - main() - # launch with: python -m experiencemaker.storage.chroma_vector_store diff --git a/experiencemaker/experiencemaker/vector_store/es_vector_store.py b/experiencemaker/experiencemaker/vector_store/es_vector_store.py deleted file mode 100644 index 2a1b7ff8..00000000 --- a/experiencemaker/experiencemaker/vector_store/es_vector_store.py +++ /dev/null @@ -1,329 +0,0 @@ -import os -from typing import List, Tuple, Iterable - -from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk -from elasticsearch import NotFoundError -from loguru import logger -from pydantic import Field, PrivateAttr, model_validator - -from experiencemaker.embedding_model.openai_compatible_embedding_model import OpenAICompatibleEmbeddingModel -from experiencemaker.schema.vector_node import VectorNode -from experiencemaker.vector_store import VECTOR_STORE_REGISTRY -from experiencemaker.vector_store.base_vector_store import BaseVectorStore - - -@VECTOR_STORE_REGISTRY.register("elasticsearch") -class EsVectorStore(BaseVectorStore): - hosts: str | List[str] = Field(default_factory=lambda: os.getenv("ES_HOSTS", "http://localhost:9200")) - basic_auth: str | Tuple[str, str] | None = Field(default=None) - retrieve_filters: List[dict] = [] - _client: Elasticsearch = PrivateAttr() - - @model_validator(mode="after") - def init_client(self): - if isinstance(self.hosts, str): - self.hosts = [self.hosts] - self._client = Elasticsearch(hosts=self.hosts, basic_auth=self.basic_auth) - return self - - def exist_workspace(self, workspace_id: str, **kwargs) -> bool: - return self._client.indices.exists(index=workspace_id) - - def delete_workspace(self, workspace_id: str, **kwargs): - return self._client.indices.delete(index=workspace_id, **kwargs) - - def create_workspace(self, workspace_id: str, **kwargs): - body = { - "mappings": { - "properties": { - "workspace_id": {"type": "keyword"}, - "content": {"type": "text"}, - "metadata": {"type": "object"}, - "vector": { - "type": "dense_vector", - "dims": self.embedding_model.dimensions - } - } - } - } - return self._client.indices.create(index=workspace_id, body=body) - - def _iter_workspace_nodes(self, workspace_id: str, max_size: int = 10000, **kwargs) -> Iterable[VectorNode]: - response = self._client.search(index=workspace_id, body={"query": {"match_all": {}}, "size": max_size}) - for doc in response['hits']['hits']: - yield self.doc2node(doc, workspace_id) - - def refresh(self, workspace_id: str): - self._client.indices.refresh(index=workspace_id) - - @staticmethod - def doc2node(doc, workspace_id: str) -> VectorNode: - node = VectorNode(**doc["_source"]) - node.workspace_id = workspace_id - node.unique_id = doc["_id"] - if "_score" in doc: - node.metadata["_score"] = doc["_score"] - 1 - return node - - def add_term_filter(self, key: str, value): - if key: - self.retrieve_filters.append({"term": {key: value}}) - return self - - def add_range_filter(self, key: str, gte=None, lte=None): - if key: - if gte is not None and lte is not None: - self.retrieve_filters.append({"range": {key: {"gte": gte, "lte": lte}}}) - elif gte is not None: - self.retrieve_filters.append({"range": {key: {"gte": gte}}}) - elif lte is not None: - self.retrieve_filters.append({"range": {key: {"lte": lte}}}) - return self - - def clear_filter(self): - self.retrieve_filters.clear() - return self - - def search(self, query: str, workspace_id: str, top_k: int = 1, **kwargs) -> List[VectorNode]: - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return [] - - query_vector = self.embedding_model.get_embeddings(query) - body = { - "query": { - "script_score": { - "query": {"bool": {"must": self.retrieve_filters}}, - "script": { - "source": "cosineSimilarity(params.query_vector, 'vector') + 1.0", - "params": {"query_vector": query_vector}, - } - } - }, - "size": top_k - } - response = self._client.search(index=workspace_id, body=body, **kwargs) - - nodes: List[VectorNode] = [] - for doc in response['hits']['hits']: - nodes.append(self.doc2node(doc, workspace_id)) - - self.retrieve_filters.clear() - return nodes - - def insert(self, nodes: VectorNode | List[VectorNode], workspace_id: str, refresh: bool = False, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - self.create_workspace(workspace_id=workspace_id) - - if isinstance(nodes, VectorNode): - nodes = [nodes] - - embedded_nodes = [node for node in nodes if node.vector] - not_embedded_nodes = [node for node in nodes if not node.vector] - now_embedded_nodes = self.embedding_model.get_node_embeddings(not_embedded_nodes) - - docs = [ - { - "_op_type": "index", - "_index": workspace_id, - "_id": node.unique_id, - "_source": { - "workspace_id": workspace_id, - "content": node.content, - "metadata": node.metadata, - "vector": node.vector, - "freq": node.freq, - "utility": node.utility - } - } for node in embedded_nodes + now_embedded_nodes] - status, error = bulk(self._client, docs, chunk_size=self.batch_size, **kwargs) - logger.info(f"insert docs.size={len(docs)} status={status} error={error}") - - if refresh: - self.refresh(workspace_id=workspace_id) - - def delete(self, node_ids: str | List[str], workspace_id: str, refresh: bool = False, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - actions = [ - { - "_op_type": "delete", - "_index": workspace_id, - "_id": node_id - } for node_id in node_ids] - status, error = bulk(self._client, actions, chunk_size=self.batch_size, **kwargs) - logger.info(f"delete actions.size={len(actions)} status={status} error={error}") - - if refresh: - self.refresh(workspace_id=workspace_id) - - def update_freq(self, node_ids: str | List[str], workspace_id: str, refresh: bool = False, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - actions = [ - { - "_op_type": "update", - "_index": workspace_id, - "_id": node_id, - "script": { - "source": "ctx._source.freq += 1", - "lang": "painless" - } - } for node_id in node_ids - ] - status, error = bulk(self._client, actions, chunk_size=self.batch_size, **kwargs) - logger.info(f"update exp.size={len(node_ids)} status={status} error={error}") - - if refresh: - self.refresh(workspace_id=workspace_id) - - - def update_utility(self, node_ids: str | List[str], workspace_id: str, refresh: bool = False, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - existing_nodes = [] - for node_id in node_ids: - try: - self._client.get(index=workspace_id, id=node_id) - existing_nodes.append(node_id) - except NotFoundError: - logger.warning(f"Experience_id={node_id} not found in workspace_id={workspace_id}") - - actions = [ - { - "_op_type": "update", - "_index": workspace_id, - "_id": node_id, - "script": { - "source": "ctx._source.utility += 1", - "lang": "painless" - } - } for node_id in existing_nodes - ] - status, error = bulk(self._client, actions, chunk_size=self.batch_size, **kwargs) - logger.info(f"when updating utility, exp.size={len(node_ids)}, status={status}, error={error}") - - if refresh: - self.refresh(workspace_id=workspace_id) - - def utility_based_delete(self, workspace_id: str, freq_threshold: int, utility_threshold: float, refresh: bool = False, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - query = { - "query": { - "bool": { - "must": [{ - "script": { - "script": { - "source": "doc.freq.value > params.freq_threshold && (doc.utility.value * 1.0 / doc.freq.value) < params.utility_threshold", - "params": { - "freq_threshold": freq_threshold, - "utility_threshold": utility_threshold - } - } - } - }] - } - } - } - - # response = self._client.search(index=workspace_id, body=query, *kwargs) - # delete_node_ids = [] - # for doc in response['hits']['hits']: - # delete_node_ids.append(doc["_id"]) - from elasticsearch.helpers import scan - results = scan(self._client, index=workspace_id, query=query, _source=False) - delete_node_ids = [hit['_id'] for hit in results] - - if delete_node_ids: - logger.info(f"Found {len(delete_node_ids)} nodes to delete with freq>={freq_threshold} and utility/freq<{utility_threshold}") - self.delete(node_ids=delete_node_ids, workspace_id=workspace_id, refresh=refresh, **kwargs) - else: - logger.info("No nodes found matching the delete criteria") - - if refresh: - self.refresh(workspace_id=workspace_id) - - -def main(): - from dotenv import load_dotenv - load_dotenv() - - embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - workspace_id = "rag_nodes_index" - hosts = "http://11.160.132.46:8200" - es = EsVectorStore(hosts=hosts, embedding_model=embedding_model) - if es.exist_workspace(workspace_id=workspace_id): - es.delete_workspace(workspace_id=workspace_id) - es.create_workspace(workspace_id=workspace_id) - - sample_nodes = [ - VectorNode( - workspace_id=workspace_id, - content="Artificial intelligence is a technology that simulates human intelligence.", - metadata={ - "node_type": "n1", - } - ), - VectorNode( - workspace_id=workspace_id, - content="AI is the future of mankind.", - metadata={ - "node_type": "n1", - } - ), - VectorNode( - workspace_id=workspace_id, - content="I want to eat fish!", - metadata={ - "node_type": "n2", - } - ), - VectorNode( - workspace_id=workspace_id, - content="The bigger the storm, the more expensive the fish.", - metadata={ - "node_type": "n1", - } - ), - ] - - es.insert(sample_nodes, workspace_id=workspace_id, refresh=True) - - logger.info("=" * 20) - results = es.add_term_filter(key="metadata.node_type", value="n1") \ - .search("What is AI?", top_k=5, workspace_id=workspace_id) - for r in results: - logger.info(r.model_dump(exclude={"vector"})) - logger.info("=" * 20) - - logger.info("=" * 20) - results = es.search("What is AI?", top_k=5, workspace_id=workspace_id) - for r in results: - logger.info(r.model_dump(exclude={"vector"})) - logger.info("=" * 20) - es.dump_workspace(workspace_id=workspace_id) - es.delete_workspace(workspace_id=workspace_id) - - -if __name__ == "__main__": - main() - # launch with: python -m experiencemaker.storage.es_vector_store diff --git a/experiencemaker/experiencemaker/vector_store/file_vector_store.py b/experiencemaker/experiencemaker/vector_store/file_vector_store.py deleted file mode 100644 index 0d9af9f7..00000000 --- a/experiencemaker/experiencemaker/vector_store/file_vector_store.py +++ /dev/null @@ -1,222 +0,0 @@ -import math -from pathlib import Path -from typing import List, Iterable - -from loguru import logger -from pydantic import Field, model_validator - -from experiencemaker.embedding_model.openai_compatible_embedding_model import OpenAICompatibleEmbeddingModel -from experiencemaker.schema.vector_node import VectorNode -from experiencemaker.vector_store import VECTOR_STORE_REGISTRY -from experiencemaker.vector_store.base_vector_store import BaseVectorStore - - -@VECTOR_STORE_REGISTRY.register("local_file") -class FileVectorStore(BaseVectorStore): - store_dir: str = Field(default="./file_vector_store") - - @model_validator(mode="after") - def init_client(self): - store_path = Path(self.store_dir) - store_path.mkdir(parents=True, exist_ok=True) - return self - - @property - def store_path(self) -> Path: - return Path(self.store_dir) - - def exist_workspace(self, workspace_id: str, **kwargs) -> bool: - workspace_path = self.store_path / f"{workspace_id}.jsonl" - return workspace_path.exists() - - def delete_workspace(self, workspace_id: str, **kwargs): - workspace_path = self.store_path / f"{workspace_id}.jsonl" - if workspace_path.is_file(): - workspace_path.unlink() - - def create_workspace(self, workspace_id: str, **kwargs): - self._dump_to_path(nodes=[], workspace_id=workspace_id, path=self.store_path, **kwargs) - - def _iter_workspace_nodes(self, workspace_id: str, **kwargs) -> Iterable[VectorNode]: - for i, node in enumerate(self._load_from_path(path=self.store_path, workspace_id=workspace_id, **kwargs)): - yield node - - @staticmethod - def calculate_similarity(query_vector: List[float], node_vector: List[float]): - assert query_vector, f"query_vector is empty!" - assert node_vector, f"node_vector is empty!" - assert len(query_vector) == len(node_vector), \ - f"query_vector.size={len(query_vector)} node_vector.size={len(node_vector)}" - - dot_product = sum(x * y for x, y in zip(query_vector, node_vector)) - norm_v1 = math.sqrt(sum(x ** 2 for x in query_vector)) - norm_v2 = math.sqrt(sum(y ** 2 for y in node_vector)) - return dot_product / (norm_v1 * norm_v2) - - def search(self, query: str, workspace_id: str, top_k: int = 1, **kwargs) -> List[VectorNode]: - query_vector = self.embedding_model.get_embeddings(query) - nodes: List[VectorNode] = [] - for node in self._load_from_path(path=self.store_path, workspace_id=workspace_id, **kwargs): - node.metadata["score"] = self.calculate_similarity(query_vector, node.vector) - nodes.append(node) - - nodes = sorted(nodes, key=lambda x: x.metadata["score"], reverse=True) - return nodes[:top_k] - - def insert(self, nodes: VectorNode | List[VectorNode], workspace_id: str, **kwargs): - if isinstance(nodes, VectorNode): - nodes = [nodes] - - all_node_dict = {} - nodes: List[VectorNode] = self.embedding_model.get_node_embeddings(nodes) - exist_nodes: List[VectorNode] = list(self._load_from_path(path=self.store_path, workspace_id=workspace_id)) - for node in exist_nodes: - all_node_dict[node.unique_id] = node - - update_cnt = 0 - for node in nodes: - if node.unique_id in all_node_dict: - update_cnt += 1 - - all_node_dict[node.unique_id] = node - - self._dump_to_path(nodes=list(all_node_dict.values()), - workspace_id=workspace_id, - path=self.store_path, - **kwargs) - - logger.info(f"update workspace_id={workspace_id} nodes.size={len(nodes)} all.size={len(all_node_dict)} " - f"update_cnt={update_cnt}") - - def delete(self, node_ids: str | List[str], workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - all_nodes: List[VectorNode] = list(self._load_from_path(path=self.store_path, workspace_id=workspace_id)) - before_size = len(all_nodes) - all_nodes = [n for n in all_nodes if n.unique_id not in node_ids] - after_size = len(all_nodes) - - self._dump_to_path(nodes=all_nodes, workspace_id=workspace_id, path=self.store_path, **kwargs) - logger.info(f"delete workspace_id={workspace_id} before_size={before_size} after_size={after_size}") - - def update_freq(self, node_ids: str | List[str], workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - all_nodes: List[VectorNode] = list(self._load_from_path(path=self.store_path, workspace_id=workspace_id)) - all_new_nodes = [] - freq_counter = [] - for n in all_nodes: - if n.unique_id in node_ids: - n.freq += 1 - - if n.freq not in freq_counter: - freq_counter[n.freq] = 0 - freq_counter[n.freq] += 1 - - all_new_nodes.append(n) - - self._dump_to_path(nodes=all_new_nodes, workspace_id=workspace_id, path=self.store_path, **kwargs) - logger.info(f"update workspace_id={workspace_id} update_cnt={len(node_ids)}") - - return freq_counter - - def update_utility(self, node_ids: str | List[str], workspace_id: str, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - if isinstance(node_ids, str): - node_ids = [node_ids] - - all_nodes: List[VectorNode] = list(self._load_from_path(path=self.store_path, workspace_id=workspace_id)) - all_new_nodes = [] - for n in all_nodes: - if n.unique_id in node_ids: - n.utility += 1 - all_new_nodes.append(n) - - self._dump_to_path(nodes=all_new_nodes, workspace_id=workspace_id, path=self.store_path, **kwargs) - logger.info(f"update workspace_id={workspace_id} update_utility_cnt={len(node_ids)}") - - def utility_based_delete(self, workspace_id: str, freq_threshold: int, utility_threshold: float, **kwargs): - if not self.exist_workspace(workspace_id=workspace_id): - logger.warning(f"workspace_id={workspace_id} is not exists!") - return - - all_nodes: List[VectorNode] = list(self._load_from_path(path=self.store_path, workspace_id=workspace_id)) - delete_node_ids = [] - for n in all_nodes: - if n.freq >= freq_threshold: - if n.utility*1.0/n.freq < utility_threshold: - delete_node_ids.append(n.unique_id) - - logger.info(f"delete when freq>={freq_threshold} and utility/freq<{utility_threshold}") - self.delete(node_ids=delete_node_ids, workspace_id=workspace_id) - - -def main(): - from dotenv import load_dotenv - load_dotenv() - - embedding_model = OpenAICompatibleEmbeddingModel(dimensions=64, model_name="text-embedding-v4") - workspace_id = "rag_nodes_index" - client = FileVectorStore(embedding_model=embedding_model) - client.delete_workspace(workspace_id) - client.create_workspace(workspace_id) - - sample_nodes = [ - VectorNode( - workspace_id=workspace_id, - content="Artificial intelligence is a technology that simulates human intelligence.", - metadata={ - "node_type": "n1", - } - ), - VectorNode( - workspace_id=workspace_id, - content="AI is the future of mankind.", - metadata={ - "node_type": "n1", - } - ), - VectorNode( - workspace_id=workspace_id, - content="I want to eat fish!", - metadata={ - "node_type": "n2", - } - ), - VectorNode( - workspace_id=workspace_id, - content="The bigger the storm, the more expensive the fish.", - metadata={ - "node_type": "n1", - } - ), - ] - - client.insert(sample_nodes, workspace_id) - - logger.info("=" * 20) - results = client.search("What is AI?", workspace_id=workspace_id, top_k=5) - for r in results: - logger.info(r.model_dump(exclude={"vector"})) - logger.info("=" * 20) - client.dump_workspace(workspace_id) - - client.delete_workspace(workspace_id) - - -if __name__ == "__main__": - main() - # launch with: python -m experiencemaker.storage.file_vector_store diff --git a/experiencemaker/pyproject.toml b/experiencemaker/pyproject.toml deleted file mode 100644 index c4ec1be8..00000000 --- a/experiencemaker/pyproject.toml +++ /dev/null @@ -1,49 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "ExperienceMaker" -version = "0.1.1" -description = "make experience from trajectory" -authors = [{ name = "experiencemaker team", email = "experiencemaker@alibaba-inc.com" }] -license = { file = "LICENSE" } -readme = "README.md" -requires-python = ">=3.12" - -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", -] - -dependencies = [ - "dashscope>=1.19.1", - "elasticsearch>=8.14.0", - "fastapi>=0.115.13", - "fastmcp>=2.10.6", - "loguru>=0.7.3", - "mcp>=1.9.4", - "numpy>=2.3.0", - "openai>=1.88.0", - "pydantic>=2.11.7", - "PyYAML>=6.0.2", - "Requests>=2.32.4", - "uvicorn>=0.34.3", - "setuptools>=75.0", -] - -[tool.setuptools.packages.find] -where = ["."] -include = ["experiencemaker*"] -exclude = ["cookbook*"] - -[tool.setuptools.package-data] -experiencemaker = [ - "config/*.yaml", - "op/**/*.yaml", -] - -[project.scripts] -experiencemaker = "experiencemaker.app:main" -experiencemaker_mcp = "experiencemaker.mcp_server:main" diff --git a/experiencemaker/library/appworld_v1.jsonl b/library/appworld_v1.jsonl similarity index 100% rename from experiencemaker/library/appworld_v1.jsonl rename to library/appworld_v1.jsonl diff --git a/experiencemaker/library/bfcl_v1.jsonl b/library/bfcl_v1.jsonl similarity index 100% rename from experiencemaker/library/bfcl_v1.jsonl rename to library/bfcl_v1.jsonl diff --git a/experiencemaker/library/index.html b/library/index.html similarity index 100% rename from experiencemaker/library/index.html rename to library/index.html diff --git a/experiencemaker/experiencemaker/config/default_config.yaml b/reme_ai/config/default_config.yaml similarity index 100% rename from experiencemaker/experiencemaker/config/default_config.yaml rename to reme_ai/config/default_config.yaml diff --git a/reme_ai/config/demo_config.yaml b/reme_ai/config/demo_config.yaml new file mode 100644 index 00000000..2e54eac8 --- /dev/null +++ b/reme_ai/config/demo_config.yaml @@ -0,0 +1,179 @@ +global: + language: en + thread_pool_max_workers: 5 + enable_ranker: false + enable_today_contra_repeat: true + enable_long_contra_repeat: false + output_memory_max_count: 20 + +memory_chat: + cli_memory_chat: + class: core.chat.cli_memory_chat + memory_service: memoryscope_service + generation_model: generation_model + stream: true + +memory_service: + memoryscope_service: + class: core.service.memory_scope_service + human_name: user + assistant_name: AI + memory_operations: + read_message: + class: core.operation.frontend_operation + workflow: read_message + description: "read short memory" + + retrieve_memory: + class: core.operation.frontend_operation + workflow: set_query,[extract_time|retrieve_obs_ins,semantic_rank],fuse_rerank + description: "retrieve long-term memory" + + list_memory: + class: core.operation.frontend_operation + workflow: set_query,retrieve_top_memory,print_memory + description: "read all long-term memory of the user, use `refresh_time=5` to refresh screen every 5 seconds." + + delete_memory: + class: core.operation.frontend_operation + workflow: set_query,retrieve_all_memory,delete_memory + description: "delete a single long-term memory" + + delete_all: + class: core.operation.frontend_operation + workflow: set_query,retrieve_all_memory,delete_all + description: "delete all long-term memory" + + add_memory: + class: core.operation.frontend_operation + workflow: add_memory + description: "add a single observation" + + consolidate_memory: + class: core.operation.consolidate_memory_op + workflow: info_filter,[get_observation|get_observation_with_time|load_today_memory],contra_repeat,store_memory + description: "summary user's observation memory, run backend." + interval_time: 1 + + reflect_and_reconsolidate: + class: core.operation.backend_operation + workflow: load_obs_and_insight,get_reflection_subject,update_insight,long_contra_repeat,store_memory + description: "summary user's insight memory, run backend." + interval_time: 15 + +worker: + dummy: + class: core.worker.dummy_worker + generation_model: generation_model + embedding_model: embedding_model + rank_model: rank_model + read_message: + class: core.worker.frontend.read_message_worker + set_query: + class: core.worker.frontend.set_query_worker + retrieve_obs_ins: + class: core.worker.frontend.retrieve_memory_worker + retrieve_obs_top_k: 100 + retrieve_ins_top_k: 100 + extract_time: + class: core.worker.frontend.extract_time_worker + generation_model: generation_model + semantic_rank: + class: core.worker.frontend.semantic_rank_worker + rank_model: rank_model + fuse_rerank: + class: core.worker.frontend.fuse_rerank_worker + fuse_score_threshold: 0.01 + fuse_ratio_dict: + conversation: 0.5 + observation: 1 + obs_customized: 1.2 + insight: 2.0 + fuse_time_ratio: 2.0 + retrieve_top_memory: + class: core.worker.frontend.retrieve_memory_worker + retrieve_obs_top_k: 100 + retrieve_ins_top_k: 100 + retrieve_expired_top_k: 100 + print_memory: + class: core.worker.frontend.print_memory_worker + retrieve_all_memory: + class: core.worker.frontend.retrieve_memory_worker + retrieve_obs_top_k: 1000 + retrieve_ins_top_k: 1000 + retrieve_expired_top_k: 1000 + delete_memory: + class: core.worker.backend.update_memory_worker + method: delete_memory + delete_all: + class: core.worker.backend.update_memory_worker + method: delete_all + add_memory: + class: core.worker.backend.update_memory_worker + method: from_query + info_filter: + class: core.worker.backend.info_filter_worker + generation_model: generation_model + load_today_memory: + class: core.worker.backend.load_memory_worker + retrieve_today_top_k: 100 + get_observation: + class: core.worker.backend.get_observation_worker + generation_model: generation_model + get_observation_with_time: + class: core.worker.backend.get_observation_with_time_worker + generation_model: generation_model + contra_repeat: + class: core.worker.backend.contra_repeat_worker + generation_model: generation_model + store_memory: + class: core.worker.backend.update_memory_worker + method: from_memory_key + memory_key: all + load_obs_and_insight: + class: core.worker.backend.load_memory_worker + retrieve_not_reflected_top_k: 100 + retrieve_not_updated_top_k: 100 + retrieve_insight_top_k: 100 + get_reflection_subject: + class: core.worker.backend.get_reflection_subject_worker + generation_model: generation_model + reflect_obs_cnt_threshold: 6 + update_insight: + class: core.worker.backend.update_insight_worker + generation_model: generation_model + rank_model: rank_model + embedding_model: embedding_model + update_insight_threshold: 0.01 + enable_parallel: false + long_contra_repeat: + class: core.worker.backend.long_contra_repeat_worker + generation_model: generation_model + long_contra_repeat_threshold: 0.5 + +model: + generation_model: + class: core.models.llama_index_generation_model + module_name: openai_generation + model_name: gpt-4o + max_tokens: 2000 + temperature: 0.01 + embedding_model: + class: core.models.llama_index_embedding_model + module_name: openai_embedding + model_name: text-embedding-3-small + rank_model: + class: core.models.llama_index_rank_model + module_name: dashscope_rank + model_name: gte-rerank + top_n: 500 + +memory_store: + class: core.storage.llama_index_es_memory_store + embedding_model: embedding_model + index_name: memory_index + es_url: http://localhost:9200 + retrieve_mode: dense + +monitor: + class: core.storage.dummy_monitor \ No newline at end of file diff --git a/experiencemaker/experiencemaker/config/simple_config.yaml b/reme_ai/config/simple_config.yaml similarity index 100% rename from experiencemaker/experiencemaker/config/simple_config.yaml rename to reme_ai/config/simple_config.yaml diff --git a/reme_ai/react/react_v1_op.py b/reme_ai/react/react_v1_op.py index 49ab8540..489b5d68 100644 --- a/reme_ai/react/react_v1_op.py +++ b/reme_ai/react/react_v1_op.py @@ -4,26 +4,20 @@ from typing import List, Dict from loguru import logger -from experiencemaker.enumeration.role import Role -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.message import Message -from experiencemaker.schema.request import AgentRequest -from experiencemaker.schema.response import AgentResponse +from flowllm import C, BaseLLMOp +from reme_ai.schema.message import Message, Role from experiencemaker.tool import TOOL_REGISTRY from experiencemaker.tool.base_tool import BaseTool -@OP_REGISTRY.register() -class ReactV1Op(BaseOp): +@C.register_op() +class ReactV1Op(BaseLLMOp): current_path: str = __file__ def execute(self): - request: AgentRequest = self.context.request - response: AgentResponse = self.context.response + query: str = self.context.query max_steps: int = int(self.op_params.get("max_steps", 10)) - # dashscope_search_tool tavily_search_tool tool_names = self.op_params.get("tool_names", "code_tool,tavily_search_tool,terminate_tool") tools: List[BaseTool] = [TOOL_REGISTRY[x.strip()]() for x in tool_names.split(",") if x] tool_dict: Dict[str, BaseTool] = {x.name: x for x in tools} @@ -33,7 +27,7 @@ class ReactV1Op(BaseOp): user_prompt = self.prompt_format(prompt_name="role_prompt", time=now_time, tools=",".join([x.name for x in tools]), - query=request.query) + query=query) messages: List[Message] = [Message(role=Role.USER, content=user_prompt)] logger.info(f"step.0 user_prompt={user_prompt}") @@ -84,5 +78,6 @@ class ReactV1Op(BaseOp): assistant_message.tool_calls.clear() messages.append(Message(role=Role.USER, content=self.prompt_format(prompt_name="final_prompt"))) - response.messages = messages - response.answer = response.messages[-1].content + # Store results in context instead of response + self.context.messages = messages + self.context.answer = messages[-1].content diff --git a/reme_ai/retrieve/personal/__init__.py b/reme_ai/retrieve/personal/__init__.py new file mode 100644 index 00000000..24d17f43 --- /dev/null +++ b/reme_ai/retrieve/personal/__init__.py @@ -0,0 +1,17 @@ +from .extract_time_worker import ExtractTimeWorker +from .fuse_rerank_worker import FuseRerankWorker +from .print_memory_worker import PrintMemoryWorker +from .read_message_worker import ReadMessageWorker +from .retrieve_memory_worker import RetrieveMemoryWorker +from .semantic_rank_worker import SemanticRankWorker +from .set_query_worker import SetQueryWorker + +__all__ = [ + "ExtractTimeWorker", + "FuseRerankWorker", + "PrintMemoryWorker", + "ReadMessageWorker", + "RetrieveMemoryWorker", + "SemanticRankWorker", + "SetQueryWorker" +] diff --git a/reme_ai/retrieve/personal/extract_time_worker.py b/reme_ai/retrieve/personal/extract_time_worker.py new file mode 100644 index 00000000..70e1ba00 --- /dev/null +++ b/reme_ai/retrieve/personal/extract_time_worker.py @@ -0,0 +1,65 @@ +import re +from typing import Dict + +from memoryscope.constants.common_constants import QUERY_WITH_TS, EXTRACT_TIME_DICT +from memoryscope.constants.language_constants import DATATIME_KEY_MAP +from memoryscope.core.utils.datetime_handler import DatetimeHandler +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker + + +class ExtractTimeWorker(MemoryBaseWorker): + """ + A specialized worker class designed to identify and extract time-related information + from text generated by an LLM, translating date-time keywords based on the set language, + and storing this extracted data within a shared context. + """ + + EXTRACT_TIME_PATTERN = r"-\s*(\S+)[：:]\s*(\S+)" + FILE_PATH: str = __file__ + + def _parse_params(self, **kwargs): + self.generation_model_kwargs: dict = kwargs.get("generation_model_kwargs", {}) + + def _run(self): + """ + Executes the primary logic of identifying and extracting time data from an LLM's response. + + This method first checks if the input query contains any datetime keywords. If not, it logs and returns. + It then constructs a prompt with contextual information including formatted timestamps and calls the LLM. + The response is parsed for time-related data using regex, translated via a language-specific key map, + and the resulting time data is stored in the shared context. + """ + query, query_timestamp = self.get_workflow_context(QUERY_WITH_TS) + + # Identify if the query contains datetime keywords + contain_datetime = DatetimeHandler.has_time_word(query, self.language) + if not contain_datetime: + self.logger.info(f"contain_datetime={contain_datetime}") + return + + # Prepare the prompt with necessary contextual details + query_time_str = DatetimeHandler(dt=query_timestamp).string_format(self.prompt_handler.time_string_format, + self.language) + system_prompt = self.prompt_handler.extract_time_system + few_shot = self.prompt_handler.extract_time_few_shot + user_query = self.prompt_handler.extract_time_user_query.format(query=query, query_time_str=query_time_str) + extract_time_message = self.prompt_to_msg(system_prompt=system_prompt, few_shot=few_shot, user_query=user_query) + self.logger.info(f"extract_time_message={extract_time_message}") + + # Invoke the LLM to generate a response + response = self.generation_model.call(messages=extract_time_message, **self.generation_model_kwargs) + + # Handle empty or unsuccessful responses + if not response.status or not response.message.content: + return + response_text = response.message.content + + # Extract time information from the LLM's response using regex + extract_time_dict: Dict[str, str] = {} + matches = re.findall(self.EXTRACT_TIME_PATTERN, response_text) + key_map: dict = self.get_language_value(DATATIME_KEY_MAP) + for key, value in matches: + if key in key_map.keys(): + extract_time_dict[key_map[key]] = value + self.logger.info(f"response_text={response_text} matches={matches} filters={extract_time_dict}") + self.set_workflow_context(EXTRACT_TIME_DICT, extract_time_dict) diff --git a/reme_ai/retrieve/personal/extract_time_worker.yaml b/reme_ai/retrieve/personal/extract_time_worker.yaml new file mode 100644 index 00000000..85a01ee3 --- /dev/null +++ b/reme_ai/retrieve/personal/extract_time_worker.yaml @@ -0,0 +1,133 @@ +time_string_format: + cn: | + {year}年{month}{day}日，{year}年第{week}周，{weekday}，{hour}时。 + en: | + {month} {day}, {year}, {week}th week of {year}, {weekday}, at {hour}. + + +extract_time_system: + cn: | + 任务：从语句与语句发生的时间，推断并提取语句内容中指向的时间段。 + 回答尽可能完整的时间段。 + 回答的格式严格遵照示例中的已有格式规范。 + 若语句不涉及时间则回答无。 + en: | + Task: From the sentences and the time when they occurred, infer and extract the time periods indicated in the content of the sentences. + Answer with the most complete time periods possible. + The format of the answers must strictly adhere to the specifications in the examples provided. + If the sentence does not involve time, respond with "none." + + +extract_time_few_shot: + cn: | + 示例1: + 句子：我记得你前年四月份去了阿联酋，阿联酋有哪些好玩的地方？迪拜和阿布扎比你更喜欢哪个？沙漠的景色壮观吗？ + 时间：1992年8月20日，1992年第34周，周一，18时。 + 回答： + - 年：1990 - 月：4月 + + 示例2: + 句子：后天下午三点的会议记得参加。我在日历上仔细标注了这个重要的日子，提醒自己不要错过。会议将在公司会议室举行，这是一个讨论未来发展方向的重要机会。 + 时间：2024年6月19日，2024年第25周，周二，13时。 + 回答： + - 年：2024 - 月：6月 - 日：21 - 时：15 + + 示例3: + 句子：下个月第一个周六去杭州玩。 + 时间：2005年7月15日，2005年第28周，周六，0时。 + 回答： + - 年：2005 - 月：8月 - 周：31 - 星期几：周六 + + 示例4: + 句子：上周末我们去的那个小镇真是太美了。 + 时间：1999年12月2日，1999年第48周，周二，8时。 + 回答： + - 年：1999 - 周：47 - 星期几：周六,周日 + + 示例5: + 句子：再过半小时就要宣讲了，记得准备材料。 + 时间：2020年6月22日，2020年第25周，周一，9时。 + 回答： + - 年：2020 - 月：6月 - 日：22 - 时：10 + + 示例6: + 句子：10000米长跑比赛的开始时间是3分47秒前。 + 时间：1987年2月17日，1987年第7周，周三，19时。 + 回答： + - 年：1987 - 月：2 - 日：17 - 时：19 + + 示例7: + 句子：上个月的这个时候我们还在筹备音乐会。每天都是忙碌而充实的日子，我们为音乐会的顺利举办而努力奋斗着。彩排、布景、节目安排，每一个细节都需要精心安排和准备。 + 时间：1995年11月24日，1995年第48周，周二，17时。 + 回答： + - 年：1995 - 月：10 - 日：24 + + 示例8: + 句子：我的朋友非常喜欢运动，他认为运动有助于增强身体素质。 + 时间：2015年1月23日，2015年第4周，周四，7时。 + 回答： + 无 + + en: | + Example 1: + Sentence: I remember you went to the UAE in April the year before last. Which places in the UAE are fun? Which do you prefer, Dubai or Abu Dhabi? Are the desert views spectacular? + Time: August 20, 1992, 34th week of 1992, Monday, at 18. + Answer: + - Year: 1990 - Month: 4 + + Example 2: + Sentence: Remember to attend the meeting at 3 PM the day after tomorrow. I carefully marked this important day on my calendar to remind myself not to miss it. The meeting will be held in the company conference room, and it's an important opportunity to discuss future development directions. + Time: June 19, 2024, 25th week of 2024, Tuesday, at 13. + Answer: + - Year: 2024 - Month: 6 - Day: 21 - Hour: 15 + + Example 3: + Sentence: Next month on the first Saturday, let's go to Hangzhou. + Time: July 15, 2005, 28th week of 2005, Saturday, at 0. + Answer: + - Year: 2005 - Month: 8 - Week: 31 - Day of Week: 6 + + Example 4: + Sentence: The small town we visited last weekend was truly beautiful. + Time: December 2, 1999, 48th week of 1999, Tuesday, at 8. + Answer: + - Year: 1999 - Week: 47 - Day of Week: 6, 7 + + Example 5: + Sentence: The presentation will start in half an hour, remember to prepare the materials. + Time: June 22, 2020, 25th week of 2020, Monday, at 9. + Answer: + - Year: 2020 - Month: 6 - Day: 22 - Hour: 10 + + Example 6: + Sentence: The start time for the 10,000-meter race was 3 minutes and 47 seconds ago. + Time: February 17, 1987, 7th week of 1987, Wednesday, at 19. + Answer: + - Year: 1987 - Month: 2 - Day: 17 - Hour: 19 + + Example 7: + Sentence: At this time last month, we were still preparing for the concert. Every day was busy and fulfilling, and we worked hard for the successful holding of the concert. Rehearsals, set design, and program arrangements - every detail needed careful planning and preparation. + Time: November 24, 1995, 48th week of 1995, Tuesday, at 17. + Answer: + - Year: 1995 - Month: 10 - Day: 24 + + Example 8: + Sentence: My friend loves sports very much and believes that exercise helps improve physical fitness. + Time: January 23, 2015, 4th week of 2015, Thursday, at 7. + Answer: + None + + +extract_time_user_query: + cn: | + 句子：{query} + 时间：{query_time_str} + 回答： + + en: | + Sentence: {query} + Time: {query_time_str} + Answer: + + + diff --git a/reme_ai/retrieve/personal/fuse_rerank_worker.py b/reme_ai/retrieve/personal/fuse_rerank_worker.py new file mode 100644 index 00000000..b137f354 --- /dev/null +++ b/reme_ai/retrieve/personal/fuse_rerank_worker.py @@ -0,0 +1,109 @@ +from typing import Dict, List + +from memoryscope.constants.common_constants import EXTRACT_TIME_DICT, RANKED_MEMORY_NODES, RESULT +from memoryscope.core.utils.datetime_handler import DatetimeHandler +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode + + +class FuseRerankWorker(MemoryBaseWorker): + """ + Reranks the memory nodes by scores, types, and temporal relevance. Formats the top-K reranked nodes to print. + """ + + def _parse_params(self, **kwargs): + self.fuse_score_threshold: float = kwargs.get("fuse_score_threshold", 0.1) + self.fuse_ratio_dict: Dict[str, float] = kwargs.get("fuse_ratio_dict", {}) + self.fuse_time_ratio: float = kwargs.get("fuse_time_ratio", 2.0) + self.output_memory_max_count: int = self.memoryscope_context.meta_data["output_memory_max_count"] + + @staticmethod + def match_node_time(extract_time_dict: Dict[str, str], node: MemoryNode): + """ + Determines whether the node is relevant. + """ + if extract_time_dict: + match_event_flag = True + for k, v in extract_time_dict.items(): + event_value = node.meta_data.get(f"event_{k}", "") + if event_value in ["-1", v]: + continue + else: + match_event_flag = False + break + + match_msg_flag = True + for k, v in extract_time_dict.items(): + msg_value = node.meta_data.get(f"msg_{k}", "") + if msg_value == v: + continue + else: + match_msg_flag = False + break + else: + match_event_flag = False + match_msg_flag = False + + node.meta_data["match_event_flag"] = str(int(match_event_flag)) + node.meta_data["match_msg_flag"] = str(int(match_msg_flag)) + return match_event_flag, match_msg_flag + + def _run(self): + """ + Executes the reranking process on memory nodes considering their scores, types, and temporal relevance. + + This method performs the following steps: + 1. Retrieves extraction time data and a list of ranked memory nodes from the worker's context. + 2. Reranks nodes based on a combination of their original rank score, type, + and temporal alignment with extracted events/messages. + 3. Selects the top-K reranked nodes according to the predefined threshold. + 4. Optionally infuses inferred time information into the content of selected nodes. + 5. Logs reranking details and formats the final list of memories for output. + """ + # Parse input parameters from the worker's context + extract_time_dict: Dict[str, str] = self.get_workflow_context(EXTRACT_TIME_DICT) + memory_node_list: List[MemoryNode] = self.memory_manager.get_memories(RANKED_MEMORY_NODES) + + # Check if memory nodes are available; warn and return if not + if not memory_node_list: + self.logger.warning("Ranked memory nodes list is empty.") + return + + # Perform reranking based on score, type, and time relevance + reranked_memory_nodes = [] + for node in memory_node_list: + # Skip nodes below the fuse score threshold + if node.score_rank < self.fuse_score_threshold: + continue + + # Calculate type-based adjustment factor + if node.memory_type not in self.fuse_ratio_dict: + self.logger.warning(f"{node.memory_type} 'factor is not configured!") + type_ratio: float = self.fuse_ratio_dict.get(node.memory_type, 0.1) + + # Determine time relevance adjustment factor + match_event_flag, match_msg_flag = self.match_node_time(extract_time_dict=extract_time_dict, node=node) + fuse_time_ratio: float = self.fuse_time_ratio if match_event_flag or match_msg_flag else 1.0 + + # Apply reranking score adjustments + node.score_rerank = node.score_rank * type_ratio * fuse_time_ratio + reranked_memory_nodes.append(node) + + # build result + memories: List[str] = [] + reranked_memory_nodes = sorted(reranked_memory_nodes, + key=lambda x: x.score_rerank, + reverse=True)[: self.output_memory_max_count] + for node in reranked_memory_nodes: + # Log reranking details including flags for event and message matches + self.logger.info(f"Rerank Stage: Content={node.content}, Score={node.score_rerank}, " + f"Event Flag={node.meta_data['match_event_flag']}, " + f"Message Flag={node.meta_data['match_msg_flag']}") + + dt_handler = DatetimeHandler(node.timestamp) + datetime = dt_handler.datetime_format("%Y-%m-%d %H:%M:%S") + weekday = dt_handler.get_dt_info_dict(self.language)["weekday"] + memories.append(f"[{datetime} {weekday}] {node.content}") + + # Set the final list of formatted memories back into the worker's context + self.set_workflow_context(RESULT, "\n".join(memories)) diff --git a/reme_ai/retrieve/personal/print_memory_worker.py b/reme_ai/retrieve/personal/print_memory_worker.py new file mode 100644 index 00000000..7421614d --- /dev/null +++ b/reme_ai/retrieve/personal/print_memory_worker.py @@ -0,0 +1,66 @@ +from typing import List + +from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, RESULT +from memoryscope.core.utils.datetime_handler import DatetimeHandler +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.scheme.memory_node import MemoryNode + + +class PrintMemoryWorker(MemoryBaseWorker): + """ + Formats the memories to print. + """ + FILE_PATH: str = __file__ + + def _run(self): + """ + Executes the primary function, it involves: + 1. Fetches the memories. + 2. Formats them by 'print_template'. + 3. Set the formatted string back into the worker's context + """ + # get long-term memory + memory_node_list: List[MemoryNode] = self.memory_manager.get_memories(RETRIEVE_MEMORY_NODES) + memory_node_list = sorted(memory_node_list, key=lambda x: x.timestamp, reverse=True) + + observation_memory_list: List[str] = [] + insight_memory_list: List[str] = [] + expired_memory_list: List[str] = [] + + i = 0 + j = 0 + k = 0 + # remove duplicate content + expired_content_set = set() + for node in memory_node_list: + if not node.content: + continue + + dt_handler = DatetimeHandler(node.timestamp) + dt = dt_handler.datetime_format("%Y%m%d %H:%M:%S") + if StoreStatusEnum(node.store_status) is StoreStatusEnum.EXPIRED: + if node.content in expired_content_set: + continue + else: + expired_content_set.add(node.content) + i += 1 + expired_memory_list.append(f"{dt}] {i}. {node.content}") + + elif MemoryTypeEnum(node.memory_type) in [MemoryTypeEnum.OBSERVATION, MemoryTypeEnum.OBS_CUSTOMIZED]: + j += 1 + observation_memory_list.append(f"{dt}] {j}. {node.content} " + f"[status({node.obs_reflected},{node.obs_updated})") + + elif MemoryTypeEnum(node.memory_type) is MemoryTypeEnum.INSIGHT: + k += 1 + insight_memory_list.append(f"{dt}] {k}. {node.content}") + + result: str = self.prompt_handler.print_template.format( + user_name=self.user_name, + target_name=self.target_name, + observation_memory="\n".join(observation_memory_list), + insight_memory="\n".join(insight_memory_list), + expired_memory="\n".join(expired_memory_list)).strip() + self.set_workflow_context(RESULT, result) diff --git a/reme_ai/retrieve/personal/print_memory_worker.yaml b/reme_ai/retrieve/personal/print_memory_worker.yaml new file mode 100644 index 00000000..d102a10c --- /dev/null +++ b/reme_ai/retrieve/personal/print_memory_worker.yaml @@ -0,0 +1,22 @@ +print_template: + cn: | + ========== {user_name}关于{target_name}的长期记忆 ========== + ----- 观察记忆 ----- + {observation_memory} + + ----- 洞察记忆 ----- + {insight_memory} + + ----- 过期记忆 ----- + {expired_memory} + + en: | + ========== The {user_name}'s long-term memory about {target_name} ========== + ----- observation memory ----- + {observation_memory} + + ----- insight memory ----- + {insight_memory} + + ----- expired memory ----- + {expired_memory} \ No newline at end of file diff --git a/reme_ai/retrieve/personal/read_message_worker.py b/reme_ai/retrieve/personal/read_message_worker.py new file mode 100644 index 00000000..2f378eef --- /dev/null +++ b/reme_ai/retrieve/personal/read_message_worker.py @@ -0,0 +1,40 @@ +from typing import List + +from memoryscope.constants.common_constants import RESULT +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.message import Message + + +class ReadMessageWorker(MemoryBaseWorker): + """ + Fetches unmemorized chat messages. + """ + + def _run(self): + """ + Executes the primary function to fetch unmemorized chat messages. + """ + chat_messages_not_memorized: List[List[Message]] = [] + for messages in self.chat_messages: + if not messages: + continue + + if messages[0].memorized: + continue + + contain_flag = False + + for msg in messages: + if msg.role_name == self.target_name: + contain_flag = True + break + + if contain_flag: + chat_messages_not_memorized.append(messages) + + contextual_msg_max_count: int = self.chat_kwargs["contextual_msg_max_count"] + chat_message_scatter = [] + for messages in chat_messages_not_memorized[-contextual_msg_max_count:]: + chat_message_scatter.extend(messages) + chat_message_scatter.sort(key=lambda _: _.time_created) + self.set_workflow_context(RESULT, chat_message_scatter) diff --git a/reme_ai/retrieve/personal/retrieve_memory_worker.py b/reme_ai/retrieve/personal/retrieve_memory_worker.py new file mode 100644 index 00000000..a539c7f4 --- /dev/null +++ b/reme_ai/retrieve/personal/retrieve_memory_worker.py @@ -0,0 +1,143 @@ +from typing import List + +from memoryscope.constants.common_constants import QUERY_WITH_TS, RETRIEVE_MEMORY_NODES +from memoryscope.core.utils.timer import timer +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.scheme.memory_node import MemoryNode + + +class RetrieveMemoryWorker(MemoryBaseWorker): + """ + Retrieves memories based on specified criteria such as status, type, and timestamp. + Processes these memories concurrently, sorts them by similarity, and logs the activity, + facilitating efficient memory retrieval operations within a given scope. + """ + + def _parse_params(self, **kwargs): + self.retrieve_obs_top_k: int = kwargs.get("retrieve_obs_top_k", 0) + self.retrieve_ins_top_k: int = kwargs.get("retrieve_ins_top_k", 0) + self.retrieve_expired_top_k: int = kwargs.get("retrieve_expired_top_k", 0) + + @timer + def retrieve_from_observation(self, query: str) -> List[MemoryNode]: + """ + Retrieves memory nodes from observation based on a query, considering active memories + with specific types. If the retrieval limit is not set, an empty list is returned. + + Args: + query (str): The query string used to filter and rank the memory nodes. + + Returns: + List[MemoryNode]: A list of MemoryNode objects that match the query criteria, + sorted by their relevance. Returns an empty list if no retrieval limit is configured. + """ + if not self.retrieve_obs_top_k: + return [] + + filter_dict = { + "user_name": self.user_name, + "target_name": self.target_name, + "store_status": StoreStatusEnum.VALID.value, + "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], + } + # Retrieve memories matching the query, filtered by the specified conditions, + # limited to a certain number, and sorted by relevance. + return self.memory_store.retrieve_memories(query=query, + top_k=self.retrieve_obs_top_k, + filter_dict=filter_dict) + + @timer + def retrieve_from_insight(self, query: str) -> List[MemoryNode]: + """ + Retrieves memories marked as insights from the store based on a query, filtered by user, target, + and set to active status. + + Args: + query (str): The search query to match against the insights. + + Returns: + List[MemoryNode]: A list of MemoryNode objects that match the query criteria, + limited by 'retrieve_ins_pf_top_k'. + Returns an empty list if 'retrieve_ins_pf_top_k' is not set. + """ + if not self.retrieve_ins_top_k: + return [] + + filter_dict = { + "user_name": self.user_name, + "target_name": self.target_name, + "store_status": StoreStatusEnum.VALID.value, + "memory_type": MemoryTypeEnum.INSIGHT.value, + } + # ⭐ Retrieve insights matching the query, filtered, and limited by top_k + return self.memory_store.retrieve_memories(query=query, + top_k=self.retrieve_ins_top_k, + filter_dict=filter_dict) + + @timer + def retrieve_expired_memory(self, query: str) -> List[MemoryNode]: + """ + Retrieves expired memories marked as observation from the store based on a query, filtered by user, target, + and set to active status. + + Args: + query (str): The search query to match against the memories. + + Returns: + List[MemoryNode]: A list of MemoryNode objects that match the query criteria, + limited by 'retrieve_expired_top_k'. + Returns an empty list if 'retrieve_expired_top_k' is not set. + """ + if not self.retrieve_expired_top_k: + return [] + + filter_dict = { + "user_name": self.user_name, + "target_name": self.target_name, + "store_status": StoreStatusEnum.EXPIRED.value, + "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], + } + return self.memory_store.retrieve_memories(query=query, + top_k=self.retrieve_expired_top_k, + filter_dict=filter_dict) + + def _run(self): + """ + Executes the main retrieval for memories. It fetches the query from the context, initiates concurrent tasks + to retrieve memories from observations, insights, and expired sources, collects the results, sorts them by + similarity score, logs the details, and finally sets the retrieved memory nodes. + + The method follows these steps: + 1. Retrieves the query from the worker's context. + 2. Submits tasks to asynchronously retrieve memories from various sources. + 3. Gathers the results from all submitted tasks. + 4. Logs the total number of collected memory nodes. + 5. Sorts the memory nodes based on their similarity scores in descending order. + 6. Logs detailed information about each memory node. + 7. Stores the processed memory nodes for further use. + """ + query, _ = self.get_workflow_context(QUERY_WITH_TS) + self.logger.info(f"retrieve memory with query={query}.") + self.submit_thread_task(self.retrieve_from_observation, query=query) + self.submit_thread_task(self.retrieve_from_insight, query=query) + self.submit_thread_task(self.retrieve_expired_memory, query=query) + + memory_node_list: List[MemoryNode] = [] + for result in self.gather_thread_result(): + if result: + memory_node_list.extend(result) + self.logger.info(f"memory_node_list.size={len(memory_node_list)}") + + if not memory_node_list: + return + + memory_node_list = sorted(memory_node_list, key=lambda x: x.score_recall, reverse=True) + for node in memory_node_list: + node.action_status = ActionStatusEnum.NONE.value + self.logger.info(f"recall_stage: content={node.content} score={node.score_recall} type={node.memory_type} " + f"store_status={node.store_status} action_status={node.action_status}") + + self.memory_manager.set_memories(RETRIEVE_MEMORY_NODES, memory_node_list) diff --git a/reme_ai/retrieve/personal/semantic_rank_worker.py b/reme_ai/retrieve/personal/semantic_rank_worker.py new file mode 100644 index 00000000..894785fd --- /dev/null +++ b/reme_ai/retrieve/personal/semantic_rank_worker.py @@ -0,0 +1,72 @@ +from typing import List, Dict + +from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, QUERY_WITH_TS, RANKED_MEMORY_NODES +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode + + +class SemanticRankWorker(MemoryBaseWorker): + """ + The `SemanticRankWorker` class processes queries by retrieving memory nodes, + removing duplicates, ranking them based on semantic relevance using a model, + assigning scores, sorting the nodes, and storing the ranked nodes back, + while logging relevant information. + """ + + def _parse_params(self, **kwargs): + self.enable_ranker: bool = self.memoryscope_context.meta_data["enable_ranker"] + self.output_memory_max_count: int = self.memoryscope_context.meta_data["output_memory_max_count"] + + def _run(self): + """ + Executes the primary workflow of the SemanticRankWorker which includes: + - Retrieves query and timestamp from context. + - Fetches memory nodes. + - Removes duplicate nodes. + - Ranks nodes semantically. + - Assigns scores to nodes. + - Sorts nodes by score. + - Saves the ranked nodes back with logging. + + If no memory nodes are retrieved or if the ranking model fails, + appropriate warnings are logged. + """ + # query + query, _ = self.get_workflow_context(QUERY_WITH_TS) + memory_node_list: List[MemoryNode] = self.memory_manager.get_memories(RETRIEVE_MEMORY_NODES) + if not memory_node_list: + self.logger.warning("Retrieve memory nodes is empty!") + return + + if not self.enable_ranker or len(memory_node_list) <= self.output_memory_max_count: + for node in memory_node_list: + node.score_rank = node.score_recall + self.logger.warning("use score_recall instead of score_rank!") + + else: + # drop repeated + memory_node_dict: Dict[str, MemoryNode] = {n.content.strip(): n for n in memory_node_list if + n.content.strip()} + memory_node_list = list(memory_node_dict.values()) + + response = self.rank_model.call(query=query, documents=[n.content for n in memory_node_list]) + if not response.status or not response.rank_scores: + return + + # set score + for idx, score in response.rank_scores.items(): + if idx >= len(memory_node_list): + self.logger.warning(f"Idx={idx} exceeds the maximum length of rank_scores!") + continue + memory_node_list[idx].score_rank = score + + # sort by score + memory_node_list = sorted(memory_node_list, key=lambda n: n.score_rank, reverse=True) + + # log ranked nodes + self.logger.info(f"Rank stage: query={query}") + for node in memory_node_list: + self.logger.info(f"Rank stage: Content={node.content}, Score={node.score_rank}") + + # save ranked nodes back to memory + self.memory_manager.set_memories(RANKED_MEMORY_NODES, memory_node_list, log_repeat=False) diff --git a/reme_ai/retrieve/personal/set_query_worker.py b/reme_ai/retrieve/personal/set_query_worker.py new file mode 100644 index 00000000..fe541c11 --- /dev/null +++ b/reme_ai/retrieve/personal/set_query_worker.py @@ -0,0 +1,39 @@ +import datetime + +from memoryscope.constants.common_constants import QUERY_WITH_TS +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker + + +class SetQueryWorker(MemoryBaseWorker): + """ + The `SetQueryWorker` class is responsible for setting a query and its associated timestamp + into the context, utilizing either provided chat parameters or details from the most recent + chat message. + """ + + def _run(self): + """ + Executes the worker's primary function, which involves determining the query and its + timestamp, then storing these values within the context. + + If 'query' is found within `self.chat_kwargs`, it is considered as the query input. + Otherwise, the content of the last message in `self.chat_messages` is used as the query, + along with its creation timestamp. + """ + query = "" # Default query value + timestamp = int(datetime.datetime.now().timestamp()) # Current timestamp as default + + if "query" in self.chat_kwargs: + # set query if exists + query = self.chat_kwargs["query"] + if not query: + query = "" + query = query.strip() + + # set ts if exists + _timestamp = self.chat_kwargs.get("timestamp") + if _timestamp and isinstance(_timestamp, int): + timestamp = _timestamp + + # Store the determined query and its timestamp in the context + self.set_workflow_context(QUERY_WITH_TS, (query, timestamp)) diff --git a/reme_ai/retrieve/task/rerank_memory_op.py b/reme_ai/retrieve/task/rerank_memory_op.py index a9b9811a..9b617fd1 100644 --- a/reme_ai/retrieve/task/rerank_memory_op.py +++ b/reme_ai/retrieve/task/rerank_memory_op.py @@ -1,30 +1,26 @@ import json import re from typing import List + +from flowllm import C, BaseLLMOp +from flowllm.enumeration.role import Role +from flowllm.schema.message import Message from loguru import logger -from pydantic import Field -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.experience import BaseExperience -from experiencemaker.schema.message import Message -from experiencemaker.schema.vector_node import VectorNode -from experiencemaker.enumeration.role import Role -from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp +from reme_ai.schema.memory import BaseMemory -@OP_REGISTRY.register() -class RerankExperienceOp(BaseOp): + +@C.register_op() +class RerankMemoryOp(BaseLLMOp): """ Rerank and filter recalled experiences using LLM and score-based filtering """ current_path: str = __file__ def execute(self): - """Execute rerank operation""" - - experiences: List[BaseExperience] = self.context.response.experience_list - retrieval_query: str = self.context.get_context(RecallVectorStoreOp.SEARCH_QUERY, "") + memory_list: List[BaseMemory] = self.context.response.metadata["memory_list"] + retrieval_query: str = self.context.query enable_llm_rerank = self.op_params.get("enable_llm_rerank", True) enable_score_filter = self.op_params.get("enable_score_filter", False) min_score_threshold = self.op_params.get("min_score_threshold", 0.3) @@ -32,98 +28,89 @@ class RerankExperienceOp(BaseOp): logger.info(f"top_k: {top_k}") - if not experiences: - logger.info("No recalled experiences to rerank") - self.context.response.experience_list = [] + if not memory_list: + logger.info("No recalled memory_list to rerank") return - try: - logger.info(f"Reranking {len(experiences)} experiences") + logger.info(f"Reranking {len(memory_list)} memories") - # Step 1: LLM reranking (optional) - if enable_llm_rerank: - experiences = self._llm_rerank(retrieval_query, experiences) - logger.info(f"After LLM reranking: {len(experiences)} experiences") + # Step 1: LLM reranking (optional) + if enable_llm_rerank: + memory_list = self._llm_rerank(retrieval_query, memory_list) + logger.info(f"After LLM reranking: {len(memory_list)} memories") - # Step 2: Score-based filtering (optional) - if enable_score_filter: - experiences = self._score_based_filter(experiences, min_score_threshold) - logger.info(f"After score filtering: {len(experiences)} experiences") + # Step 2: Score-based filtering (optional) + if enable_score_filter: + memory_list = self._score_based_filter(memory_list, min_score_threshold) + logger.info(f"After score filtering: {len(memory_list)} memories") - # Step 3: Return top-k results - renranked_experiences = experiences[:top_k] - logger.info(f"Final reranked results: {len(renranked_experiences)} experiences") + # Step 3: Return top-k results + reranked_memories = memory_list[:top_k] + logger.info(f"Final reranked results: {len(reranked_memories)} memories") - # Store results in context - self.context.response.experience_list = renranked_experiences + # Store results in context + self.context.response.metadata["memory_list"] = reranked_memories - except Exception as e: - logger.error(f"Error in rerank operation: {e}") - self.context.response.experience_list = renranked_experiences[:top_k] - - def _llm_rerank(self, query: str, candidates: List[BaseExperience]) -> List[BaseExperience]: + def _llm_rerank(self, query: str, candidates: List[BaseMemory]) -> List[BaseMemory]: """LLM-based reranking of candidate experiences""" if not candidates: return candidates - try: - # Format candidates for LLM evaluation - candidates_text = self._format_candidates_for_rerank(candidates) + # Format candidates for LLM evaluation + candidates_text = self._format_candidates_for_rerank(candidates) - prompt = self.prompt_format( - prompt_name="experience_rerank_prompt", - query=query, - candidates=candidates_text, - num_candidates=len(candidates) - ) + prompt = self.prompt_format( + prompt_name="memory_rerank_prompt", + query=query, + candidates=candidates_text, + num_candidates=len(candidates) + ) - response = self.llm.chat([Message(role=Role.USER, content=prompt)]) + response = self.llm.chat([Message(role=Role.USER, content=prompt)]) - # Parse reranking results - reranked_indices = self._parse_rerank_response(response.content) + # Parse reranking results + reranked_indices = self._parse_rerank_response(response.content) - # Reorder candidates based on LLM ranking - if reranked_indices: - reranked_candidates = [] - for idx in reranked_indices: - if 0 <= idx < len(candidates): - reranked_candidates.append(candidates[idx]) + # Reorder candidates based on LLM ranking + if reranked_indices: + reranked_candidates = [] + for idx in reranked_indices: + if 0 <= idx < len(candidates): + reranked_candidates.append(candidates[idx]) - # Add any remaining candidates that weren't explicitly ranked - ranked_indices_set = set(reranked_indices) - for i, candidate in enumerate(candidates): - if i not in ranked_indices_set: - reranked_candidates.append(candidate) + # Add any remaining candidates that weren't explicitly ranked + ranked_indices_set = set(reranked_indices) + for i, candidate in enumerate(candidates): + if i not in ranked_indices_set: + reranked_candidates.append(candidate) - return reranked_candidates + return reranked_candidates - return candidates + return candidates - except Exception as e: - logger.error(f"Error in LLM reranking: {e}") - return candidates + @staticmethod + def _score_based_filter(memories: List[BaseMemory], min_score: float) -> List[BaseMemory]: + """Filter memories based on quality scores""" + filtered_memories = [] - def _score_based_filter(self, experiences: List[BaseExperience], min_score: float) -> List[BaseExperience]: - """Filter experiences based on quality scores""" - filtered_experiences = [] - - for exp in experiences: + for memory in memories: # Get confidence score from metadata - confidence = exp.metadata.get("confidence", 0.5) - validation_score = exp.score + confidence = memory.metadata.get("confidence", 0.5) + validation_score = memory.score or 0.5 # Calculate combined score combined_score = (confidence + validation_score) / 2 if combined_score >= min_score: - filtered_experiences.append(exp) + filtered_memories.append(memory) else: - logger.debug(f"Filtered out experience with score {combined_score:.2f}") + logger.debug(f"Filtered out memory with score {combined_score:.2f}") - logger.info(f"Score filtering: {len(filtered_experiences)}/{len(experiences)} experiences retained") - return filtered_experiences + logger.info(f"Score filtering: {len(filtered_memories)}/{len(memories)} memories retained") + return filtered_memories - def _format_candidates_for_rerank(self, candidates: List[BaseExperience]) -> str: + @staticmethod + def _format_candidates_for_rerank(candidates: List[BaseMemory]) -> str: """Format candidates for LLM reranking""" formatted_candidates = [] @@ -139,7 +126,8 @@ class RerankExperienceOp(BaseOp): return "\n---\n".join(formatted_candidates) - def _parse_rerank_response(self, response: str) -> List[int]: + @staticmethod + def _parse_rerank_response(response: str) -> List[int]: """Parse LLM reranking response to extract ranked indices""" try: # Try to extract JSON format diff --git a/reme_ai/retrieve/task/rewrite_memory_op.py b/reme_ai/retrieve/task/rewrite_memory_op.py index 9b61a92b..b9c7cc9a 100644 --- a/reme_ai/retrieve/task/rewrite_memory_op.py +++ b/reme_ai/retrieve/task/rewrite_memory_op.py @@ -1,21 +1,17 @@ import json import re from typing import List + +from flowllm import C, BaseLLMOp +from flowllm.enumeration.role import Role +from flowllm.schema.message import Message from loguru import logger -from pydantic import Field -from experiencemaker.op import OP_REGISTRY -from experiencemaker.op.base_op import BaseOp -from experiencemaker.schema.experience import TextExperience, BaseExperience -from experiencemaker.schema.message import Message -from experiencemaker.schema.vector_node import VectorNode -from experiencemaker.enumeration.role import Role -from experiencemaker.schema.response import RetrieverResponse -from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp +from reme_ai.schema.memory import BaseMemory -@OP_REGISTRY.register() -class RewriteExperienceOp(BaseOp): +@C.register_op() +class RewriteMemoryOp(BaseLLMOp): """ Generate and rewrite context messages from reranked experiences """ @@ -23,47 +19,44 @@ class RewriteExperienceOp(BaseOp): def execute(self): """Execute rewrite operation""" - experiences: List[BaseExperience] = self.context.response.experience_list - query: str = self.context.get_context(RecallVectorStoreOp.SEARCH_QUERY, "") - messages: List[Message] = self.context.get_context("messages", []) + memory_list: List[BaseMemory] = self.context.response.metadata["memory_list"] + query: str = self.context.query + messages: List[Message] = \ + [Message(**x) if isinstance(x, dict) else x for x in self.context.get('messages', [])] - if not experiences: - logger.info("No reranked experiences to rewrite") - self.context.response.experience_merged = "" + if not memory_list: + logger.info("No reranked memories to rewrite") + self.context.response.answer = "" return - logger.info(f"Generating context from {len(experiences)} experiences") + logger.info(f"Generating context from {len(memory_list)} memories") # Generate initial context message - context_message = self._generate_context_message(query, messages, experiences) + rewritten_memory = self._generate_context_message(query, messages, memory_list) # Store results in context - self.context.set_context("context_message", context_message) + self.context.response.answer = rewritten_memory - response: RetrieverResponse = self.context.response - response.experience_merged = context_message - - - def _generate_context_message(self, query: str, messages: List[Message], experiences: List[BaseExperience], + def _generate_context_message(self, query: str, messages: List[Message], memories: List[BaseMemory], ) -> str: - """Generate context message from retrieved experiences""" - if not experiences: + """Generate context message from retrieved memories""" + if not memories: return "" try: - # Format retrieved experiences - formatted_experiences = self._format_experiences_for_context(experiences) + # Format retrieved memories + formatted_memories = self._format_memories_for_context(memories) if self.op_params.get("enable_llm_rewrite", True): - context_content = self._rewrite_context(query, formatted_experiences, messages) + context_content = self._rewrite_context(query, formatted_memories, messages) else: - context_content = formatted_experiences + context_content = formatted_memories return context_content except Exception as e: logger.error(f"Error generating context message: {e}") - return self._format_experiences_for_context(experiences) + return self._format_memories_for_context(memories) def _rewrite_context(self, query: str, context_content: str, messages: List[Message]) -> str: """LLM-based context rewriting to make experiences more relevant and actionable""" @@ -75,7 +68,7 @@ class RewriteExperienceOp(BaseOp): current_context = self._extract_context(messages) prompt = self.prompt_format( - prompt_name="experience_rewrite_prompt", + prompt_name="memory_rewrite_prompt", current_query=query, current_context=current_context, original_context=context_content @@ -96,18 +89,18 @@ class RewriteExperienceOp(BaseOp): logger.error(f"Error in context rewriting: {e}") return context_content - def _format_experiences_for_context(self, experiences: List[BaseExperience]) -> str: - """Format experiences for context generation""" - formatted_experiences = [] + def _format_memories_for_context(self, memories: List[BaseMemory]) -> str: + """Format memories for context generation""" + formatted_memories = [] - for i, exp in enumerate(experiences, 1): - condition = exp.when_to_use - experience_content = exp.content - exp_text = f"Experience {i} :\n When to use: {condition}\n Content: {experience_content}\n" + for i, memory in enumerate(memories, 1): + condition = memory.when_to_use + memory_content = memory.content + memory_text = f"Memory {i} :\n When to use: {condition}\n Content: {memory_content}\n" - formatted_experiences.append(exp_text) + formatted_memories.append(memory_text) - return "\n".join(formatted_experiences) + return "\n".join(formatted_memories) def _extract_context(self, messages: List[Message]) -> str: """Extract relevant context from messages""" diff --git a/reme_ai/summary/personal/__init__.py b/reme_ai/summary/personal/__init__.py new file mode 100644 index 00000000..01d3682e --- /dev/null +++ b/reme_ai/summary/personal/__init__.py @@ -0,0 +1,21 @@ +from .contra_repeat_worker import ContraRepeatWorker +from .get_observation_with_time_worker import GetObservationWithTimeWorker +from .get_observation_worker import GetObservationWorker +from .get_reflection_subject_worker import GetReflectionSubjectWorker +from .info_filter_worker import InfoFilterWorker +from .load_memory_worker import LoadMemoryWorker +from .long_contra_repeat_worker import LongContraRepeatWorker +from .update_insight_worker import UpdateInsightWorker +from .update_memory_worker import UpdateMemoryWorker + +__all__ = [ + "ContraRepeatWorker", + "GetObservationWithTimeWorker", + "GetObservationWorker", + "GetReflectionSubjectWorker", + "InfoFilterWorker", + "LoadMemoryWorker", + "LongContraRepeatWorker", + "UpdateInsightWorker", + "UpdateMemoryWorker" +] diff --git a/reme_ai/summary/personal/contra_repeat_worker.py b/reme_ai/summary/personal/contra_repeat_worker.py new file mode 100644 index 00000000..9507b23b --- /dev/null +++ b/reme_ai/summary/personal/contra_repeat_worker.py @@ -0,0 +1,129 @@ +from typing import List + +from memoryscope.constants.common_constants import NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, MERGE_OBS_NODES, TODAY_NODES +from memoryscope.constants.language_constants import NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD +from memoryscope.core.utils.response_text_parser import ResponseTextParser +from memoryscope.core.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.scheme.memory_node import MemoryNode + + +class ContraRepeatWorker(MemoryBaseWorker): + """ + The `ContraRepeatWorker` class specializes in processing memory nodes to identify and handle + contradictory and repetitive information. It extends the base functionality of `MemoryBaseWorker`. + + Responsibilities: + - Collects observation nodes from various memory categories. + - Constructs a prompt with these observations for language model analysis. + - Parses the model's response to detect contradictions or redundancies. + - Adjusts the status of memory nodes based on the analysis. + - Persists the updated node statuses back into memory. + """ + FILE_PATH: str = __file__ + + def _parse_params(self, **kwargs): + self.generation_model_kwargs: dict = kwargs.get("generation_model_kwargs", {}) + self.retrieve_top_k: int = kwargs.get("retrieve_top_k", 30) + self.contra_repeat_max_count: int = kwargs.get("contra_repeat_max_count", 50) + self.enable_today_contra_repeat: bool = self.memoryscope_context.meta_data["enable_today_contra_repeat"] + + def _run(self): + """ + Executes the primary routine of the ContraRepeatWorker which involves fetching memory nodes, + constructing a prompt, querying a language model, parsing the response to identify nodes for merging, + updating node statuses, and saving the updated nodes back to memory. + + Steps: + 1. Retrieves new observation nodes and nodes observed on the current day. + 2. Optionally combines today's nodes with the new ones, sorts, and limits the list by a predefined count. + 3. Constructs a prompt using the combined nodes, system prompt, and a few-shot example. + 4. Queries a language model with the constructed prompt. + 5. Parses the model's response to identify nodes to merge or exclude based on contradiction or redundancy. + 6. Updates the status of nodes accordingly. + 7. Persists the changes back to memory storage. + """ + if not self.enable_today_contra_repeat: + self.logger.warning("today_contra_repeat is not enabled!") + return + + all_obs_nodes: List[MemoryNode] = self.memory_manager.get_memories([NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES]) + if not all_obs_nodes: + self.logger.info("all_obs_nodes is empty!") + # self.continue_run = False + return + + today_obs_nodes: List[MemoryNode] = self.memory_manager.get_memories(TODAY_NODES) + + if today_obs_nodes: + all_obs_nodes.extend(today_obs_nodes) + all_obs_nodes = sorted(all_obs_nodes, key=lambda x: x.timestamp, reverse=True)[:self.contra_repeat_max_count] + + if len(all_obs_nodes) == 1: + self.logger.info("all_obs_nodes.size=1, stop.") + return + + # build prompt + user_query_list = [] + for i, n in enumerate(all_obs_nodes): + user_query_list.append(f"{i + 1} {n.content}") + + system_prompt = self.prompt_handler.contra_repeat_system.format(num_obs=len(user_query_list), + user_name=self.target_name) + few_shot = self.prompt_handler.contra_repeat_few_shot.format(user_name=self.target_name) + user_query = self.prompt_handler.contra_repeat_user_query.format(user_query="\n".join(user_query_list)) + contra_repeat_message = self.prompt_to_msg(system_prompt=system_prompt, few_shot=few_shot, + user_query=user_query) + self.logger.info(f"contra_repeat_message={contra_repeat_message}") + + # call LLM + response = self.generation_model.call(messages=contra_repeat_message, **self.generation_model_kwargs) + + # return if empty + if not response.status or not response.message.content: + return + response_text = response.message.content + + # parse text + idx_merge_obs_list = ResponseTextParser(response_text, self.language, self.__class__.__name__).parse_v1() + if len(idx_merge_obs_list) <= 0: + self.logger.warning("idx_merge_obs_list is empty!") + return + + # add merged obs + merge_obs_nodes: List[MemoryNode] = [] + for obs_content_list in idx_merge_obs_list: + if not obs_content_list: + continue + + # Expecting a pair [index, flag] + if len(obs_content_list) != 2: + self.logger.warning(f"obs_content_list={obs_content_list} is invalid!") + continue + + idx, keep_flag = obs_content_list + + if not idx.isdigit(): + self.logger.warning(f"idx={idx} is invalid!") + continue + + # index number needs to be corrected to -1 + idx = int(idx) - 1 + if idx >= len(all_obs_nodes): + self.logger.warning(f"idx={idx} is invalid!") + continue + + # judge flag + keep_flag = keep_flag.lower() + if keep_flag not in self.get_language_value([NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD]): + self.logger.warning(f"keep_flag={keep_flag} is invalid!") + continue + + node: MemoryNode = all_obs_nodes[idx] + if keep_flag != self.get_language_value(NONE_WORD): + node.store_status = StoreStatusEnum.EXPIRED.value + self.logger.info(f"contra_repeat stage: {node.content} {node.store_status} {node.action_status}") + merge_obs_nodes.append(node) + + # save context + self.memory_manager.set_memories(MERGE_OBS_NODES, merge_obs_nodes, log_repeat=False) diff --git a/reme_ai/summary/personal/contra_repeat_worker.yaml b/reme_ai/summary/personal/contra_repeat_worker.yaml new file mode 100644 index 00000000..3be7d3f8 --- /dev/null +++ b/reme_ai/summary/personal/contra_repeat_worker.yaml @@ -0,0 +1,126 @@ +contra_repeat_system: + cn: | + 任务：对下面的{num_obs}句句子，逐一判断是否与“前面序号”的任意句子存在信息的矛盾，或者句子的主要信息被“前面序号”的任意句子中的信息包含。 + 注意：对每句句子，只判断与“前面序号”的句子的关系，不要判断与“后面序号”的句子的关系。 + 其中矛盾的形式可以有很多种，可以是逻辑上的矛盾，可以是属性上的变化导致的矛盾，比如不能同时在两个地方工作，同一个时刻不能在两个地点，同一个时刻不能干两件事情等等。 + 对每个句子都做一个判断，最后一共输出{num_obs}条判断。 + 请一步步思考，并按如下格式输出: + 思考：思考的依据和过程，30字以内。 + 判断：<句子序号> <矛盾，被包含，无>，一定加<> + + en: | + Task: For the following {num_obs} sentences, determine whether each sentence contradicts any of the previous numbered sentences or if the main information in the sentence is contained within the information from any of the previous numbered sentences. + Note: Only determine the relationship with the "previous numbered" sentences, do not judge the "later numbered" sentences. + The forms of contradiction can be varied. It can be a logical contradiction, or a contradiction due to changes in attributes, for example, not being able to work in two places at once, not being able to be in two places at the same time, not being able to do two things at the same time, etc. + Make a judgment for each sentence and output a total of {num_obs} judgments. + Think step by step and output in the following format: + Thought: Basis and process of thinking, within 30 words. + Judgment: , must be enclosed in <>. + +contra_repeat_few_shot: + cn: | + 示例1 + 句子： + 1 {user_name}经常失眠，对安眠药的效果感兴趣，暗示可能考虑使用。 + 2 {user_name}经常失眠，寻求缓解方法。 + 3 陈伟业是{user_name}的领导 + 4 陈伟业是{user_name}的领导 + 5 陈伟业是{user_name}的领导，是银行分行行长 + 6 {user_name}喜欢吃西瓜 + 7 {user_name}喜欢吃苹果 + + 思考：第1句不会存在与前面序号句子的矛盾或者完全重复。 + 判断：<1> <无> + 思考：第2句中所有信息都被前面序号中第1句的信息完全包含。 + 判断：<2> <被包含> + 思考：第3句信息没有在前面序号句子中出现 + 判断：<3> <无> + 思考：第4句与前面序号中第3句的信息完全重复，即被完全包含。 + 判断：<4> <被包含> + 思考：第5句中陈伟业是{user_name}的领导的信息被前面序号中第3句的信息包含，但新增了陈伟业是银行分行行长的信息，故不是被完全包含。 + 判断：<5> <无> + 思考：第6句中表达了{user_name}的水果偏好，喜欢吃西瓜，信息没有在前面序号句子中出现。 + 判断：<6> <无> + 思考：第7句也表达了{user_name}的水果偏好，喜欢吃桃子，和前面序号中的第6句不冲突，喜好可以同时存在。 + 判断：<7> <无> + + 示例2 + 句子： + 1 {user_name}的孩子成绩不太好。 + 2 {user_name}的孩子在学校经常逃课。 + 3 {user_name}的父亲生日在2024年6月2日，{user_name}打算准备礼物。 + 4 {user_name}的父亲生日在2024年5月1日。 + 5 {user_name}很喜欢和同班同学打篮球。 + 6 {user_name}喜欢打篮球。 + + 思考：第1句不会存在与前面序号句子的矛盾或者完全重复。 + 判断：<1> <无> + 思考：第2句与前面序号句子既不矛盾也不重复。 + 判断：<2> <无> + 思考：第3句与前面序号句子既不矛盾也不重复。 + 判断：<3> <无> + 思考：第4句关于{user_name}父亲生日的日期信息与前面序号句子第3句矛盾了。 + 判断：<4> <矛盾> + 思考：第5句与前面序号句子既不矛盾也不重复。 + 判断：<5> <无> + 思考：第6句中所有信息都被前面序号中第5句的信息完全包含。 + 判断：<2> <被包含> + + en: | + Example 1 + Sentences: + 1 {user_name} suffers from insomnia frequently and is interested in the effects of sleeping pills, suggesting a possible consideration of their use. + 2 {user_name} suffers from insomnia frequently and seeks remedies. + 3 Charles is {user_name}'s supervisor. + 4 Charles is {user_name}'s supervisor. + 5 Charles is {user_name}'s supervisor and the branch manager of a bank. + 6 {user_name} loves playing basketball with classmates. + 7 {user_name} likes playing basketball. + + Thought: The first sentence does not have any contradictions or complete repetitions with the previously numbered sentences. + Judgment: <1> + Thought: All information in the second sentence is completely contained within the information of the first sentence. + Judgment: <2> + Thought: The information in the third sentence does not appear in the previously numbered sentences. + Judgment: <3> + Thought: The fourth sentence is completely repetitive of the information in the third sentence, i.e., it is completely contained. + Judgment: <4> + Thought: The information that Charles is {user_name}'s supervisor in the fifth sentence is contained within the information of the third sentence, but the new information that Charles is the branch manager of a bank is not, so it is not contained. + Judgment: <5> + Thought: Sentence 6 expresses {user_name}'s fruit preference, liking to eat watermelon, which is information not present in any preceding sentences. + Judgment: <6> + Thought: Sentence 7 also expresses {user_name}'s fruit preference, liking to eat apples; it does not conflict with sentence 6, and both preferences can coexist. + Judgment: <7> + + Example 2 + Sentences: + 1 {user_name}'s child does not perform well academically. + 2 {user_name}'s child often skips school. + 3 {user_name}'s father's birthday is on June 2, 2024, and {user_name} plans to prepare a gift. + 4 {user_name}'s father's birthday is on May 1, 2024. + 5 {user_name} loves playing basketball with classmates. + 6 {user_name} likes playing basketball. + + Thought: The first sentence does not have any contradictions or complete repetitions with the previously numbered sentences. + Judgment: <1> + Thought: The second sentence neither contradicts nor repeats any of the previously numbered sentences. + Judgment: <2> + Thought: The third sentence neither contradicts nor repeats any of the previously numbered sentences. + Judgment: <3> + Thought: The date of {user_name}'s father's birthday in the fourth sentence contradicts the information in the third sentence. + Judgment: <4> + Thought: The fifth sentence neither contradicts nor repeats any of the previously numbered sentences. + Judgment: <5> + Thought: All information in the sixth sentence is completely contained within the information of the fifth sentence. + Judgment: <6> + + + +contra_repeat_user_query: + cn: | + 句子： + {user_query} + + en: | + Sentences: + {user_query} diff --git a/reme_ai/summary/personal/get_observation_with_time_worker.py b/reme_ai/summary/personal/get_observation_with_time_worker.py new file mode 100644 index 00000000..4bea129a --- /dev/null +++ b/reme_ai/summary/personal/get_observation_with_time_worker.py @@ -0,0 +1,76 @@ +from typing import List + +from memoryscope.constants.common_constants import NEW_OBS_WITH_TIME_NODES +from memoryscope.constants.language_constants import COLON_WORD +from memoryscope.core.utils.datetime_handler import DatetimeHandler +from memoryscope.core.worker.backend.get_observation_worker import GetObservationWorker +from memoryscope.scheme.message import Message + + +class GetObservationWithTimeWorker(GetObservationWorker): + """ + A specialized worker class that extends GetObservationWorker functionality to handle + retrieval of observations which include associated timestamp information from chat messages. + """ + FILE_PATH: str = __file__ + OBS_STORE_KEY: str = NEW_OBS_WITH_TIME_NODES + + def filter_messages(self) -> List[Message]: + """ + Filters the chat messages to only include those which contain time-related keywords. + + Returns: + List[Message]: A list of filtered messages that mention time. + """ + filter_messages = [] + for msg in self.chat_messages_scatter: + # Checks if the message content has any time reference words + if DatetimeHandler.has_time_word(query=msg.content, language=self.language): + filter_messages.append(msg) + return filter_messages + + def build_message(self, filter_messages: List[Message]) -> List[Message]: + """ + Constructs a prompt message for obtaining observations with timestamp information + based on filtered chat messages. + + This method processes each filtered message with the timestamp information. + It then organizes these timestamped messages into a structured prompt that includes a system prompt, + few-shot examples, and the concatenated user queries. + + Args: + filter_messages (List[Message]): A list of Message objects that have been filtered for processing. + + Returns: + List[Message]: A list containing the newly constructed Message object for further interaction. + """ + user_query_list = [] + for i, msg in enumerate(filter_messages): + # Create a DatetimeHandler instance for each message's timestamp and format it + dt_handler = DatetimeHandler(dt=msg.time_created) + dt = dt_handler.string_format(string_format=self.prompt_handler.time_string_format, language=self.language) + # Append formatted timestamp-query pairs to the user_query_list + user_query_list.append(f"{i + 1} {dt} {self.target_name}{self.get_language_value(COLON_WORD)}{msg.content}") + + # Construct the system prompt with the count of observations + system_prompt = self.prompt_handler.get_observation_with_time_system.format(num_obs=len(user_query_list), + user_name=self.target_name) + + # Retrieve the few-shot examples for the prompt + few_shot = self.prompt_handler.get_observation_with_time_few_shot.format(user_name=self.target_name) + + # Format the user query section with the concatenated list of timestamped queries + user_query = self.prompt_handler.get_observation_with_time_user_query.format( + user_query="\n".join(user_query_list), + user_name=self.target_name) + + # Assemble the final message for observation retrieval + get_observation_message_wt = self.prompt_to_msg(system_prompt=system_prompt, + few_shot=few_shot, + user_query=user_query) + + # Log the constructed message for debugging purposes + self.logger.info(f"get_observation_message_wt={get_observation_message_wt}") + + # Return the newly created message + return get_observation_message_wt diff --git a/reme_ai/summary/personal/get_observation_with_time_worker.yaml b/reme_ai/summary/personal/get_observation_with_time_worker.yaml new file mode 100644 index 00000000..97320c23 --- /dev/null +++ b/reme_ai/summary/personal/get_observation_with_time_worker.yaml @@ -0,0 +1,156 @@ +time_string_format: + cn: | + {year}年{month}{day}日{weekday}{hour}点 + en: | + {month} {day}, {year}, {weekday}, at {hour} + +get_observation_with_time_system: + cn: | + 任务：从下面的{num_obs}句{user_name}句子中依次提取出关于{user_name}的重要信息，相应的关键词与时间信息。如果没有重要信息则回答“无”，最多提取{num_obs}条信息。 + 每一句{user_name}句子的格式是：<序号> <对话时间> {user_name}：<句子> + {user_name}的重要信息可以包含用户基本信息，用户画像信息，用户兴趣偏好信息，用户性格，用户价值观，用户人际关系，用户重大事件转折点等等重要信息。 + 如果句子中只包含{user_name}假设的信息或者{user_name}虚构的内容比如{user_name}创作的小说或剧本，回答“无”。 + 如果{user_name}信息涉及时间，则结合对话时间推断{user_name}信息的时间信息，没有则不输出。 + 对每个句子都做一次信息提取，最后一共输出{num_obs}条信息。 + 请一步步思考，并一定要按如下格式依次输出，最后的结果一定要加<>: + 思考：思考的依据和过程，50字以内。 + 信息：<句子序号> <时间信息或不输出> <明确的重要信息或“无”> <关键词> + + en: | + Task: Extract important information about {user_name} from the following {num_obs} sentences of {user_name}, including relevant keywords and time information. If there is no important information, answer "none", with a maximum of {num_obs} pieces of information extracted. + Each sentence from {user_name} is formatted as follows: {user_name}: . + Important information about {user_name} can include basic information, user profile information, interest preferences, personality, values, human relationships, significant life events, etc. + If a sentence only contains hypothetical information or fictional content created by {user_name} (e.g., novels or scripts), answer "none". + If {user_name}'s information involves time, infer the time information based on the conversation time; if not, do not output. + Analyze each sentence once to extract information and output a total of {num_obs} pieces of information. + Please think step-by-step and be sure to output in the following format, with the final results enclosed in <>: + Thought: Basis and process of thought, within 50 words. + Information: