[dev] add memory base worker

config/model/dash_embedding.json

@@ -1,4 +1,5 @@
 {
-    "method": "DashScopeEmbedding",
-    "model_name": "text-embedding-v2"
+    "model_name": "text-embedding-v2",
+    "method_type": "DashScopeEmbedding",
+    "clazz": "models.base_embedding_model"
 }

config/worker.json

@@ -51,7 +51,7 @@
     },
     "ExtractTimeWorker": {
         "name": "ExtractTimeWorker",
-        "path": "memory_scope/worker",
+        "clazz": "worker.summary_long.get_insight",
         "parse_time_model": "qwen_1_8_parse_time_service"
     },
     "InfoFilterWorker": {

memory_scope/cli.py

@@ -1,14 +1,31 @@
 import fire
 
-from chat.memory_chat import MemoryChat
-from config import init
+from memory_scope.chat.memory_chat import MemoryChat
+from memory_scope.handler.config_handler import ConfigHandler
+
+"""
+1. fire read config
+2. init config
+    1. global configs: global + db + llm + monitor
+    2. worker config list: worker + llm
+3. init db
+4. init workers, global+worker
+5. init llms
+6. init monitor
+7. new Agent,add db workers llms monitor
+    1. new memory service
+        1. new pipeline
+    2. chat
+    3. 
+"""
 
 
-def main(config_path:str):
-    init(config_path)
-    
+def main(config_path: str):
+    config_handler = ConfigHandler(config_path)
+
     agent = MemoryChat()
     agent.run()
 
+
 if __name__ == "__main__":
     fire.Fire(main)

memory_scope/db/base_db.py

@@ -0,0 +1,38 @@
+from abc import ABCMeta, abstractmethod
+
+from memory_scope.models.base_model import BaseModel
+
+
+class BaseDBClient(metaclass=ABCMeta):
+
+    def __init__(self, index_name: str, embedding_model: BaseModel, content_key: str = "text", **kwargs):
+        self.index_name: str = index_name
+        self.embedding_model: BaseModel = embedding_model
+        self.content_key: str = content_key
+        self.kwargs: dict = kwargs
+
+    @abstractmethod
+    def retrieve(self, text: str, limit_size: int):
+        """
+        :param text:
+        :param limit_size:
+        :return:
+        """
+
+    @abstractmethod
+    def insert(self, text: str):
+        """ TODO 是否overwrite
+        :return:
+        """
+
+    @abstractmethod
+    def insert_batch(self):
+        """
+        :return:
+        """
+
+    @abstractmethod
+    def delete(self):
+        """
+        :return:
+        """

memory_scope/handler/config_handler.py

@@ -0,0 +1,73 @@
+import json
+import os.path
+from typing import Dict
+
+from memory_scope.models.base_model import BaseModel
+from memory_scope.utils.tool_functions import init_instance_by_config_v2
+from memory_scope.worker.base_worker import BaseWorker
+
+
+class ConfigHandler(object):
+
+    def __init__(self, path: str):
+        self.config_name: str = os.path.basename(path)
+        self.config_base_dir: str = os.path.dirname(path)
+
+        with open(path) as f:
+            config = json.load(f)
+
+        self.global_configs: Dict[str, str] = {}
+        self.worker_dict: Dict[str, BaseWorker] = {}
+        self.model_dict: Dict[str, BaseModel] = {}
+
+        self._init_global_config(config["global"])
+        self.worker_base_dir = self.global_configs.get("worker_base_dir", "config")
+        self.model_base_dir = self.global_configs.get("model_base_dir", "config/model")
+        self._init_workers(config["workers"])
+        self._init_db(config["db"])
+        self._init_chat_model(config["chat_model"])
+        self._init_monitor(config["monitor"])
+
+    def _init_global_config(self, global_configs: Dict[str, str]):
+        """set global_configs & set apikey into env
+        """
+
+    def _init_workers(self, worker_config_name: str):
+        """ load worker config & init workers
+        """
+        with open(os.path.join(self.config_base_dir, worker_config_name)) as f:
+            worker_config_dict = json.load(f)
+
+        for worker_name, worker_config in worker_config_dict.items():
+            if worker_name in self.worker_dict:
+                continue
+            self.worker_dict[worker_name] = init_instance_by_config_v2(worker_config,
+                                                                       default_clazz_path=self.worker_base_dir,
+                                                                       suffix_name="worker",
+                                                                       **self.global_configs,
+                                                                       **worker_config)
+
+            self._init_model(worker_config.get("embedding_model"))
+            self._init_model(worker_config.get("generation_model"))
+            self._init_model(worker_config.get("rank_model"))
+
+    def _init_model(self, model_name: str):
+        if not model_name or model_name in self.model_dict:
+            return
+
+        with open(os.path.join(self.model_base_dir, model_name)) as f:
+            model_config = json.load(f)
+            self.model_dict[model_name] = init_instance_by_config_v2(model_config,
+                                                                     default_clazz_path=self.model_base_dir,
+                                                                     suffix_name="",
+                                                                     **model_config)
+
+    def _init_db(self, db_config: dict):
+        pass
+
+    def _init_chat_model(self, chat_model_config: dict):
+        chat_model_name = chat_model_config["name"]
+        self._init_model(chat_model_name)
+
+    def _init_monitor(self, monitor_config: dict):
+        pass

memory_scope/models/base_model.py

@@ -19,6 +19,7 @@ class BaseModel(metaclass=ABCMeta):
                  retry_interval: float = 1.0,
                  kwargs_filter: bool = True,
                  **kwargs):
+
         self.model_name: str = model_name
         self.method_type: str = method_type
         self.timeout: int = timeout

memory_scope/models/response.py

@@ -8,7 +8,9 @@ class ModelResponse(BaseModel):
 
     embedding_results: Dict[int, List[float]] | List[float] = Field([], description="")
 
-    rank_scores: List[float] = Field([], description="")
+    rank_scores: Dict[int, float] = Field({}, description="")
+
+    model_type: str = Field("", description="")
 
     status: bool = Field(True, description="")
 

memory_scope/utils/tool_functions.py

@@ -1,10 +1,10 @@
-import os
 import re
 from datetime import datetime
+from importlib import import_module
 from typing import Dict, List
 
 from constants.common_constants import WEEKDAYS
-from importlib import import_module
+
 
 def under_line_to_hump(underline_str):
     sub = re.sub(r'(_\w)', lambda x: x.group(1)[1].upper(), underline_str)
@@ -153,4 +153,25 @@ def init_instance_by_config(config: dict|object, default_module_path: str = None
     try:
         return clazz(**config, **try_kwargs)
     except:
-        return clazz(**config)
+        return clazz(**config)
+
+
+def init_instance_by_config_v2(config: dict, default_clazz_path: str = "", suffix_name: str = "", **kwargs):
+    clazz_path = config.pop("clazz")
+    if not clazz_path:
+        raise RuntimeError("empty clazz_path!")
+    clazz_name_split = clazz_path.split(".")
+    clazz_name: str = clazz_name_split[-1]
+    if suffix_name and not clazz_name.endswith(suffix_name):
+        clazz_name = f"{clazz_name}_{suffix_name}"
+
+    # 构造path
+    clazz_paths = []
+    if default_clazz_path:
+        clazz_paths.append(default_clazz_path)
+    clazz_paths.extend(clazz_name_split[:-1])
+    clazz_paths.append(clazz_name)
+    module = import_module(".".join(clazz_paths))
+
+    cls_name = under_line_to_hump(clazz_name)
+    return getattr(module, cls_name)(**kwargs)