diff --git a/config/model/dash_embedding.json b/config/model/dash_embedding.json index 8ceaf9b5..70408e30 100644 --- a/config/model/dash_embedding.json +++ b/config/model/dash_embedding.json @@ -1,4 +1,5 @@ { - "method": "DashScopeEmbedding", - "model_name": "text-embedding-v2" + "model_name": "text-embedding-v2", + "method_type": "DashScopeEmbedding", + "clazz": "models.base_embedding_model" } \ No newline at end of file diff --git a/config/worker.json b/config/worker.json index a55f33e6..32c31749 100644 --- a/config/worker.json +++ b/config/worker.json @@ -51,7 +51,7 @@ }, "ExtractTimeWorker": { "name": "ExtractTimeWorker", - "path": "memory_scope/worker", + "clazz": "worker.summary_long.get_insight", "parse_time_model": "qwen_1_8_parse_time_service" }, "InfoFilterWorker": { diff --git a/memory_scope/cli.py b/memory_scope/cli.py index 46ab0af2..3de177eb 100644 --- a/memory_scope/cli.py +++ b/memory_scope/cli.py @@ -1,14 +1,31 @@ import fire -from chat.memory_chat import MemoryChat -from config import init +from memory_scope.chat.memory_chat import MemoryChat +from memory_scope.handler.config_handler import ConfigHandler + +""" +1. fire read config +2. init config + 1. global configs: global + db + llm + monitor + 2. worker config list: worker + llm +3. init db +4. init workers, global+worker +5. init llms +6. init monitor +7. new Agent,add db workers llms monitor + 1. new memory service + 1. new pipeline + 2. chat + 3. +""" -def main(config_path:str): - init(config_path) - +def main(config_path: str): + config_handler = ConfigHandler(config_path) + agent = MemoryChat() agent.run() + if __name__ == "__main__": fire.Fire(main) diff --git a/memory_scope/db/base_db.py b/memory_scope/db/base_db.py new file mode 100644 index 00000000..59272d0a --- /dev/null +++ b/memory_scope/db/base_db.py @@ -0,0 +1,38 @@ +from abc import ABCMeta, abstractmethod + +from memory_scope.models.base_model import BaseModel + + +class BaseDBClient(metaclass=ABCMeta): + + def __init__(self, index_name: str, embedding_model: BaseModel, content_key: str = "text", **kwargs): + self.index_name: str = index_name + self.embedding_model: BaseModel = embedding_model + self.content_key: str = content_key + self.kwargs: dict = kwargs + + @abstractmethod + def retrieve(self, text: str, limit_size: int): + """ + :param text: + :param limit_size: + :return: + """ + + @abstractmethod + def insert(self, text: str): + """ TODO 是否overwrite + :return: + """ + + @abstractmethod + def insert_batch(self): + """ + :return: + """ + + @abstractmethod + def delete(self): + """ + :return: + """ \ No newline at end of file diff --git a/memory_scope/handler/__init__.py b/memory_scope/handler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/memory_scope/handler/config_handler.py b/memory_scope/handler/config_handler.py new file mode 100644 index 00000000..96251033 --- /dev/null +++ b/memory_scope/handler/config_handler.py @@ -0,0 +1,73 @@ +import json +import os.path +from typing import Dict + +from memory_scope.models.base_model import BaseModel +from memory_scope.utils.tool_functions import init_instance_by_config_v2 +from memory_scope.worker.base_worker import BaseWorker + + +class ConfigHandler(object): + + def __init__(self, path: str): + self.config_name: str = os.path.basename(path) + self.config_base_dir: str = os.path.dirname(path) + + with open(path) as f: + config = json.load(f) + + self.global_configs: Dict[str, str] = {} + self.worker_dict: Dict[str, BaseWorker] = {} + self.model_dict: Dict[str, BaseModel] = {} + + self._init_global_config(config["global"]) + self.worker_base_dir = self.global_configs.get("worker_base_dir", "config") + self.model_base_dir = self.global_configs.get("model_base_dir", "config/model") + self._init_workers(config["workers"]) + self._init_db(config["db"]) + self._init_chat_model(config["chat_model"]) + self._init_monitor(config["monitor"]) + + def _init_global_config(self, global_configs: Dict[str, str]): + """set global_configs & set apikey into env + """ + + def _init_workers(self, worker_config_name: str): + """ load worker config & init workers + """ + with open(os.path.join(self.config_base_dir, worker_config_name)) as f: + worker_config_dict = json.load(f) + + for worker_name, worker_config in worker_config_dict.items(): + if worker_name in self.worker_dict: + continue + self.worker_dict[worker_name] = init_instance_by_config_v2(worker_config, + default_clazz_path=self.worker_base_dir, + suffix_name="worker", + **self.global_configs, + **worker_config) + + self._init_model(worker_config.get("embedding_model")) + self._init_model(worker_config.get("generation_model")) + self._init_model(worker_config.get("rank_model")) + + def _init_model(self, model_name: str): + if not model_name or model_name in self.model_dict: + return + + with open(os.path.join(self.model_base_dir, model_name)) as f: + model_config = json.load(f) + self.model_dict[model_name] = init_instance_by_config_v2(model_config, + default_clazz_path=self.model_base_dir, + suffix_name="", + **model_config) + + def _init_db(self, db_config: dict): + pass + + def _init_chat_model(self, chat_model_config: dict): + chat_model_name = chat_model_config["name"] + self._init_model(chat_model_name) + + def _init_monitor(self, monitor_config: dict): + pass diff --git a/memory_scope/pipeline/pipeline.py b/memory_scope/handler/pipeline.py similarity index 100% rename from memory_scope/pipeline/pipeline.py rename to memory_scope/handler/pipeline.py diff --git a/memory_scope/models/base_embeddding_model.py b/memory_scope/models/base_embedding_model.py similarity index 100% rename from memory_scope/models/base_embeddding_model.py rename to memory_scope/models/base_embedding_model.py diff --git a/memory_scope/models/base_model.py b/memory_scope/models/base_model.py index bdb1ee46..af415976 100644 --- a/memory_scope/models/base_model.py +++ b/memory_scope/models/base_model.py @@ -19,6 +19,7 @@ class BaseModel(metaclass=ABCMeta): retry_interval: float = 1.0, kwargs_filter: bool = True, **kwargs): + self.model_name: str = model_name self.method_type: str = method_type self.timeout: int = timeout diff --git a/memory_scope/models/response.py b/memory_scope/models/response.py index d99debd5..11890183 100644 --- a/memory_scope/models/response.py +++ b/memory_scope/models/response.py @@ -8,7 +8,9 @@ class ModelResponse(BaseModel): embedding_results: Dict[int, List[float]] | List[float] = Field([], description="") - rank_scores: List[float] = Field([], description="") + rank_scores: Dict[int, float] = Field({}, description="") + + model_type: str = Field("", description="") status: bool = Field(True, description="") diff --git a/memory_scope/utils/tool_functions.py b/memory_scope/utils/tool_functions.py index fdf0d8ef..6d8aeb41 100644 --- a/memory_scope/utils/tool_functions.py +++ b/memory_scope/utils/tool_functions.py @@ -1,10 +1,10 @@ -import os import re from datetime import datetime +from importlib import import_module from typing import Dict, List from constants.common_constants import WEEKDAYS -from importlib import import_module + def under_line_to_hump(underline_str): sub = re.sub(r'(_\w)', lambda x: x.group(1)[1].upper(), underline_str) @@ -153,4 +153,25 @@ def init_instance_by_config(config: dict|object, default_module_path: str = None try: return clazz(**config, **try_kwargs) except: - return clazz(**config) \ No newline at end of file + return clazz(**config) + + +def init_instance_by_config_v2(config: dict, default_clazz_path: str = "", suffix_name: str = "", **kwargs): + clazz_path = config.pop("clazz") + if not clazz_path: + raise RuntimeError("empty clazz_path!") + clazz_name_split = clazz_path.split(".") + clazz_name: str = clazz_name_split[-1] + if suffix_name and not clazz_name.endswith(suffix_name): + clazz_name = f"{clazz_name}_{suffix_name}" + + # 构造path + clazz_paths = [] + if default_clazz_path: + clazz_paths.append(default_clazz_path) + clazz_paths.extend(clazz_name_split[:-1]) + clazz_paths.append(clazz_name) + module = import_module(".".join(clazz_paths)) + + cls_name = under_line_to_hump(clazz_name) + return getattr(module, cls_name)(**kwargs)