[dev] modify default json config

config/config.json

@@ -1,27 +1,85 @@
 {
-    "global_configs": {
-        "thread_pool_max_count": 5,
-        "dash_scope_apikey": "",
-        "open_ai_apikey": "",
-        "language": "en",
-        "chat_list": [
-            "memory_chat"
-        ]
+  "global_config": {
+    "language": "en",
+    "max_workers": 5,
+    "dash_scope_apikey": null,
+    "open_ai_apikey": null
+  },
+  "memory_chat": {
+    "cli_memory_chat": {
+      "class": "chat_v2.cli_memory_chat",
+      "memory_service": "memory_chat_service",
+      "generation_model": "dashscope_generation"
+    }
+  },
+  "memory_service": {
+    "memory_chat_service": {
+      "class": "memory.service.chat_memory_service",
+      "history_msg_count": 32,
+      "contextual_msg_count": 6,
+      "read_memory_key": "read_memory",
+      "memory_operations": {
+        "read_message": {
+          "class": "memory.operation.read_memory",
+          "workflow": "dummy_worker",
+          "description": "read session messages of the user",
+          "contextual_msg_count": 0
+        },
+        "read_memory": {
+          "class": "memory.operation.read_memory",
+          "workflow": "dummy_worker",
+          "description": "read related memories of the user"
+        },
+        "list_memory": {
+          "class": "memory.operation.read_memory",
+          "workflow": "dummy_worker",
+          "description": "read all memories of the user"
+        },
+        "write_memory": {
+          "class": "memory.operation.write_memory",
+          "workflow": "dummy_worker",
+          "description": "write observation memories of the user",
+          "interval_time": 60
+        },
+        "summary_memory": {
+          "class": "memory.operation.summary_memory",
+          "workflow": "dummy_worker",
+          "description": "summary observation memories of the user",
+          "interval_time": 300
+        }
+      }
+    }
+  },
+  "models": {
+    "dashscope_generation": {
+      "clazz": "models.llama_index_generation_model",
+      "module_name": "dashscope_generation",
+      "model_name": "qwen-max"
     },
-    "memory_chat": {
-        "clazz": "chat.memory_chat",
-        "retrieve": "parse_params,load_profile,extract_time,es_similar,es_keyword,semantic_rank,fuse_rerank",
-        "generation_model": "dashscope_generation",
-        "history_msg_count": 3
+    "dashscope_embedding": {
+      "clazz": "models.llama_index_embedding_model",
+      "module_name": "dashscope_embedding",
+      "model_name": "text-embedding-v2"
     },
-    "vector_store": {
-        "clazz": "storage.base_vector_store",
-        "index_name": "memory_test",
-        "password": ""
-    },
-    "monitor": {
-        "clazz": "storage.base_monitor",
-        "index_name": "memory_test"
-    },
-    "workers": "workers"
+    "dashscope_rank": {
+      "clazz": "models.llama_index_rank_model",
+      "module_name": "dashscope_rank",
+      "model_name": "gte-rerank"
+    }
+  },
+  "vector_store": {
+    "clazz": "storage.dummy_vector_store",
+    "embedding_model": "dashscope_embedding"
+  },
+  "monitor": {
+    "clazz": "storage.dummy_monitor"
+  },
+  "workers": {
+    "dummy_worker": {
+      "clazz": "memory.worker.dummy_worker",
+      "generation_model": "dashscope_generation",
+      "embedding_model": "dashscope_embedding",
+      "rank_model": "dashscope_rank"
+    }
+  }
 }

config/model/dashscope_embedding.json

@@ -1,5 +0,0 @@
-{
-    "clazz": "models.base_embedding_model",
-    "model_name": "text-embedding-v2",
-    "method_type": "DashScopeEmbedding"
-}

config/model/dashscope_generation.json

@@ -1,5 +0,0 @@
-{
-    "clazz": "models.llama_index_generation_model",
-    "model_name": "qwen-max",
-    "method_type": "DashScope"
-}

config/model/dashscope_rank.json

@@ -1,5 +0,0 @@
-{
-    "clazz": "models.base_rank_model",
-    "model_name": "gte-rerank",
-    "method_type": "DashScopeRerank"
-}

config/workers.json

@@ -1,8 +0,0 @@
-{
-    "update_insight": {
-        "clazz": "worker.summary_long.update_insight",
-        "generation_model": "dashscope_generation",
-        "embedding_model": "dashscope_embedding",
-        "rank_model": "dashscope_rank"
-    }
-}

memory_scope/cli.py

@@ -1,67 +1,54 @@
+import json
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Any
 
-import yaml
 import fire
+import yaml
 
-from .chat_v2.global_context import G_CONTEXT
-from .enumeration.language_enum import LanguageEnum
-from .utils.logger import Logger
-from .utils.tool_functions import init_instance_by_config
+from memory_scope.chat_v2.global_context import G_CONTEXT
+from memory_scope.enumeration.language_enum import LanguageEnum
+from memory_scope.utils.logger import Logger
+from memory_scope.utils.tool_functions import init_instance_by_config
 
 
 class CliJob(object):
 
-    def __init__(self, config_path: str, config_suffix: str = ".yaml"):
-        self.config_path: str = config_path
-        self.config_suffix: str = config_suffix
+    def __init__(self):
         self.config: Dict[str, Any] = {}
-
         self.logger: Logger = Logger.get_logger("cli_job")
 
-    @staticmethod
-    def set_global_config(global_config: Dict[str, Any]):
-        """set global_configs & set apikey into env
-        :return:
-        TODO at sen
-        """
-        G_CONTEXT.global_config = global_config
+    def load_config(self, path: str):
+        with open(path) as f:
+            if path.endswith("yaml"):
+                self.config = yaml.load(f, yaml.FullLoader)
+            elif path.endswith("json"):
+                self.config = json.load(f)
+            else:
+                raise RuntimeError("not supported config file type!")
+
+    def set_global_config(self):
+        G_CONTEXT.global_config = global_config = self.config["global_config"]
         G_CONTEXT.language = LanguageEnum(global_config["language"])
-        G_CONTEXT.thread_pool = ThreadPoolExecutor(
-            max_workers=int(global_config["max_workers"])
-        )
+        G_CONTEXT.thread_pool = ThreadPoolExecutor(max_workers=int(global_config["max_workers"]))
 
     def init_global_content_by_config(self):
-        # load config
-        config_path = self.config_path
-        if not self.config_path.endswith(self.config_suffix):
-            config_path += self.config_suffix
-        with open(config_path) as f:
-            self.config = yaml.load(f, yaml.FullLoader)
-
-        # set global_config
-        self.set_global_config(self.config["global_config"])
+        # set global config
+        self.set_global_config()
 
         # init memory_chat
         for name, conf in self.config["memory_chat"].items():
-            G_CONTEXT.memory_chat_dict[name] = init_instance_by_config(
-                conf, name=name
-            )
+            G_CONTEXT.memory_chat_dict[name] = init_instance_by_config(conf, name=name)
 
         # set memory_service
         for name, conf in self.config["memory_service"].items():
-            G_CONTEXT.memory_service_dict[name] = init_instance_by_config(
-                conf, name=name
-            )
+            G_CONTEXT.memory_service_dict[name] = init_instance_by_config(conf, name=name)
 
         # init models
         for name, conf in self.config["models"].items():
             G_CONTEXT.model_dict[name] = init_instance_by_config(conf, name=name)
 
         # init vector_store
-        G_CONTEXT.vector_store = init_instance_by_config(
-            self.config["vector_store"]
-        )
+        G_CONTEXT.vector_store = init_instance_by_config(self.config["vector_store"])
 
         # init monitor
         G_CONTEXT.monitor = init_instance_by_config(self.config["monitor"])
@@ -69,8 +56,14 @@ class CliJob(object):
         # set worker config
         G_CONTEXT.worker_config = self.config["workers"]
 
-    @staticmethod
-    def run():
+    def run(self, config: str):
+        self.load_config(config)
+
         with G_CONTEXT.thread_pool:
             memory_chat = list(G_CONTEXT.memory_chat_dict.values())[0]
             memory_chat.run()
+
+
+if __name__ == "__main__":
+    cli_job = CliJob()
+    fire.Fire(cli_job.run)