From 84c147740eb5da8deaf5557dced6d45c465d5923 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 19:28:40 +0800 Subject: [PATCH 1/8] [features] rename package name & fix es bug --- {memory_scope => memoryscope}/__init__.py | 0 .../chat/__init__.py | 0 .../chat/base_memory_chat.py | 2 +- .../chat/cli_memory_chat.py | 22 +++++++++---------- .../chat/cli_memory_chat.yaml | 0 {memory_scope => memoryscope}/cli.py | 12 +++++----- .../constants/__init__.py | 0 .../constants/common_constants.py | 0 .../constants/language_constants.py | 2 +- .../enumeration/__init__.py | 0 .../enumeration/action_status_enum.py | 0 .../enumeration/language_enum.py | 0 .../enumeration/memory_type_enum.py | 0 .../enumeration/message_role_enum.py | 0 .../enumeration/model_enum.py | 0 .../enumeration/store_status_enum.py | 0 .../memory/__init__.py | 0 .../memory/operation/__init__.py | 0 .../memory/operation/backend_operation.py | 12 +++++----- .../memory/operation/base_operation.py | 0 .../memory/operation/base_workflow.py | 12 +++++----- .../memory/operation/frontend_operation.py | 8 +++---- .../operation/summary_observation_op.py | 6 ++--- .../memory/service/__init__.py | 0 .../memory/service/base_memory_service.py | 6 ++--- .../memory/service/memory_scope_service.py | 8 +++---- .../memory/worker/__init__.py | 0 .../memory/worker/base_worker.py | 4 ++-- .../memory/worker/dummy_worker.py | 4 ++-- .../memory/worker/frontend/__init__.py | 0 .../worker/frontend/extract_time_worker.py | 10 ++++----- .../worker/frontend/extract_time_worker.yaml | 0 .../worker/frontend/fuse_rerank_worker.py | 8 +++---- .../worker/frontend/print_memory_worker.py | 12 +++++----- .../worker/frontend/print_memory_worker.yaml | 0 .../worker/frontend/read_message_worker.py | 6 ++--- .../worker/frontend/retrieve_memory_worker.py | 14 ++++++------ .../worker/frontend/semantic_rank_worker.py | 6 ++--- .../worker/frontend/set_query_worker.py | 6 ++--- .../memory/worker/memory_base_worker.py | 18 +++++++-------- .../memory/worker/summary/__init__.py | 0 .../summary/get_reflection_subject_worker.py | 18 +++++++-------- .../get_reflection_subject_worker.yaml | 0 .../summary/long_contra_repeat_worker.py | 18 +++++++-------- .../summary/long_contra_repeat_worker.yaml | 0 .../worker/summary/update_insight_worker.py | 16 +++++++------- .../worker/summary/update_insight_worker.yaml | 0 .../memory/worker/write/__init__.py | 0 .../worker/write/contra_repeat_worker.py | 14 ++++++------ .../worker/write/contra_repeat_worker.yaml | 0 .../write/get_observation_with_time_worker.py | 12 +++++----- .../get_observation_with_time_worker.yaml | 0 .../worker/write/get_observation_worker.py | 20 ++++++++--------- .../worker/write/get_observation_worker.yaml | 0 .../memory/worker/write/info_filter_worker.py | 12 +++++----- .../worker/write/info_filter_worker.yaml | 0 .../memory/worker/write/load_memory_worker.py | 14 ++++++------ .../worker/write/update_memory_worker.py | 10 ++++----- .../models/__init__.py | 0 .../models/base_model.py | 10 ++++----- .../models/dummy_generation_model.py | 10 ++++----- .../models/llama_index_embedding_model.py | 6 ++--- .../models/llama_index_generation_model.py | 10 ++++----- .../models/llama_index_rank_model.py | 6 ++--- .../scheme/__init__.py | 0 .../scheme/memory_node.py | 0 .../scheme/message.py | 0 .../scheme/model_response.py | 4 ++-- .../storage/__init__.py | 0 .../storage/base_memory_store.py | 2 +- .../storage/base_monitor.py | 0 .../storage/dummy_memory_store.py | 6 ++--- .../storage/dummy_monitor.py | 2 +- .../storage/llama_index_es_memory_store.py | 10 ++++----- .../storage/llama_index_sync_elasticsearch.py | 0 .../utils/__init__.py | 0 .../utils/datetime_handler.py | 6 ++--- .../utils/global_context.py | 12 +++++----- {memory_scope => memoryscope}/utils/logger.py | 0 .../utils/memory_handler.py | 12 +++++----- .../utils/prompt_handler.py | 2 +- .../utils/registry.py | 0 .../utils/response_text_parser.py | 6 ++--- {memory_scope => memoryscope}/utils/timer.py | 2 +- .../utils/tool_functions.py | 6 ++--- tests/models/test_models_lli_embedding.py | 4 ++-- tests/models/test_models_lli_generation.py | 6 ++--- tests/models/test_models_lli_rank.py | 2 +- tests/operations/test_interface.py | 4 ++-- tests/storages/test_storages_lli_es.py | 6 ++--- tests/storages/test_storages_lli_synces.py | 6 ++--- tests/worker/test_workers_cn.py | 18 +++++++-------- tests/worker/test_workers_en.py | 18 +++++++-------- 93 files changed, 244 insertions(+), 244 deletions(-) rename {memory_scope => memoryscope}/__init__.py (100%) rename {memory_scope => memoryscope}/chat/__init__.py (100%) rename {memory_scope => memoryscope}/chat/base_memory_chat.py (94%) rename {memory_scope => memoryscope}/chat/cli_memory_chat.py (95%) rename {memory_scope => memoryscope}/chat/cli_memory_chat.yaml (100%) rename {memory_scope => memoryscope}/cli.py (89%) rename {memory_scope => memoryscope}/constants/__init__.py (100%) rename {memory_scope => memoryscope}/constants/common_constants.py (100%) rename {memory_scope => memoryscope}/constants/language_constants.py (98%) rename {memory_scope => memoryscope}/enumeration/__init__.py (100%) rename {memory_scope => memoryscope}/enumeration/action_status_enum.py (100%) rename {memory_scope => memoryscope}/enumeration/language_enum.py (100%) rename {memory_scope => memoryscope}/enumeration/memory_type_enum.py (100%) rename {memory_scope => memoryscope}/enumeration/message_role_enum.py (100%) rename {memory_scope => memoryscope}/enumeration/model_enum.py (100%) rename {memory_scope => memoryscope}/enumeration/store_status_enum.py (100%) rename {memory_scope => memoryscope}/memory/__init__.py (100%) rename {memory_scope => memoryscope}/memory/operation/__init__.py (100%) rename {memory_scope => memoryscope}/memory/operation/backend_operation.py (91%) rename {memory_scope => memoryscope}/memory/operation/base_operation.py (100%) rename {memory_scope => memoryscope}/memory/operation/base_workflow.py (95%) rename {memory_scope => memoryscope}/memory/operation/frontend_operation.py (84%) rename {memory_scope => memoryscope}/memory/operation/summary_observation_op.py (90%) rename {memory_scope => memoryscope}/memory/service/__init__.py (100%) rename {memory_scope => memoryscope}/memory/service/base_memory_service.py (96%) rename {memory_scope => memoryscope}/memory/service/memory_scope_service.py (94%) rename {memory_scope => memoryscope}/memory/worker/__init__.py (100%) rename {memory_scope => memoryscope}/memory/worker/base_worker.py (98%) rename {memory_scope => memoryscope}/memory/worker/dummy_worker.py (84%) rename {memory_scope => memoryscope}/memory/worker/frontend/__init__.py (100%) rename {memory_scope => memoryscope}/memory/worker/frontend/extract_time_worker.py (89%) rename {memory_scope => memoryscope}/memory/worker/frontend/extract_time_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/frontend/fuse_rerank_worker.py (94%) rename {memory_scope => memoryscope}/memory/worker/frontend/print_memory_worker.py (84%) rename {memory_scope => memoryscope}/memory/worker/frontend/print_memory_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/frontend/read_message_worker.py (75%) rename {memory_scope => memoryscope}/memory/worker/frontend/retrieve_memory_worker.py (92%) rename {memory_scope => memoryscope}/memory/worker/frontend/semantic_rank_worker.py (90%) rename {memory_scope => memoryscope}/memory/worker/frontend/set_query_worker.py (88%) rename {memory_scope => memoryscope}/memory/worker/memory_base_worker.py (92%) rename {memory_scope => memoryscope}/memory/worker/summary/__init__.py (100%) rename {memory_scope => memoryscope}/memory/worker/summary/get_reflection_subject_worker.py (88%) rename {memory_scope => memoryscope}/memory/worker/summary/get_reflection_subject_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/summary/long_contra_repeat_worker.py (91%) rename {memory_scope => memoryscope}/memory/worker/summary/long_contra_repeat_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/summary/update_insight_worker.py (94%) rename {memory_scope => memoryscope}/memory/worker/summary/update_insight_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/write/__init__.py (100%) rename {memory_scope => memoryscope}/memory/worker/write/contra_repeat_worker.py (90%) rename {memory_scope => memoryscope}/memory/worker/write/contra_repeat_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/write/get_observation_with_time_worker.py (89%) rename {memory_scope => memoryscope}/memory/worker/write/get_observation_with_time_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/write/get_observation_worker.py (92%) rename {memory_scope => memoryscope}/memory/worker/write/get_observation_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/write/info_filter_worker.py (92%) rename {memory_scope => memoryscope}/memory/worker/write/info_filter_worker.yaml (100%) rename {memory_scope => memoryscope}/memory/worker/write/load_memory_worker.py (91%) rename {memory_scope => memoryscope}/memory/worker/write/update_memory_worker.py (91%) rename {memory_scope => memoryscope}/models/__init__.py (100%) rename {memory_scope => memoryscope}/models/base_model.py (94%) rename {memory_scope => memoryscope}/models/dummy_generation_model.py (93%) rename {memory_scope => memoryscope}/models/llama_index_embedding_model.py (94%) rename {memory_scope => memoryscope}/models/llama_index_generation_model.py (93%) rename {memory_scope => memoryscope}/models/llama_index_rank_model.py (94%) rename {memory_scope => memoryscope}/scheme/__init__.py (100%) rename {memory_scope => memoryscope}/scheme/memory_node.py (100%) rename {memory_scope => memoryscope}/scheme/message.py (100%) rename {memory_scope => memoryscope}/scheme/model_response.py (94%) rename {memory_scope => memoryscope}/storage/__init__.py (100%) rename {memory_scope => memoryscope}/storage/base_memory_store.py (98%) rename {memory_scope => memoryscope}/storage/base_monitor.py (100%) rename {memory_scope => memoryscope}/storage/dummy_memory_store.py (88%) rename {memory_scope => memoryscope}/storage/dummy_monitor.py (93%) rename {memory_scope => memoryscope}/storage/llama_index_es_memory_store.py (94%) rename {memory_scope => memoryscope}/storage/llama_index_sync_elasticsearch.py (100%) rename {memory_scope => memoryscope}/utils/__init__.py (100%) rename {memory_scope => memoryscope}/utils/datetime_handler.py (98%) rename {memory_scope => memoryscope}/utils/global_context.py (68%) rename {memory_scope => memoryscope}/utils/logger.py (100%) rename {memory_scope => memoryscope}/utils/memory_handler.py (95%) rename {memory_scope => memoryscope}/utils/prompt_handler.py (98%) rename {memory_scope => memoryscope}/utils/registry.py (100%) rename {memory_scope => memoryscope}/utils/response_text_parser.py (93%) rename {memory_scope => memoryscope}/utils/timer.py (98%) rename {memory_scope => memoryscope}/utils/tool_functions.py (97%) diff --git a/memory_scope/__init__.py b/memoryscope/__init__.py similarity index 100% rename from memory_scope/__init__.py rename to memoryscope/__init__.py diff --git a/memory_scope/chat/__init__.py b/memoryscope/chat/__init__.py similarity index 100% rename from memory_scope/chat/__init__.py rename to memoryscope/chat/__init__.py diff --git a/memory_scope/chat/base_memory_chat.py b/memoryscope/chat/base_memory_chat.py similarity index 94% rename from memory_scope/chat/base_memory_chat.py rename to memoryscope/chat/base_memory_chat.py index 2e44e316..2be6ad6b 100644 --- a/memory_scope/chat/base_memory_chat.py +++ b/memoryscope/chat/base_memory_chat.py @@ -1,6 +1,6 @@ from abc import ABCMeta, abstractmethod -from memory_scope.memory.service.base_memory_service import BaseMemoryService +from memoryscope.memory.service.base_memory_service import BaseMemoryService class BaseMemoryChat(metaclass=ABCMeta): diff --git a/memory_scope/chat/cli_memory_chat.py b/memoryscope/chat/cli_memory_chat.py similarity index 95% rename from memory_scope/chat/cli_memory_chat.py rename to memoryscope/chat/cli_memory_chat.py index 7a95c6c3..e06fa0fc 100644 --- a/memory_scope/chat/cli_memory_chat.py +++ b/memoryscope/chat/cli_memory_chat.py @@ -4,17 +4,17 @@ from typing import List import questionary -from memory_scope.chat.base_memory_chat import BaseMemoryChat -from memory_scope.constants.language_constants import DEFAULT_HUMAN_NAME -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.service.base_memory_service import BaseMemoryService -from memory_scope.models.base_model import BaseModel -from memory_scope.scheme.message import Message -from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger -from memory_scope.utils.prompt_handler import PromptHandler -from memory_scope.utils.tool_functions import char_logo +from memoryscope.chat.base_memory_chat import BaseMemoryChat +from memoryscope.constants.language_constants import DEFAULT_HUMAN_NAME +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.service.base_memory_service import BaseMemoryService +from memoryscope.models.base_model import BaseModel +from memoryscope.scheme.message import Message +from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger +from memoryscope.utils.prompt_handler import PromptHandler +from memoryscope.utils.tool_functions import char_logo class CliMemoryChat(BaseMemoryChat): diff --git a/memory_scope/chat/cli_memory_chat.yaml b/memoryscope/chat/cli_memory_chat.yaml similarity index 100% rename from memory_scope/chat/cli_memory_chat.yaml rename to memoryscope/chat/cli_memory_chat.yaml diff --git a/memory_scope/cli.py b/memoryscope/cli.py similarity index 89% rename from memory_scope/cli.py rename to memoryscope/cli.py index 02fb5736..14b82099 100644 --- a/memory_scope/cli.py +++ b/memoryscope/cli.py @@ -13,12 +13,12 @@ import fire import yaml import atexit -from memory_scope.enumeration.language_enum import LanguageEnum -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger -from memory_scope.utils.timer import timer -from memory_scope.utils.tool_functions import init_instance_by_config, camelcase_to_underscore +from memoryscope.enumeration.language_enum import LanguageEnum +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger +from memoryscope.utils.timer import timer +from memoryscope.utils.tool_functions import init_instance_by_config, camelcase_to_underscore class MemoryScope(object): diff --git a/memory_scope/constants/__init__.py b/memoryscope/constants/__init__.py similarity index 100% rename from memory_scope/constants/__init__.py rename to memoryscope/constants/__init__.py diff --git a/memory_scope/constants/common_constants.py b/memoryscope/constants/common_constants.py similarity index 100% rename from memory_scope/constants/common_constants.py rename to memoryscope/constants/common_constants.py diff --git a/memory_scope/constants/language_constants.py b/memoryscope/constants/language_constants.py similarity index 98% rename from memory_scope/constants/language_constants.py rename to memoryscope/constants/language_constants.py index 14d92113..94fbcd47 100644 --- a/memory_scope/constants/language_constants.py +++ b/memoryscope/constants/language_constants.py @@ -1,4 +1,4 @@ -from memory_scope.enumeration.language_enum import LanguageEnum +from memoryscope.enumeration.language_enum import LanguageEnum # This dictionary maps languages to lists of words related to datetime expressions. # It aids in recognizing and processing datetime mentions in text, enhancing the system's ability to understand diff --git a/memory_scope/enumeration/__init__.py b/memoryscope/enumeration/__init__.py similarity index 100% rename from memory_scope/enumeration/__init__.py rename to memoryscope/enumeration/__init__.py diff --git a/memory_scope/enumeration/action_status_enum.py b/memoryscope/enumeration/action_status_enum.py similarity index 100% rename from memory_scope/enumeration/action_status_enum.py rename to memoryscope/enumeration/action_status_enum.py diff --git a/memory_scope/enumeration/language_enum.py b/memoryscope/enumeration/language_enum.py similarity index 100% rename from memory_scope/enumeration/language_enum.py rename to memoryscope/enumeration/language_enum.py diff --git a/memory_scope/enumeration/memory_type_enum.py b/memoryscope/enumeration/memory_type_enum.py similarity index 100% rename from memory_scope/enumeration/memory_type_enum.py rename to memoryscope/enumeration/memory_type_enum.py diff --git a/memory_scope/enumeration/message_role_enum.py b/memoryscope/enumeration/message_role_enum.py similarity index 100% rename from memory_scope/enumeration/message_role_enum.py rename to memoryscope/enumeration/message_role_enum.py diff --git a/memory_scope/enumeration/model_enum.py b/memoryscope/enumeration/model_enum.py similarity index 100% rename from memory_scope/enumeration/model_enum.py rename to memoryscope/enumeration/model_enum.py diff --git a/memory_scope/enumeration/store_status_enum.py b/memoryscope/enumeration/store_status_enum.py similarity index 100% rename from memory_scope/enumeration/store_status_enum.py rename to memoryscope/enumeration/store_status_enum.py diff --git a/memory_scope/memory/__init__.py b/memoryscope/memory/__init__.py similarity index 100% rename from memory_scope/memory/__init__.py rename to memoryscope/memory/__init__.py diff --git a/memory_scope/memory/operation/__init__.py b/memoryscope/memory/operation/__init__.py similarity index 100% rename from memory_scope/memory/operation/__init__.py rename to memoryscope/memory/operation/__init__.py diff --git a/memory_scope/memory/operation/backend_operation.py b/memoryscope/memory/operation/backend_operation.py similarity index 91% rename from memory_scope/memory/operation/backend_operation.py rename to memoryscope/memory/operation/backend_operation.py index f406c929..2fcbe398 100644 --- a/memory_scope/memory/operation/backend_operation.py +++ b/memoryscope/memory/operation/backend_operation.py @@ -1,12 +1,12 @@ import time from typing import List -from memory_scope.constants.common_constants import CHAT_KWARGS, RESULT, CHAT_MESSAGES -from memory_scope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE -from memory_scope.memory.operation.base_workflow import BaseWorkflow -from memory_scope.scheme.message import Message -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger +from memoryscope.constants.common_constants import CHAT_KWARGS, RESULT, CHAT_MESSAGES +from memoryscope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE +from memoryscope.memory.operation.base_workflow import BaseWorkflow +from memoryscope.scheme.message import Message +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger class BackendOperation(BaseWorkflow, BaseOperation): diff --git a/memory_scope/memory/operation/base_operation.py b/memoryscope/memory/operation/base_operation.py similarity index 100% rename from memory_scope/memory/operation/base_operation.py rename to memoryscope/memory/operation/base_operation.py diff --git a/memory_scope/memory/operation/base_workflow.py b/memoryscope/memory/operation/base_workflow.py similarity index 95% rename from memory_scope/memory/operation/base_workflow.py rename to memoryscope/memory/operation/base_workflow.py index 1b5d7119..926b59a1 100644 --- a/memory_scope/memory/operation/base_workflow.py +++ b/memoryscope/memory/operation/base_workflow.py @@ -4,12 +4,12 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from itertools import zip_longest from typing import Dict, Any, List -from memory_scope.constants.common_constants import WORKFLOW_NAME -from memory_scope.memory.worker.base_worker import BaseWorker -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger -from memory_scope.utils.timer import Timer -from memory_scope.utils.tool_functions import init_instance_by_config +from memoryscope.constants.common_constants import WORKFLOW_NAME +from memoryscope.memory.worker.base_worker import BaseWorker +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger +from memoryscope.utils.timer import Timer +from memoryscope.utils.tool_functions import init_instance_by_config class BaseWorkflow(object): diff --git a/memory_scope/memory/operation/frontend_operation.py b/memoryscope/memory/operation/frontend_operation.py similarity index 84% rename from memory_scope/memory/operation/frontend_operation.py rename to memoryscope/memory/operation/frontend_operation.py index 14471f75..8cde7977 100644 --- a/memory_scope/memory/operation/frontend_operation.py +++ b/memoryscope/memory/operation/frontend_operation.py @@ -1,9 +1,9 @@ from typing import List -from memory_scope.constants.common_constants import RESULT, CHAT_MESSAGES, CHAT_KWARGS -from memory_scope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE -from memory_scope.memory.operation.base_workflow import BaseWorkflow -from memory_scope.scheme.message import Message +from memoryscope.constants.common_constants import RESULT, CHAT_MESSAGES, CHAT_KWARGS +from memoryscope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE +from memoryscope.memory.operation.base_workflow import BaseWorkflow +from memoryscope.scheme.message import Message class FrontendOperation(BaseWorkflow, BaseOperation): diff --git a/memory_scope/memory/operation/summary_observation_op.py b/memoryscope/memory/operation/summary_observation_op.py similarity index 90% rename from memory_scope/memory/operation/summary_observation_op.py rename to memoryscope/memory/operation/summary_observation_op.py index 47100423..1da2e65c 100644 --- a/memory_scope/memory/operation/summary_observation_op.py +++ b/memoryscope/memory/operation/summary_observation_op.py @@ -1,6 +1,6 @@ -from memory_scope.constants.common_constants import CHAT_KWARGS, CHAT_MESSAGES, RESULT -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.operation.backend_operation import BackendOperation +from memoryscope.constants.common_constants import CHAT_KWARGS, CHAT_MESSAGES, RESULT +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.operation.backend_operation import BackendOperation class SummaryObservationOp(BackendOperation): diff --git a/memory_scope/memory/service/__init__.py b/memoryscope/memory/service/__init__.py similarity index 100% rename from memory_scope/memory/service/__init__.py rename to memoryscope/memory/service/__init__.py diff --git a/memory_scope/memory/service/base_memory_service.py b/memoryscope/memory/service/base_memory_service.py similarity index 96% rename from memory_scope/memory/service/base_memory_service.py rename to memoryscope/memory/service/base_memory_service.py index e4a7310e..31b2df5e 100644 --- a/memory_scope/memory/service/base_memory_service.py +++ b/memoryscope/memory/service/base_memory_service.py @@ -2,9 +2,9 @@ import threading from abc import ABCMeta, abstractmethod from typing import List, Dict -from memory_scope.memory.operation.base_operation import BaseOperation -from memory_scope.scheme.message import Message -from memory_scope.utils.logger import Logger +from memoryscope.memory.operation.base_operation import BaseOperation +from memoryscope.scheme.message import Message +from memoryscope.utils.logger import Logger class BaseMemoryService(metaclass=ABCMeta): diff --git a/memory_scope/memory/service/memory_scope_service.py b/memoryscope/memory/service/memory_scope_service.py similarity index 94% rename from memory_scope/memory/service/memory_scope_service.py rename to memoryscope/memory/service/memory_scope_service.py index 754fa7f6..da4f4b76 100644 --- a/memory_scope/memory/service/memory_scope_service.py +++ b/memoryscope/memory/service/memory_scope_service.py @@ -1,9 +1,9 @@ from typing import List -from memory_scope.memory.operation.base_operation import BaseOperation -from memory_scope.memory.service.base_memory_service import BaseMemoryService -from memory_scope.scheme.message import Message -from memory_scope.utils.tool_functions import init_instance_by_config +from memoryscope.memory.operation.base_operation import BaseOperation +from memoryscope.memory.service.base_memory_service import BaseMemoryService +from memoryscope.scheme.message import Message +from memoryscope.utils.tool_functions import init_instance_by_config class MemoryScopeService(BaseMemoryService): diff --git a/memory_scope/memory/worker/__init__.py b/memoryscope/memory/worker/__init__.py similarity index 100% rename from memory_scope/memory/worker/__init__.py rename to memoryscope/memory/worker/__init__.py diff --git a/memory_scope/memory/worker/base_worker.py b/memoryscope/memory/worker/base_worker.py similarity index 98% rename from memory_scope/memory/worker/base_worker.py rename to memoryscope/memory/worker/base_worker.py index 07eeb7d7..e5bf691c 100644 --- a/memory_scope/memory/worker/base_worker.py +++ b/memoryscope/memory/worker/base_worker.py @@ -3,8 +3,8 @@ from abc import ABCMeta, abstractmethod from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict -from memory_scope.utils.logger import Logger -from memory_scope.utils.timer import Timer +from memoryscope.utils.logger import Logger +from memoryscope.utils.timer import Timer class BaseWorker(metaclass=ABCMeta): diff --git a/memory_scope/memory/worker/dummy_worker.py b/memoryscope/memory/worker/dummy_worker.py similarity index 84% rename from memory_scope/memory/worker/dummy_worker.py rename to memoryscope/memory/worker/dummy_worker.py index e4a4b1c8..b8eecb7b 100644 --- a/memory_scope/memory/worker/dummy_worker.py +++ b/memoryscope/memory/worker/dummy_worker.py @@ -1,7 +1,7 @@ import datetime -from memory_scope.constants.common_constants import RESULT, WORKFLOW_NAME, CHAT_KWARGS -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.constants.common_constants import RESULT, WORKFLOW_NAME, CHAT_KWARGS +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker class DummyWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/__init__.py b/memoryscope/memory/worker/frontend/__init__.py similarity index 100% rename from memory_scope/memory/worker/frontend/__init__.py rename to memoryscope/memory/worker/frontend/__init__.py diff --git a/memory_scope/memory/worker/frontend/extract_time_worker.py b/memoryscope/memory/worker/frontend/extract_time_worker.py similarity index 89% rename from memory_scope/memory/worker/frontend/extract_time_worker.py rename to memoryscope/memory/worker/frontend/extract_time_worker.py index 44eba917..087431c5 100644 --- a/memory_scope/memory/worker/frontend/extract_time_worker.py +++ b/memoryscope/memory/worker/frontend/extract_time_worker.py @@ -1,11 +1,11 @@ import re from typing import Dict -from memory_scope.constants.common_constants import QUERY_WITH_TS, EXTRACT_TIME_DICT -from memory_scope.constants.language_constants import DATATIME_KEY_MAP -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import QUERY_WITH_TS, EXTRACT_TIME_DICT +from memoryscope.constants.language_constants import DATATIME_KEY_MAP +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.tool_functions import prompt_to_msg class ExtractTimeWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/extract_time_worker.yaml b/memoryscope/memory/worker/frontend/extract_time_worker.yaml similarity index 100% rename from memory_scope/memory/worker/frontend/extract_time_worker.yaml rename to memoryscope/memory/worker/frontend/extract_time_worker.yaml diff --git a/memory_scope/memory/worker/frontend/fuse_rerank_worker.py b/memoryscope/memory/worker/frontend/fuse_rerank_worker.py similarity index 94% rename from memory_scope/memory/worker/frontend/fuse_rerank_worker.py rename to memoryscope/memory/worker/frontend/fuse_rerank_worker.py index dc46b4ba..373d82b0 100644 --- a/memory_scope/memory/worker/frontend/fuse_rerank_worker.py +++ b/memoryscope/memory/worker/frontend/fuse_rerank_worker.py @@ -1,9 +1,9 @@ from typing import Dict, List -from memory_scope.constants.common_constants import EXTRACT_TIME_DICT, RANKED_MEMORY_NODES, RESULT -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler +from memoryscope.constants.common_constants import EXTRACT_TIME_DICT, RANKED_MEMORY_NODES, RESULT +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler class FuseRerankWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/print_memory_worker.py b/memoryscope/memory/worker/frontend/print_memory_worker.py similarity index 84% rename from memory_scope/memory/worker/frontend/print_memory_worker.py rename to memoryscope/memory/worker/frontend/print_memory_worker.py index c1f034b1..6617c49c 100644 --- a/memory_scope/memory/worker/frontend/print_memory_worker.py +++ b/memoryscope/memory/worker/frontend/print_memory_worker.py @@ -1,11 +1,11 @@ from typing import List -from memory_scope.constants.common_constants import RETRIEVE_MEMORY_NODES, RESULT -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler +from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, RESULT +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler class PrintMemoryWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/print_memory_worker.yaml b/memoryscope/memory/worker/frontend/print_memory_worker.yaml similarity index 100% rename from memory_scope/memory/worker/frontend/print_memory_worker.yaml rename to memoryscope/memory/worker/frontend/print_memory_worker.yaml diff --git a/memory_scope/memory/worker/frontend/read_message_worker.py b/memoryscope/memory/worker/frontend/read_message_worker.py similarity index 75% rename from memory_scope/memory/worker/frontend/read_message_worker.py rename to memoryscope/memory/worker/frontend/read_message_worker.py index 2841ccde..798f1697 100644 --- a/memory_scope/memory/worker/frontend/read_message_worker.py +++ b/memoryscope/memory/worker/frontend/read_message_worker.py @@ -1,6 +1,6 @@ -from memory_scope.constants.common_constants import RESULT -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.constants.common_constants import RESULT +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker class ReadMessageWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/retrieve_memory_worker.py b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py similarity index 92% rename from memory_scope/memory/worker/frontend/retrieve_memory_worker.py rename to memoryscope/memory/worker/frontend/retrieve_memory_worker.py index 67a45b2e..178e870c 100644 --- a/memory_scope/memory/worker/frontend/retrieve_memory_worker.py +++ b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py @@ -1,12 +1,12 @@ from typing import List -from memory_scope.constants.common_constants import QUERY_WITH_TS, RETRIEVE_MEMORY_NODES -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.timer import timer +from memoryscope.constants.common_constants import QUERY_WITH_TS, RETRIEVE_MEMORY_NODES +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.timer import timer class RetrieveMemoryWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/semantic_rank_worker.py b/memoryscope/memory/worker/frontend/semantic_rank_worker.py similarity index 90% rename from memory_scope/memory/worker/frontend/semantic_rank_worker.py rename to memoryscope/memory/worker/frontend/semantic_rank_worker.py index f2521f46..3ea4801a 100644 --- a/memory_scope/memory/worker/frontend/semantic_rank_worker.py +++ b/memoryscope/memory/worker/frontend/semantic_rank_worker.py @@ -1,8 +1,8 @@ from typing import List, Dict -from memory_scope.constants.common_constants import RETRIEVE_MEMORY_NODES, QUERY_WITH_TS, RANKED_MEMORY_NODES -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode +from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, QUERY_WITH_TS, RANKED_MEMORY_NODES +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode class SemanticRankWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/frontend/set_query_worker.py b/memoryscope/memory/worker/frontend/set_query_worker.py similarity index 88% rename from memory_scope/memory/worker/frontend/set_query_worker.py rename to memoryscope/memory/worker/frontend/set_query_worker.py index 80d10937..a94c5675 100644 --- a/memory_scope/memory/worker/frontend/set_query_worker.py +++ b/memoryscope/memory/worker/frontend/set_query_worker.py @@ -1,8 +1,8 @@ import datetime -from memory_scope.constants.common_constants import QUERY_WITH_TS -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.constants.common_constants import QUERY_WITH_TS +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker class SetQueryWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/memory_base_worker.py b/memoryscope/memory/worker/memory_base_worker.py similarity index 92% rename from memory_scope/memory/worker/memory_base_worker.py rename to memoryscope/memory/worker/memory_base_worker.py index aee86d52..8ef58c79 100644 --- a/memory_scope/memory/worker/memory_base_worker.py +++ b/memoryscope/memory/worker/memory_base_worker.py @@ -1,15 +1,15 @@ from abc import ABCMeta from typing import List, Dict, Any -from memory_scope.constants.common_constants import CHAT_MESSAGES, CHAT_KWARGS, MEMORY_HANDLER -from memory_scope.memory.worker.base_worker import BaseWorker -from memory_scope.models.base_model import BaseModel -from memory_scope.scheme.message import Message -from memory_scope.storage.base_memory_store import BaseMemoryStore -from memory_scope.storage.base_monitor import BaseMonitor -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.memory_handler import MemoryHandler -from memory_scope.utils.prompt_handler import PromptHandler +from memoryscope.constants.common_constants import CHAT_MESSAGES, CHAT_KWARGS, MEMORY_HANDLER +from memoryscope.memory.worker.base_worker import BaseWorker +from memoryscope.models.base_model import BaseModel +from memoryscope.scheme.message import Message +from memoryscope.storage.base_memory_store import BaseMemoryStore +from memoryscope.storage.base_monitor import BaseMonitor +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.memory_handler import MemoryHandler +from memoryscope.utils.prompt_handler import PromptHandler class MemoryBaseWorker(BaseWorker, metaclass=ABCMeta): diff --git a/memory_scope/memory/worker/summary/__init__.py b/memoryscope/memory/worker/summary/__init__.py similarity index 100% rename from memory_scope/memory/worker/summary/__init__.py rename to memoryscope/memory/worker/summary/__init__.py diff --git a/memory_scope/memory/worker/summary/get_reflection_subject_worker.py b/memoryscope/memory/worker/summary/get_reflection_subject_worker.py similarity index 88% rename from memory_scope/memory/worker/summary/get_reflection_subject_worker.py rename to memoryscope/memory/worker/summary/get_reflection_subject_worker.py index ee0da56a..1dbb9d6f 100644 --- a/memory_scope/memory/worker/summary/get_reflection_subject_worker.py +++ b/memoryscope/memory/worker/summary/get_reflection_subject_worker.py @@ -1,14 +1,14 @@ from typing import List -from memory_scope.constants.common_constants import NOT_REFLECTED_NODES, INSIGHT_NODES -from memory_scope.constants.language_constants import COMMA_WORD -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import NOT_REFLECTED_NODES, INSIGHT_NODES +from memoryscope.constants.language_constants import COMMA_WORD +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class GetReflectionSubjectWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/summary/get_reflection_subject_worker.yaml b/memoryscope/memory/worker/summary/get_reflection_subject_worker.yaml similarity index 100% rename from memory_scope/memory/worker/summary/get_reflection_subject_worker.yaml rename to memoryscope/memory/worker/summary/get_reflection_subject_worker.yaml diff --git a/memory_scope/memory/worker/summary/long_contra_repeat_worker.py b/memoryscope/memory/worker/summary/long_contra_repeat_worker.py similarity index 91% rename from memory_scope/memory/worker/summary/long_contra_repeat_worker.py rename to memoryscope/memory/worker/summary/long_contra_repeat_worker.py index ca46bf5c..34ce6853 100644 --- a/memory_scope/memory/worker/summary/long_contra_repeat_worker.py +++ b/memoryscope/memory/worker/summary/long_contra_repeat_worker.py @@ -1,14 +1,14 @@ from typing import List, Dict -from memory_scope.constants.common_constants import NOT_UPDATED_NODES, MERGE_OBS_NODES -from memory_scope.constants.language_constants import NONE_WORD, CONTAINED_WORD, CONTRADICTORY_WORD -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import NOT_UPDATED_NODES, MERGE_OBS_NODES +from memoryscope.constants.language_constants import NONE_WORD, CONTAINED_WORD, CONTRADICTORY_WORD +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class LongContraRepeatWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/summary/long_contra_repeat_worker.yaml b/memoryscope/memory/worker/summary/long_contra_repeat_worker.yaml similarity index 100% rename from memory_scope/memory/worker/summary/long_contra_repeat_worker.yaml rename to memoryscope/memory/worker/summary/long_contra_repeat_worker.yaml diff --git a/memory_scope/memory/worker/summary/update_insight_worker.py b/memoryscope/memory/worker/summary/update_insight_worker.py similarity index 94% rename from memory_scope/memory/worker/summary/update_insight_worker.py rename to memoryscope/memory/worker/summary/update_insight_worker.py index 995a4020..498c62e1 100644 --- a/memory_scope/memory/worker/summary/update_insight_worker.py +++ b/memoryscope/memory/worker/summary/update_insight_worker.py @@ -1,14 +1,14 @@ import time from typing import List -from memory_scope.constants.common_constants import INSIGHT_NODES, NOT_UPDATED_NODES, NOT_REFLECTED_NODES -from memory_scope.constants.language_constants import COLON_WORD, NONE_WORD, REPEATED_WORD -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import INSIGHT_NODES, NOT_UPDATED_NODES, NOT_REFLECTED_NODES +from memoryscope.constants.language_constants import COLON_WORD, NONE_WORD, REPEATED_WORD +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class UpdateInsightWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/summary/update_insight_worker.yaml b/memoryscope/memory/worker/summary/update_insight_worker.yaml similarity index 100% rename from memory_scope/memory/worker/summary/update_insight_worker.yaml rename to memoryscope/memory/worker/summary/update_insight_worker.yaml diff --git a/memory_scope/memory/worker/write/__init__.py b/memoryscope/memory/worker/write/__init__.py similarity index 100% rename from memory_scope/memory/worker/write/__init__.py rename to memoryscope/memory/worker/write/__init__.py diff --git a/memory_scope/memory/worker/write/contra_repeat_worker.py b/memoryscope/memory/worker/write/contra_repeat_worker.py similarity index 90% rename from memory_scope/memory/worker/write/contra_repeat_worker.py rename to memoryscope/memory/worker/write/contra_repeat_worker.py index e9ca2094..05008c9c 100644 --- a/memory_scope/memory/worker/write/contra_repeat_worker.py +++ b/memoryscope/memory/worker/write/contra_repeat_worker.py @@ -1,12 +1,12 @@ from typing import List -from memory_scope.constants.common_constants import NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, MERGE_OBS_NODES, TODAY_NODES -from memory_scope.constants.language_constants import NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, MERGE_OBS_NODES, TODAY_NODES +from memoryscope.constants.language_constants import NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class ContraRepeatWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/write/contra_repeat_worker.yaml b/memoryscope/memory/worker/write/contra_repeat_worker.yaml similarity index 100% rename from memory_scope/memory/worker/write/contra_repeat_worker.yaml rename to memoryscope/memory/worker/write/contra_repeat_worker.yaml diff --git a/memory_scope/memory/worker/write/get_observation_with_time_worker.py b/memoryscope/memory/worker/write/get_observation_with_time_worker.py similarity index 89% rename from memory_scope/memory/worker/write/get_observation_with_time_worker.py rename to memoryscope/memory/worker/write/get_observation_with_time_worker.py index 83e45eff..18a77380 100644 --- a/memory_scope/memory/worker/write/get_observation_with_time_worker.py +++ b/memoryscope/memory/worker/write/get_observation_with_time_worker.py @@ -1,11 +1,11 @@ from typing import List -from memory_scope.constants.common_constants import NEW_OBS_WITH_TIME_NODES -from memory_scope.constants.language_constants import COLON_WORD -from memory_scope.memory.worker.write.get_observation_worker import GetObservationWorker -from memory_scope.scheme.message import Message -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import NEW_OBS_WITH_TIME_NODES +from memoryscope.constants.language_constants import COLON_WORD +from memoryscope.memory.worker.write.get_observation_worker import GetObservationWorker +from memoryscope.scheme.message import Message +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.tool_functions import prompt_to_msg class GetObservationWithTimeWorker(GetObservationWorker): diff --git a/memory_scope/memory/worker/write/get_observation_with_time_worker.yaml b/memoryscope/memory/worker/write/get_observation_with_time_worker.yaml similarity index 100% rename from memory_scope/memory/worker/write/get_observation_with_time_worker.yaml rename to memoryscope/memory/worker/write/get_observation_with_time_worker.yaml diff --git a/memory_scope/memory/worker/write/get_observation_worker.py b/memoryscope/memory/worker/write/get_observation_worker.py similarity index 92% rename from memory_scope/memory/worker/write/get_observation_worker.py rename to memoryscope/memory/worker/write/get_observation_worker.py index 6ff801ae..9fe97e34 100644 --- a/memory_scope/memory/worker/write/get_observation_worker.py +++ b/memoryscope/memory/worker/write/get_observation_worker.py @@ -1,15 +1,15 @@ from typing import List -from memory_scope.constants.common_constants import NEW_OBS_NODES, TIME_INFER -from memory_scope.constants.language_constants import REPEATED_WORD, NONE_WORD, COLON_WORD, TIME_INFER_WORD -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.scheme.message import Message -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.common_constants import NEW_OBS_NODES, TIME_INFER +from memoryscope.constants.language_constants import REPEATED_WORD, NONE_WORD, COLON_WORD, TIME_INFER_WORD +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.scheme.message import Message +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class GetObservationWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/write/get_observation_worker.yaml b/memoryscope/memory/worker/write/get_observation_worker.yaml similarity index 100% rename from memory_scope/memory/worker/write/get_observation_worker.yaml rename to memoryscope/memory/worker/write/get_observation_worker.yaml diff --git a/memory_scope/memory/worker/write/info_filter_worker.py b/memoryscope/memory/worker/write/info_filter_worker.py similarity index 92% rename from memory_scope/memory/worker/write/info_filter_worker.py rename to memoryscope/memory/worker/write/info_filter_worker.py index d3ebcc34..29a84a86 100644 --- a/memory_scope/memory/worker/write/info_filter_worker.py +++ b/memoryscope/memory/worker/write/info_filter_worker.py @@ -1,11 +1,11 @@ from typing import List -from memory_scope.constants.language_constants import COLON_WORD -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.message import Message -from memory_scope.utils.response_text_parser import ResponseTextParser -from memory_scope.utils.tool_functions import prompt_to_msg +from memoryscope.constants.language_constants import COLON_WORD +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.message import Message +from memoryscope.utils.response_text_parser import ResponseTextParser +from memoryscope.utils.tool_functions import prompt_to_msg class InfoFilterWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/write/info_filter_worker.yaml b/memoryscope/memory/worker/write/info_filter_worker.yaml similarity index 100% rename from memory_scope/memory/worker/write/info_filter_worker.yaml rename to memoryscope/memory/worker/write/info_filter_worker.yaml diff --git a/memory_scope/memory/worker/write/load_memory_worker.py b/memoryscope/memory/worker/write/load_memory_worker.py similarity index 91% rename from memory_scope/memory/worker/write/load_memory_worker.py rename to memoryscope/memory/worker/write/load_memory_worker.py index 7d39943e..4a7e60c0 100644 --- a/memory_scope/memory/worker/write/load_memory_worker.py +++ b/memoryscope/memory/worker/write/load_memory_worker.py @@ -1,12 +1,12 @@ from typing import List -from memory_scope.constants.common_constants import NOT_REFLECTED_NODES, NOT_UPDATED_NODES, INSIGHT_NODES, TODAY_NODES -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler -from memory_scope.utils.timer import timer +from memoryscope.constants.common_constants import NOT_REFLECTED_NODES, NOT_UPDATED_NODES, INSIGHT_NODES, TODAY_NODES +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler +from memoryscope.utils.timer import timer class LoadMemoryWorker(MemoryBaseWorker): diff --git a/memory_scope/memory/worker/write/update_memory_worker.py b/memoryscope/memory/worker/write/update_memory_worker.py similarity index 91% rename from memory_scope/memory/worker/write/update_memory_worker.py rename to memoryscope/memory/worker/write/update_memory_worker.py index 4c8a8fba..87b182a7 100644 --- a/memory_scope/memory/worker/write/update_memory_worker.py +++ b/memoryscope/memory/worker/write/update_memory_worker.py @@ -1,10 +1,10 @@ from typing import List -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.utils.datetime_handler import DatetimeHandler +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.utils.datetime_handler import DatetimeHandler class UpdateMemoryWorker(MemoryBaseWorker): diff --git a/memory_scope/models/__init__.py b/memoryscope/models/__init__.py similarity index 100% rename from memory_scope/models/__init__.py rename to memoryscope/models/__init__.py diff --git a/memory_scope/models/base_model.py b/memoryscope/models/base_model.py similarity index 94% rename from memory_scope/models/base_model.py rename to memoryscope/models/base_model.py index 8e0197d1..83b5aa50 100644 --- a/memory_scope/models/base_model.py +++ b/memoryscope/models/base_model.py @@ -3,11 +3,11 @@ import time from abc import abstractmethod, ABCMeta from typing import Any -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen -from memory_scope.utils.logger import Logger -from memory_scope.utils.registry import Registry -from memory_scope.utils.timer import Timer +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen +from memoryscope.utils.logger import Logger +from memoryscope.utils.registry import Registry +from memoryscope.utils.timer import Timer MODEL_REGISTRY = Registry("models") diff --git a/memory_scope/models/dummy_generation_model.py b/memoryscope/models/dummy_generation_model.py similarity index 93% rename from memory_scope/models/dummy_generation_model.py rename to memoryscope/models/dummy_generation_model.py index c56c5549..5949bf8e 100644 --- a/memory_scope/models/dummy_generation_model.py +++ b/memoryscope/models/dummy_generation_model.py @@ -3,11 +3,11 @@ from typing import List from llama_index.core.base.llms.types import ChatMessage -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY -from memory_scope.scheme.message import Message -from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY +from memoryscope.scheme.message import Message +from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen class DummyGenerationModel(BaseModel): diff --git a/memory_scope/models/llama_index_embedding_model.py b/memoryscope/models/llama_index_embedding_model.py similarity index 94% rename from memory_scope/models/llama_index_embedding_model.py rename to memoryscope/models/llama_index_embedding_model.py index 6ed455bd..822f619d 100644 --- a/memory_scope/models/llama_index_embedding_model.py +++ b/memoryscope/models/llama_index_embedding_model.py @@ -2,9 +2,9 @@ from typing import List from llama_index.embeddings.dashscope import DashScopeEmbedding -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY -from memory_scope.scheme.model_response import ModelResponse +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY +from memoryscope.scheme.model_response import ModelResponse class LlamaIndexEmbeddingModel(BaseModel): diff --git a/memory_scope/models/llama_index_generation_model.py b/memoryscope/models/llama_index_generation_model.py similarity index 93% rename from memory_scope/models/llama_index_generation_model.py rename to memoryscope/models/llama_index_generation_model.py index cc0d15b4..63a249d9 100644 --- a/memory_scope/models/llama_index_generation_model.py +++ b/memoryscope/models/llama_index_generation_model.py @@ -3,11 +3,11 @@ from typing import List from llama_index.core.base.llms.types import ChatMessage, ChatResponse, CompletionResponse from llama_index.llms.dashscope import DashScope -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY -from memory_scope.scheme.message import Message -from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY +from memoryscope.scheme.message import Message +from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen class LlamaIndexGenerationModel(BaseModel): diff --git a/memory_scope/models/llama_index_rank_model.py b/memoryscope/models/llama_index_rank_model.py similarity index 94% rename from memory_scope/models/llama_index_rank_model.py rename to memoryscope/models/llama_index_rank_model.py index 62210183..9e74cdbd 100644 --- a/memory_scope/models/llama_index_rank_model.py +++ b/memoryscope/models/llama_index_rank_model.py @@ -4,9 +4,9 @@ from llama_index.core.data_structs import Node from llama_index.core.schema import NodeWithScore from llama_index.postprocessor.dashscope_rerank import DashScopeRerank -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY -from memory_scope.scheme.model_response import ModelResponse +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY +from memoryscope.scheme.model_response import ModelResponse class LlamaIndexRankModel(BaseModel): diff --git a/memory_scope/scheme/__init__.py b/memoryscope/scheme/__init__.py similarity index 100% rename from memory_scope/scheme/__init__.py rename to memoryscope/scheme/__init__.py diff --git a/memory_scope/scheme/memory_node.py b/memoryscope/scheme/memory_node.py similarity index 100% rename from memory_scope/scheme/memory_node.py rename to memoryscope/scheme/memory_node.py diff --git a/memory_scope/scheme/message.py b/memoryscope/scheme/message.py similarity index 100% rename from memory_scope/scheme/message.py rename to memoryscope/scheme/message.py diff --git a/memory_scope/scheme/model_response.py b/memoryscope/scheme/model_response.py similarity index 94% rename from memory_scope/scheme/model_response.py rename to memoryscope/scheme/model_response.py index 294b6a8a..b5a1d84f 100644 --- a/memory_scope/scheme/model_response.py +++ b/memoryscope/scheme/model_response.py @@ -3,8 +3,8 @@ from typing import Generator, List, Dict, Any from pydantic import BaseModel, Field -from memory_scope.enumeration.model_enum import ModelEnum -from memory_scope.scheme.message import Message +from memoryscope.enumeration.model_enum import ModelEnum +from memoryscope.scheme.message import Message class ModelResponse(BaseModel): diff --git a/memory_scope/storage/__init__.py b/memoryscope/storage/__init__.py similarity index 100% rename from memory_scope/storage/__init__.py rename to memoryscope/storage/__init__.py diff --git a/memory_scope/storage/base_memory_store.py b/memoryscope/storage/base_memory_store.py similarity index 98% rename from memory_scope/storage/base_memory_store.py rename to memoryscope/storage/base_memory_store.py index f1335e53..f39305fb 100644 --- a/memory_scope/storage/base_memory_store.py +++ b/memoryscope/storage/base_memory_store.py @@ -1,7 +1,7 @@ from abc import ABCMeta, abstractmethod from typing import Dict, List -from memory_scope.scheme.memory_node import MemoryNode +from memoryscope.scheme.memory_node import MemoryNode class BaseMemoryStore(metaclass=ABCMeta): diff --git a/memory_scope/storage/base_monitor.py b/memoryscope/storage/base_monitor.py similarity index 100% rename from memory_scope/storage/base_monitor.py rename to memoryscope/storage/base_monitor.py diff --git a/memory_scope/storage/dummy_memory_store.py b/memoryscope/storage/dummy_memory_store.py similarity index 88% rename from memory_scope/storage/dummy_memory_store.py rename to memoryscope/storage/dummy_memory_store.py index d7b1c42a..fa6ab1fd 100644 --- a/memory_scope/storage/dummy_memory_store.py +++ b/memoryscope/storage/dummy_memory_store.py @@ -1,8 +1,8 @@ from typing import Dict, List -from memory_scope.models.base_model import BaseModel -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.storage.base_memory_store import BaseMemoryStore +from memoryscope.models.base_model import BaseModel +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.storage.base_memory_store import BaseMemoryStore class DummyMemoryStore(BaseMemoryStore): diff --git a/memory_scope/storage/dummy_monitor.py b/memoryscope/storage/dummy_monitor.py similarity index 93% rename from memory_scope/storage/dummy_monitor.py rename to memoryscope/storage/dummy_monitor.py index 735f643e..4818e0fb 100644 --- a/memory_scope/storage/dummy_monitor.py +++ b/memoryscope/storage/dummy_monitor.py @@ -1,4 +1,4 @@ -from memory_scope.storage.base_monitor import BaseMonitor +from memoryscope.storage.base_monitor import BaseMonitor class DummyMonitor(BaseMonitor): diff --git a/memory_scope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py similarity index 94% rename from memory_scope/storage/llama_index_es_memory_store.py rename to memoryscope/storage/llama_index_es_memory_store.py index 6fb6e510..b8e87fd3 100644 --- a/memory_scope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -5,11 +5,11 @@ from typing import Dict, List, Any, Optional, cast from llama_index.core import VectorStoreIndex from llama_index.core.schema import TextNode, NodeWithScore, QueryBundle -from memory_scope.models.base_model import BaseModel -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.storage.base_memory_store import BaseMemoryStore -from memory_scope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, _to_elasticsearch_filter -from memory_scope.utils.logger import Logger +from memoryscope.models.base_model import BaseModel +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.storage.base_memory_store import BaseMemoryStore +from memoryscope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, _to_elasticsearch_filter +from memoryscope.utils.logger import Logger class LlamaIndexEsMemoryStore(BaseMemoryStore): diff --git a/memory_scope/storage/llama_index_sync_elasticsearch.py b/memoryscope/storage/llama_index_sync_elasticsearch.py similarity index 100% rename from memory_scope/storage/llama_index_sync_elasticsearch.py rename to memoryscope/storage/llama_index_sync_elasticsearch.py diff --git a/memory_scope/utils/__init__.py b/memoryscope/utils/__init__.py similarity index 100% rename from memory_scope/utils/__init__.py rename to memoryscope/utils/__init__.py diff --git a/memory_scope/utils/datetime_handler.py b/memoryscope/utils/datetime_handler.py similarity index 98% rename from memory_scope/utils/datetime_handler.py rename to memoryscope/utils/datetime_handler.py index 81ea671c..6f2d6d2f 100644 --- a/memory_scope/utils/datetime_handler.py +++ b/memoryscope/utils/datetime_handler.py @@ -1,9 +1,9 @@ import datetime import re -from memory_scope.constants.language_constants import WEEKDAYS, DATATIME_WORD_LIST, MONTH_DICT -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger +from memoryscope.constants.language_constants import WEEKDAYS, DATATIME_WORD_LIST, MONTH_DICT +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger class DatetimeHandler(object): diff --git a/memory_scope/utils/global_context.py b/memoryscope/utils/global_context.py similarity index 68% rename from memory_scope/utils/global_context.py rename to memoryscope/utils/global_context.py index 5bab0afb..81ac9e07 100644 --- a/memory_scope/utils/global_context.py +++ b/memoryscope/utils/global_context.py @@ -1,12 +1,12 @@ from concurrent.futures import ThreadPoolExecutor from typing import Dict, Any -from memory_scope.chat.base_memory_chat import BaseMemoryChat -from memory_scope.enumeration.language_enum import LanguageEnum -from memory_scope.memory.service.base_memory_service import BaseMemoryService -from memory_scope.models.base_model import BaseModel -from memory_scope.storage.base_memory_store import BaseMemoryStore -from memory_scope.storage.base_monitor import BaseMonitor +from memoryscope.chat.base_memory_chat import BaseMemoryChat +from memoryscope.enumeration.language_enum import LanguageEnum +from memoryscope.memory.service.base_memory_service import BaseMemoryService +from memoryscope.models.base_model import BaseModel +from memoryscope.storage.base_memory_store import BaseMemoryStore +from memoryscope.storage.base_monitor import BaseMonitor class GlobalContext(object): diff --git a/memory_scope/utils/logger.py b/memoryscope/utils/logger.py similarity index 100% rename from memory_scope/utils/logger.py rename to memoryscope/utils/logger.py diff --git a/memory_scope/utils/memory_handler.py b/memoryscope/utils/memory_handler.py similarity index 95% rename from memory_scope/utils/memory_handler.py rename to memoryscope/utils/memory_handler.py index 80c41adf..b554401c 100644 --- a/memory_scope/utils/memory_handler.py +++ b/memoryscope/utils/memory_handler.py @@ -1,11 +1,11 @@ from typing import Dict, List -from memory_scope.enumeration.action_status_enum import ActionStatusEnum -from memory_scope.enumeration.store_status_enum import StoreStatusEnum -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.storage.base_memory_store import BaseMemoryStore -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger +from memoryscope.enumeration.action_status_enum import ActionStatusEnum +from memoryscope.enumeration.store_status_enum import StoreStatusEnum +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.storage.base_memory_store import BaseMemoryStore +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger class MemoryHandler(object): diff --git a/memory_scope/utils/prompt_handler.py b/memoryscope/utils/prompt_handler.py similarity index 98% rename from memory_scope/utils/prompt_handler.py rename to memoryscope/utils/prompt_handler.py index 8fdecdeb..f2e9b380 100644 --- a/memory_scope/utils/prompt_handler.py +++ b/memoryscope/utils/prompt_handler.py @@ -4,7 +4,7 @@ from typing import Dict import yaml -from memory_scope.utils.global_context import G_CONTEXT +from memoryscope.utils.global_context import G_CONTEXT class PromptHandler(object): diff --git a/memory_scope/utils/registry.py b/memoryscope/utils/registry.py similarity index 100% rename from memory_scope/utils/registry.py rename to memoryscope/utils/registry.py diff --git a/memory_scope/utils/response_text_parser.py b/memoryscope/utils/response_text_parser.py similarity index 93% rename from memory_scope/utils/response_text_parser.py rename to memoryscope/utils/response_text_parser.py index 52d5b931..aa064528 100644 --- a/memory_scope/utils/response_text_parser.py +++ b/memoryscope/utils/response_text_parser.py @@ -1,9 +1,9 @@ from typing import List import re -from memory_scope.constants.language_constants import NONE_WORD -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger +from memoryscope.constants.language_constants import NONE_WORD +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger class ResponseTextParser(object): diff --git a/memory_scope/utils/timer.py b/memoryscope/utils/timer.py similarity index 98% rename from memory_scope/utils/timer.py rename to memoryscope/utils/timer.py index 1a7cefa3..f63ff880 100644 --- a/memory_scope/utils/timer.py +++ b/memoryscope/utils/timer.py @@ -1,7 +1,7 @@ import time from typing import Literal -from memory_scope.utils.logger import Logger +from memoryscope.utils.logger import Logger TIME_LOG_TYPE = Literal["end", "wrap", "none"] diff --git a/memory_scope/utils/tool_functions.py b/memoryscope/utils/tool_functions.py similarity index 97% rename from memory_scope/utils/tool_functions.py rename to memoryscope/utils/tool_functions.py index b3f5cd28..2fd432c1 100644 --- a/memory_scope/utils/tool_functions.py +++ b/memoryscope/utils/tool_functions.py @@ -9,8 +9,8 @@ from typing import List import pyfiglet from termcolor import colored -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.scheme.message import Message +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.scheme.message import Message ALL_COLORS = ["red", "green", "yellow", "blue", "magenta", "cyan", "light_grey", "light_red", "light_green", "light_yellow", "light_blue", "light_magenta", "light_cyan", "white"] @@ -48,7 +48,7 @@ def camelcase_to_underscore(name: str) -> str: def init_instance_by_config(config: dict, - default_class_path: str = "memory_scope", + default_class_path: str = "memoryscope", suffix_name: str = "", **kwargs): """ diff --git a/tests/models/test_models_lli_embedding.py b/tests/models/test_models_lli_embedding.py index c71236f5..f21d0bc7 100644 --- a/tests/models/test_models_lli_embedding.py +++ b/tests/models/test_models_lli_embedding.py @@ -5,8 +5,8 @@ sys.path.append(".") # noqa: E402 import asyncio import unittest -from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel -from memory_scope.utils.logger import Logger +from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel +from memoryscope.utils.logger import Logger class TestLLIEmbedding(unittest.TestCase): diff --git a/tests/models/test_models_lli_generation.py b/tests/models/test_models_lli_generation.py index 8757e13f..474851b4 100644 --- a/tests/models/test_models_lli_generation.py +++ b/tests/models/test_models_lli_generation.py @@ -5,9 +5,9 @@ sys.path.append(".") # noqa: E402 import unittest import time import asyncio -from memory_scope.scheme.message import Message -from memory_scope.models.llama_index_generation_model import LlamaIndexGenerationModel -from memory_scope.utils.logger import Logger +from memoryscope.scheme.message import Message +from memoryscope.models.llama_index_generation_model import LlamaIndexGenerationModel +from memoryscope.utils.logger import Logger class TestLLILLM(unittest.TestCase): diff --git a/tests/models/test_models_lli_rank.py b/tests/models/test_models_lli_rank.py index 7c5efa7e..25c59fde 100644 --- a/tests/models/test_models_lli_rank.py +++ b/tests/models/test_models_lli_rank.py @@ -1,7 +1,7 @@ import asyncio import unittest -from memory_scope.models.llama_index_rank_model import LlamaIndexRankModel +from memoryscope.models.llama_index_rank_model import LlamaIndexRankModel class TestLLIReRank(unittest.TestCase): diff --git a/tests/operations/test_interface.py b/tests/operations/test_interface.py index 5a92f587..d89efb93 100644 --- a/tests/operations/test_interface.py +++ b/tests/operations/test_interface.py @@ -1,5 +1,5 @@ -from memory_scope.cli import MemoryScope -from memory_scope.scheme.message import Message +from memoryscope.cli import MemoryScope +from memoryscope.scheme.message import Message ms = MemoryScope().load_config("config/demo_config_no_stream.yaml") memory_service = ms.default_service diff --git a/tests/storages/test_storages_lli_es.py b/tests/storages/test_storages_lli_es.py index 2033f54c..791286aa 100644 --- a/tests/storages/test_storages_lli_es.py +++ b/tests/storages/test_storages_lli_es.py @@ -1,8 +1,8 @@ import unittest -from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore +from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore class TestLlamaIndexElasticSearchStore(unittest.TestCase): diff --git a/tests/storages/test_storages_lli_synces.py b/tests/storages/test_storages_lli_synces.py index 89d7f15d..8186179a 100644 --- a/tests/storages/test_storages_lli_synces.py +++ b/tests/storages/test_storages_lli_synces.py @@ -1,8 +1,8 @@ import unittest -from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore +from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore class TestLlamaIndexElasticSearchStore(unittest.TestCase): diff --git a/tests/worker/test_workers_cn.py b/tests/worker/test_workers_cn.py index 28843299..a8114b35 100644 --- a/tests/worker/test_workers_cn.py +++ b/tests/worker/test_workers_cn.py @@ -1,16 +1,16 @@ import datetime import unittest -from memory_scope.cli import MemoryScope -from memory_scope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \ +from memoryscope.cli import MemoryScope +from memoryscope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \ MERGE_OBS_NODES, QUERY_WITH_TS, EXTRACT_TIME_DICT, NOT_REFLECTED_NODES, INSIGHT_NODES, NOT_UPDATED_NODES -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.scheme.message import Message -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger -from memory_scope.utils.tool_functions import init_instance_by_config +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.scheme.message import Message +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger +from memoryscope.utils.tool_functions import init_instance_by_config class TestWorkersCn(unittest.TestCase): diff --git a/tests/worker/test_workers_en.py b/tests/worker/test_workers_en.py index 7d7fd14b..6b1c012a 100644 --- a/tests/worker/test_workers_en.py +++ b/tests/worker/test_workers_en.py @@ -1,16 +1,16 @@ import datetime import unittest -from memory_scope.cli import MemoryScope -from memory_scope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \ +from memoryscope.cli import MemoryScope +from memoryscope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \ MERGE_OBS_NODES, QUERY_WITH_TS, EXTRACT_TIME_DICT, NOT_REFLECTED_NODES, INSIGHT_NODES, NOT_UPDATED_NODES -from memory_scope.enumeration.message_role_enum import MessageRoleEnum -from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker -from memory_scope.scheme.memory_node import MemoryNode -from memory_scope.scheme.message import Message -from memory_scope.utils.global_context import G_CONTEXT -from memory_scope.utils.logger import Logger -from memory_scope.utils.tool_functions import init_instance_by_config +from memoryscope.enumeration.message_role_enum import MessageRoleEnum +from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker +from memoryscope.scheme.memory_node import MemoryNode +from memoryscope.scheme.message import Message +from memoryscope.utils.global_context import G_CONTEXT +from memoryscope.utils.logger import Logger +from memoryscope.utils.tool_functions import init_instance_by_config class TestWorkersEn(unittest.TestCase): From c437553ff69fc7c5a5d9e53fd477ecfa306126b0 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 19:31:44 +0800 Subject: [PATCH 2/8] format code --- .../worker/frontend/read_message_worker.py | 1 + .../storage/llama_index_es_memory_store.py | 26 +++++++++---------- .../storage/llama_index_sync_elasticsearch.py | 14 +++++----- memoryscope/utils/logger.py | 1 + memoryscope/utils/response_text_parser.py | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/memoryscope/memory/worker/frontend/read_message_worker.py b/memoryscope/memory/worker/frontend/read_message_worker.py index 798f1697..dfbe4a12 100644 --- a/memoryscope/memory/worker/frontend/read_message_worker.py +++ b/memoryscope/memory/worker/frontend/read_message_worker.py @@ -7,6 +7,7 @@ class ReadMessageWorker(MemoryBaseWorker): """ Fetches unmemorized chat messages. """ + def _run(self): """ Executes the primary function to fetch unmemorized chat messages. diff --git a/memoryscope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py index b8e87fd3..7d9dbd1e 100644 --- a/memoryscope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -1,6 +1,5 @@ -import warnings import random -from typing import Dict, List, Any, Optional, cast +from typing import Dict, List, Optional from llama_index.core import VectorStoreIndex from llama_index.core.schema import TextNode, NodeWithScore, QueryBundle @@ -8,7 +7,8 @@ from llama_index.core.schema import TextNode, NodeWithScore, QueryBundle from memoryscope.models.base_model import BaseModel from memoryscope.scheme.memory_node import MemoryNode from memoryscope.storage.base_memory_store import BaseMemoryStore -from memoryscope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, _to_elasticsearch_filter +from memoryscope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, \ + _to_elasticsearch_filter from memoryscope.utils.logger import Logger @@ -30,7 +30,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): **kwargs) # TODO The llamaIndex utilizes some deprecated functions, hence langchain logs warning messages. By # adding the following lines of code, the display of deprecated information is suppressed. - + self.index = VectorStoreIndex.from_vector_store(vector_store=self.es_store, embed_model=self.embedding_model.model) @@ -44,18 +44,18 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): exists = self.es_store._store.client.indices.exists(index=self.index_name) if not exists: return [] - + if filter_dict is None: filter_dict = {} es_filter = _to_elasticsearch_filter(filter_dict) - retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']}, + retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']}, similarity_top_k=top_k, sparse_top_k=top_k, ) if query is None: query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector()) - + text_nodes = retriever.retrieve(query) return [self._text_node_2_memory_node(n) for n in text_nodes] @@ -71,11 +71,11 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): retriever = self.index.as_retriever( vector_store_kwargs={"es_filter": es_filter}, similarity_top_k=top_k) - + if query is None: query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector()) - + text_nodes: List[NodeWithScore] = await retriever.aretrieve(query) return [self._text_node_2_memory_node(n) for n in text_nodes] @@ -114,11 +114,11 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): Closes the Elasticsearch store, releasing any resources associated with it. """ self.es_store.close() - - def dummy_query_vector(self): + + def dummy_query_vector(self): random_floats = [random.uniform(0, 1) for _ in range(self.emb_dims)] return random_floats - + @staticmethod def _memory_node_2_text_node(memory_node: MemoryNode) -> TextNode: """ @@ -129,7 +129,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): Returns: TextNode: The converted TextNode with content and metadata from the MemoryNode. - """ + """ embedding = memory_node.vector if not embedding: embedding = None diff --git a/memoryscope/storage/llama_index_sync_elasticsearch.py b/memoryscope/storage/llama_index_sync_elasticsearch.py index 1532d9f5..2c525920 100644 --- a/memoryscope/storage/llama_index_sync_elasticsearch.py +++ b/memoryscope/storage/llama_index_sync_elasticsearch.py @@ -18,7 +18,6 @@ from llama_index.core.bridge.pydantic import PrivateAttr from llama_index.core.schema import BaseNode, MetadataMode, TextNode from llama_index.core.vector_stores.types import ( BasePydanticVectorStore, - MetadataFilters, VectorStoreQuery, VectorStoreQueryMode, VectorStoreQueryResult, @@ -141,10 +140,10 @@ class _AsyncDenseVectorStrategy(AsyncDenseVectorStrategy): if query == "**--**": query_body = { "query": { - "bool": { - "filter": filter, - } - }, + "bool": { + "filter": filter, + } + }, } else: query_body = { @@ -262,7 +261,6 @@ def _to_elasticsearch_filter(standard_filters: Dict[str, List[str]]) -> Dict[str return result - class SyncElasticsearchStore(BasePydanticVectorStore): """ Elasticsearch vector store. @@ -676,9 +674,9 @@ class SyncElasticsearchStore(BasePydanticVectorStore): isinstance(self.retrieval_strategy, AsyncDenseVectorStrategy) and self.retrieval_strategy.hybrid ): - total_rank = sum(top_k_scores) + total_rank = sum(top_k_scores) top_k_scores = [rank for rank in top_k_scores] - #top_k_scores = [(total_rank - rank) / total_rank for rank in top_k_scores] + # top_k_scores = [(total_rank - rank) / total_rank for rank in top_k_scores] # top_k_scores = [total_rank - rank / total_rank for rank in top_k_scores] return VectorStoreQueryResult( diff --git a/memoryscope/utils/logger.py b/memoryscope/utils/logger.py index a3d5ea89..9558b3b0 100644 --- a/memoryscope/utils/logger.py +++ b/memoryscope/utils/logger.py @@ -12,6 +12,7 @@ class Logger(logging.Logger): """ The `Logger` class handle the stream of information or errors in activities. """ + def __init__(self, name: str, level: int = logging.INFO, diff --git a/memoryscope/utils/response_text_parser.py b/memoryscope/utils/response_text_parser.py index aa064528..4081cbbf 100644 --- a/memoryscope/utils/response_text_parser.py +++ b/memoryscope/utils/response_text_parser.py @@ -1,5 +1,5 @@ -from typing import List import re +from typing import List from memoryscope.constants.language_constants import NONE_WORD from memoryscope.utils.global_context import G_CONTEXT From 83de295eff5403a89a21815c4f6630b8496942c9 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 20:05:27 +0800 Subject: [PATCH 3/8] fix score_similar -> score_recall --- .../worker/frontend/retrieve_memory_worker.py | 4 ++-- .../summary/get_reflection_subject_worker.py | 2 +- .../summary/long_contra_repeat_worker.py | 4 ++-- .../worker/summary/update_insight_worker.py | 3 ++- .../worker/write/contra_repeat_worker.py | 2 +- .../worker/write/get_observation_worker.py | 2 +- .../memory/worker/write/info_filter_worker.py | 2 +- .../storage/llama_index_es_memory_store.py | 7 ++---- memoryscope/utils/response_text_parser.py | 24 +++++++++++-------- 9 files changed, 26 insertions(+), 24 deletions(-) diff --git a/memoryscope/memory/worker/frontend/retrieve_memory_worker.py b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py index 178e870c..459db4ee 100644 --- a/memoryscope/memory/worker/frontend/retrieve_memory_worker.py +++ b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py @@ -133,10 +133,10 @@ class RetrieveMemoryWorker(MemoryBaseWorker): if not memory_node_list: return - memory_node_list = sorted(memory_node_list, key=lambda x: x.score_similar, reverse=True) + memory_node_list = sorted(memory_node_list, key=lambda x: x.score_recall, reverse=True) for node in memory_node_list: node.action_status = ActionStatusEnum.NONE.value - self.logger.info(f"recall_stage: content={node.content} score={node.score_similar} type={node.memory_type} " + self.logger.info(f"recall_stage: content={node.content} score={node.score_rerank} type={node.memory_type} " f"store_status={node.store_status} action_status={node.action_status}") self.memory_handler.set_memories(RETRIEVE_MEMORY_NODES, memory_node_list) diff --git a/memoryscope/memory/worker/summary/get_reflection_subject_worker.py b/memoryscope/memory/worker/summary/get_reflection_subject_worker.py index 1dbb9d6f..4c9d7b27 100644 --- a/memoryscope/memory/worker/summary/get_reflection_subject_worker.py +++ b/memoryscope/memory/worker/summary/get_reflection_subject_worker.py @@ -101,7 +101,7 @@ class GetReflectionSubjectWorker(MemoryBaseWorker): return # Parse LLM response for new insight keys and update memory - new_insight_keys = ResponseTextParser(response.message.content).parse_v2(self.__class__.__name__) + new_insight_keys = ResponseTextParser(response.message.content, self.__class__.__name__).parse_v2() if new_insight_keys: for insight_key in new_insight_keys: self.memory_handler.add_memories(INSIGHT_NODES, self.new_insight_node(insight_key)) diff --git a/memoryscope/memory/worker/summary/long_contra_repeat_worker.py b/memoryscope/memory/worker/summary/long_contra_repeat_worker.py index 34ce6853..94b306b4 100644 --- a/memoryscope/memory/worker/summary/long_contra_repeat_worker.py +++ b/memoryscope/memory/worker/summary/long_contra_repeat_worker.py @@ -49,7 +49,7 @@ class LongContraRepeatWorker(MemoryBaseWorker): top_k=self.long_contra_repeat_top_k, filter_dict=filter_dict) # Filter retrieved nodes based on the similarity threshold - return node, [n for n in retrieve_nodes if n.score_similar >= self.long_contra_repeat_threshold] + return node, [n for n in retrieve_nodes if n.score_recall >= self.long_contra_repeat_threshold] def _run(self): """ @@ -111,7 +111,7 @@ class LongContraRepeatWorker(MemoryBaseWorker): return # Parses the model's response text to identify updates for memory nodes - idx_obs_info_list = ResponseTextParser(response.message.content).parse_v1(self.__class__.__name__) + idx_obs_info_list = ResponseTextParser(response.message.content, self.__class__.__name__).parse_v1() if len(idx_obs_info_list) <= 0: self.logger.warning("idx_obs_info_list is empty!") return diff --git a/memoryscope/memory/worker/summary/update_insight_worker.py b/memoryscope/memory/worker/summary/update_insight_worker.py index 498c62e1..b56ed6f7 100644 --- a/memoryscope/memory/worker/summary/update_insight_worker.py +++ b/memoryscope/memory/worker/summary/update_insight_worker.py @@ -136,7 +136,8 @@ class UpdateInsightWorker(MemoryBaseWorker): if not response.status or not response.message.content: return insight_node - insight_value_list = ResponseTextParser(response.message.content).parse_v1(f"update_{insight_node.key}") + insight_value_list = ResponseTextParser(response.message.content, + f"update_{insight_node.key}").parse_v1() if not insight_value_list: self.logger.warning(f"update_{insight_node.key} insight_value_list is empty!") return insight_node diff --git a/memoryscope/memory/worker/write/contra_repeat_worker.py b/memoryscope/memory/worker/write/contra_repeat_worker.py index 05008c9c..5146887f 100644 --- a/memoryscope/memory/worker/write/contra_repeat_worker.py +++ b/memoryscope/memory/worker/write/contra_repeat_worker.py @@ -80,7 +80,7 @@ class ContraRepeatWorker(MemoryBaseWorker): response_text = response.message.content # parse text - idx_merge_obs_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__) + idx_merge_obs_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1() if len(idx_merge_obs_list) <= 0: self.logger.warning("idx_merge_obs_list is empty!") return diff --git a/memoryscope/memory/worker/write/get_observation_worker.py b/memoryscope/memory/worker/write/get_observation_worker.py index 9fe97e34..b36d2b6f 100644 --- a/memoryscope/memory/worker/write/get_observation_worker.py +++ b/memoryscope/memory/worker/write/get_observation_worker.py @@ -139,7 +139,7 @@ class GetObservationWorker(MemoryBaseWorker): response_text = response.message.content # Parses the generated text to extract observation indices, times, contents, and keywords - idx_obs_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__) + idx_obs_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1() if len(idx_obs_list) <= 0: self.logger.warning("idx_obs_list is empty!") return diff --git a/memoryscope/memory/worker/write/info_filter_worker.py b/memoryscope/memory/worker/write/info_filter_worker.py index 29a84a86..ae28c13d 100644 --- a/memoryscope/memory/worker/write/info_filter_worker.py +++ b/memoryscope/memory/worker/write/info_filter_worker.py @@ -76,7 +76,7 @@ class InfoFilterWorker(MemoryBaseWorker): response_text = response.message.content # parse text - info_score_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__) + info_score_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1() if len(info_score_list) != len(info_messages): self.logger.warning(f"score_size != messages_size, {len(info_score_list)} vs {len(info_messages)}") diff --git a/memoryscope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py index 7d9dbd1e..02e0c085 100644 --- a/memoryscope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -149,8 +149,5 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): Returns: MemoryNode: The converted MemoryNode with text and metadata from the NodeWithScore. """ - embedding = text_node.embedding - print("textnode embedding", embedding) - if not embedding: - embedding = [] - return MemoryNode(content=text_node.text, vector=embedding, **text_node.metadata) + text_node.metadata["vector"] = text_node.embedding + return MemoryNode(content=text_node.text, **text_node.metadata) diff --git a/memoryscope/utils/response_text_parser.py b/memoryscope/utils/response_text_parser.py index 4081cbbf..b7de8982 100644 --- a/memoryscope/utils/response_text_parser.py +++ b/memoryscope/utils/response_text_parser.py @@ -14,23 +14,27 @@ class ResponseTextParser(object): pattern_v1 = re.compile(r"<(.*?)>") # Regular expression pattern to match content within angle brackets - def __init__(self, response_text: str): + def __init__(self, response_text: str, logger_prefix: str = ""): """ Initializes the `ResponseTextParser` instance with the provided response text and sets up a logger. Args: response_text (str): The raw response text that needs to be parsed and processed. """ - self.response_text: str = response_text.strip() # Strips leading and trailing whitespace from the response text - self.logger: Logger = Logger.get_logger() # Initializes a logger instance for logging parsing activities - def parse_v1(self, prefix: str = "") -> List[str]: + # Strips leading and trailing whitespace from the response text + self.response_text: str = response_text.strip() + + # The prefix of log. Defaults to "". + self.logger_prefix: str = logger_prefix + + # Initializes a logger instance for logging parsing activities + self.logger: Logger = Logger.get_logger() + + def parse_v1(self) -> List[List[str]]: """ Extract specific patterns from the text which match content within angle brackets. - Args: - prefix (str): The prefix of log. Defaults to "". - Returns: Contents match the specific patterns. """ @@ -42,10 +46,10 @@ class ResponseTextParser(object): matches = [match.group(1) for match in self.pattern_v1.finditer(line)] if matches: result.append(matches) - self.logger.info(f"{prefix} response_text={self.response_text} result={result}", stacklevel=2) + self.logger.info(f"{self.logger_prefix} response_text={self.response_text} result={result}", stacklevel=2) return result - def parse_v2(self, prefix: str = "") -> List[str]: + def parse_v2(self) -> List[str]: """ Extract lines which contain NONE_WORD in Chinese or English. @@ -61,5 +65,5 @@ class ResponseTextParser(object): if not line or line.lower() == NONE_WORD.get(G_CONTEXT.language): continue result.append(line) - self.logger.info(f"{prefix} response_text={self.response_text} result={result}", stacklevel=2) + self.logger.info(f"{self.logger_prefix} response_text={self.response_text} result={result}", stacklevel=2) return result From 85962e34bf74a309a8e0d5e91a5c1137134db92d Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 20:21:32 +0800 Subject: [PATCH 4/8] fix _text_node_2_memory_node --- memoryscope/storage/llama_index_es_memory_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memoryscope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py index 02e0c085..4bd7730e 100644 --- a/memoryscope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -149,5 +149,5 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): Returns: MemoryNode: The converted MemoryNode with text and metadata from the NodeWithScore. """ - text_node.metadata["vector"] = text_node.embedding + text_node.metadata["vector"] = text_node.embedding if text_node.embedding else [] return MemoryNode(content=text_node.text, **text_node.metadata) From c0eae3f802a5ab0d0ea5e33088cdbc54d54df0fa Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 20:53:50 +0800 Subject: [PATCH 5/8] fix es update emb problem --- .../storage/llama_index_es_memory_store.py | 44 +++++++++---------- memoryscope/utils/memory_handler.py | 4 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/memoryscope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py index 4bd7730e..8a6bd771 100644 --- a/memoryscope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -28,9 +28,9 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): es_url=es_url, retrieval_strategy=_AsyncDenseVectorStrategy(hybrid=use_hybrid), **kwargs) + # TODO The llamaIndex utilizes some deprecated functions, hence langchain logs warning messages. By # adding the following lines of code, the display of deprecated information is suppressed. - self.index = VectorStoreIndex.from_vector_store(vector_store=self.es_store, embed_model=self.embedding_model.model) @@ -41,7 +41,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): top_k: int = 3, filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]: # if index is not created, return [] - exists = self.es_store._store.client.indices.exists(index=self.index_name) + exists = self.es_store.client.indices.exists(index=self.index_name) if not exists: return [] @@ -51,30 +51,32 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): es_filter = _to_elasticsearch_filter(filter_dict) retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']}, similarity_top_k=top_k, - sparse_top_k=top_k, ) - if query is None: - query = QueryBundle(query_str='**--**', - embedding=self.dummy_query_vector()) + sparse_top_k=top_k) + if not query: + query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector()) text_nodes = retriever.retrieve(query) return [self._text_node_2_memory_node(n) for n in text_nodes] async def a_retrieve_memories(self, - query: str, - top_k: int, + query: Optional[str] = None, + top_k: int = 3, filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]: - self.logger.info(f"query={query} top_k={top_k} filter_dict={filter_dict}") + # if index is not created, return [] + exists = self.es_store.client.indices.exists(index=self.index_name) + if not exists: + return [] if filter_dict is None: filter_dict = {} - es_filter = _to_elasticsearch_filter(filter_dict) - retriever = self.index.as_retriever( - vector_store_kwargs={"es_filter": es_filter}, - similarity_top_k=top_k) - if query is None: - query = QueryBundle(query_str='**--**', - embedding=self.dummy_query_vector()) + es_filter = _to_elasticsearch_filter(filter_dict) + retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']}, + similarity_top_k=top_k, + sparse_top_k=top_k) + + if not query: + query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector()) text_nodes: List[NodeWithScore] = await retriever.aretrieve(query) return [self._text_node_2_memory_node(n) for n in text_nodes] @@ -85,7 +87,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): self.insert(node) def batch_update(self, nodes: List[MemoryNode], update_embedding: bool = True): - # TODO batch_update & update_embedding + # TODO batch_update for node in nodes: self.update(node, update_embedding=update_embedding) @@ -100,12 +102,10 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): def delete(self, node: MemoryNode): return self.es_store.delete(node.memory_id) - def delete_conditional(self, filter_dict: Dict = {}): - nodes = self.retrieve_memories(filter_dict=filter_dict, top_k=10000) - self.batch_delete(nodes) - def update(self, node: MemoryNode, update_embedding: bool = True): - # TODO update without embedding? + if update_embedding: + node.vector = [] + self.delete(node) self.insert(node) diff --git a/memoryscope/utils/memory_handler.py b/memoryscope/utils/memory_handler.py index b554401c..36ec9b9d 100644 --- a/memoryscope/utils/memory_handler.py +++ b/memoryscope/utils/memory_handler.py @@ -78,6 +78,7 @@ class MemoryHandler(object): Args: key (str): The key mapping to memory nodes. nodes (List[MemoryNode]): A single memory node or a list of memory nodes to be updated. + log_repeat: if log_repeat=True, print log info """ if nodes is None: nodes = [] @@ -102,7 +103,7 @@ class MemoryHandler(object): Fetch the memories by keys. Args: - key (str): The key mapping to memory nodes. + keys (str | List[str]): The key mapping to memory nodes. Returns: List[MemoryNode]: Memories mapped to the key. @@ -157,7 +158,6 @@ class MemoryHandler(object): Args: keys (str): The memories. nodes (List[MemoryNode]): A single memory node or a list of memory nodes to be updated. - : """ update_memories: Dict[str, MemoryNode] = {n.memory_id: n for n in self.get_memories(keys=keys)} From 6a5e830d46234ac613d1e4113e0a112a2a4f4482 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 21:19:35 +0800 Subject: [PATCH 6/8] fix es interface params --- config/cli_chat_dash_cn.yaml | 20 +++++----- .../worker/frontend/retrieve_memory_worker.py | 4 +- .../worker/frontend/set_query_worker.py | 2 +- .../memory/worker/memory_base_worker.py | 2 +- .../contra_repeat_worker.py | 0 .../contra_repeat_worker.yaml | 0 .../get_observation_with_time_worker.py | 0 .../get_observation_with_time_worker.yaml | 0 .../get_observation_worker.py | 0 .../get_observation_worker.yaml | 0 .../{write => summary}/info_filter_worker.py | 0 .../info_filter_worker.yaml | 0 .../{write => summary}/load_memory_worker.py | 37 ++++++------------- .../update_memory_worker.py | 0 memoryscope/memory/worker/write/__init__.py | 0 memoryscope/storage/base_memory_store.py | 10 ++++- memoryscope/storage/dummy_memory_store.py | 18 ++++++--- .../storage/llama_index_es_memory_store.py | 4 +- 18 files changed, 47 insertions(+), 50 deletions(-) rename memoryscope/memory/worker/{write => summary}/contra_repeat_worker.py (100%) rename memoryscope/memory/worker/{write => summary}/contra_repeat_worker.yaml (100%) rename memoryscope/memory/worker/{write => summary}/get_observation_with_time_worker.py (100%) rename memoryscope/memory/worker/{write => summary}/get_observation_with_time_worker.yaml (100%) rename memoryscope/memory/worker/{write => summary}/get_observation_worker.py (100%) rename memoryscope/memory/worker/{write => summary}/get_observation_worker.yaml (100%) rename memoryscope/memory/worker/{write => summary}/info_filter_worker.py (100%) rename memoryscope/memory/worker/{write => summary}/info_filter_worker.yaml (100%) rename memoryscope/memory/worker/{write => summary}/load_memory_worker.py (79%) rename memoryscope/memory/worker/{write => summary}/update_memory_worker.py (100%) delete mode 100644 memoryscope/memory/worker/write/__init__.py diff --git a/config/cli_chat_dash_cn.yaml b/config/cli_chat_dash_cn.yaml index 1d8269c5..ada734da 100644 --- a/config/cli_chat_dash_cn.yaml +++ b/config/cli_chat_dash_cn.yaml @@ -103,43 +103,43 @@ worker: retrieve_ins_top_k: 100 retrieve_expired_top_k: 100 delete_memory: - class: memory.worker.write.update_memory_worker + class: memory.worker.summary.update_memory_worker method: delete_memory delete_all: - class: memory.worker.write.update_memory_worker + class: memory.worker.summary.update_memory_worker method: delete_all add_memory: - class: memory.worker.write.update_memory_worker + class: memory.worker.summary.update_memory_worker method: from_query info_filter: - class: memory.worker.write.info_filter_worker + class: memory.worker.summary.info_filter_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 load_today_memory: - class: memory.worker.write.load_memory_worker + class: memory.worker.summary.load_memory_worker retrieve_today_top_k: 100 get_observation: - class: memory.worker.write.get_observation_worker + class: memory.worker.summary.get_observation_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 get_observation_with_time: - class: memory.worker.write.get_observation_with_time_worker + class: memory.worker.summary.get_observation_with_time_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 contra_repeat: - class: memory.worker.write.contra_repeat_worker + class: memory.worker.summary.contra_repeat_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 store_memory: - class: memory.worker.write.update_memory_worker + class: memory.worker.summary.update_memory_worker method: from_memory_key memory_key: all load_obs_and_insight: - class: memory.worker.write.load_memory_worker + class: memory.worker.summary.load_memory_worker retrieve_not_reflected_top_k: 100 retrieve_not_updated_top_k: 100 retrieve_insight_top_k: 100 diff --git a/memoryscope/memory/worker/frontend/retrieve_memory_worker.py b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py index 459db4ee..fd928f7d 100644 --- a/memoryscope/memory/worker/frontend/retrieve_memory_worker.py +++ b/memoryscope/memory/worker/frontend/retrieve_memory_worker.py @@ -43,8 +43,8 @@ class RetrieveMemoryWorker(MemoryBaseWorker): "store_status": StoreStatusEnum.VALID.value, "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], } - # ⭐ Retrieve memories matching the query, filtered by the specified conditions, - # limited to a certain number, and sorted by relevance. + # Retrieve memories matching the query, filtered by the specified conditions, + # limited to a certain number, and sorted by relevance. return self.memory_store.retrieve_memories(query=query, top_k=self.retrieve_obs_top_k, filter_dict=filter_dict) diff --git a/memoryscope/memory/worker/frontend/set_query_worker.py b/memoryscope/memory/worker/frontend/set_query_worker.py index a94c5675..552baf5e 100644 --- a/memoryscope/memory/worker/frontend/set_query_worker.py +++ b/memoryscope/memory/worker/frontend/set_query_worker.py @@ -21,7 +21,7 @@ class SetQueryWorker(MemoryBaseWorker): Otherwise, the content of the last message in `self.chat_messages` is used as the query, along with its creation timestamp. """ - query = "_" # Default query value + query = "" # Default query value query_timestamp = int(datetime.datetime.now().timestamp()) # Current timestamp as default if "query" in self.chat_kwargs: diff --git a/memoryscope/memory/worker/memory_base_worker.py b/memoryscope/memory/worker/memory_base_worker.py index 8ef58c79..5d0ee99a 100644 --- a/memoryscope/memory/worker/memory_base_worker.py +++ b/memoryscope/memory/worker/memory_base_worker.py @@ -194,7 +194,7 @@ class MemoryBaseWorker(BaseWorker, metaclass=ABCMeta): return self.get_context(MEMORY_HANDLER) @staticmethod - def get_language_value(languages: dict | list[dict]) -> Any | list[Any]: + def get_language_value(languages: dict | List[dict]) -> Any | List[Any]: """ Retrieves the value(s) corresponding to the current language context. diff --git a/memoryscope/memory/worker/write/contra_repeat_worker.py b/memoryscope/memory/worker/summary/contra_repeat_worker.py similarity index 100% rename from memoryscope/memory/worker/write/contra_repeat_worker.py rename to memoryscope/memory/worker/summary/contra_repeat_worker.py diff --git a/memoryscope/memory/worker/write/contra_repeat_worker.yaml b/memoryscope/memory/worker/summary/contra_repeat_worker.yaml similarity index 100% rename from memoryscope/memory/worker/write/contra_repeat_worker.yaml rename to memoryscope/memory/worker/summary/contra_repeat_worker.yaml diff --git a/memoryscope/memory/worker/write/get_observation_with_time_worker.py b/memoryscope/memory/worker/summary/get_observation_with_time_worker.py similarity index 100% rename from memoryscope/memory/worker/write/get_observation_with_time_worker.py rename to memoryscope/memory/worker/summary/get_observation_with_time_worker.py diff --git a/memoryscope/memory/worker/write/get_observation_with_time_worker.yaml b/memoryscope/memory/worker/summary/get_observation_with_time_worker.yaml similarity index 100% rename from memoryscope/memory/worker/write/get_observation_with_time_worker.yaml rename to memoryscope/memory/worker/summary/get_observation_with_time_worker.yaml diff --git a/memoryscope/memory/worker/write/get_observation_worker.py b/memoryscope/memory/worker/summary/get_observation_worker.py similarity index 100% rename from memoryscope/memory/worker/write/get_observation_worker.py rename to memoryscope/memory/worker/summary/get_observation_worker.py diff --git a/memoryscope/memory/worker/write/get_observation_worker.yaml b/memoryscope/memory/worker/summary/get_observation_worker.yaml similarity index 100% rename from memoryscope/memory/worker/write/get_observation_worker.yaml rename to memoryscope/memory/worker/summary/get_observation_worker.yaml diff --git a/memoryscope/memory/worker/write/info_filter_worker.py b/memoryscope/memory/worker/summary/info_filter_worker.py similarity index 100% rename from memoryscope/memory/worker/write/info_filter_worker.py rename to memoryscope/memory/worker/summary/info_filter_worker.py diff --git a/memoryscope/memory/worker/write/info_filter_worker.yaml b/memoryscope/memory/worker/summary/info_filter_worker.yaml similarity index 100% rename from memoryscope/memory/worker/write/info_filter_worker.yaml rename to memoryscope/memory/worker/summary/info_filter_worker.yaml diff --git a/memoryscope/memory/worker/write/load_memory_worker.py b/memoryscope/memory/worker/summary/load_memory_worker.py similarity index 79% rename from memoryscope/memory/worker/write/load_memory_worker.py rename to memoryscope/memory/worker/summary/load_memory_worker.py index 4a7e60c0..24972900 100644 --- a/memoryscope/memory/worker/write/load_memory_worker.py +++ b/memoryscope/memory/worker/summary/load_memory_worker.py @@ -17,12 +17,9 @@ class LoadMemoryWorker(MemoryBaseWorker): self.retrieve_today_top_k: int = kwargs.get("retrieve_today_top_k", 0) @timer - def retrieve_not_reflected_memory(self, query: str): + def retrieve_not_reflected_memory(self): """ Retrieves top-K not reflected memories based on the query and stores them in the memory handler. - - Args: - query (str): The search query for retrieving memories. """ if not self.retrieve_not_reflected_top_k: return @@ -34,18 +31,14 @@ class LoadMemoryWorker(MemoryBaseWorker): "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], "obs_reflected": 0, } - nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query, - top_k=self.retrieve_not_reflected_top_k, + nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_not_reflected_top_k, filter_dict=filter_dict) self.memory_handler.set_memories(NOT_REFLECTED_NODES, nodes) @timer - def retrieve_not_updated_memory(self, query: str): + def retrieve_not_updated_memory(self): """ Retrieves top-K not updated memories based on the query and stores them in the memory handler. - - Args: - query (str): The search query for retrieving memories. """ if not self.retrieve_not_updated_top_k: return @@ -57,8 +50,7 @@ class LoadMemoryWorker(MemoryBaseWorker): "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], "obs_updated": 0, } - nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query, - top_k=self.retrieve_not_updated_top_k, + nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_not_updated_top_k, filter_dict=filter_dict) self.memory_handler.set_memories(NOT_UPDATED_NODES, nodes) @@ -66,9 +58,6 @@ class LoadMemoryWorker(MemoryBaseWorker): def retrieve_insight_memory(self, query: str): """ Retrieves top-K insight memories based on the query and stores them in the memory handler. - - Args: - query (str): The search query for retrieving memories. """ if not self.retrieve_insight_top_k: return @@ -79,18 +68,16 @@ class LoadMemoryWorker(MemoryBaseWorker): "store_status": StoreStatusEnum.VALID.value, "memory_type": MemoryTypeEnum.INSIGHT.value, } - nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query, - top_k=self.retrieve_insight_top_k, + nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_insight_top_k, filter_dict=filter_dict) self.memory_handler.set_memories(INSIGHT_NODES, nodes) @timer - def retrieve_today_memory(self, query: str, dt: str): + def retrieve_today_memory(self, dt: str): """ Retrieves top-K memories from today based on the query and stores them in the memory handler. Args: - query (str): The search query for retrieving memories. dt (str): The date string to filter today's memories. """ if not self.retrieve_today_top_k: @@ -103,8 +90,7 @@ class LoadMemoryWorker(MemoryBaseWorker): "memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value], "dt": dt, } - nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query, - top_k=self.retrieve_today_top_k, + nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_today_top_k, filter_dict=filter_dict) self.memory_handler.set_memories(TODAY_NODES, nodes) @@ -120,12 +106,11 @@ class LoadMemoryWorker(MemoryBaseWorker): """ # Placeholder query - query = "-" dt = DatetimeHandler().datetime_format() - self.submit_thread_task(self.retrieve_not_reflected_memory, query=query) - self.submit_thread_task(self.retrieve_not_updated_memory, query=query) - self.submit_thread_task(self.retrieve_insight_memory, query=query) - self.submit_thread_task(self.retrieve_today_memory, query=query, dt=dt) + self.submit_thread_task(self.retrieve_not_reflected_memory) + self.submit_thread_task(self.retrieve_not_updated_memory) + self.submit_thread_task(self.retrieve_insight_memory) + self.submit_thread_task(self.retrieve_today_memory, dt=dt) # Waits for all submitted tasks to complete for _ in self.gather_thread_result(): diff --git a/memoryscope/memory/worker/write/update_memory_worker.py b/memoryscope/memory/worker/summary/update_memory_worker.py similarity index 100% rename from memoryscope/memory/worker/write/update_memory_worker.py rename to memoryscope/memory/worker/summary/update_memory_worker.py diff --git a/memoryscope/memory/worker/write/__init__.py b/memoryscope/memory/worker/write/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/memoryscope/storage/base_memory_store.py b/memoryscope/storage/base_memory_store.py index f39305fb..a06167ac 100644 --- a/memoryscope/storage/base_memory_store.py +++ b/memoryscope/storage/base_memory_store.py @@ -11,7 +11,10 @@ class BaseMemoryStore(metaclass=ABCMeta): """ @abstractmethod - def retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]: + def retrieve_memories(self, + query: str = "", + top_k: int = 3, + filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]: """ Retrieves a list of MemoryNode objects that are most relevant to the query, considering a filter dictionary for additional constraints. The number of nodes returned @@ -30,7 +33,10 @@ class BaseMemoryStore(metaclass=ABCMeta): pass @abstractmethod - async def a_retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]: + async def a_retrieve_memories(self, + query: str = "", + top_k: int = 3, + filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]: """ Asynchronously retrieves a list of MemoryNode objects that best match the query, respecting a filter dictionary, with the result size capped at top_k. diff --git a/memoryscope/storage/dummy_memory_store.py b/memoryscope/storage/dummy_memory_store.py index fa6ab1fd..304818bb 100644 --- a/memoryscope/storage/dummy_memory_store.py +++ b/memoryscope/storage/dummy_memory_store.py @@ -12,6 +12,18 @@ class DummyMemoryStore(BaseMemoryStore): semantic retrieval. Actual storage operations are not implemented. """ + def retrieve_memories(self, + query: str = "", + top_k: int = 3, + filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]: + pass + + async def a_retrieve_memories(self, + query: str = "", + top_k: int = 3, + filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]: + pass + def __init__(self, embedding_model: BaseModel, **kwargs): """ Initializes the DummyMemoryStore with an embedding model and additional keyword arguments. @@ -23,12 +35,6 @@ class DummyMemoryStore(BaseMemoryStore): self.embedding_model: BaseModel = embedding_model self.kwargs = kwargs - def retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]: - pass - - async def a_retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]: - pass - def batch_insert(self, nodes: List[MemoryNode]): pass diff --git a/memoryscope/storage/llama_index_es_memory_store.py b/memoryscope/storage/llama_index_es_memory_store.py index 8a6bd771..b7ce3b21 100644 --- a/memoryscope/storage/llama_index_es_memory_store.py +++ b/memoryscope/storage/llama_index_es_memory_store.py @@ -37,7 +37,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): self.logger = Logger.get_logger() def retrieve_memories(self, - query: Optional[str] = None, + query: str = "", top_k: int = 3, filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]: # if index is not created, return [] @@ -59,7 +59,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore): return [self._text_node_2_memory_node(n) for n in text_nodes] async def a_retrieve_memories(self, - query: Optional[str] = None, + query: str = "", top_k: int = 3, filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]: # if index is not created, return [] From 8515af0528781377f780048262e31fb49e59587d Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 21:25:13 +0800 Subject: [PATCH 7/8] fix useless params --- .../memory/worker/summary/get_observation_with_time_worker.py | 2 +- memoryscope/memory/worker/summary/load_memory_worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/memoryscope/memory/worker/summary/get_observation_with_time_worker.py b/memoryscope/memory/worker/summary/get_observation_with_time_worker.py index 18a77380..3b451a56 100644 --- a/memoryscope/memory/worker/summary/get_observation_with_time_worker.py +++ b/memoryscope/memory/worker/summary/get_observation_with_time_worker.py @@ -2,7 +2,7 @@ from typing import List from memoryscope.constants.common_constants import NEW_OBS_WITH_TIME_NODES from memoryscope.constants.language_constants import COLON_WORD -from memoryscope.memory.worker.write.get_observation_worker import GetObservationWorker +from memoryscope.memory.worker.summary.get_observation_worker import GetObservationWorker from memoryscope.scheme.message import Message from memoryscope.utils.datetime_handler import DatetimeHandler from memoryscope.utils.tool_functions import prompt_to_msg diff --git a/memoryscope/memory/worker/summary/load_memory_worker.py b/memoryscope/memory/worker/summary/load_memory_worker.py index 24972900..1f5e4233 100644 --- a/memoryscope/memory/worker/summary/load_memory_worker.py +++ b/memoryscope/memory/worker/summary/load_memory_worker.py @@ -55,7 +55,7 @@ class LoadMemoryWorker(MemoryBaseWorker): self.memory_handler.set_memories(NOT_UPDATED_NODES, nodes) @timer - def retrieve_insight_memory(self, query: str): + def retrieve_insight_memory(self): """ Retrieves top-K insight memories based on the query and stores them in the memory handler. """ From 2728d88abac2df895f0470507282e8b06e377105 Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Wed, 24 Jul 2024 22:27:47 +0800 Subject: [PATCH 8/8] rename summary -> backend --- config/cli_chat_dash_cn.yaml | 29 ++++++++++--------- .../worker/{summary => backend}/__init__.py | 0 .../contra_repeat_worker.py | 0 .../contra_repeat_worker.yaml | 0 .../get_observation_with_time_worker.py | 2 +- .../get_observation_with_time_worker.yaml | 0 .../get_observation_worker.py | 0 .../get_observation_worker.yaml | 0 .../get_reflection_subject_worker.py | 0 .../get_reflection_subject_worker.yaml | 0 .../info_filter_worker.py | 0 .../info_filter_worker.yaml | 0 .../load_memory_worker.py | 0 .../long_contra_repeat_worker.py | 0 .../long_contra_repeat_worker.yaml | 0 .../update_insight_worker.py | 0 .../update_insight_worker.yaml | 0 .../update_memory_worker.py | 0 .../worker/frontend/semantic_rank_worker.py | 1 + 19 files changed, 17 insertions(+), 15 deletions(-) rename memoryscope/memory/worker/{summary => backend}/__init__.py (100%) rename memoryscope/memory/worker/{summary => backend}/contra_repeat_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/contra_repeat_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/get_observation_with_time_worker.py (98%) rename memoryscope/memory/worker/{summary => backend}/get_observation_with_time_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/get_observation_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/get_observation_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/get_reflection_subject_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/get_reflection_subject_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/info_filter_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/info_filter_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/load_memory_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/long_contra_repeat_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/long_contra_repeat_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/update_insight_worker.py (100%) rename memoryscope/memory/worker/{summary => backend}/update_insight_worker.yaml (100%) rename memoryscope/memory/worker/{summary => backend}/update_memory_worker.py (100%) diff --git a/config/cli_chat_dash_cn.yaml b/config/cli_chat_dash_cn.yaml index ada734da..dd04463e 100644 --- a/config/cli_chat_dash_cn.yaml +++ b/config/cli_chat_dash_cn.yaml @@ -82,7 +82,7 @@ worker: rank_model: dashscope_rank fuse_rerank: class: memory.worker.frontend.fuse_rerank_worker - fuse_score_threshold: 0.05 + fuse_score_threshold: 0.01 fuse_ratio_dict: conversation: 0.5 observation: 1 @@ -103,60 +103,60 @@ worker: retrieve_ins_top_k: 100 retrieve_expired_top_k: 100 delete_memory: - class: memory.worker.summary.update_memory_worker + class: memory.worker.backend.update_memory_worker method: delete_memory delete_all: - class: memory.worker.summary.update_memory_worker + class: memory.worker.backend.update_memory_worker method: delete_all add_memory: - class: memory.worker.summary.update_memory_worker + class: memory.worker.backend.update_memory_worker method: from_query info_filter: - class: memory.worker.summary.info_filter_worker + class: memory.worker.backend.info_filter_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 load_today_memory: - class: memory.worker.summary.load_memory_worker + class: memory.worker.backend.load_memory_worker retrieve_today_top_k: 100 get_observation: - class: memory.worker.summary.get_observation_worker + class: memory.worker.backend.get_observation_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 get_observation_with_time: - class: memory.worker.summary.get_observation_with_time_worker + class: memory.worker.backend.get_observation_with_time_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 contra_repeat: - class: memory.worker.summary.contra_repeat_worker + class: memory.worker.backend.contra_repeat_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 store_memory: - class: memory.worker.summary.update_memory_worker + class: memory.worker.backend.update_memory_worker method: from_memory_key memory_key: all load_obs_and_insight: - class: memory.worker.summary.load_memory_worker + class: memory.worker.backend.load_memory_worker retrieve_not_reflected_top_k: 100 retrieve_not_updated_top_k: 100 retrieve_insight_top_k: 100 get_reflection_subject: - class: memory.worker.summary.get_reflection_subject_worker + class: memory.worker.backend.get_reflection_subject_worker generation_model: dashscope_generation reflect_obs_cnt_threshold: 10 generation_model_kwargs: top_k: 1 update_insight: - class: memory.worker.summary.update_insight_worker + class: memory.worker.backend.update_insight_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 rank_model: dashscope_rank long_contra_repeat: - class: memory.worker.summary.long_contra_repeat_worker + class: memory.worker.backend.long_contra_repeat_worker generation_model: dashscope_generation generation_model_kwargs: top_k: 1 @@ -175,6 +175,7 @@ models: class: models.llama_index_rank_model module_name: dashscope_rank model_name: gte-rerank + top_n: 10 dummy_generation: class: models.dummy_generation_model module_name: dummy_generation diff --git a/memoryscope/memory/worker/summary/__init__.py b/memoryscope/memory/worker/backend/__init__.py similarity index 100% rename from memoryscope/memory/worker/summary/__init__.py rename to memoryscope/memory/worker/backend/__init__.py diff --git a/memoryscope/memory/worker/summary/contra_repeat_worker.py b/memoryscope/memory/worker/backend/contra_repeat_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/contra_repeat_worker.py rename to memoryscope/memory/worker/backend/contra_repeat_worker.py diff --git a/memoryscope/memory/worker/summary/contra_repeat_worker.yaml b/memoryscope/memory/worker/backend/contra_repeat_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/contra_repeat_worker.yaml rename to memoryscope/memory/worker/backend/contra_repeat_worker.yaml diff --git a/memoryscope/memory/worker/summary/get_observation_with_time_worker.py b/memoryscope/memory/worker/backend/get_observation_with_time_worker.py similarity index 98% rename from memoryscope/memory/worker/summary/get_observation_with_time_worker.py rename to memoryscope/memory/worker/backend/get_observation_with_time_worker.py index 3b451a56..9655fce0 100644 --- a/memoryscope/memory/worker/summary/get_observation_with_time_worker.py +++ b/memoryscope/memory/worker/backend/get_observation_with_time_worker.py @@ -2,7 +2,7 @@ from typing import List from memoryscope.constants.common_constants import NEW_OBS_WITH_TIME_NODES from memoryscope.constants.language_constants import COLON_WORD -from memoryscope.memory.worker.summary.get_observation_worker import GetObservationWorker +from memoryscope.memory.worker.backend.get_observation_worker import GetObservationWorker from memoryscope.scheme.message import Message from memoryscope.utils.datetime_handler import DatetimeHandler from memoryscope.utils.tool_functions import prompt_to_msg diff --git a/memoryscope/memory/worker/summary/get_observation_with_time_worker.yaml b/memoryscope/memory/worker/backend/get_observation_with_time_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/get_observation_with_time_worker.yaml rename to memoryscope/memory/worker/backend/get_observation_with_time_worker.yaml diff --git a/memoryscope/memory/worker/summary/get_observation_worker.py b/memoryscope/memory/worker/backend/get_observation_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/get_observation_worker.py rename to memoryscope/memory/worker/backend/get_observation_worker.py diff --git a/memoryscope/memory/worker/summary/get_observation_worker.yaml b/memoryscope/memory/worker/backend/get_observation_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/get_observation_worker.yaml rename to memoryscope/memory/worker/backend/get_observation_worker.yaml diff --git a/memoryscope/memory/worker/summary/get_reflection_subject_worker.py b/memoryscope/memory/worker/backend/get_reflection_subject_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/get_reflection_subject_worker.py rename to memoryscope/memory/worker/backend/get_reflection_subject_worker.py diff --git a/memoryscope/memory/worker/summary/get_reflection_subject_worker.yaml b/memoryscope/memory/worker/backend/get_reflection_subject_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/get_reflection_subject_worker.yaml rename to memoryscope/memory/worker/backend/get_reflection_subject_worker.yaml diff --git a/memoryscope/memory/worker/summary/info_filter_worker.py b/memoryscope/memory/worker/backend/info_filter_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/info_filter_worker.py rename to memoryscope/memory/worker/backend/info_filter_worker.py diff --git a/memoryscope/memory/worker/summary/info_filter_worker.yaml b/memoryscope/memory/worker/backend/info_filter_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/info_filter_worker.yaml rename to memoryscope/memory/worker/backend/info_filter_worker.yaml diff --git a/memoryscope/memory/worker/summary/load_memory_worker.py b/memoryscope/memory/worker/backend/load_memory_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/load_memory_worker.py rename to memoryscope/memory/worker/backend/load_memory_worker.py diff --git a/memoryscope/memory/worker/summary/long_contra_repeat_worker.py b/memoryscope/memory/worker/backend/long_contra_repeat_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/long_contra_repeat_worker.py rename to memoryscope/memory/worker/backend/long_contra_repeat_worker.py diff --git a/memoryscope/memory/worker/summary/long_contra_repeat_worker.yaml b/memoryscope/memory/worker/backend/long_contra_repeat_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/long_contra_repeat_worker.yaml rename to memoryscope/memory/worker/backend/long_contra_repeat_worker.yaml diff --git a/memoryscope/memory/worker/summary/update_insight_worker.py b/memoryscope/memory/worker/backend/update_insight_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/update_insight_worker.py rename to memoryscope/memory/worker/backend/update_insight_worker.py diff --git a/memoryscope/memory/worker/summary/update_insight_worker.yaml b/memoryscope/memory/worker/backend/update_insight_worker.yaml similarity index 100% rename from memoryscope/memory/worker/summary/update_insight_worker.yaml rename to memoryscope/memory/worker/backend/update_insight_worker.yaml diff --git a/memoryscope/memory/worker/summary/update_memory_worker.py b/memoryscope/memory/worker/backend/update_memory_worker.py similarity index 100% rename from memoryscope/memory/worker/summary/update_memory_worker.py rename to memoryscope/memory/worker/backend/update_memory_worker.py diff --git a/memoryscope/memory/worker/frontend/semantic_rank_worker.py b/memoryscope/memory/worker/frontend/semantic_rank_worker.py index 3ea4801a..be5d67dc 100644 --- a/memoryscope/memory/worker/frontend/semantic_rank_worker.py +++ b/memoryscope/memory/worker/frontend/semantic_rank_worker.py @@ -53,6 +53,7 @@ class SemanticRankWorker(MemoryBaseWorker): memory_node_list = sorted(memory_node_list, key=lambda n: n.score_rank, reverse=True) # log ranked nodes + self.logger.info(f"Rank stage: query={query}") for node in memory_node_list: self.logger.info(f"Rank stage: Content={node.content}, Score={node.score_rank}")