mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-05 08:06:15 +00:00
[features] rename package name & fix es bug
This commit is contained in:
jinli.yl
commit
84166dfc1e
94 changed files with 366 additions and 365 deletions
|
|
@ -82,7 +82,7 @@ worker:
|
|||
rank_model: dashscope_rank
|
||||
fuse_rerank:
|
||||
class: memory.worker.frontend.fuse_rerank_worker
|
||||
fuse_score_threshold: 0.05
|
||||
fuse_score_threshold: 0.01
|
||||
fuse_ratio_dict:
|
||||
conversation: 0.5
|
||||
observation: 1
|
||||
|
|
@ -103,60 +103,60 @@ worker:
|
|||
retrieve_ins_top_k: 100
|
||||
retrieve_expired_top_k: 100
|
||||
delete_memory:
|
||||
class: memory.worker.write.update_memory_worker
|
||||
class: memory.worker.backend.update_memory_worker
|
||||
method: delete_memory
|
||||
delete_all:
|
||||
class: memory.worker.write.update_memory_worker
|
||||
class: memory.worker.backend.update_memory_worker
|
||||
method: delete_all
|
||||
add_memory:
|
||||
class: memory.worker.write.update_memory_worker
|
||||
class: memory.worker.backend.update_memory_worker
|
||||
method: from_query
|
||||
info_filter:
|
||||
class: memory.worker.write.info_filter_worker
|
||||
class: memory.worker.backend.info_filter_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
load_today_memory:
|
||||
class: memory.worker.write.load_memory_worker
|
||||
class: memory.worker.backend.load_memory_worker
|
||||
retrieve_today_top_k: 100
|
||||
get_observation:
|
||||
class: memory.worker.write.get_observation_worker
|
||||
class: memory.worker.backend.get_observation_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
get_observation_with_time:
|
||||
class: memory.worker.write.get_observation_with_time_worker
|
||||
class: memory.worker.backend.get_observation_with_time_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
contra_repeat:
|
||||
class: memory.worker.write.contra_repeat_worker
|
||||
class: memory.worker.backend.contra_repeat_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
store_memory:
|
||||
class: memory.worker.write.update_memory_worker
|
||||
class: memory.worker.backend.update_memory_worker
|
||||
method: from_memory_key
|
||||
memory_key: all
|
||||
load_obs_and_insight:
|
||||
class: memory.worker.write.load_memory_worker
|
||||
class: memory.worker.backend.load_memory_worker
|
||||
retrieve_not_reflected_top_k: 100
|
||||
retrieve_not_updated_top_k: 100
|
||||
retrieve_insight_top_k: 100
|
||||
get_reflection_subject:
|
||||
class: memory.worker.summary.get_reflection_subject_worker
|
||||
class: memory.worker.backend.get_reflection_subject_worker
|
||||
generation_model: dashscope_generation
|
||||
reflect_obs_cnt_threshold: 10
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
update_insight:
|
||||
class: memory.worker.summary.update_insight_worker
|
||||
class: memory.worker.backend.update_insight_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
rank_model: dashscope_rank
|
||||
long_contra_repeat:
|
||||
class: memory.worker.summary.long_contra_repeat_worker
|
||||
class: memory.worker.backend.long_contra_repeat_worker
|
||||
generation_model: dashscope_generation
|
||||
generation_model_kwargs:
|
||||
top_k: 1
|
||||
|
|
@ -175,6 +175,7 @@ models:
|
|||
class: models.llama_index_rank_model
|
||||
module_name: dashscope_rank
|
||||
model_name: gte-rerank
|
||||
top_n: 10
|
||||
dummy_generation:
|
||||
class: models.dummy_generation_model
|
||||
module_name: dummy_generation
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
from memory_scope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memoryscope.memory.service.base_memory_service import BaseMemoryService
|
||||
|
||||
|
||||
class BaseMemoryChat(metaclass=ABCMeta):
|
||||
|
|
@ -4,17 +4,17 @@ from typing import List
|
|||
|
||||
import questionary
|
||||
|
||||
from memory_scope.chat.base_memory_chat import BaseMemoryChat
|
||||
from memory_scope.constants.language_constants import DEFAULT_HUMAN_NAME
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memory_scope.models.base_model import BaseModel
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.prompt_handler import PromptHandler
|
||||
from memory_scope.utils.tool_functions import char_logo
|
||||
from memoryscope.chat.base_memory_chat import BaseMemoryChat
|
||||
from memoryscope.constants.language_constants import DEFAULT_HUMAN_NAME
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memoryscope.models.base_model import BaseModel
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.prompt_handler import PromptHandler
|
||||
from memoryscope.utils.tool_functions import char_logo
|
||||
|
||||
|
||||
class CliMemoryChat(BaseMemoryChat):
|
||||
|
|
@ -13,12 +13,12 @@ import fire
|
|||
import yaml
|
||||
import atexit
|
||||
|
||||
from memory_scope.enumeration.language_enum import LanguageEnum
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.timer import timer
|
||||
from memory_scope.utils.tool_functions import init_instance_by_config, camelcase_to_underscore
|
||||
from memoryscope.enumeration.language_enum import LanguageEnum
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.timer import timer
|
||||
from memoryscope.utils.tool_functions import init_instance_by_config, camelcase_to_underscore
|
||||
|
||||
|
||||
class MemoryScope(object):
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from memory_scope.enumeration.language_enum import LanguageEnum
|
||||
from memoryscope.enumeration.language_enum import LanguageEnum
|
||||
|
||||
# This dictionary maps languages to lists of words related to datetime expressions.
|
||||
# It aids in recognizing and processing datetime mentions in text, enhancing the system's ability to understand
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
import time
|
||||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import CHAT_KWARGS, RESULT, CHAT_MESSAGES
|
||||
from memory_scope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE
|
||||
from memory_scope.memory.operation.base_workflow import BaseWorkflow
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.constants.common_constants import CHAT_KWARGS, RESULT, CHAT_MESSAGES
|
||||
from memoryscope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE
|
||||
from memoryscope.memory.operation.base_workflow import BaseWorkflow
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class BackendOperation(BaseWorkflow, BaseOperation):
|
||||
|
|
@ -4,12 +4,12 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
|
|||
from itertools import zip_longest
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from memory_scope.constants.common_constants import WORKFLOW_NAME
|
||||
from memory_scope.memory.worker.base_worker import BaseWorker
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.timer import Timer
|
||||
from memory_scope.utils.tool_functions import init_instance_by_config
|
||||
from memoryscope.constants.common_constants import WORKFLOW_NAME
|
||||
from memoryscope.memory.worker.base_worker import BaseWorker
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.timer import Timer
|
||||
from memoryscope.utils.tool_functions import init_instance_by_config
|
||||
|
||||
|
||||
class BaseWorkflow(object):
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import RESULT, CHAT_MESSAGES, CHAT_KWARGS
|
||||
from memory_scope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE
|
||||
from memory_scope.memory.operation.base_workflow import BaseWorkflow
|
||||
from memory_scope.scheme.message import Message
|
||||
from memoryscope.constants.common_constants import RESULT, CHAT_MESSAGES, CHAT_KWARGS
|
||||
from memoryscope.memory.operation.base_operation import BaseOperation, OPERATION_TYPE
|
||||
from memoryscope.memory.operation.base_workflow import BaseWorkflow
|
||||
from memoryscope.scheme.message import Message
|
||||
|
||||
|
||||
class FrontendOperation(BaseWorkflow, BaseOperation):
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from memory_scope.constants.common_constants import CHAT_KWARGS, CHAT_MESSAGES, RESULT
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.operation.backend_operation import BackendOperation
|
||||
from memoryscope.constants.common_constants import CHAT_KWARGS, CHAT_MESSAGES, RESULT
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.operation.backend_operation import BackendOperation
|
||||
|
||||
|
||||
class SummaryObservationOp(BackendOperation):
|
||||
|
|
@ -2,9 +2,9 @@ import threading
|
|||
from abc import ABCMeta, abstractmethod
|
||||
from typing import List, Dict
|
||||
|
||||
from memory_scope.memory.operation.base_operation import BaseOperation
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.memory.operation.base_operation import BaseOperation
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class BaseMemoryService(metaclass=ABCMeta):
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.memory.operation.base_operation import BaseOperation
|
||||
from memory_scope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.tool_functions import init_instance_by_config
|
||||
from memoryscope.memory.operation.base_operation import BaseOperation
|
||||
from memoryscope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.tool_functions import init_instance_by_config
|
||||
|
||||
|
||||
class MemoryScopeService(BaseMemoryService):
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, MERGE_OBS_NODES, TODAY_NODES
|
||||
from memory_scope.constants.language_constants import NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, MERGE_OBS_NODES, TODAY_NODES
|
||||
from memoryscope.constants.language_constants import NONE_WORD, CONTRADICTORY_WORD, CONTAINED_WORD
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class ContraRepeatWorker(MemoryBaseWorker):
|
||||
|
|
@ -80,7 +80,7 @@ class ContraRepeatWorker(MemoryBaseWorker):
|
|||
response_text = response.message.content
|
||||
|
||||
# parse text
|
||||
idx_merge_obs_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__)
|
||||
idx_merge_obs_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1()
|
||||
if len(idx_merge_obs_list) <= 0:
|
||||
self.logger.warning("idx_merge_obs_list is empty!")
|
||||
return
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import NEW_OBS_WITH_TIME_NODES
|
||||
from memory_scope.constants.language_constants import COLON_WORD
|
||||
from memory_scope.memory.worker.write.get_observation_worker import GetObservationWorker
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import NEW_OBS_WITH_TIME_NODES
|
||||
from memoryscope.constants.language_constants import COLON_WORD
|
||||
from memoryscope.memory.worker.backend.get_observation_worker import GetObservationWorker
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class GetObservationWithTimeWorker(GetObservationWorker):
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import NEW_OBS_NODES, TIME_INFER
|
||||
from memory_scope.constants.language_constants import REPEATED_WORD, NONE_WORD, COLON_WORD, TIME_INFER_WORD
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import NEW_OBS_NODES, TIME_INFER
|
||||
from memoryscope.constants.language_constants import REPEATED_WORD, NONE_WORD, COLON_WORD, TIME_INFER_WORD
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class GetObservationWorker(MemoryBaseWorker):
|
||||
|
|
@ -139,7 +139,7 @@ class GetObservationWorker(MemoryBaseWorker):
|
|||
response_text = response.message.content
|
||||
|
||||
# Parses the generated text to extract observation indices, times, contents, and keywords
|
||||
idx_obs_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__)
|
||||
idx_obs_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1()
|
||||
if len(idx_obs_list) <= 0:
|
||||
self.logger.warning("idx_obs_list is empty!")
|
||||
return
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import NOT_REFLECTED_NODES, INSIGHT_NODES
|
||||
from memory_scope.constants.language_constants import COMMA_WORD
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import NOT_REFLECTED_NODES, INSIGHT_NODES
|
||||
from memoryscope.constants.language_constants import COMMA_WORD
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class GetReflectionSubjectWorker(MemoryBaseWorker):
|
||||
|
|
@ -101,7 +101,7 @@ class GetReflectionSubjectWorker(MemoryBaseWorker):
|
|||
return
|
||||
|
||||
# Parse LLM response for new insight keys and update memory
|
||||
new_insight_keys = ResponseTextParser(response.message.content).parse_v2(self.__class__.__name__)
|
||||
new_insight_keys = ResponseTextParser(response.message.content, self.__class__.__name__).parse_v2()
|
||||
if new_insight_keys:
|
||||
for insight_key in new_insight_keys:
|
||||
self.memory_handler.add_memories(INSIGHT_NODES, self.new_insight_node(insight_key))
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.language_constants import COLON_WORD
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.language_constants import COLON_WORD
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class InfoFilterWorker(MemoryBaseWorker):
|
||||
|
|
@ -76,7 +76,7 @@ class InfoFilterWorker(MemoryBaseWorker):
|
|||
response_text = response.message.content
|
||||
|
||||
# parse text
|
||||
info_score_list = ResponseTextParser(response_text).parse_v1(self.__class__.__name__)
|
||||
info_score_list = ResponseTextParser(response_text, self.__class__.__name__).parse_v1()
|
||||
if len(info_score_list) != len(info_messages):
|
||||
self.logger.warning(f"score_size != messages_size, {len(info_score_list)} vs {len(info_messages)}")
|
||||
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import NOT_REFLECTED_NODES, NOT_UPDATED_NODES, INSIGHT_NODES, TODAY_NODES
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.timer import timer
|
||||
from memoryscope.constants.common_constants import NOT_REFLECTED_NODES, NOT_UPDATED_NODES, INSIGHT_NODES, TODAY_NODES
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.timer import timer
|
||||
|
||||
|
||||
class LoadMemoryWorker(MemoryBaseWorker):
|
||||
|
|
@ -17,12 +17,9 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
self.retrieve_today_top_k: int = kwargs.get("retrieve_today_top_k", 0)
|
||||
|
||||
@timer
|
||||
def retrieve_not_reflected_memory(self, query: str):
|
||||
def retrieve_not_reflected_memory(self):
|
||||
"""
|
||||
Retrieves top-K not reflected memories based on the query and stores them in the memory handler.
|
||||
|
||||
Args:
|
||||
query (str): The search query for retrieving memories.
|
||||
"""
|
||||
if not self.retrieve_not_reflected_top_k:
|
||||
return
|
||||
|
|
@ -34,18 +31,14 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
"memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value],
|
||||
"obs_reflected": 0,
|
||||
}
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query,
|
||||
top_k=self.retrieve_not_reflected_top_k,
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_not_reflected_top_k,
|
||||
filter_dict=filter_dict)
|
||||
self.memory_handler.set_memories(NOT_REFLECTED_NODES, nodes)
|
||||
|
||||
@timer
|
||||
def retrieve_not_updated_memory(self, query: str):
|
||||
def retrieve_not_updated_memory(self):
|
||||
"""
|
||||
Retrieves top-K not updated memories based on the query and stores them in the memory handler.
|
||||
|
||||
Args:
|
||||
query (str): The search query for retrieving memories.
|
||||
"""
|
||||
if not self.retrieve_not_updated_top_k:
|
||||
return
|
||||
|
|
@ -57,18 +50,14 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
"memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value],
|
||||
"obs_updated": 0,
|
||||
}
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query,
|
||||
top_k=self.retrieve_not_updated_top_k,
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_not_updated_top_k,
|
||||
filter_dict=filter_dict)
|
||||
self.memory_handler.set_memories(NOT_UPDATED_NODES, nodes)
|
||||
|
||||
@timer
|
||||
def retrieve_insight_memory(self, query: str):
|
||||
def retrieve_insight_memory(self):
|
||||
"""
|
||||
Retrieves top-K insight memories based on the query and stores them in the memory handler.
|
||||
|
||||
Args:
|
||||
query (str): The search query for retrieving memories.
|
||||
"""
|
||||
if not self.retrieve_insight_top_k:
|
||||
return
|
||||
|
|
@ -79,18 +68,16 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
"store_status": StoreStatusEnum.VALID.value,
|
||||
"memory_type": MemoryTypeEnum.INSIGHT.value,
|
||||
}
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query,
|
||||
top_k=self.retrieve_insight_top_k,
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_insight_top_k,
|
||||
filter_dict=filter_dict)
|
||||
self.memory_handler.set_memories(INSIGHT_NODES, nodes)
|
||||
|
||||
@timer
|
||||
def retrieve_today_memory(self, query: str, dt: str):
|
||||
def retrieve_today_memory(self, dt: str):
|
||||
"""
|
||||
Retrieves top-K memories from today based on the query and stores them in the memory handler.
|
||||
|
||||
Args:
|
||||
query (str): The search query for retrieving memories.
|
||||
dt (str): The date string to filter today's memories.
|
||||
"""
|
||||
if not self.retrieve_today_top_k:
|
||||
|
|
@ -103,8 +90,7 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
"memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value],
|
||||
"dt": dt,
|
||||
}
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(query=query,
|
||||
top_k=self.retrieve_today_top_k,
|
||||
nodes: List[MemoryNode] = self.memory_store.retrieve_memories(top_k=self.retrieve_today_top_k,
|
||||
filter_dict=filter_dict)
|
||||
|
||||
self.memory_handler.set_memories(TODAY_NODES, nodes)
|
||||
|
|
@ -120,12 +106,11 @@ class LoadMemoryWorker(MemoryBaseWorker):
|
|||
"""
|
||||
|
||||
# Placeholder query
|
||||
query = "-"
|
||||
dt = DatetimeHandler().datetime_format()
|
||||
self.submit_thread_task(self.retrieve_not_reflected_memory, query=query)
|
||||
self.submit_thread_task(self.retrieve_not_updated_memory, query=query)
|
||||
self.submit_thread_task(self.retrieve_insight_memory, query=query)
|
||||
self.submit_thread_task(self.retrieve_today_memory, query=query, dt=dt)
|
||||
self.submit_thread_task(self.retrieve_not_reflected_memory)
|
||||
self.submit_thread_task(self.retrieve_not_updated_memory)
|
||||
self.submit_thread_task(self.retrieve_insight_memory)
|
||||
self.submit_thread_task(self.retrieve_today_memory, dt=dt)
|
||||
|
||||
# Waits for all submitted tasks to complete
|
||||
for _ in self.gather_thread_result():
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
from typing import List, Dict
|
||||
|
||||
from memory_scope.constants.common_constants import NOT_UPDATED_NODES, MERGE_OBS_NODES
|
||||
from memory_scope.constants.language_constants import NONE_WORD, CONTAINED_WORD, CONTRADICTORY_WORD
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import NOT_UPDATED_NODES, MERGE_OBS_NODES
|
||||
from memoryscope.constants.language_constants import NONE_WORD, CONTAINED_WORD, CONTRADICTORY_WORD
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class LongContraRepeatWorker(MemoryBaseWorker):
|
||||
|
|
@ -49,7 +49,7 @@ class LongContraRepeatWorker(MemoryBaseWorker):
|
|||
top_k=self.long_contra_repeat_top_k,
|
||||
filter_dict=filter_dict)
|
||||
# Filter retrieved nodes based on the similarity threshold
|
||||
return node, [n for n in retrieve_nodes if n.score_similar >= self.long_contra_repeat_threshold]
|
||||
return node, [n for n in retrieve_nodes if n.score_recall >= self.long_contra_repeat_threshold]
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
|
|
@ -111,7 +111,7 @@ class LongContraRepeatWorker(MemoryBaseWorker):
|
|||
return
|
||||
|
||||
# Parses the model's response text to identify updates for memory nodes
|
||||
idx_obs_info_list = ResponseTextParser(response.message.content).parse_v1(self.__class__.__name__)
|
||||
idx_obs_info_list = ResponseTextParser(response.message.content, self.__class__.__name__).parse_v1()
|
||||
if len(idx_obs_info_list) <= 0:
|
||||
self.logger.warning("idx_obs_info_list is empty!")
|
||||
return
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
import time
|
||||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import INSIGHT_NODES, NOT_UPDATED_NODES, NOT_REFLECTED_NODES
|
||||
from memory_scope.constants.language_constants import COLON_WORD, NONE_WORD, REPEATED_WORD
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.response_text_parser import ResponseTextParser
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import INSIGHT_NODES, NOT_UPDATED_NODES, NOT_REFLECTED_NODES
|
||||
from memoryscope.constants.language_constants import COLON_WORD, NONE_WORD, REPEATED_WORD
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.response_text_parser import ResponseTextParser
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class UpdateInsightWorker(MemoryBaseWorker):
|
||||
|
|
@ -136,7 +136,8 @@ class UpdateInsightWorker(MemoryBaseWorker):
|
|||
if not response.status or not response.message.content:
|
||||
return insight_node
|
||||
|
||||
insight_value_list = ResponseTextParser(response.message.content).parse_v1(f"update_{insight_node.key}")
|
||||
insight_value_list = ResponseTextParser(response.message.content,
|
||||
f"update_{insight_node.key}").parse_v1()
|
||||
if not insight_value_list:
|
||||
self.logger.warning(f"update_{insight_node.key} insight_value_list is empty!")
|
||||
return insight_node
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
|
||||
|
||||
class UpdateMemoryWorker(MemoryBaseWorker):
|
||||
|
|
@ -3,8 +3,8 @@ from abc import ABCMeta, abstractmethod
|
|||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Dict
|
||||
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.timer import Timer
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.timer import Timer
|
||||
|
||||
|
||||
class BaseWorker(metaclass=ABCMeta):
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import datetime
|
||||
|
||||
from memory_scope.constants.common_constants import RESULT, WORKFLOW_NAME, CHAT_KWARGS
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.constants.common_constants import RESULT, WORKFLOW_NAME, CHAT_KWARGS
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
|
||||
|
||||
class DummyWorker(MemoryBaseWorker):
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
import re
|
||||
from typing import Dict
|
||||
|
||||
from memory_scope.constants.common_constants import QUERY_WITH_TS, EXTRACT_TIME_DICT
|
||||
from memory_scope.constants.language_constants import DATATIME_KEY_MAP
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memory_scope.utils.tool_functions import prompt_to_msg
|
||||
from memoryscope.constants.common_constants import QUERY_WITH_TS, EXTRACT_TIME_DICT
|
||||
from memoryscope.constants.language_constants import DATATIME_KEY_MAP
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.utils.tool_functions import prompt_to_msg
|
||||
|
||||
|
||||
class ExtractTimeWorker(MemoryBaseWorker):
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
from typing import Dict, List
|
||||
|
||||
from memory_scope.constants.common_constants import EXTRACT_TIME_DICT, RANKED_MEMORY_NODES, RESULT
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.constants.common_constants import EXTRACT_TIME_DICT, RANKED_MEMORY_NODES, RESULT
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
|
||||
|
||||
class FuseRerankWorker(MemoryBaseWorker):
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import RETRIEVE_MEMORY_NODES, RESULT
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.datetime_handler import DatetimeHandler
|
||||
from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, RESULT
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.datetime_handler import DatetimeHandler
|
||||
|
||||
|
||||
class PrintMemoryWorker(MemoryBaseWorker):
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
from memory_scope.constants.common_constants import RESULT
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.constants.common_constants import RESULT
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
|
||||
|
||||
class ReadMessageWorker(MemoryBaseWorker):
|
||||
"""
|
||||
Fetches unmemorized chat messages.
|
||||
"""
|
||||
|
||||
def _run(self):
|
||||
"""
|
||||
Executes the primary function to fetch unmemorized chat messages.
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
from typing import List
|
||||
|
||||
from memory_scope.constants.common_constants import QUERY_WITH_TS, RETRIEVE_MEMORY_NODES
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.utils.timer import timer
|
||||
from memoryscope.constants.common_constants import QUERY_WITH_TS, RETRIEVE_MEMORY_NODES
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.memory_type_enum import MemoryTypeEnum
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.utils.timer import timer
|
||||
|
||||
|
||||
class RetrieveMemoryWorker(MemoryBaseWorker):
|
||||
|
|
@ -43,8 +43,8 @@ class RetrieveMemoryWorker(MemoryBaseWorker):
|
|||
"store_status": StoreStatusEnum.VALID.value,
|
||||
"memory_type": [MemoryTypeEnum.OBSERVATION.value, MemoryTypeEnum.OBS_CUSTOMIZED.value],
|
||||
}
|
||||
# ⭐ Retrieve memories matching the query, filtered by the specified conditions,
|
||||
# limited to a certain number, and sorted by relevance.
|
||||
# Retrieve memories matching the query, filtered by the specified conditions,
|
||||
# limited to a certain number, and sorted by relevance.
|
||||
return self.memory_store.retrieve_memories(query=query,
|
||||
top_k=self.retrieve_obs_top_k,
|
||||
filter_dict=filter_dict)
|
||||
|
|
@ -133,10 +133,10 @@ class RetrieveMemoryWorker(MemoryBaseWorker):
|
|||
if not memory_node_list:
|
||||
return
|
||||
|
||||
memory_node_list = sorted(memory_node_list, key=lambda x: x.score_similar, reverse=True)
|
||||
memory_node_list = sorted(memory_node_list, key=lambda x: x.score_recall, reverse=True)
|
||||
for node in memory_node_list:
|
||||
node.action_status = ActionStatusEnum.NONE.value
|
||||
self.logger.info(f"recall_stage: content={node.content} score={node.score_similar} type={node.memory_type} "
|
||||
self.logger.info(f"recall_stage: content={node.content} score={node.score_rerank} type={node.memory_type} "
|
||||
f"store_status={node.store_status} action_status={node.action_status}")
|
||||
|
||||
self.memory_handler.set_memories(RETRIEVE_MEMORY_NODES, memory_node_list)
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
from typing import List, Dict
|
||||
|
||||
from memory_scope.constants.common_constants import RETRIEVE_MEMORY_NODES, QUERY_WITH_TS, RANKED_MEMORY_NODES
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.constants.common_constants import RETRIEVE_MEMORY_NODES, QUERY_WITH_TS, RANKED_MEMORY_NODES
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
|
||||
|
||||
class SemanticRankWorker(MemoryBaseWorker):
|
||||
|
|
@ -53,6 +53,7 @@ class SemanticRankWorker(MemoryBaseWorker):
|
|||
memory_node_list = sorted(memory_node_list, key=lambda n: n.score_rank, reverse=True)
|
||||
|
||||
# log ranked nodes
|
||||
self.logger.info(f"Rank stage: query={query}")
|
||||
for node in memory_node_list:
|
||||
self.logger.info(f"Rank stage: Content={node.content}, Score={node.score_rank}")
|
||||
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import datetime
|
||||
|
||||
from memory_scope.constants.common_constants import QUERY_WITH_TS
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.constants.common_constants import QUERY_WITH_TS
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
|
||||
|
||||
class SetQueryWorker(MemoryBaseWorker):
|
||||
|
|
@ -21,7 +21,7 @@ class SetQueryWorker(MemoryBaseWorker):
|
|||
Otherwise, the content of the last message in `self.chat_messages` is used as the query,
|
||||
along with its creation timestamp.
|
||||
"""
|
||||
query = "_" # Default query value
|
||||
query = "" # Default query value
|
||||
query_timestamp = int(datetime.datetime.now().timestamp()) # Current timestamp as default
|
||||
|
||||
if "query" in self.chat_kwargs:
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
from abc import ABCMeta
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from memory_scope.constants.common_constants import CHAT_MESSAGES, CHAT_KWARGS, MEMORY_HANDLER
|
||||
from memory_scope.memory.worker.base_worker import BaseWorker
|
||||
from memory_scope.models.base_model import BaseModel
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.storage.base_memory_store import BaseMemoryStore
|
||||
from memory_scope.storage.base_monitor import BaseMonitor
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.memory_handler import MemoryHandler
|
||||
from memory_scope.utils.prompt_handler import PromptHandler
|
||||
from memoryscope.constants.common_constants import CHAT_MESSAGES, CHAT_KWARGS, MEMORY_HANDLER
|
||||
from memoryscope.memory.worker.base_worker import BaseWorker
|
||||
from memoryscope.models.base_model import BaseModel
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.storage.base_memory_store import BaseMemoryStore
|
||||
from memoryscope.storage.base_monitor import BaseMonitor
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.memory_handler import MemoryHandler
|
||||
from memoryscope.utils.prompt_handler import PromptHandler
|
||||
|
||||
|
||||
class MemoryBaseWorker(BaseWorker, metaclass=ABCMeta):
|
||||
|
|
@ -194,7 +194,7 @@ class MemoryBaseWorker(BaseWorker, metaclass=ABCMeta):
|
|||
return self.get_context(MEMORY_HANDLER)
|
||||
|
||||
@staticmethod
|
||||
def get_language_value(languages: dict | list[dict]) -> Any | list[Any]:
|
||||
def get_language_value(languages: dict | List[dict]) -> Any | List[Any]:
|
||||
"""
|
||||
Retrieves the value(s) corresponding to the current language context.
|
||||
|
||||
|
|
@ -3,11 +3,11 @@ import time
|
|||
from abc import abstractmethod, ABCMeta
|
||||
from typing import Any
|
||||
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.registry import Registry
|
||||
from memory_scope.utils.timer import Timer
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.registry import Registry
|
||||
from memoryscope.utils.timer import Timer
|
||||
|
||||
MODEL_REGISTRY = Registry("models")
|
||||
|
||||
|
|
@ -3,11 +3,11 @@ from typing import List
|
|||
|
||||
from llama_index.core.base.llms.types import ChatMessage
|
||||
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
|
||||
|
||||
class DummyGenerationModel(BaseModel):
|
||||
|
|
@ -2,9 +2,9 @@ from typing import List
|
|||
|
||||
from llama_index.embeddings.dashscope import DashScopeEmbedding
|
||||
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memory_scope.scheme.model_response import ModelResponse
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memoryscope.scheme.model_response import ModelResponse
|
||||
|
||||
|
||||
class LlamaIndexEmbeddingModel(BaseModel):
|
||||
|
|
@ -3,11 +3,11 @@ from typing import List
|
|||
from llama_index.core.base.llms.types import ChatMessage, ChatResponse, CompletionResponse
|
||||
from llama_index.llms.dashscope import DashScope
|
||||
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.scheme.model_response import ModelResponse, ModelResponseGen
|
||||
|
||||
|
||||
class LlamaIndexGenerationModel(BaseModel):
|
||||
|
|
@ -4,9 +4,9 @@ from llama_index.core.data_structs import Node
|
|||
from llama_index.core.schema import NodeWithScore
|
||||
from llama_index.postprocessor.dashscope_rerank import DashScopeRerank
|
||||
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memory_scope.scheme.model_response import ModelResponse
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.models.base_model import BaseModel, MODEL_REGISTRY
|
||||
from memoryscope.scheme.model_response import ModelResponse
|
||||
|
||||
|
||||
class LlamaIndexRankModel(BaseModel):
|
||||
|
|
@ -3,8 +3,8 @@ from typing import Generator, List, Dict, Any
|
|||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from memory_scope.enumeration.model_enum import ModelEnum
|
||||
from memory_scope.scheme.message import Message
|
||||
from memoryscope.enumeration.model_enum import ModelEnum
|
||||
from memoryscope.scheme.message import Message
|
||||
|
||||
|
||||
class ModelResponse(BaseModel):
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Dict, List
|
||||
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
|
||||
|
||||
class BaseMemoryStore(metaclass=ABCMeta):
|
||||
|
|
@ -11,7 +11,10 @@ class BaseMemoryStore(metaclass=ABCMeta):
|
|||
"""
|
||||
|
||||
@abstractmethod
|
||||
def retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]:
|
||||
def retrieve_memories(self,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]:
|
||||
"""
|
||||
Retrieves a list of MemoryNode objects that are most relevant to the query,
|
||||
considering a filter dictionary for additional constraints. The number of nodes returned
|
||||
|
|
@ -30,7 +33,10 @@ class BaseMemoryStore(metaclass=ABCMeta):
|
|||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def a_retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]:
|
||||
async def a_retrieve_memories(self,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]:
|
||||
"""
|
||||
Asynchronously retrieves a list of MemoryNode objects that best match the query,
|
||||
respecting a filter dictionary, with the result size capped at top_k.
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
from typing import Dict, List
|
||||
|
||||
from memory_scope.models.base_model import BaseModel
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.storage.base_memory_store import BaseMemoryStore
|
||||
from memoryscope.models.base_model import BaseModel
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.storage.base_memory_store import BaseMemoryStore
|
||||
|
||||
|
||||
class DummyMemoryStore(BaseMemoryStore):
|
||||
|
|
@ -12,6 +12,18 @@ class DummyMemoryStore(BaseMemoryStore):
|
|||
semantic retrieval. Actual storage operations are not implemented.
|
||||
"""
|
||||
|
||||
def retrieve_memories(self,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]:
|
||||
pass
|
||||
|
||||
async def a_retrieve_memories(self,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] = None) -> List[MemoryNode]:
|
||||
pass
|
||||
|
||||
def __init__(self, embedding_model: BaseModel, **kwargs):
|
||||
"""
|
||||
Initializes the DummyMemoryStore with an embedding model and additional keyword arguments.
|
||||
|
|
@ -23,12 +35,6 @@ class DummyMemoryStore(BaseMemoryStore):
|
|||
self.embedding_model: BaseModel = embedding_model
|
||||
self.kwargs = kwargs
|
||||
|
||||
def retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]:
|
||||
pass
|
||||
|
||||
async def a_retrieve_memories(self, query: str, top_k: int, filter_dict: Dict[str, List[str]]) -> List[MemoryNode]:
|
||||
pass
|
||||
|
||||
def batch_insert(self, nodes: List[MemoryNode]):
|
||||
pass
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from memory_scope.storage.base_monitor import BaseMonitor
|
||||
from memoryscope.storage.base_monitor import BaseMonitor
|
||||
|
||||
|
||||
class DummyMonitor(BaseMonitor):
|
||||
|
|
@ -1,15 +1,15 @@
|
|||
import warnings
|
||||
import random
|
||||
from typing import Dict, List, Any, Optional, cast
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from llama_index.core import VectorStoreIndex
|
||||
from llama_index.core.schema import TextNode, NodeWithScore, QueryBundle
|
||||
|
||||
from memory_scope.models.base_model import BaseModel
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.storage.base_memory_store import BaseMemoryStore
|
||||
from memory_scope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, _to_elasticsearch_filter
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.models.base_model import BaseModel
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.storage.base_memory_store import BaseMemoryStore
|
||||
from memoryscope.storage.llama_index_sync_elasticsearch import SyncElasticsearchStore, _AsyncDenseVectorStrategy, \
|
||||
_to_elasticsearch_filter
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
||||
|
|
@ -28,54 +28,56 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
es_url=es_url,
|
||||
retrieval_strategy=_AsyncDenseVectorStrategy(hybrid=use_hybrid),
|
||||
**kwargs)
|
||||
|
||||
# TODO The llamaIndex utilizes some deprecated functions, hence langchain logs warning messages. By
|
||||
# adding the following lines of code, the display of deprecated information is suppressed.
|
||||
|
||||
self.index = VectorStoreIndex.from_vector_store(vector_store=self.es_store,
|
||||
embed_model=self.embedding_model.model)
|
||||
|
||||
self.logger = Logger.get_logger()
|
||||
|
||||
def retrieve_memories(self,
|
||||
query: Optional[str] = None,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]:
|
||||
# if index is not created, return []
|
||||
exists = self.es_store._store.client.indices.exists(index=self.index_name)
|
||||
exists = self.es_store.client.indices.exists(index=self.index_name)
|
||||
if not exists:
|
||||
return []
|
||||
|
||||
|
||||
if filter_dict is None:
|
||||
filter_dict = {}
|
||||
|
||||
es_filter = _to_elasticsearch_filter(filter_dict)
|
||||
retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']},
|
||||
retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']},
|
||||
similarity_top_k=top_k,
|
||||
sparse_top_k=top_k, )
|
||||
if query is None:
|
||||
query = QueryBundle(query_str='**--**',
|
||||
embedding=self.dummy_query_vector())
|
||||
|
||||
sparse_top_k=top_k)
|
||||
if not query:
|
||||
query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector())
|
||||
|
||||
text_nodes = retriever.retrieve(query)
|
||||
return [self._text_node_2_memory_node(n) for n in text_nodes]
|
||||
|
||||
async def a_retrieve_memories(self,
|
||||
query: str,
|
||||
top_k: int,
|
||||
query: str = "",
|
||||
top_k: int = 3,
|
||||
filter_dict: Dict[str, List[str]] | Dict[str, str] = None) -> List[MemoryNode]:
|
||||
self.logger.info(f"query={query} top_k={top_k} filter_dict={filter_dict}")
|
||||
# if index is not created, return []
|
||||
exists = self.es_store.client.indices.exists(index=self.index_name)
|
||||
if not exists:
|
||||
return []
|
||||
|
||||
if filter_dict is None:
|
||||
filter_dict = {}
|
||||
|
||||
es_filter = _to_elasticsearch_filter(filter_dict)
|
||||
retriever = self.index.as_retriever(
|
||||
vector_store_kwargs={"es_filter": es_filter},
|
||||
similarity_top_k=top_k)
|
||||
|
||||
if query is None:
|
||||
query = QueryBundle(query_str='**--**',
|
||||
embedding=self.dummy_query_vector())
|
||||
|
||||
retriever = self.index.as_retriever(vector_store_kwargs={"es_filter": es_filter, "fields": ['embedding']},
|
||||
similarity_top_k=top_k,
|
||||
sparse_top_k=top_k)
|
||||
|
||||
if not query:
|
||||
query = QueryBundle(query_str='**--**', embedding=self.dummy_query_vector())
|
||||
|
||||
text_nodes: List[NodeWithScore] = await retriever.aretrieve(query)
|
||||
return [self._text_node_2_memory_node(n) for n in text_nodes]
|
||||
|
||||
|
|
@ -85,7 +87,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
self.insert(node)
|
||||
|
||||
def batch_update(self, nodes: List[MemoryNode], update_embedding: bool = True):
|
||||
# TODO batch_update & update_embedding
|
||||
# TODO batch_update
|
||||
for node in nodes:
|
||||
self.update(node, update_embedding=update_embedding)
|
||||
|
||||
|
|
@ -100,12 +102,10 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
def delete(self, node: MemoryNode):
|
||||
return self.es_store.delete(node.memory_id)
|
||||
|
||||
def delete_conditional(self, filter_dict: Dict = {}):
|
||||
nodes = self.retrieve_memories(filter_dict=filter_dict, top_k=10000)
|
||||
self.batch_delete(nodes)
|
||||
|
||||
def update(self, node: MemoryNode, update_embedding: bool = True):
|
||||
# TODO update without embedding?
|
||||
if update_embedding:
|
||||
node.vector = []
|
||||
|
||||
self.delete(node)
|
||||
self.insert(node)
|
||||
|
||||
|
|
@ -114,11 +114,11 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
Closes the Elasticsearch store, releasing any resources associated with it.
|
||||
"""
|
||||
self.es_store.close()
|
||||
|
||||
def dummy_query_vector(self):
|
||||
|
||||
def dummy_query_vector(self):
|
||||
random_floats = [random.uniform(0, 1) for _ in range(self.emb_dims)]
|
||||
return random_floats
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _memory_node_2_text_node(memory_node: MemoryNode) -> TextNode:
|
||||
"""
|
||||
|
|
@ -129,7 +129,7 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
|
||||
Returns:
|
||||
TextNode: The converted TextNode with content and metadata from the MemoryNode.
|
||||
"""
|
||||
"""
|
||||
embedding = memory_node.vector
|
||||
if not embedding:
|
||||
embedding = None
|
||||
|
|
@ -149,8 +149,5 @@ class LlamaIndexEsMemoryStore(BaseMemoryStore):
|
|||
Returns:
|
||||
MemoryNode: The converted MemoryNode with text and metadata from the NodeWithScore.
|
||||
"""
|
||||
embedding = text_node.embedding
|
||||
print("textnode embedding", embedding)
|
||||
if not embedding:
|
||||
embedding = []
|
||||
return MemoryNode(content=text_node.text, vector=embedding, **text_node.metadata)
|
||||
text_node.metadata["vector"] = text_node.embedding if text_node.embedding else []
|
||||
return MemoryNode(content=text_node.text, **text_node.metadata)
|
||||
|
|
@ -18,7 +18,6 @@ from llama_index.core.bridge.pydantic import PrivateAttr
|
|||
from llama_index.core.schema import BaseNode, MetadataMode, TextNode
|
||||
from llama_index.core.vector_stores.types import (
|
||||
BasePydanticVectorStore,
|
||||
MetadataFilters,
|
||||
VectorStoreQuery,
|
||||
VectorStoreQueryMode,
|
||||
VectorStoreQueryResult,
|
||||
|
|
@ -141,10 +140,10 @@ class _AsyncDenseVectorStrategy(AsyncDenseVectorStrategy):
|
|||
if query == "**--**":
|
||||
query_body = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": filter,
|
||||
}
|
||||
},
|
||||
"bool": {
|
||||
"filter": filter,
|
||||
}
|
||||
},
|
||||
}
|
||||
else:
|
||||
query_body = {
|
||||
|
|
@ -262,7 +261,6 @@ def _to_elasticsearch_filter(standard_filters: Dict[str, List[str]]) -> Dict[str
|
|||
return result
|
||||
|
||||
|
||||
|
||||
class SyncElasticsearchStore(BasePydanticVectorStore):
|
||||
"""
|
||||
Elasticsearch vector store.
|
||||
|
|
@ -676,9 +674,9 @@ class SyncElasticsearchStore(BasePydanticVectorStore):
|
|||
isinstance(self.retrieval_strategy, AsyncDenseVectorStrategy)
|
||||
and self.retrieval_strategy.hybrid
|
||||
):
|
||||
total_rank = sum(top_k_scores)
|
||||
total_rank = sum(top_k_scores)
|
||||
top_k_scores = [rank for rank in top_k_scores]
|
||||
#top_k_scores = [(total_rank - rank) / total_rank for rank in top_k_scores]
|
||||
# top_k_scores = [(total_rank - rank) / total_rank for rank in top_k_scores]
|
||||
# top_k_scores = [total_rank - rank / total_rank for rank in top_k_scores]
|
||||
|
||||
return VectorStoreQueryResult(
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import datetime
|
||||
import re
|
||||
|
||||
from memory_scope.constants.language_constants import WEEKDAYS, DATATIME_WORD_LIST, MONTH_DICT
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.constants.language_constants import WEEKDAYS, DATATIME_WORD_LIST, MONTH_DICT
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class DatetimeHandler(object):
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Dict, Any
|
||||
|
||||
from memory_scope.chat.base_memory_chat import BaseMemoryChat
|
||||
from memory_scope.enumeration.language_enum import LanguageEnum
|
||||
from memory_scope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memory_scope.models.base_model import BaseModel
|
||||
from memory_scope.storage.base_memory_store import BaseMemoryStore
|
||||
from memory_scope.storage.base_monitor import BaseMonitor
|
||||
from memoryscope.chat.base_memory_chat import BaseMemoryChat
|
||||
from memoryscope.enumeration.language_enum import LanguageEnum
|
||||
from memoryscope.memory.service.base_memory_service import BaseMemoryService
|
||||
from memoryscope.models.base_model import BaseModel
|
||||
from memoryscope.storage.base_memory_store import BaseMemoryStore
|
||||
from memoryscope.storage.base_monitor import BaseMonitor
|
||||
|
||||
|
||||
class GlobalContext(object):
|
||||
|
|
@ -12,6 +12,7 @@ class Logger(logging.Logger):
|
|||
"""
|
||||
The `Logger` class handle the stream of information or errors in activities.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
name: str,
|
||||
level: int = logging.INFO,
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
from typing import Dict, List
|
||||
|
||||
from memory_scope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memory_scope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.storage.base_memory_store import BaseMemoryStore
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.enumeration.action_status_enum import ActionStatusEnum
|
||||
from memoryscope.enumeration.store_status_enum import StoreStatusEnum
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.storage.base_memory_store import BaseMemoryStore
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class MemoryHandler(object):
|
||||
|
|
@ -78,6 +78,7 @@ class MemoryHandler(object):
|
|||
Args:
|
||||
key (str): The key mapping to memory nodes.
|
||||
nodes (List[MemoryNode]): A single memory node or a list of memory nodes to be updated.
|
||||
log_repeat: if log_repeat=True, print log info
|
||||
"""
|
||||
if nodes is None:
|
||||
nodes = []
|
||||
|
|
@ -102,7 +103,7 @@ class MemoryHandler(object):
|
|||
Fetch the memories by keys.
|
||||
|
||||
Args:
|
||||
key (str): The key mapping to memory nodes.
|
||||
keys (str | List[str]): The key mapping to memory nodes.
|
||||
|
||||
Returns:
|
||||
List[MemoryNode]: Memories mapped to the key.
|
||||
|
|
@ -157,7 +158,6 @@ class MemoryHandler(object):
|
|||
Args:
|
||||
keys (str): The memories.
|
||||
nodes (List[MemoryNode]): A single memory node or a list of memory nodes to be updated.
|
||||
:
|
||||
"""
|
||||
update_memories: Dict[str, MemoryNode] = {n.memory_id: n for n in self.get_memories(keys=keys)}
|
||||
|
||||
|
|
@ -4,7 +4,7 @@ from typing import Dict
|
|||
|
||||
import yaml
|
||||
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
|
||||
|
||||
class PromptHandler(object):
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
from typing import List
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
from memory_scope.constants.language_constants import NONE_WORD
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.constants.language_constants import NONE_WORD
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class ResponseTextParser(object):
|
||||
|
|
@ -14,23 +14,27 @@ class ResponseTextParser(object):
|
|||
|
||||
pattern_v1 = re.compile(r"<(.*?)>") # Regular expression pattern to match content within angle brackets
|
||||
|
||||
def __init__(self, response_text: str):
|
||||
def __init__(self, response_text: str, logger_prefix: str = ""):
|
||||
"""
|
||||
Initializes the `ResponseTextParser` instance with the provided response text and sets up a logger.
|
||||
|
||||
Args:
|
||||
response_text (str): The raw response text that needs to be parsed and processed.
|
||||
"""
|
||||
self.response_text: str = response_text.strip() # Strips leading and trailing whitespace from the response text
|
||||
self.logger: Logger = Logger.get_logger() # Initializes a logger instance for logging parsing activities
|
||||
|
||||
def parse_v1(self, prefix: str = "") -> List[str]:
|
||||
# Strips leading and trailing whitespace from the response text
|
||||
self.response_text: str = response_text.strip()
|
||||
|
||||
# The prefix of log. Defaults to "".
|
||||
self.logger_prefix: str = logger_prefix
|
||||
|
||||
# Initializes a logger instance for logging parsing activities
|
||||
self.logger: Logger = Logger.get_logger()
|
||||
|
||||
def parse_v1(self) -> List[List[str]]:
|
||||
"""
|
||||
Extract specific patterns from the text which match content within angle brackets.
|
||||
|
||||
Args:
|
||||
prefix (str): The prefix of log. Defaults to "".
|
||||
|
||||
Returns:
|
||||
Contents match the specific patterns.
|
||||
"""
|
||||
|
|
@ -42,10 +46,10 @@ class ResponseTextParser(object):
|
|||
matches = [match.group(1) for match in self.pattern_v1.finditer(line)]
|
||||
if matches:
|
||||
result.append(matches)
|
||||
self.logger.info(f"{prefix} response_text={self.response_text} result={result}", stacklevel=2)
|
||||
self.logger.info(f"{self.logger_prefix} response_text={self.response_text} result={result}", stacklevel=2)
|
||||
return result
|
||||
|
||||
def parse_v2(self, prefix: str = "") -> List[str]:
|
||||
def parse_v2(self) -> List[str]:
|
||||
"""
|
||||
Extract lines which contain NONE_WORD in Chinese or English.
|
||||
|
||||
|
|
@ -61,5 +65,5 @@ class ResponseTextParser(object):
|
|||
if not line or line.lower() == NONE_WORD.get(G_CONTEXT.language):
|
||||
continue
|
||||
result.append(line)
|
||||
self.logger.info(f"{prefix} response_text={self.response_text} result={result}", stacklevel=2)
|
||||
self.logger.info(f"{self.logger_prefix} response_text={self.response_text} result={result}", stacklevel=2)
|
||||
return result
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import time
|
||||
from typing import Literal
|
||||
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
TIME_LOG_TYPE = Literal["end", "wrap", "none"]
|
||||
|
||||
|
|
@ -9,8 +9,8 @@ from typing import List
|
|||
import pyfiglet
|
||||
from termcolor import colored
|
||||
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.scheme.message import Message
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.scheme.message import Message
|
||||
|
||||
ALL_COLORS = ["red", "green", "yellow", "blue", "magenta", "cyan", "light_grey", "light_red", "light_green",
|
||||
"light_yellow", "light_blue", "light_magenta", "light_cyan", "white"]
|
||||
|
|
@ -48,7 +48,7 @@ def camelcase_to_underscore(name: str) -> str:
|
|||
|
||||
|
||||
def init_instance_by_config(config: dict,
|
||||
default_class_path: str = "memory_scope",
|
||||
default_class_path: str = "memoryscope",
|
||||
suffix_name: str = "",
|
||||
**kwargs):
|
||||
"""
|
||||
|
|
@ -5,8 +5,8 @@ sys.path.append(".") # noqa: E402
|
|||
import asyncio
|
||||
import unittest
|
||||
|
||||
from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class TestLLIEmbedding(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ sys.path.append(".") # noqa: E402
|
|||
import unittest
|
||||
import time
|
||||
import asyncio
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.models.llama_index_generation_model import LlamaIndexGenerationModel
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.models.llama_index_generation_model import LlamaIndexGenerationModel
|
||||
from memoryscope.utils.logger import Logger
|
||||
|
||||
|
||||
class TestLLILLM(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import unittest
|
||||
|
||||
from memory_scope.models.llama_index_rank_model import LlamaIndexRankModel
|
||||
from memoryscope.models.llama_index_rank_model import LlamaIndexRankModel
|
||||
|
||||
|
||||
class TestLLIReRank(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from memory_scope.cli import MemoryScope
|
||||
from memory_scope.scheme.message import Message
|
||||
from memoryscope.cli import MemoryScope
|
||||
from memoryscope.scheme.message import Message
|
||||
|
||||
ms = MemoryScope().load_config("config/demo_config_no_stream.yaml")
|
||||
memory_service = ms.default_service
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import unittest
|
||||
|
||||
from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore
|
||||
from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore
|
||||
|
||||
|
||||
class TestLlamaIndexElasticSearchStore(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import unittest
|
||||
|
||||
from memory_scope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore
|
||||
from memoryscope.models.llama_index_embedding_model import LlamaIndexEmbeddingModel
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.storage.llama_index_es_memory_store import LlamaIndexEsMemoryStore
|
||||
|
||||
|
||||
class TestLlamaIndexElasticSearchStore(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
import datetime
|
||||
import unittest
|
||||
|
||||
from memory_scope.cli import MemoryScope
|
||||
from memory_scope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \
|
||||
from memoryscope.cli import MemoryScope
|
||||
from memoryscope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \
|
||||
MERGE_OBS_NODES, QUERY_WITH_TS, EXTRACT_TIME_DICT, NOT_REFLECTED_NODES, INSIGHT_NODES, NOT_UPDATED_NODES
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.tool_functions import init_instance_by_config
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.tool_functions import init_instance_by_config
|
||||
|
||||
|
||||
class TestWorkersCn(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
import datetime
|
||||
import unittest
|
||||
|
||||
from memory_scope.cli import MemoryScope
|
||||
from memory_scope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \
|
||||
from memoryscope.cli import MemoryScope
|
||||
from memoryscope.constants.common_constants import CHAT_MESSAGES, NEW_OBS_NODES, NEW_OBS_WITH_TIME_NODES, \
|
||||
MERGE_OBS_NODES, QUERY_WITH_TS, EXTRACT_TIME_DICT, NOT_REFLECTED_NODES, INSIGHT_NODES, NOT_UPDATED_NODES
|
||||
from memory_scope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memory_scope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memory_scope.scheme.memory_node import MemoryNode
|
||||
from memory_scope.scheme.message import Message
|
||||
from memory_scope.utils.global_context import G_CONTEXT
|
||||
from memory_scope.utils.logger import Logger
|
||||
from memory_scope.utils.tool_functions import init_instance_by_config
|
||||
from memoryscope.enumeration.message_role_enum import MessageRoleEnum
|
||||
from memoryscope.memory.worker.memory_base_worker import MemoryBaseWorker
|
||||
from memoryscope.scheme.memory_node import MemoryNode
|
||||
from memoryscope.scheme.message import Message
|
||||
from memoryscope.utils.global_context import G_CONTEXT
|
||||
from memoryscope.utils.logger import Logger
|
||||
from memoryscope.utils.tool_functions import init_instance_by_config
|
||||
|
||||
|
||||
class TestWorkersEn(unittest.TestCase):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue