mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
feat(reme_ai): implement memory retrieval and merging functionality
- Add BuildQueryOp to construct query for memory retrieval-Implement MergeMemoryOp to combine retrieved memories - Create RecallVectorStoreOp to fetch memories from vector store - Develop memory representation and conversion methods - Establish initial project structure and dependencies
This commit is contained in:
parent
a3f281f365
commit
4739d39f3f
24 changed files with 500 additions and 211 deletions
10
.gitignore
vendored
10
.gitignore
vendored
|
|
@ -29,12 +29,4 @@ cookbook/appworld/experiments/*
|
|||
cookbook/appworld/exp_result/*
|
||||
file_vector_store/*
|
||||
cookbook/appworld/file_vector_store/*
|
||||
experiencemaker/tool/web_search_cach/*
|
||||
experiencemaker/cookbook/bfcl/exp_result
|
||||
experiencemaker/cookbook/bfcl/old_exp_result
|
||||
experiencemaker/cookbook/bfcl/data
|
||||
experiencemaker/cookbook/bfcl/gorilla
|
||||
experiencemaker/*.sh
|
||||
experiencemaker/file_vector_store
|
||||
experiencemaker/*.egg-info
|
||||
experiencemaker/experiment_library
|
||||
experiencemaker/tool/web_search_cach/*
|
||||
|
|
@ -186,7 +186,7 @@
|
|||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2024 Alibaba Group
|
||||
Copyright 2025 Alibaba Group
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
0
README.md
Normal file
0
README.md
Normal file
10
doc/ROADMAP.md
Normal file
10
doc/ROADMAP.md
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# 代码框架
|
||||
1. flowllm: 通过op pipeline的配置实现mcp接口的生成。
|
||||
2. 重写Remy readme.
|
||||
3. 迁移memoryscope/experiencemaker到flowllm的框架下
|
||||
4. 迁移到新的op框架下
|
||||
5. 重写memoryscope的cli-chat前端
|
||||
6. 迁移memoryscope的文档
|
||||
7. 迁移experiencemaker的文档
|
||||
|
||||
#
|
||||
11
example.env
Normal file
11
example.env
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
OPENAI_API_KEY=sk-xxxx
|
||||
OPENAI_BASE_URL=https://xxxx/v1
|
||||
|
||||
EMBEDDING_API_KEY=sk-xxxx
|
||||
EMBEDDING_BASE_URL=https://xxxx/v1
|
||||
|
||||
LLM_API_KEY=sk-xxxx
|
||||
LLM_BASE_URL=https://xxxx/v1
|
||||
|
||||
ES_HOSTS=http://0.0.0.0:9200
|
||||
DASHSCOPE_API_KEY=sk-xxxx
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2024 Alibaba Group
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
48
pyproject.toml
Normal file
48
pyproject.toml
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
[build-system]
|
||||
requires = ["setuptools", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "Reme-AI"
|
||||
version = "0.1.0"
|
||||
description = "Manage Experience, Navigate Tasks & Optimize Reuse"
|
||||
authors = [{ name = "reme-ai team", email = "reme-ai@alibaba-inc.com" }]
|
||||
license = { file = "LICENSE" }
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
classifiers = [
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: OS Independent",
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"dashscope>=1.19.1",
|
||||
"elasticsearch>=8.14.0",
|
||||
"fastapi>=0.115.13",
|
||||
"fastmcp>=2.10.6",
|
||||
"loguru>=0.7.3",
|
||||
"mcp>=1.9.4",
|
||||
"numpy>=2.3.0",
|
||||
"openai>=1.88.0",
|
||||
"pydantic>=2.11.7",
|
||||
"PyYAML>=6.0.2",
|
||||
"Requests>=2.32.4",
|
||||
"uvicorn>=0.34.3",
|
||||
"setuptools>=75.0",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["experiencemaker*"]
|
||||
exclude = ["memoryscope*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
experiencemaker = [
|
||||
"config/*.yaml",
|
||||
"op/**/*.yaml",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
reme = "experiencemaker.app:main"
|
||||
1
reme_ai/__init__.py
Normal file
1
reme_ai/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
__version__ = "0.1.0"
|
||||
15
reme_ai/app.py
Normal file
15
reme_ai/app.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import sys
|
||||
|
||||
from flowllm.service.base_service import BaseService
|
||||
|
||||
|
||||
def main():
|
||||
with BaseService.get_service(*sys.argv[1:]) as service:
|
||||
service()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
# python -m build
|
||||
# twine upload dist/*
|
||||
0
reme_ai/config/__init__.py
Normal file
0
reme_ai/config/__init__.py
Normal file
6
reme_ai/config/config_parser.py
Normal file
6
reme_ai/config/config_parser.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
from flowllm.config.pydantic_config_parser import PydanticConfigParser
|
||||
|
||||
|
||||
class ConfigParser(PydanticConfigParser):
|
||||
current_file: str = __file__
|
||||
default_config_name: str = "default"
|
||||
67
reme_ai/config/default.yaml
Normal file
67
reme_ai/config/default.yaml
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# default config.yaml
|
||||
backend: mcp
|
||||
language: ""
|
||||
thread_pool_max_workers: 32
|
||||
ray_max_workers: 1
|
||||
|
||||
mcp:
|
||||
transport: sse
|
||||
host: "0.0.0.0"
|
||||
port: 8002
|
||||
|
||||
http:
|
||||
host: "0.0.0.0"
|
||||
port: 8002
|
||||
timeout_keep_alive: 600
|
||||
limit_concurrency: 64
|
||||
|
||||
flow:
|
||||
task_retrieve:
|
||||
flow_content: build_query_op->recall_vector_store_op->merge_experience_op
|
||||
# task_summarizer: simple_summary_op->update_vector_store_op
|
||||
# vector_store: vector_store_action_op
|
||||
# agent: react_op
|
||||
|
||||
mock_expression_flow:
|
||||
flow_content: mock1_op>>((mock4_op>>mock2_op)|mock5_op)>>(mock3_op|mock6_op)
|
||||
description: "mock flow"
|
||||
input_schema:
|
||||
a:
|
||||
type: "str"
|
||||
description: "mock attr a"
|
||||
required: true
|
||||
b:
|
||||
type: "str"
|
||||
description: "mock attr b"
|
||||
required: true
|
||||
|
||||
op:
|
||||
mock1_op:
|
||||
backend: mock1_op
|
||||
llm: default
|
||||
vector_store: default
|
||||
|
||||
llm:
|
||||
default:
|
||||
backend: openai_compatible
|
||||
model_name: qwen3-30b-a3b-thinking-2507
|
||||
params:
|
||||
temperature: 0.6
|
||||
|
||||
qwen3_30b_instruct:
|
||||
backend: openai_compatible
|
||||
model_name: qwen3-30b-a3b-instruct-2507
|
||||
|
||||
embedding_model:
|
||||
default:
|
||||
backend: openai_compatible
|
||||
model_name: text-embedding-v4
|
||||
params:
|
||||
dimensions: 1024
|
||||
|
||||
vector_store:
|
||||
default:
|
||||
backend: elasticsearch
|
||||
embedding_model: default
|
||||
params:
|
||||
hosts: "http://localhost:9200"
|
||||
1
reme_ai/retrieve/__init__.py
Normal file
1
reme_ai/retrieve/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from reme_ai.retrieve import task
|
||||
2
reme_ai/retrieve/task/__init__.py
Normal file
2
reme_ai/retrieve/task/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
from reme_ai.retrieve.task.build_query_op import BuildQueryOp
|
||||
from reme_ai.retrieve.task.merge_memory_op import MergeMemoryOp
|
||||
34
reme_ai/retrieve/task/build_query_op.py
Normal file
34
reme_ai/retrieve/task/build_query_op.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from flowllm import C, BaseLLMOp
|
||||
from flowllm.utils.llm_utils import merge_messages_content
|
||||
from loguru import logger
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class BuildQueryOp(BaseLLMOp):
|
||||
current_path: str = __file__
|
||||
|
||||
def execute(self):
|
||||
if "query" in self.context:
|
||||
query = self.context.query
|
||||
|
||||
elif "messages" in self.context:
|
||||
if self.op_params.get("enable_llm_build", True):
|
||||
execution_process = merge_messages_content(self.context.messages)
|
||||
query = self.prompt_format(prompt_name="query_build", execution_process=execution_process)
|
||||
|
||||
else:
|
||||
context_parts = []
|
||||
message_summaries = []
|
||||
for message in self.context.messages[-3:]: # Last 3 messages
|
||||
content = message.content[:200] + "..." if len(message.content) > 200 else message.content
|
||||
message_summaries.append(f"- {message.role.value}: {content}")
|
||||
if message_summaries:
|
||||
context_parts.append("Recent messages:\n" + "\n".join(message_summaries))
|
||||
|
||||
query = "\n\n".join(context_parts)
|
||||
|
||||
else:
|
||||
raise RuntimeError("query or messages is required!")
|
||||
|
||||
logger.info(f"build.query={query}")
|
||||
self.context.query = query
|
||||
6
reme_ai/retrieve/task/build_query_prompt.yaml
Normal file
6
reme_ai/retrieve/task/build_query_prompt.yaml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
query_build: |
|
||||
# Execution Process
|
||||
{execution_process}
|
||||
|
||||
Read through the entire execution process to understand which part is currently being executed.
|
||||
Generate a `query` that reflects the current state, which will later be used to search for similar problems in the database and help resolve the issue at hand.
|
||||
28
reme_ai/retrieve/task/merge_memory_op.py
Normal file
28
reme_ai/retrieve/task/merge_memory_op.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
from typing import List
|
||||
|
||||
from flowllm import C, BaseOp
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class MergeMemoryOp(BaseOp):
|
||||
|
||||
def execute(self):
|
||||
memory_list: List[BaseMemory] = self.context.response.metadata["memory_list"]
|
||||
|
||||
if not memory_list:
|
||||
return
|
||||
|
||||
content_collector = ["Previous Memory"]
|
||||
for memory in memory_list:
|
||||
if not memory.content:
|
||||
continue
|
||||
|
||||
content_collector.append(f"- when_to_use: {memory.when_to_use}\n"
|
||||
f"content: {memory.content}\n")
|
||||
content_collector.append("Please consider the helpful parts from these in answering the question, "
|
||||
"to make the response more comprehensive and substantial.")
|
||||
self.context.response.answer = "\n".join(content_collector)
|
||||
logger.info(f"response.answer={self.context.response.answer}")
|
||||
0
reme_ai/schema/__init__.py
Normal file
0
reme_ai/schema/__init__.py
Normal file
141
reme_ai/schema/memory.py
Normal file
141
reme_ai/schema/memory.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import datetime
|
||||
from abc import ABC
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class BaseMemory(BaseModel, ABC):
|
||||
workspace_id: str = Field(default="")
|
||||
memory_id: str = Field(default_factory=lambda: uuid4().hex)
|
||||
memory_type: str = Field(default=...)
|
||||
|
||||
when_to_use: str = Field(default="")
|
||||
content: str | bytes = Field(default="")
|
||||
score: float | None = Field(default=None)
|
||||
|
||||
created_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
modified_time: str = Field(default_factory=lambda: datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
author: str = Field(default="")
|
||||
|
||||
metadata: dict = Field(default_factory=dict)
|
||||
|
||||
def update_modified_time(self):
|
||||
self.modified_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def to_vector_node(self) -> VectorNode:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def from_vector_node(cls, node: VectorNode):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TaskMemory(BaseMemory):
|
||||
memory_type: str = Field(default="task")
|
||||
|
||||
def to_vector_node(self) -> VectorNode:
|
||||
return VectorNode(unique_id=self.memory_id,
|
||||
workspace_id=self.workspace_id,
|
||||
content=self.when_to_use,
|
||||
metadata={
|
||||
"memory_type": self.memory_type,
|
||||
"content": self.content,
|
||||
"score": self.score,
|
||||
"created_time": self.created_time,
|
||||
"modified_time": self.modified_time,
|
||||
"author": self.author,
|
||||
"metadata": self.metadata,
|
||||
})
|
||||
|
||||
@classmethod
|
||||
def from_vector_node(cls, node: VectorNode) -> "TaskMemory":
|
||||
return cls(workspace_id=node.workspace_id,
|
||||
memory_id=node.unique_id,
|
||||
memory_type=node.metadata.get("memory_type"),
|
||||
when_to_use=node.content,
|
||||
content=node.metadata.get("content"),
|
||||
score=node.metadata.get("score"),
|
||||
created_time=node.metadata.get("created_time"),
|
||||
modified_time=node.metadata.get("modified_time"),
|
||||
author=node.metadata.get("author"),
|
||||
metadata=node.metadata.get("metadata"))
|
||||
|
||||
|
||||
class FunctionArg(BaseModel):
|
||||
arg_name: str = Field(default=...)
|
||||
arg_type: str = Field(default=...)
|
||||
required: bool = Field(default=True)
|
||||
|
||||
|
||||
class Function(BaseModel):
|
||||
func_code: str = Field(default=..., description="function code")
|
||||
func_name: str = Field(default=..., description="function name")
|
||||
func_args: List[FunctionArg] = Field(default_factory=list)
|
||||
|
||||
|
||||
class FuncMemory(BaseMemory):
|
||||
memory_type: str = Field(default="function")
|
||||
functions: List[Function] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PersonalMemory(BaseMemory):
|
||||
memory_type: str = Field(default="personal")
|
||||
target: str = Field(default="")
|
||||
|
||||
|
||||
class PersonalTopicMemory(PersonalMemory):
|
||||
memory_type: str = Field(default="personal_topic")
|
||||
|
||||
|
||||
def vector_node_to_memory(node: VectorNode) -> BaseMemory:
|
||||
memory_type = node.metadata.get("memory_type")
|
||||
if memory_type == "task":
|
||||
return TaskMemory.from_vector_node(node)
|
||||
|
||||
elif memory_type == "function":
|
||||
return FuncMemory.from_vector_node(node)
|
||||
|
||||
elif memory_type == "personal":
|
||||
return PersonalMemory.from_vector_node(node)
|
||||
|
||||
elif memory_type == "personal_topic":
|
||||
return PersonalTopicMemory.from_vector_node(node)
|
||||
|
||||
else:
|
||||
raise RuntimeError(f"memory_type={memory_type} not supported!")
|
||||
|
||||
|
||||
def dict_to_experience(memory_dict: dict):
|
||||
memory_type = memory_dict.get("memory_type", "task")
|
||||
if memory_type == "task":
|
||||
return TaskMemory(**memory_dict)
|
||||
|
||||
elif memory_type == "function":
|
||||
return FuncMemory(**memory_dict)
|
||||
|
||||
elif memory_type == "personal":
|
||||
return PersonalMemory(**memory_dict)
|
||||
|
||||
elif memory_type == "personal_topic":
|
||||
return PersonalTopicMemory(**memory_dict)
|
||||
|
||||
else:
|
||||
raise RuntimeError(f"memory_type={memory_type} not supported!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
e1 = TaskMemory(
|
||||
workspace_id="w_1024",
|
||||
memory_id="123",
|
||||
when_to_use="test case use",
|
||||
content="test content",
|
||||
score=0.99,
|
||||
metadata={})
|
||||
print(e1.model_dump_json(indent=2))
|
||||
v1 = e1.to_vector_node()
|
||||
print(v1.model_dump_json(indent=2))
|
||||
e2 = vector_node_to_memory(v1)
|
||||
print(e2.model_dump_json(indent=2))
|
||||
0
reme_ai/summary/__init__.py
Normal file
0
reme_ai/summary/__init__.py
Normal file
0
reme_ai/utils/__init__.py
Normal file
0
reme_ai/utils/__init__.py
Normal file
79
reme_ai/utils/op_utils.py
Normal file
79
reme_ai/utils/op_utils.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import json
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
from flowllm.schema.message import Message, Trajectory
|
||||
from loguru import logger
|
||||
|
||||
|
||||
def parse_json_experience_response(response: str) -> List[dict]:
|
||||
"""Parse JSON formatted experience response"""
|
||||
try:
|
||||
# Extract JSON blocks
|
||||
json_pattern = r'```json\s*([\s\S]*?)\s*```'
|
||||
json_blocks = re.findall(json_pattern, response)
|
||||
|
||||
if json_blocks:
|
||||
parsed = json.loads(json_blocks[0])
|
||||
|
||||
# Handle array format
|
||||
if isinstance(parsed, list):
|
||||
experiences = []
|
||||
for exp_data in parsed:
|
||||
if isinstance(exp_data, dict) and (
|
||||
("when_to_use" in exp_data and "experience" in exp_data) or
|
||||
("condition" in exp_data and "experience" in exp_data)
|
||||
):
|
||||
experiences.append(exp_data)
|
||||
|
||||
return experiences
|
||||
|
||||
|
||||
# Handle single object
|
||||
elif isinstance(parsed, dict) and (
|
||||
("when_to_use" in parsed and "experience" in parsed) or
|
||||
("condition" in parsed and "experience" in parsed)
|
||||
):
|
||||
return [parsed]
|
||||
|
||||
# Fallback: try to parse entire response
|
||||
parsed = json.loads(response)
|
||||
if isinstance(parsed, list):
|
||||
return parsed
|
||||
elif isinstance(parsed, dict):
|
||||
return [parsed]
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse JSON experience response: {e}")
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def get_trajectory_context(trajectory: Trajectory, step_sequence: List[Message]) -> str:
|
||||
"""Get context of step sequence within trajectory"""
|
||||
try:
|
||||
# Find position of step sequence in trajectory
|
||||
start_idx = 0
|
||||
for i, step in enumerate(trajectory.messages):
|
||||
if step == step_sequence[0]:
|
||||
start_idx = i
|
||||
break
|
||||
|
||||
# Extract before and after context
|
||||
context_before = trajectory.messages[max(0, start_idx - 2):start_idx]
|
||||
context_after = trajectory.messages[start_idx + len(step_sequence):start_idx + len(step_sequence) + 2]
|
||||
|
||||
context = f"Query: {trajectory.metadata.get('query', 'N/A')}\n"
|
||||
|
||||
if context_before:
|
||||
context += "Previous steps:\n" + "\n".join(
|
||||
[f"- {step.content[:100]}..." for step in context_before]) + "\n"
|
||||
|
||||
if context_after:
|
||||
context += "Following steps:\n" + "\n".join([f"- {step.content[:100]}..." for step in context_after])
|
||||
|
||||
return context
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting trajectory context: {e}")
|
||||
return f"Query: {trajectory.metadata.get('query', 'N/A')}"
|
||||
13
reme_ai/vector_store/__init__.py
Normal file
13
reme_ai/vector_store/__init__.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
"""
|
||||
1. retrieve:
|
||||
search: query(context), workspace_id(request), top_k(request)
|
||||
2. summary:
|
||||
insert: nodes(context), workspace_id(request)
|
||||
delete: ids(context), workspace_id(request)
|
||||
search: query(context), workspace_id(request), top_k(request.config.op)
|
||||
3. vector:
|
||||
dump: workspace_id(request), path(str), max_size(int)
|
||||
load: workspace_id(request), path(str)
|
||||
delete: workspace_id(request)
|
||||
copy: source_id, target_id, max_size(int)
|
||||
"""
|
||||
36
reme_ai/vector_store/recall_vector_store_op.py
Normal file
36
reme_ai/vector_store/recall_vector_store_op.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
from typing import List
|
||||
|
||||
from flowllm import C, BaseLLMOp
|
||||
from flowllm.schema.vector_node import VectorNode
|
||||
from loguru import logger
|
||||
|
||||
from reme_ai.schema.memory import BaseMemory, vector_node_to_memory
|
||||
|
||||
|
||||
@C.register_op()
|
||||
class RecallVectorStoreOp(BaseLLMOp):
|
||||
|
||||
def execute(self):
|
||||
recall_key: str = self.op_params.get("recall_key", "query")
|
||||
query: str = self.context[recall_key]
|
||||
assert query, "query should be not empty!"
|
||||
|
||||
top_k: int = self.context.top_k
|
||||
workspace_id: str = self.context.workspace_id
|
||||
nodes: List[VectorNode] = self.vector_store.search(query=query, workspace_id=workspace_id, top_k=top_k)
|
||||
|
||||
memory_list: List[BaseMemory] = []
|
||||
memory_content_list: List[str] = []
|
||||
for node in nodes:
|
||||
memory: BaseMemory = vector_node_to_memory(node)
|
||||
if memory.content not in memory_content_list:
|
||||
memory_list.append(memory)
|
||||
memory_content_list.append(memory.content)
|
||||
logger.info(f"retrieve memory.size={len(memory_list)}")
|
||||
|
||||
threshold_score: float | None = self.op_params.get("threshold_score", None)
|
||||
if threshold_score is not None:
|
||||
memory_list = [mem for mem in memory_list if mem.score >= threshold_score or mem.score is None]
|
||||
logger.info(f"after filter by threshold_score size={len(memory_list)}")
|
||||
|
||||
self.context.response.metadata["memory_list"] = memory_list
|
||||
Loading…
Add table
Reference in a new issue