scriptoriumadmin/ReMe
1
0
Fork 0
mirror of https://github.com/agentscope-ai/ReMe.git synced 2026-09-07 08:26:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

bugfix

Browse source
This commit is contained in:
jinli.yl 2025-07-11 19:44:07 +08:00
parent 12f93937fe
commit 7832f20d41
13 changed files with 78 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
experiencemaker/config/config_parser.py
View file

@ -29,7 +29,7 @@ class ConfigParser:
self.app_config = OmegaConf.merge(self.app_config, cli_config)
app_config_dict = OmegaConf.to_container(self.app_config, resolve=True)
logger.info(f"app_config_str={json.dumps(app_config_dict, indent=2, ensure_ascii=False)}")
logger.info(f"app_config=\n{json.dumps(app_config_dict, indent=2, ensure_ascii=False)}")
def get_app_config(self, **kwargs) -> AppConfig:
app_config = self.app_config.copy()

45
experiencemaker/config/demo_config.yaml
View file

@ -7,33 +7,35 @@ http_service:
limit_concurrency: 64
thread_pool:
max_workers: 10
max_workers: 64
api:
retriever: mock1_op->[mock4_op->mock2_op|mock5_op]->[mock3_op|mock6_op]
summarizer: mock1_op->[mock4_op->mock2_op|mock5_op]->mock3_op
vector_store: mock6_op
retriever: build_query_op->recall_vector_store_op->merge_experience_op
summarizer: simple_summary_op->update_vector_store_op
vector_store: vector_store_action_op
op:
mock1_op:
backend: mock1_op
build_query_op:
backend: build_query_op
llm: default
params:
enable_llm_build: false
recall_vector_store_op:
backend: recall_vector_store_op
vector_store: default
merge_experience_op:
backend: merge_experience_op
simple_summary_op:
backend: simple_summary_op
llm: default
params:
a: 1
b: 2
mock2_op:
backend: mock2_op
params:
a: 1
mock3_op:
backend: mock3_op
mock4_op:
backend: mock4_op
mock5_op:
backend: mock5_op
mock6_op:
backend: mock6_op
success_score_threshold: 0.9
update_vector_store_op:
backend: update_vector_store_op
vector_store: default
vector_store_action_op:
backend: vector_store_action_op
vector_store: default
llm:
default:
@ -53,6 +55,3 @@ vector_store:
default:
backend: elasticsearch
embedding_model: default
params:
hosts: "http://localhost:9200"

6
experiencemaker/op/__init__.py
View file

@ -3,3 +3,9 @@ from experiencemaker.utils.registry import Registry
OP_REGISTRY = Registry()
from experiencemaker.op.mock_op import Mock1Op, Mock2Op, Mock3Op, Mock4Op, Mock5Op, Mock6Op
from experiencemaker.op.retriever.build_query_op import BuildQueryOp
from experiencemaker.op.retriever.merge_experience_op import MergeExperienceOp
from experiencemaker.op.summarizer.simple_summary_op import SimpleSummaryOp
from experiencemaker.op.vector_store.update_vector_store_op import UpdateVectorStoreOp
from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp
from experiencemaker.op.vector_store.vector_store_action_op import VectorStoreActionOp

7
experiencemaker/op/base_op.py
View file

@ -19,6 +19,7 @@ from experiencemaker.vector_store.base_vector_store import BaseVectorStore
class BaseOp(PromptMixin, ABC):
current_path: str = __file__
def __init__(self, context: PipelineContext, op_config: OpConfig):
super().__init__()
@ -39,7 +40,7 @@ class BaseOp(PromptMixin, ABC):
prompt_file_path = self.op_config.prompt_file_path
else:
prompt_name = self.simple_name.replace("_op", "_prompt.yaml")
prompt_file_path = Path(__file__).parent / prompt_name
prompt_file_path = Path(self.current_path).parent / prompt_name
# Load custom prompts from prompt file
self.load_prompt_by_file(prompt_file_path=prompt_file_path)
@ -116,8 +117,4 @@ class BaseOp(PromptMixin, ABC):
f"vector_store={vector_store_name} not found in vector_store_dict!"
self._vector_store = self.context.vector_store_dict[vector_store_name]
if self._vector_store.embedding_model is None:
logger.info(f"set embedding_model for vector_store={self.op_config.vector_store}")
self._vector_store.embedding_model = self.embedding_model
return self._vector_store

5
experiencemaker/op/retriever/build_query_op.py
View file

@ -8,7 +8,7 @@ from experiencemaker.utils.op_utils import merge_messages_content
@OP_REGISTRY.register()
class BuildQueryOp(BaseOp):
RETRIEVE_QUERY = "retrieve_query"
current_path: str = __file__
def execute(self):
request: RetrieverRequest = self.context.request
@ -28,4 +28,5 @@ class BuildQueryOp(BaseOp):
logger.info(f"build.query={query}")
self.context.set_context(self.RETRIEVE_QUERY, query)
from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp
self.context.set_context(RecallVectorStoreOp.SEARCH_QUERY, query)

5
experiencemaker/op/retriever/merge_experience_op.py
View file

@ -1,5 +1,7 @@
from typing import List
from loguru import logger
from experiencemaker.op import OP_REGISTRY
from experiencemaker.op.base_op import BaseOp
from experiencemaker.schema.experience import BaseExperience
@ -22,7 +24,8 @@ class MergeExperienceOp(BaseOp):
continue
content_collector.append(f"- when_to_use: {experience.when_to_use}\n"
f"content: {experience.experience_content}\n")
f"content: {experience.content}\n")
content_collector.append("Please consider the helpful parts from these in answering the question, "
"to make the response more comprehensive and substantial.")
response.experience_merged = "\n".join(content_collector)
logger.info(f"experience_merged={response.experience_merged}")

22
experiencemaker/op/summarizer/simple_summary_op.py
View file

@ -14,10 +14,14 @@ from experiencemaker.utils.op_utils import merge_messages_content
@OP_REGISTRY.register()
class SimpleSummaryOp(BaseOp):
current_path: str = __file__
def summary_trajectory(self, trajectory: Trajectory) -> List[BaseExperience]:
execution_process = merge_messages_content(trajectory.messages)
execution_result = "success" if trajectory.score > 0.9 else "fail"
success_score_threshold: float = self.op_params.get("success_score_threshold", 0.9)
logger.info(f"success_score_threshold={success_score_threshold}")
execution_result = "success" if trajectory.score > success_score_threshold else "fail"
summary_prompt = self.prompt_format(prompt_name="summary_prompt",
execution_process=execution_process,
execution_result=execution_result,
@ -29,7 +33,7 @@ class SimpleSummaryOp(BaseOp):
try:
content = content.split("```")[1].strip()
if content.startswith("json"):
content.strip("json")
content = content.strip("json")
for exp_dict in json.loads(content):
when_to_use = exp_dict.get("when_to_use", "").strip()
@ -51,13 +55,7 @@ class SimpleSummaryOp(BaseOp):
def execute(self):
request: SummarizerRequest = self.context.request
for trajectory in request.traj_list:
execution_process = merge_messages_content(trajectory.messages)
execution_result = "success" if trajectory.score > 0.9 else "fail"
summary_prompt = self.prompt_format(prompt_name="summary_prompt",
execution_process=execution_process,
execution_result=execution_result,
summary_example=self.get_prompt("summary_example"))
self.submit_task(self.llm.chat, messages=[Message(content=summary_prompt)])
self.submit_task(self.summary_trajectory, trajectory=trajectory)
experience_list: List[BaseExperience] = []
for task_result in self.join_task():
@ -67,7 +65,7 @@ class SimpleSummaryOp(BaseOp):
response: SummarizerResponse = self.context.response
response.experience_list = experience_list
for e in experience_list:
logger.info(f"add experience {e.when_to_use}\n{e.content}")
logger.info(f"add experience when_to_use={e.when_to_use}\ncontent={e.content}")
from experiencemaker.op.summarizer.insert_database_op import InsertDatabaseOp
self.context.set_context(InsertDatabaseOp.INSERT_NODES, [x.to_vector_node() for x in experience_list])
from experiencemaker.op.vector_store.update_vector_store_op import UpdateVectorStoreOp
self.context.set_context(UpdateVectorStoreOp.INSERT_NODES, [x.to_vector_node() for x in experience_list])

8
experiencemaker/op/vector_store/recall_vector_store_op.py
View file

@ -1,5 +1,7 @@
from typing import List
from loguru import logger
from experiencemaker.op import OP_REGISTRY
from experiencemaker.op.base_op import BaseOp
from experiencemaker.schema.experience import BaseExperience, vector_node_to_experience
@ -10,10 +12,11 @@ from experiencemaker.schema.vector_node import VectorNode
@OP_REGISTRY.register()
class RecallVectorStoreOp(BaseOp):
SEARCH_QUERY = "search_query"
def execute(self):
# get query
query = self.context.get_context("search_query")
query = self.context.get_context(self.SEARCH_QUERY)
assert query, "query should be not empty!"
# retrieve from vector store
@ -30,11 +33,14 @@ class RecallVectorStoreOp(BaseOp):
if experience.content not in experience_content_list:
experience_list.append(experience)
experience_content_list.append(experience.content)
experience_size = len(experience_list)
logger.info(f"retrieve experience size={experience_size}")
# filter by score
threshold_score: float | None = self.op_params.get("threshold_score", None)
if threshold_score is not None:
experience_list = [e for e in experience_list if e.score >= threshold_score or e.score is None]
logger.info(f"after filter by threshold_score size={len(experience_list)}")
# set response
request: RetrieverResponse = self.context.response

3
experiencemaker/op/vector_store/update_vector_store.py → experiencemaker/op/vector_store/update_vector_store_op.py
View file

@ -25,5 +25,4 @@ class UpdateVectorStoreOp(BaseOp):
insert_nodes: List[VectorNode] | None = self.context.get_context(self.INSERT_NODES)
if insert_nodes:
self.vector_store.insert(nodes=insert_nodes, workspace_id=request.workspace_id)
for node in insert_nodes:
logger.info(f"insert insert_node={node.model_dump_json(indent=2)}")
logger.info(f"insert insert_node.size={len(insert_nodes)}")

2
experiencemaker/pipeline/pipeline.py
View file

@ -68,7 +68,7 @@ class Pipeline:
elif isinstance(pipeline, list):
parallel_pipeline = [self._parse_sub_pipeline(x) for x in pipeline]
for op_list in zip_longest(*parallel_pipeline, fillvalue="-"):
for op_list in zip_longest(parallel_pipeline, fillvalue="-"):
i += 1
logger.info(f"stage{i}: {' | '.join(op_list)}")

2
experiencemaker/pipeline/pipeline_context.py
View file

@ -2,8 +2,6 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Dict
from experiencemaker.schema.app_config import AppConfig
from experiencemaker.schema.request import BaseRequest
from experiencemaker.schema.response import BaseResponse
from experiencemaker.vector_store.base_vector_store import BaseVectorStore

8
experiencemaker/service/experience_maker_client.py
View file

@ -1,5 +1,6 @@
from pydantic import Field
from experiencemaker.schema.message import Trajectory, Message
from experiencemaker.schema.request import RetrieverRequest, SummarizerRequest, VectorStoreRequest, AgentRequest
from experiencemaker.schema.response import RetrieverResponse, SummarizerResponse, VectorStoreResponse, AgentResponse
from experiencemaker.utils.http_client import HttpClient
@ -27,5 +28,10 @@ class ExperienceMakerClient(HttpClient):
if __name__ == "__main__":
client = ExperienceMakerClient(base_url="http://0.0.0.0:8001")
response = client.call_retriever(RetrieverRequest(workspace_id="123", query="hello world"))
workspace_id = "t123"
response = client.call_summarizer(
SummarizerRequest(workspace_id=workspace_id,
traj_list=[Trajectory(messages=[Message(content="hello world!")])]))
print(response.model_dump())
response = client.call_retriever(RetrieverRequest(workspace_id=workspace_id, query="hello world"))
print(response.model_dump())

16
experiencemaker/service/experience_maker_service.py
View file

@ -4,9 +4,10 @@ from typing import List
from loguru import logger
from experiencemaker.config.config_parser import ConfigParser
from experiencemaker.embedding_model import EMBEDDING_MODEL_REGISTRY
from experiencemaker.pipeline.pipeline import Pipeline
from experiencemaker.pipeline.pipeline_context import PipelineContext
from experiencemaker.schema.app_config import AppConfig, HttpServiceConfig
from experiencemaker.schema.app_config import AppConfig, HttpServiceConfig, EmbeddingModelConfig
from experiencemaker.schema.request import SummarizerRequest, RetrieverRequest, VectorStoreRequest, AgentRequest, \
BaseRequest
from experiencemaker.schema.response import SummarizerResponse, RetrieverResponse, VectorStoreResponse, AgentResponse, \
@ -26,7 +27,18 @@ class ExperienceMakerService:
for name, config in self.init_app_config.vector_store.items():
assert config.backend in VECTOR_STORE_REGISTRY, f"backend={config.backend} is not existed"
vector_store_cls = VECTOR_STORE_REGISTRY[config.backend]
self.vector_store_dict[name] = vector_store_cls(**config.params)
assert config.embedding_model in self.init_app_config.embedding_model, \
f"embedding_model={config.embedding_model} is not existed"
embedding_model_config: EmbeddingModelConfig = self.init_app_config.embedding_model[config.embedding_model]
assert embedding_model_config.backend in EMBEDDING_MODEL_REGISTRY, \
f"embedding_model={embedding_model_config.backend} is not existed"
embedding_model_cls = EMBEDDING_MODEL_REGISTRY[embedding_model_config.backend]
embedding_model = embedding_model_cls(model_name=embedding_model_config.model_name,
**embedding_model_config.params)
self.vector_store_dict[name] = vector_store_cls(embedding_model=embedding_model, **config.params)
@property
def http_service_config(self) -> HttpServiceConfig: