diff --git a/experiencemaker/config/config_parser.py b/experiencemaker/config/config_parser.py index 64318fec..77179fec 100644 --- a/experiencemaker/config/config_parser.py +++ b/experiencemaker/config/config_parser.py @@ -29,7 +29,7 @@ class ConfigParser: self.app_config = OmegaConf.merge(self.app_config, cli_config) app_config_dict = OmegaConf.to_container(self.app_config, resolve=True) - logger.info(f"app_config_str={json.dumps(app_config_dict, indent=2, ensure_ascii=False)}") + logger.info(f"app_config=\n{json.dumps(app_config_dict, indent=2, ensure_ascii=False)}") def get_app_config(self, **kwargs) -> AppConfig: app_config = self.app_config.copy() diff --git a/experiencemaker/config/demo_config.yaml b/experiencemaker/config/demo_config.yaml index 9fcc4555..a17d632c 100644 --- a/experiencemaker/config/demo_config.yaml +++ b/experiencemaker/config/demo_config.yaml @@ -7,33 +7,35 @@ http_service: limit_concurrency: 64 thread_pool: - max_workers: 10 + max_workers: 64 api: - retriever: mock1_op->[mock4_op->mock2_op|mock5_op]->[mock3_op|mock6_op] - summarizer: mock1_op->[mock4_op->mock2_op|mock5_op]->mock3_op - vector_store: mock6_op + retriever: build_query_op->recall_vector_store_op->merge_experience_op + summarizer: simple_summary_op->update_vector_store_op + vector_store: vector_store_action_op op: - mock1_op: - backend: mock1_op + build_query_op: + backend: build_query_op llm: default + params: + enable_llm_build: false + recall_vector_store_op: + backend: recall_vector_store_op vector_store: default + merge_experience_op: + backend: merge_experience_op + simple_summary_op: + backend: simple_summary_op + llm: default params: - a: 1 - b: 2 - mock2_op: - backend: mock2_op - params: - a: 1 - mock3_op: - backend: mock3_op - mock4_op: - backend: mock4_op - mock5_op: - backend: mock5_op - mock6_op: - backend: mock6_op + success_score_threshold: 0.9 + update_vector_store_op: + backend: update_vector_store_op + vector_store: default + vector_store_action_op: + backend: vector_store_action_op + vector_store: default llm: default: @@ -53,6 +55,3 @@ vector_store: default: backend: elasticsearch embedding_model: default - params: - hosts: "http://localhost:9200" - diff --git a/experiencemaker/op/__init__.py b/experiencemaker/op/__init__.py index a1785ad0..c8c85319 100644 --- a/experiencemaker/op/__init__.py +++ b/experiencemaker/op/__init__.py @@ -3,3 +3,9 @@ from experiencemaker.utils.registry import Registry OP_REGISTRY = Registry() from experiencemaker.op.mock_op import Mock1Op, Mock2Op, Mock3Op, Mock4Op, Mock5Op, Mock6Op +from experiencemaker.op.retriever.build_query_op import BuildQueryOp +from experiencemaker.op.retriever.merge_experience_op import MergeExperienceOp +from experiencemaker.op.summarizer.simple_summary_op import SimpleSummaryOp +from experiencemaker.op.vector_store.update_vector_store_op import UpdateVectorStoreOp +from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp +from experiencemaker.op.vector_store.vector_store_action_op import VectorStoreActionOp diff --git a/experiencemaker/op/base_op.py b/experiencemaker/op/base_op.py index 94b5330d..e31a8ac6 100644 --- a/experiencemaker/op/base_op.py +++ b/experiencemaker/op/base_op.py @@ -19,6 +19,7 @@ from experiencemaker.vector_store.base_vector_store import BaseVectorStore class BaseOp(PromptMixin, ABC): + current_path: str = __file__ def __init__(self, context: PipelineContext, op_config: OpConfig): super().__init__() @@ -39,7 +40,7 @@ class BaseOp(PromptMixin, ABC): prompt_file_path = self.op_config.prompt_file_path else: prompt_name = self.simple_name.replace("_op", "_prompt.yaml") - prompt_file_path = Path(__file__).parent / prompt_name + prompt_file_path = Path(self.current_path).parent / prompt_name # Load custom prompts from prompt file self.load_prompt_by_file(prompt_file_path=prompt_file_path) @@ -116,8 +117,4 @@ class BaseOp(PromptMixin, ABC): f"vector_store={vector_store_name} not found in vector_store_dict!" self._vector_store = self.context.vector_store_dict[vector_store_name] - if self._vector_store.embedding_model is None: - logger.info(f"set embedding_model for vector_store={self.op_config.vector_store}") - self._vector_store.embedding_model = self.embedding_model - return self._vector_store diff --git a/experiencemaker/op/retriever/build_query_op.py b/experiencemaker/op/retriever/build_query_op.py index 012d3836..6fac6903 100644 --- a/experiencemaker/op/retriever/build_query_op.py +++ b/experiencemaker/op/retriever/build_query_op.py @@ -8,7 +8,7 @@ from experiencemaker.utils.op_utils import merge_messages_content @OP_REGISTRY.register() class BuildQueryOp(BaseOp): - RETRIEVE_QUERY = "retrieve_query" + current_path: str = __file__ def execute(self): request: RetrieverRequest = self.context.request @@ -28,4 +28,5 @@ class BuildQueryOp(BaseOp): logger.info(f"build.query={query}") - self.context.set_context(self.RETRIEVE_QUERY, query) + from experiencemaker.op.vector_store.recall_vector_store_op import RecallVectorStoreOp + self.context.set_context(RecallVectorStoreOp.SEARCH_QUERY, query) diff --git a/experiencemaker/op/retriever/merge_experience_op.py b/experiencemaker/op/retriever/merge_experience_op.py index d4356a7f..95ab9baa 100644 --- a/experiencemaker/op/retriever/merge_experience_op.py +++ b/experiencemaker/op/retriever/merge_experience_op.py @@ -1,5 +1,7 @@ from typing import List +from loguru import logger + from experiencemaker.op import OP_REGISTRY from experiencemaker.op.base_op import BaseOp from experiencemaker.schema.experience import BaseExperience @@ -22,7 +24,8 @@ class MergeExperienceOp(BaseOp): continue content_collector.append(f"- when_to_use: {experience.when_to_use}\n" - f"content: {experience.experience_content}\n") + f"content: {experience.content}\n") content_collector.append("Please consider the helpful parts from these in answering the question, " "to make the response more comprehensive and substantial.") response.experience_merged = "\n".join(content_collector) + logger.info(f"experience_merged={response.experience_merged}") diff --git a/experiencemaker/op/summarizer/simple_summary_op.py b/experiencemaker/op/summarizer/simple_summary_op.py index 4451fab1..383e9a25 100644 --- a/experiencemaker/op/summarizer/simple_summary_op.py +++ b/experiencemaker/op/summarizer/simple_summary_op.py @@ -14,10 +14,14 @@ from experiencemaker.utils.op_utils import merge_messages_content @OP_REGISTRY.register() class SimpleSummaryOp(BaseOp): + current_path: str = __file__ def summary_trajectory(self, trajectory: Trajectory) -> List[BaseExperience]: execution_process = merge_messages_content(trajectory.messages) - execution_result = "success" if trajectory.score > 0.9 else "fail" + success_score_threshold: float = self.op_params.get("success_score_threshold", 0.9) + logger.info(f"success_score_threshold={success_score_threshold}") + + execution_result = "success" if trajectory.score > success_score_threshold else "fail" summary_prompt = self.prompt_format(prompt_name="summary_prompt", execution_process=execution_process, execution_result=execution_result, @@ -29,7 +33,7 @@ class SimpleSummaryOp(BaseOp): try: content = content.split("```")[1].strip() if content.startswith("json"): - content.strip("json") + content = content.strip("json") for exp_dict in json.loads(content): when_to_use = exp_dict.get("when_to_use", "").strip() @@ -51,13 +55,7 @@ class SimpleSummaryOp(BaseOp): def execute(self): request: SummarizerRequest = self.context.request for trajectory in request.traj_list: - execution_process = merge_messages_content(trajectory.messages) - execution_result = "success" if trajectory.score > 0.9 else "fail" - summary_prompt = self.prompt_format(prompt_name="summary_prompt", - execution_process=execution_process, - execution_result=execution_result, - summary_example=self.get_prompt("summary_example")) - self.submit_task(self.llm.chat, messages=[Message(content=summary_prompt)]) + self.submit_task(self.summary_trajectory, trajectory=trajectory) experience_list: List[BaseExperience] = [] for task_result in self.join_task(): @@ -67,7 +65,7 @@ class SimpleSummaryOp(BaseOp): response: SummarizerResponse = self.context.response response.experience_list = experience_list for e in experience_list: - logger.info(f"add experience {e.when_to_use}\n{e.content}") + logger.info(f"add experience when_to_use={e.when_to_use}\ncontent={e.content}") - from experiencemaker.op.summarizer.insert_database_op import InsertDatabaseOp - self.context.set_context(InsertDatabaseOp.INSERT_NODES, [x.to_vector_node() for x in experience_list]) + from experiencemaker.op.vector_store.update_vector_store_op import UpdateVectorStoreOp + self.context.set_context(UpdateVectorStoreOp.INSERT_NODES, [x.to_vector_node() for x in experience_list]) diff --git a/experiencemaker/op/vector_store/recall_vector_store_op.py b/experiencemaker/op/vector_store/recall_vector_store_op.py index a852efd8..8f877c7c 100644 --- a/experiencemaker/op/vector_store/recall_vector_store_op.py +++ b/experiencemaker/op/vector_store/recall_vector_store_op.py @@ -1,5 +1,7 @@ from typing import List +from loguru import logger + from experiencemaker.op import OP_REGISTRY from experiencemaker.op.base_op import BaseOp from experiencemaker.schema.experience import BaseExperience, vector_node_to_experience @@ -10,10 +12,11 @@ from experiencemaker.schema.vector_node import VectorNode @OP_REGISTRY.register() class RecallVectorStoreOp(BaseOp): + SEARCH_QUERY = "search_query" def execute(self): # get query - query = self.context.get_context("search_query") + query = self.context.get_context(self.SEARCH_QUERY) assert query, "query should be not empty!" # retrieve from vector store @@ -30,11 +33,14 @@ class RecallVectorStoreOp(BaseOp): if experience.content not in experience_content_list: experience_list.append(experience) experience_content_list.append(experience.content) + experience_size = len(experience_list) + logger.info(f"retrieve experience size={experience_size}") # filter by score threshold_score: float | None = self.op_params.get("threshold_score", None) if threshold_score is not None: experience_list = [e for e in experience_list if e.score >= threshold_score or e.score is None] + logger.info(f"after filter by threshold_score size={len(experience_list)}") # set response request: RetrieverResponse = self.context.response diff --git a/experiencemaker/op/vector_store/update_vector_store.py b/experiencemaker/op/vector_store/update_vector_store_op.py similarity index 88% rename from experiencemaker/op/vector_store/update_vector_store.py rename to experiencemaker/op/vector_store/update_vector_store_op.py index 45ba9769..cc2d6a18 100644 --- a/experiencemaker/op/vector_store/update_vector_store.py +++ b/experiencemaker/op/vector_store/update_vector_store_op.py @@ -25,5 +25,4 @@ class UpdateVectorStoreOp(BaseOp): insert_nodes: List[VectorNode] | None = self.context.get_context(self.INSERT_NODES) if insert_nodes: self.vector_store.insert(nodes=insert_nodes, workspace_id=request.workspace_id) - for node in insert_nodes: - logger.info(f"insert insert_node={node.model_dump_json(indent=2)}") + logger.info(f"insert insert_node.size={len(insert_nodes)}") diff --git a/experiencemaker/pipeline/pipeline.py b/experiencemaker/pipeline/pipeline.py index 41c304ed..7d56c0f8 100644 --- a/experiencemaker/pipeline/pipeline.py +++ b/experiencemaker/pipeline/pipeline.py @@ -68,7 +68,7 @@ class Pipeline: elif isinstance(pipeline, list): parallel_pipeline = [self._parse_sub_pipeline(x) for x in pipeline] - for op_list in zip_longest(*parallel_pipeline, fillvalue="-"): + for op_list in zip_longest(parallel_pipeline, fillvalue="-"): i += 1 logger.info(f"stage{i}: {' | '.join(op_list)}") diff --git a/experiencemaker/pipeline/pipeline_context.py b/experiencemaker/pipeline/pipeline_context.py index 9687a263..29bfa49e 100644 --- a/experiencemaker/pipeline/pipeline_context.py +++ b/experiencemaker/pipeline/pipeline_context.py @@ -2,8 +2,6 @@ from concurrent.futures import ThreadPoolExecutor from typing import Dict from experiencemaker.schema.app_config import AppConfig -from experiencemaker.schema.request import BaseRequest -from experiencemaker.schema.response import BaseResponse from experiencemaker.vector_store.base_vector_store import BaseVectorStore diff --git a/experiencemaker/service/experience_maker_client.py b/experiencemaker/service/experience_maker_client.py index b2ea8e17..565b6b90 100644 --- a/experiencemaker/service/experience_maker_client.py +++ b/experiencemaker/service/experience_maker_client.py @@ -1,5 +1,6 @@ from pydantic import Field +from experiencemaker.schema.message import Trajectory, Message from experiencemaker.schema.request import RetrieverRequest, SummarizerRequest, VectorStoreRequest, AgentRequest from experiencemaker.schema.response import RetrieverResponse, SummarizerResponse, VectorStoreResponse, AgentResponse from experiencemaker.utils.http_client import HttpClient @@ -27,5 +28,10 @@ class ExperienceMakerClient(HttpClient): if __name__ == "__main__": client = ExperienceMakerClient(base_url="http://0.0.0.0:8001") - response = client.call_retriever(RetrieverRequest(workspace_id="123", query="hello world")) + workspace_id = "t123" + response = client.call_summarizer( + SummarizerRequest(workspace_id=workspace_id, + traj_list=[Trajectory(messages=[Message(content="hello world!")])])) + print(response.model_dump()) + response = client.call_retriever(RetrieverRequest(workspace_id=workspace_id, query="hello world")) print(response.model_dump()) diff --git a/experiencemaker/service/experience_maker_service.py b/experiencemaker/service/experience_maker_service.py index dd2e8b79..edb86dba 100644 --- a/experiencemaker/service/experience_maker_service.py +++ b/experiencemaker/service/experience_maker_service.py @@ -4,9 +4,10 @@ from typing import List from loguru import logger from experiencemaker.config.config_parser import ConfigParser +from experiencemaker.embedding_model import EMBEDDING_MODEL_REGISTRY from experiencemaker.pipeline.pipeline import Pipeline from experiencemaker.pipeline.pipeline_context import PipelineContext -from experiencemaker.schema.app_config import AppConfig, HttpServiceConfig +from experiencemaker.schema.app_config import AppConfig, HttpServiceConfig, EmbeddingModelConfig from experiencemaker.schema.request import SummarizerRequest, RetrieverRequest, VectorStoreRequest, AgentRequest, \ BaseRequest from experiencemaker.schema.response import SummarizerResponse, RetrieverResponse, VectorStoreResponse, AgentResponse, \ @@ -26,7 +27,18 @@ class ExperienceMakerService: for name, config in self.init_app_config.vector_store.items(): assert config.backend in VECTOR_STORE_REGISTRY, f"backend={config.backend} is not existed" vector_store_cls = VECTOR_STORE_REGISTRY[config.backend] - self.vector_store_dict[name] = vector_store_cls(**config.params) + + assert config.embedding_model in self.init_app_config.embedding_model, \ + f"embedding_model={config.embedding_model} is not existed" + embedding_model_config: EmbeddingModelConfig = self.init_app_config.embedding_model[config.embedding_model] + + assert embedding_model_config.backend in EMBEDDING_MODEL_REGISTRY, \ + f"embedding_model={embedding_model_config.backend} is not existed" + embedding_model_cls = EMBEDDING_MODEL_REGISTRY[embedding_model_config.backend] + embedding_model = embedding_model_cls(model_name=embedding_model_config.model_name, + **embedding_model_config.params) + + self.vector_store_dict[name] = vector_store_cls(embedding_model=embedding_model, **config.params) @property def http_service_config(self) -> HttpServiceConfig: