diff --git a/bench/halumem/eval_tools.py b/bench/halumem/eval_tools.py index 8377e87d..29868ca8 100644 --- a/bench/halumem/eval_tools.py +++ b/bench/halumem/eval_tools.py @@ -120,7 +120,8 @@ async def evaluation_for_question2( dialogue: The formatted dialogue history (role, content, time_created). """ - prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION2"].format( + # prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION2"].format( + prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION"].format( question=question, reference_answer=reference_answer, key_memory_points=key_memory_points, @@ -128,7 +129,7 @@ async def evaluation_for_question2( dialogue=dialogue, ) - result = await llm_request_for_json(prompt) + result = await llm_request_for_json(prompt, model_name="qwen3-max") return result diff --git a/bench/halumem/halumem.yaml b/bench/halumem/halumem.yaml index 08c77ecc..51419c37 100644 --- a/bench/halumem/halumem.yaml +++ b/bench/halumem/halumem.yaml @@ -26,6 +26,30 @@ PROMPT_MEMZERO_JSON: | }} ``` +PROMPT_MEMZERO_JSON2: | + # CONTEXT: + {context} + + # CONTEXT PRIORITY: + When the context contains information from multiple sources, follow this strict priority order: + 1. **Historical Dialogue** (highest priority) - Direct conversation content + 2. **Extracted Memories** (medium priority) - Summarized memory points + 3. **User Profile** (lowest priority) - General user information + + # Question: + {question} + + # OUTPUT FORMAT: + Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT. + Please provide your response in the following JSON format: + + ```json + {{ + "reasoning": "reasoning content", + "answer": "Provide a detailed answer" + }} + ``` + PROMPT_MEMZERO: | You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories. diff --git a/bench/halumem/llms.py b/bench/halumem/llms.py index a6a06218..24edc703 100644 --- a/bench/halumem/llms.py +++ b/bench/halumem/llms.py @@ -58,7 +58,8 @@ async def llm_request(prompt, model_name: str = "qwen3-max", **kwargs) -> str: reraise=True, before_sleep=before_sleep_log(logger, logging.WARNING), ) -async def llm_request_for_json(prompt, model_name: str = "qwen3-max", **kwargs): +async def llm_request_for_json(prompt, model_name: str = "qwen-flash", **kwargs): + # async def llm_request_for_json(prompt, model_name: str = "qwen3-max", **kwargs): """Make an LLM request expecting JSON response using ReMe's LLM. Args: diff --git a/reme_ai/core/config/default.yaml b/reme_ai/core/config/default.yaml index 2c8488a8..6ae47d6a 100644 --- a/reme_ai/core/config/default.yaml +++ b/reme_ai/core/config/default.yaml @@ -15,7 +15,8 @@ http: llm: default: backend: openai - model_name: qwen3-30b-a3b-instruct-2507 +# model_name: qwen3-30b-a3b-instruct-2507 + model_name: qwen-flash request_interval: 1 temperature: 0.0001