diff --git a/benchmark/results_md/beam.md b/benchmark/results_md/beam.md index 4ec5b71a..97adff3c 100644 --- a/benchmark/results_md/beam.md +++ b/benchmark/results_md/beam.md @@ -23,14 +23,14 @@ | 题型 | Prompted(limit=15) | Prompted Binary | Agentic | Agentic Binary | |---|---|---|---|---| -| abstention | 0.464 | 0.464 | 0.514 | 0.514 | -| contradiction_resolution | 0.079 | 0.068 | 0.373 | 0.339 | -| event_ordering | 0.455 | 0.334 | 0.547 | 0.450 | -| information_extraction | 0.653 | 0.589 | 0.818 | 0.764 | -| instruction_following | 0.541 | 0.524 | 0.765 | 0.745 | -| knowledge_update | 0.571 | 0.507 | 0.636 | 0.629 | -| multi_session_reasoning | 0.426 | 0.324 | 0.593 | 0.540 | -| preference_following | 0.718 | 0.676 | 0.838 | 0.824 | -| summarization | 0.516 | 0.303 | 0.661 | 0.478 | -| temporal_reasoning | 0.198 | 0.169 | 0.394 | 0.383 | -| **OVERALL** | **0.462** | **0.396** | **0.614** | **0.567** | \ No newline at end of file +| abstention | 0.464 | 0.464 | 0.464 | 0.464 | +| contradiction_resolution | 0.079 | 0.068 | 0.379 | 0.346 | +| event_ordering | 0.455 | 0.334 | 0.535 | 0.442 | +| information_extraction | 0.653 | 0.589 | 0.795 | 0.758 | +| instruction_following | 0.541 | 0.524 | 0.774 | 0.758 | +| knowledge_update | 0.571 | 0.507 | 0.693 | 0.679 | +| multi_session_reasoning | 0.426 | 0.324 | 0.656 | 0.602 | +| preference_following | 0.718 | 0.676 | 0.805 | 0.796 | +| summarization | 0.516 | 0.303 | 0.672 | 0.504 | +| temporal_reasoning | 0.198 | 0.169 | 0.462 | 0.448 | +| **OVERALL** | **0.462** | **0.396** | **0.623** | **0.580** | \ No newline at end of file diff --git a/reme/steps/benchmark/base/agentic_answer.py b/reme/steps/benchmark/base/agentic_answer.py index 27b9f26c..9b357d36 100644 --- a/reme/steps/benchmark/base/agentic_answer.py +++ b/reme/steps/benchmark/base/agentic_answer.py @@ -54,7 +54,7 @@ class BaseAgenticAnswerStep(BaseStep): tool_context_id = f"{self.TOOL_CONTEXT_PREFIX}_{os.getpid()}_local" wrapper_kwargs = { "system_prompt": sys_prompt, - "job_tools": ["search", "add_draft", "read_all_draft"], + "job_tools": ["search", "add_draft", "read_all_draft","read"], "react_config": {"max_iters": self.MAX_ITERATION}, "tool_context_id": tool_context_id, } diff --git a/reme/steps/benchmark/lme/agentic_answer.yaml b/reme/steps/benchmark/lme/agentic_answer.yaml index f856f0fc..769f5d74 100644 --- a/reme/steps/benchmark/lme/agentic_answer.yaml +++ b/reme/steps/benchmark/lme/agentic_answer.yaml @@ -5,6 +5,8 @@ system_prompt: | ## Search Strategy - You can call 'search' tool to search multiple times (at least once) with different queries to gather comprehensive information. - Do not stop after the first relevant one unless you are confident that the information is sufficient. + - You can use 'read' tool to read specific parts of a specific file. + ## Draft Tool - Use 'add_draft' to save key findings during search, and 'read_all_draft' to review all saved notes before answering. ## Answer Rules