feat(benchmark): add lme benchmark steps (#326)

* refactor(search): replace time module with datetime for timestamp generation

- Removed unused time import
- Added static method _now_ts using datetime.timestamp
- Updated clock parameter to use _now_ts method instead of time.time
- Maintained same timestamp precision and functionality

* test(http): add tests for HTTP client display formatting

- Add test for default metadata hiding behavior in CLI output
- Add test for metadata display when show_metadata is enabled
- Verify _format_for_display method correctly formats response text
- Test both success case and metadata inclusion scenarios

* chore(build): remove longmemeval from gitignore

- Removed longmemeval directory from gitignore list
- Kept evaluation and datasets directories in ignore list
- Updated gitignore configuration for proper version control

.github/workflows/pr-title-check.yml

@@ -0,0 +1,36 @@
+name: PR Title Check
+
+on:
+  pull_request:
+    branches: [main, master, dev, develop]
+    types: [opened, edited, synchronize, reopened]
+
+jobs:
+  check-pr-title:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check PR title format
+        uses: amannn/action-semantic-pull-request@v6.1.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          types: |
+            feat
+            fix
+            docs
+            ci
+            refactor
+            test
+            chore
+            perf
+            style
+            build
+            revert
+          requireScope: false
+          scopePattern: ^[a-z0-9_-]+$
+          scopePatternError: |
+            The scope must contain only lowercase letters, numbers, hyphens, and underscores.
+            Example: "feat(memory): add redis cache support"
+          validateSingleCommit: false
+          ignoreLabels: |
+            ignore-semantic-pull-request

.github/workflows/pre-commit.yml

@@ -13,9 +13,9 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: '3.11'
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@master
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Update setuptools

.github/workflows/python-publish.yml

@@ -33,6 +33,10 @@ jobs:
         pip install setuptools wheel build
     - name: Build package
       run: python -m build
+    - name: Test installation
+      run: |
+        pip install dist/*.whl
+        python -c "import reme; print(reme.__version__)"
     - name: Publish package to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/unittest.yml

@@ -33,11 +33,15 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install -e ".[dev,core]"
+          pip install coverage
 
       - name: Run unit tests
         run: |
-          pytest tests/unit \
+          coverage run -m pytest tests/unit \
             -v \
             --tb=long \
             -s \
             --log-cli-level=WARNING
+
+      - name: Generate coverage report
+        run: coverage report -m

.gitignore

@@ -54,6 +54,5 @@ vault/
 docs/_build/
 site/
 
-longmemeval/
 evaluation/
 datasets/

.pre-commit-config.yaml

@@ -14,10 +14,10 @@ repos:
     hooks:
       - id: add-trailing-comma
   - repo: https://github.com/psf/black
-    rev: 25.9.0
+    rev: 26.5.1
     hooks:
     - id: black
-      args: [--line-length=120]
+      args: [--line-length=120, --target-version=py311]
   - repo: https://github.com/PyCQA/flake8
     rev: 7.3.0
     hooks:
@@ -27,7 +27,7 @@ repos:
           "--max-line-length=120"
         ]
   - repo: https://github.com/pylint-dev/pylint
-    rev: v4.0.2
+    rev: v4.0.6
     hooks:
       - id: pylint
         exclude:
@@ -76,7 +76,7 @@ repos:
           --max-module-lines=1500,
         ]
   - repo: https://github.com/regebro/pyroma
-    rev: "5.0"
+    rev: "5.0.1"
     hooks:
       - id: pyroma
         args: [--min=10, .]

reme/components/client/http_client.py

@@ -22,6 +22,7 @@ class HttpClient(BaseClient):
         host: str | None = None,
         port: int | None = None,
         timeout: float = 3600.0,
+        show_metadata: bool = False,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -41,6 +42,7 @@ class HttpClient(BaseClient):
 
         self.base_url = f"http://{host}:{port}"
         self.timeout = timeout
+        self.show_metadata = show_metadata
 
     async def _start(self) -> None:
         """Initialize the HTTP client."""
@@ -101,8 +103,7 @@ class HttpClient(BaseClient):
                 actions.append({"action": path.lstrip("/"), "method": method.upper(), **op})
         return actions
 
-    @staticmethod
-    def _format_for_display(text: str) -> str:
+    def _format_for_display(self, text: str) -> str:
         """Render a JSON response as human-friendly CLI text; pass through unrecognized payloads."""
         try:
             data = json.loads(text)
@@ -118,7 +119,7 @@ class HttpClient(BaseClient):
         status_pieces = []
         if success is not None:
             status_pieces.append("✅" if success else "❌")
-        if metadata:
+        if self.show_metadata and metadata:
             status_pieces.append(json.dumps(metadata, ensure_ascii=False))
         if status_pieces:
             parts.append(" ".join(status_pieces))

reme/components/file_chunker/markdown_file_chunker.py

@@ -31,7 +31,6 @@ from ...schema import (
 )
 from ...utils.wikilink_handler import WikilinkHandler
 
-
 # -- AST node + helpers ---------------------------------------------------
 
 

reme/components/file_graph/neo4j_file_graph.py

@@ -42,7 +42,6 @@ from ...enumeration import LinkScopeEnum
 from ...schema import FileLink, FileNode
 from ...schema.file_node import FileFrontMatter
 
-
 _TYPED_FRONTMATTER_FIELDS = {"name", "description"}
 _LINK_FIELDS = {"source_path", "target_path", "target_anchor", "predicate"}
 

reme/config/jinli_lme.yaml

@@ -0,0 +1,177 @@
+service:
+  backend: http
+
+jobs:
+  update_index:
+    backend: base
+    watch_dirs: [daily_dir, digest_dir, resource_dir]
+#    watch_dirs: [daily_dir, digest_dir]
+    watch_suffixes: [md, jsonl]
+#    watch_suffixes: [md]
+    steps:
+      - backend: init_changes_step
+        monitor_type: file_store
+        monitor_name: default
+        dispatch_steps: [update_index_step]
+      - backend: watch_changes_step
+        dispatch_steps:
+          - backend: update_index_step
+            persist: False
+
+  auto_memory:
+    backend: base
+    description: "Auto-memory: record conversation facts into a daily note"
+    parameters:
+      type: object
+      properties:
+        messages:
+          type: array
+          description: "messages"
+          items:
+            type: object
+        session_id:
+          type: string
+          description: "source conversation session identifier"
+          default: ""
+        memory_hint:
+          type: string
+          description: "optional hint"
+        date:
+          type: string
+          description: "YYYY-MM-DD daily note date; empty = infer from message timestamps or today"
+          default: ""
+      required:
+        - messages
+    steps:
+      - backend: auto_memory_step
+
+  version:
+    backend: base
+    description: "return reme package version"
+    parameters:
+      type: object
+      properties: { }
+    steps:
+      - backend: version_step
+
+  search:
+    backend: base
+    description: "Hybrid workspace search (vector + BM25, RRF-fused)."
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: "search query"
+        limit:
+          type: integer
+          description: "max results"
+          default: 5
+        min_score:
+          type: number
+          description: "min fused score"
+          default: 0.0
+        start_date:
+          type: string
+          description: "optional inclusive start date filter (YYYY-MM-DD); results earlier than this date are excluded"
+        end_date:
+          type: string
+          description: "optional inclusive end date filter (YYYY-MM-DD); results later than this date are excluded"
+      required:
+        - query
+    steps:
+      - backend: search_step
+        vector_weight: 0.7
+        candidate_multiplier: 5.0
+        expand_links: true
+        max_links_per_direction: 10
+
+components:
+  tokenizer:
+    default:
+      backend: regex
+
+  as_embedding:
+    default:
+      backend: ${EMBEDDING_BACKEND:-openai}
+      model: ${EMBEDDING_MODEL_NAME:-text-embedding-v4}
+      credential:
+        api_key: ${EMBEDDING_API_KEY:-}
+        base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1}
+      parameters:
+        dimensions: 1024
+
+  embedding_store:
+    default:
+      backend: local
+      as_embedding: default
+
+  as_llm:
+    default:
+      backend: ${LLM_BACKEND:-openai}
+      model: ${LLM_MODEL_NAME:-qwen3.7-plus}
+      stream: true
+      context_size: 200000
+      max_retries: 3
+      credential:
+        api_key: ${LLM_API_KEY:-}
+        base_url: ${LLM_BASE_URL:-}
+      parameters:
+        max_tokens: 65536
+        thinking_enable: false
+
+  agent_wrapper:
+    default:
+      backend: agentscope
+      as_llm: default
+      permission_mode: bypass
+      react_config:
+        max_iters: 30
+      context_config:
+        trigger_ratio: 0.8
+        reserve_ratio: 0.1
+        tool_result_limit: 50000
+      model_config:
+        max_retries: 1
+    claude_code:
+      backend: claude_code
+      model: ${CLAUDE_CODE_MODEL_NAME:-glm-5.1}
+      api_key: ${CLAUDE_CODE_API_KEY:-}
+      base_url: ${CLAUDE_CODE_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic}
+      permission_mode: bypassPermissions
+
+  file_graph:
+    default:
+      backend: local
+
+  file_catalog:
+    default:
+      backend: local
+    resource:
+      backend: local
+    digest:
+      backend: local
+    dream:
+      backend: local
+
+  file_chunker:
+    markdown:
+      backend: markdown
+      supported_extensions: [ "md" ]
+    default:
+      backend: default
+      supported_extensions: [ "jsonl" ]
+
+  keyword_index:
+    default:
+      backend: bm25
+      tokenizer: default
+
+  file_store:
+    default:
+      backend: local
+      store_name: local
+      #  embedding_store: default
+      embedding_store: ""
+      keyword_index: default
+      file_graph: default

reme/reme.py

@@ -9,7 +9,7 @@ from .config import parse_args, resolve_app_config
 from .enumeration import ComponentEnum
 from .utils import cli_find_reme, load_env, precheck_start, running_service_config
 
-_CLIENT_KWARGS = {"host", "port", "timeout", "transport", "command", "args"}
+_CLIENT_KWARGS = {"host", "port", "timeout", "transport", "command", "args", "show_metadata"}
 
 
 class ReMe(Application):

reme/steps/__init__.py

@@ -1,10 +1,11 @@
 """steps"""
 
-from . import channel, common, evolve, file_io, index, transfer
+from . import benchmark, channel, common, evolve, file_io, index, transfer
 from .base_step import BaseStep
 
 __all__ = [
     "BaseStep",
+    "benchmark",
     "channel",
     "common",
     "evolve",

reme/steps/benchmark/__init__.py

@@ -0,0 +1,10 @@
+"""Benchmark steps."""
+
+from . import lme
+from .lme import AnswerJudgeStep, ContextAnswerStep
+
+__all__ = [
+    "AnswerJudgeStep",
+    "ContextAnswerStep",
+    "lme",
+]

reme/steps/benchmark/lme/__init__.py

@@ -0,0 +1,9 @@
+"""LongMemEval benchmark steps."""
+
+from .context_answer import ContextAnswerStep
+from .llm_judge import AnswerJudgeStep
+
+__all__ = [
+    "AnswerJudgeStep",
+    "ContextAnswerStep",
+]

reme/steps/benchmark/lme/context_answer.py

@@ -0,0 +1,45 @@
+"""Answer a query directly from the supplied session context."""
+
+from ...base_step import BaseStep
+from ....components import R
+
+
+@R.register("context_answer_step")
+class ContextAnswerStep(BaseStep):
+    """Answer a query using the LongMemEval direct-reading prompt."""
+
+    async def execute(self):
+        assert self.context is not None
+        query: str = self.context.get("query", "")
+        session_context: str = self.context.get("session_context", "")
+        current_date: str = self.context.get("current_date", "")
+
+        if not query:
+            raise ValueError("context_answer_step requires non-empty query")
+        if not session_context:
+            raise ValueError("context_answer_step requires non-empty session_context")
+        if self.agent_wrapper is None:
+            raise ValueError("context_answer_step requires agent_wrapper")
+
+        user_prompt = self.prompt_format(
+            "user_message",
+            session_context=session_context,
+            current_date=current_date,
+            query=query,
+        )
+        result = await self.agent_wrapper.reply(user_prompt)
+        answer = (result.get("result") or "").strip()
+
+        self.logger.info(f"[{self.name}] context answer: {answer}")
+        self.context["context_answer"] = answer
+        self.context.response.success = True
+        self.context.response.answer = answer
+        self.context.response.metadata.update(
+            {
+                "query": query,
+                "session_context": session_context,
+                "current_date": current_date,
+                "context_answer": answer,
+            },
+        )
+        return self.context.response

reme/steps/benchmark/lme/context_answer.yaml

@@ -0,0 +1,9 @@
+user_message: |
+  I will give you several history chats between you and a user. Please answer the question
+  based on the relevant chat history. Answer the question step by step: first extract all the
+  relevant information, and then reason over the information to get the answer.
+
+  History Chats: {session_context}
+  Current Date: {current_date}
+  Question: {query}
+  Answer (step by step):

reme/steps/benchmark/lme/llm_judge.py

@@ -0,0 +1,81 @@
+"""Judge whether an agent answer matches the golden answer."""
+
+import re
+
+from ...base_step import BaseStep
+from ....components import R
+
+
+@R.register("answer_judge_step")
+class AnswerJudgeStep(BaseStep):
+    """Evaluate whether an agent answer is correct against a golden answer."""
+
+    PROMPT_KEYS_BY_QUESTION_TYPE = {
+        "temporal_reasoning": "temporal_reasoning_system_prompt",
+        "knowledge_update": "knowledge_update_system_prompt",
+        "single_session_preference": "single_session_preference_system_prompt",
+    }
+
+    @classmethod
+    def _judge_prompt_key(cls, question_type: str) -> str:
+        normalized = question_type.strip().lower().replace("-", "_").replace(" ", "_")
+        return cls.PROMPT_KEYS_BY_QUESTION_TYPE.get(normalized, "other_question_types_system_prompt")
+
+    @staticmethod
+    def _normalize_judgement(raw_answer: str) -> str:
+        match = re.match(r"\s*(yes|no)\b", raw_answer, re.IGNORECASE)
+        if match:
+            return match.group(1).lower()
+        return raw_answer.strip().lower()
+
+    async def execute(self):
+        assert self.context is not None
+        query: str = self.context.get("query", "")
+        agent_answer: str = self.context.get("agent_answer", "")
+        golden_answer: str = self.context.get("golden_answer", "")
+        question_type: str = self.context.get("question_type", "")
+
+        if not query:
+            raise ValueError("answer_judge_step requires non-empty query")
+        if not agent_answer:
+            raise ValueError("answer_judge_step requires non-empty agent_answer")
+        if not golden_answer:
+            raise ValueError("answer_judge_step requires non-empty golden_answer")
+        if self.agent_wrapper is None:
+            raise RuntimeError("answer_judge_step requires agent_wrapper")
+
+        judge_prompt_key = self._judge_prompt_key(question_type)
+        user_prompt_key = (
+            "preference_judge_user_message"
+            if judge_prompt_key == "single_session_preference_system_prompt"
+            else "answer_judge_user_message"
+        )
+        user_prompt = self.prompt_format(
+            user_prompt_key,
+            query=query,
+            golden_answer=golden_answer,
+            agent_answer=agent_answer,
+        )
+        result = await self.agent_wrapper.reply(
+            user_prompt,
+            system_prompt=self.prompt_format(judge_prompt_key),
+        )
+
+        raw_answer = (result.get("result") or "").strip()
+        answer = self._normalize_judgement(raw_answer)
+
+        self.logger.info(f"[{self.name}] answer judgement: {answer}")
+        self.context["answer_judgement"] = answer
+        self.context.response.success = True
+        self.context.response.answer = answer
+        self.context.response.metadata.update(
+            {
+                "query": query,
+                "agent_answer": agent_answer,
+                "golden_answer": golden_answer,
+                "question_type": question_type,
+                "answer_judgement": answer,
+                "raw_answer_judgement": raw_answer,
+            },
+        )
+        return self.context.response

reme/steps/benchmark/lme/llm_judge.yaml

@@ -0,0 +1,42 @@
+temporal_reasoning_system_prompt: |
+  I will give you a question, a correct answer, and a response from a model. Please answer
+  yes if the response contains the correct answer. Otherwise, answer no. If the response is
+  equivalent to the correct answer or contains all the intermediate steps to get the correct
+  answer, you should also answer yes. If the response only contains a subset of the information
+  required by the answer, answer no. In addition, do not penalize off-by-one errors for the
+  number of days. If the question asks for the number of days/weeks/months, etc., and the
+  model makes off-by-one errors (e.g., predicting 19 days when the answer is 18), the model's
+  response is still correct.
+  Only output one lowercase word: yes or no.
+
+knowledge_update_system_prompt: |
+  I will give you a question, a correct answer, and a response from a model. Please answer
+  yes if the response contains the correct answer. Otherwise, answer no. If the response
+  contains some previous information along with an updated answer, the response should be
+  considered as correct as long as the updated answer is the required answer.
+  Only output one lowercase word: yes or no.
+
+single_session_preference_system_prompt: |
+  I will give you a question, a rubric for desired personalized response, and a response from a
+  model. Please answer yes if the response satisfies the desired response. Otherwise, answer
+  no. The model does not need to reflect all the points in the rubric. The response is correct
+  as long as it recalls and utilizes the user's personal information correctly.
+  Only output one lowercase word: yes or no.
+
+other_question_types_system_prompt: |
+  I will give you a question, a correct answer, and a response from a model. Please answer
+  yes if the response contains the correct answer. Otherwise, answer no. If the response
+  is equivalent to the correct answer or contains all the intermediate steps to get the correct
+  answer, you should also answer yes. If the response only contains a subset of the information
+  required by the answer, answer no.
+  Only output one lowercase word: yes or no.
+
+answer_judge_user_message: |
+  Question: {query}
+  Correct answer: {golden_answer}
+  Response from a model: {agent_answer}
+
+preference_judge_user_message: |
+  Question: {query}
+  Rubric for desired personalized response: {golden_answer}
+  Response from a model: {agent_answer}

reme/steps/common/health_check.py

@@ -10,7 +10,6 @@ from ... import __version__
 from ...components import R
 from ...enumeration import ComponentEnum
 
-
 # ---------------------------------------------------------------------------
 # Memory accounting
 # ---------------------------------------------------------------------------

reme/steps/index/watch_changes.py

@@ -36,7 +36,6 @@ from ._watch_rules import WatchRule, build_context_watch_rules, match_file
 from ..base_step import BaseStep
 from ...components import R
 
-
 DEFAULT_WATCH_DEBOUNCE_MS = 5_000
 DEFAULT_WATCH_STEP_MS = 1_000
 DEFAULT_LOW_POWER_POLL_MS = 5_000

reme/steps/transfer/download.py

@@ -24,7 +24,6 @@ from ..base_step import BaseStep
 
 from ...components import R
 
-
 _TEMP_ROOT: Path | None = None
 
 

tests/unit/test_base_component.py

@@ -11,7 +11,6 @@ import pytest
 from reme.components.base_component import BaseComponent, ComponentMixin, Dependency
 from reme.enumeration import ComponentEnum
 
-
 # -- Test subclasses ----------------------------------------------------------
 
 

tests/unit/test_default_file_chunker.py

@@ -7,7 +7,6 @@ import tempfile
 from reme.components.file_chunker import DefaultFileChunker
 from reme.utils.wikilink_handler import WikilinkHandler
 
-
 # Add parent path for import
 
 

tests/unit/test_http_client.py

@@ -0,0 +1,21 @@
+"""Tests for HTTP client display formatting."""
+
+# pylint: disable=protected-access
+
+from reme.components.client.http_client import HttpClient
+
+
+def test_format_for_display_hides_metadata_by_default():
+    """Response metadata is available structurally but not shown in normal CLI output."""
+    client = HttpClient()
+    text = '{"answer":"0.4.0.7","success":true,"metadata":{"version":"0.4.0.7"}}'
+
+    assert client._format_for_display(text) == "0.4.0.7\n✅"
+
+
+def test_format_for_display_shows_metadata_when_requested():
+    """show_metadata=true opts into metadata display."""
+    client = HttpClient(show_metadata=True)
+    text = '{"answer":"0.4.0.7","success":true,"metadata":{"version":"0.4.0.7"}}'
+
+    assert client._format_for_display(text) == '0.4.0.7\n✅ {"version": "0.4.0.7"}'

tests/unit/test_job.py

@@ -19,7 +19,6 @@ from reme.components.job.stream_job import StreamJob
 from reme.components.job import cron_job as cron_job_module
 from reme.schema import ComponentConfig
 
-
 # -- helpers ------------------------------------------------------------------
 
 

tests/unit/test_prompt_handler.py

@@ -11,7 +11,6 @@ import yaml
 
 from reme.components.prompt_handler import PromptHandler
 
-
 # -- init & load_prompt_dict --------------------------------------------------
 
 

tests/unit/test_reme_cli.py

@@ -25,6 +25,7 @@ def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys):
             yield "ok"
 
     monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
+    monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
 
     async def run():
         await reme_module.call_server(
@@ -44,3 +45,40 @@ def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys):
     assert seen["action"] == "search"
     assert seen["payload"] == {"query": "hello"}
     assert capsys.readouterr().out == "ok\n"
+
+
+def test_call_server_treats_show_metadata_as_client_kwarg(monkeypatch, capsys):
+    """show_metadata controls client display and is not sent as a tool argument."""
+    seen = {}
+
+    class FakeClient:
+        """Async client stub that records call arguments."""
+
+        def __init__(self, **kwargs):
+            seen["client_kwargs"] = kwargs
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, exc_type, exc_val, exc_tb):
+            return None
+
+        async def __call__(self, action: str, **kwargs):
+            seen["action"] = action
+            seen["payload"] = kwargs
+            yield "ok"
+
+    monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
+    monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
+
+    async def run():
+        await reme_module.call_server("version", backend="http", show_metadata=True)
+
+    import asyncio
+
+    asyncio.run(run())
+
+    assert seen["client_kwargs"] == {"show_metadata": True}
+    assert seen["action"] == "version"
+    assert seen["payload"] == {}
+    assert capsys.readouterr().out == "ok\n"

tests/unit/test_runtime_context.py

@@ -9,7 +9,6 @@ import pytest
 from reme.components.runtime_context import RuntimeContext
 from reme.enumeration import ChunkEnum
 
-
 # -- dict-like access ---------------------------------------------------------
 
 

tests/unit/test_service_utils.py

@@ -20,7 +20,6 @@ import psutil
 
 from reme.utils import service_utils as su
 
-
 # ----------------------------------------------------------------------
 # Fakes for mocking psutil.process_iter
 # ----------------------------------------------------------------------