.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,97 +0,0 @@
-name: Bug report
-description: Report reproducible incorrect or unexpected ReMe behavior
-title: "[Bug]: "
-labels: [bug]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thanks for helping improve ReMe. Please remove secrets, API keys, and private memory content before submitting.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Description
-      description: What happened, and what did you expect instead?
-      placeholder: Describe the observed and expected behavior.
-    validations:
-      required: true
-
-  - type: textarea
-    id: reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Provide the smallest configuration and command sequence that reproduces the problem.
-      placeholder: |
-        1. Configure ...
-        2. Run ...
-        3. Observe ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: config
-    attributes:
-      label: Relevant configuration
-      description: Include only relevant values and redact credentials, tokens, endpoints, and private paths.
-      render: yaml
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or traceback
-      description: Paste relevant output after removing secrets and private workspace content.
-      render: shell
-
-  - type: input
-    id: reme-version
-    attributes:
-      label: ReMe version
-      placeholder: e.g. 0.4.1.8 or a commit SHA
-    validations:
-      required: true
-
-  - type: input
-    id: python-version
-    attributes:
-      label: Python version
-      placeholder: e.g. 3.11.9
-    validations:
-      required: true
-
-  - type: dropdown
-    id: os
-    attributes:
-      label: Operating system
-      options:
-        - Linux
-        - macOS
-        - Windows
-        - Other
-    validations:
-      required: true
-
-  - type: dropdown
-    id: area
-    attributes:
-      label: Affected area
-      options:
-        - CLI or configuration
-        - HTTP, MCP, or local service
-        - Memory or workspace files
-        - Search, catalog, graph, or index
-        - Model or agent integration
-        - ReMe Studio
-        - Plugin or external integration
-        - Packaging or installation
-        - Other
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: safety
-    attributes:
-      label: Data safety
-      options:
-        - label: I removed credentials and private memory content from this report.
-          required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: ReMe documentation
-    url: https://reme.agentscope.io
-    about: Read the installation, configuration, and usage guides.
-  - name: Existing issues
-    url: https://github.com/agentscope-ai/ReMe/issues
-    about: Search for existing reports and discussions before opening a new issue.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,64 +0,0 @@
-name: Feature request
-description: Propose a focused enhancement to ReMe
-title: "[Feature]: "
-labels: [enhancement]
-body:
-  - type: textarea
-    id: problem
-    attributes:
-      label: Problem
-      description: What user problem or limitation should this change address?
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposal
-    attributes:
-      label: Proposed behavior
-      description: Describe the desired behavior and its user-visible contract.
-    validations:
-      required: true
-
-  - type: dropdown
-    id: area
-    attributes:
-      label: Area
-      options:
-        - CLI or configuration
-        - Jobs or steps
-        - Memory or workspace files
-        - Search, catalog, graph, or index
-        - Service or client
-        - Model or agent integration
-        - ReMe Studio
-        - Plugin or external integration
-        - Documentation
-        - Other
-    validations:
-      required: true
-
-  - type: textarea
-    id: ownership
-    attributes:
-      label: Local-first and compatibility considerations
-      description: Explain any effect on user-owned files, rebuildable state, configuration, schemas, or service interfaces.
-
-  - type: textarea
-    id: alternatives
-    attributes:
-      label: Alternatives considered
-      description: Describe workarounds or alternative designs you considered.
-
-  - type: textarea
-    id: examples
-    attributes:
-      label: Example usage
-      description: Show the proposed CLI, configuration, API, or UI behavior when useful.
-      render: shell
-
-  - type: checkboxes
-    id: contribution
-    attributes:
-      label: Contribution
-      options:
-        - label: I am willing to help implement or test this feature.

.github/ISSUE_TEMPLATE/question.yml

@@ -1,53 +0,0 @@
-name: Usage question
-description: Ask for help using or configuring ReMe
-title: "[Question]: "
-labels: [question]
-body:
-  - type: markdown
-    attributes:
-      value: Please check the documentation and existing issues before asking a new question.
-
-  - type: textarea
-    id: goal
-    attributes:
-      label: What are you trying to achieve?
-    validations:
-      required: true
-
-  - type: textarea
-    id: attempted
-    attributes:
-      label: What have you tried?
-      description: Include relevant commands or configuration, with secrets and private memory content removed.
-    validations:
-      required: true
-
-  - type: input
-    id: reme-version
-    attributes:
-      label: ReMe version
-      placeholder: e.g. 0.4.1.8 or a commit SHA
-
-  - type: dropdown
-    id: area
-    attributes:
-      label: Area
-      options:
-        - Installation
-        - Configuration
-        - CLI or service usage
-        - Memory and workspace management
-        - Search and retrieval
-        - ReMe Studio
-        - Plugin or integration
-        - Other
-
-  - type: checkboxes
-    id: checked
-    attributes:
-      label: Before submitting
-      options:
-        - label: I checked the [ReMe documentation](https://reme.agentscope.io) and searched existing issues.
-          required: true
-        - label: I removed credentials and private memory content.
-          required: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,35 +0,0 @@
-## Summary
-
-<!-- Explain the problem and the smallest coherent change that addresses it. -->
-
-## Related issue
-
-<!-- Use "Fixes #123" when applicable. -->
-
-## Contract and data impact
-
-- [ ] No public configuration, schema, CLI, endpoint, streaming, or workspace-layout contract changes
-- [ ] No user-owned memory files are deleted or rewritten
-- [ ] Derived indexes, catalogs, graphs, caches, and metadata remain rebuildable
-
-<!-- If any item is unchecked, describe the impact and migration or recovery path. -->
-
-## Validation
-
-<!-- List the exact checks run and their results. Explain relevant checks that were not run. -->
-
-- [ ] Focused tests pass
-- [ ] Unit tests pass, or omitted tests are explained below
-- [ ] `pre-commit run --all-files` passes, or omitted checks are explained below
-- [ ] Frontend checks were run when `reme_studio/` changed
-
-## Checklist
-
-- [ ] I reviewed the diff for unrelated changes and sensitive data
-- [ ] Tests cover intentional behavior changes
-- [ ] Defaults, schemas, and concise documentation were updated together when required
-- [ ] Long-lived clients, tasks, services, and executors follow the application lifecycle
-
-## Screenshots or additional notes
-
-<!-- Include UI screenshots, compatibility notes, or follow-up work when relevant. -->

.github/workflows/_build-docs.yml

@@ -1,58 +0,0 @@
-name: _Build documentation
-
-on:
-  workflow_call:
-    inputs:
-      run_tests:
-        description: Run the documentation test suite before building
-        required: false
-        default: true
-        type: boolean
-      upload_pages_artifact:
-        description: Upload the build for a later GitHub Pages deployment job
-        required: false
-        default: false
-        type: boolean
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build documentation
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: github-pages
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Node
-        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: '22.22.3'
-          cache: npm
-          cache-dependency-path: github-pages/package-lock.json
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Run tests
-        if: inputs.run_tests
-        run: npm test
-
-      - name: Build documentation
-        run: npm run build
-
-      - name: Configure Pages
-        if: inputs.upload_pages_artifact
-        uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6
-
-      - name: Upload Pages artifact
-        if: inputs.upload_pages_artifact
-        uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
-        with:
-          path: github-pages/dist

.github/workflows/_build-python-packages.yml

@@ -1,88 +0,0 @@
-name: _Build Python packages
-
-on:
-  workflow_call:
-    inputs:
-      expected_version:
-        description: Expected release version; omit for a consistency-only check
-        required: false
-        default: ''
-        type: string
-      upload_artifacts:
-        description: Upload distributions for later publish jobs
-        required: false
-        default: false
-        type: boolean
-
-permissions:
-  contents: read
-
-jobs:
-  distributions:
-    name: Build Python distributions
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: '3.11'
-
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install build packaging pytest twine
-
-      - name: Validate package versions
-        if: inputs.expected_version == ''
-        run: python scripts/bump_version.py --check
-
-      - name: Validate release version
-        if: inputs.expected_version != ''
-        env:
-          EXPECTED_VERSION: ${{ inputs.expected_version }}
-        run: python scripts/bump_version.py --check --expected-version "${EXPECTED_VERSION}"
-
-      - name: Run package tests
-        run: PYTHONPATH=. python -m pytest tests/unit/test_package_versions.py -q
-
-      - name: Build and check distributions
-        run: |
-          mkdir -p dist/reme
-          python -m build --outdir dist/reme
-          python -m twine check dist/reme/*
-
-      - name: Verify distributions and isolated installation
-        run: |
-          REME_WHEEL="$(pwd)/$(ls dist/reme/reme_ai-[0-9]*.whl)"
-          python -m zipfile -l "${REME_WHEEL}" | (! grep 'reme/web/')
-          python -m zipfile -l "${REME_WHEEL}" | (! grep 'reme_studio/')
-          python -m venv "${RUNNER_TEMP}/reme-package-smoke"
-          "${RUNNER_TEMP}/reme-package-smoke/bin/python" -m pip install "${REME_WHEEL}[as]"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-package-smoke/bin/python" -c "import reme"
-
-      - name: Verify released core dependencies
-        if: inputs.expected_version != ''
-        run: |
-          REME_WHEEL="$(pwd)/$(ls dist/reme/reme_ai-[0-9]*.whl)"
-          python -m venv "${RUNNER_TEMP}/reme-core-package-smoke"
-          "${RUNNER_TEMP}/reme-core-package-smoke/bin/python" -m pip install "${REME_WHEEL}[core]"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-core-package-smoke/bin/python" - <<'PY'
-          from reme_studio import static_dir
-
-          assert (static_dir() / "index.html").is_file()
-          PY
-
-      - name: Upload ReMe distributions
-        if: inputs.upload_artifacts
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: reme-distributions
-          path: dist/reme/
-          if-no-files-found: error

.github/workflows/ci-docs.yml

@@ -1,48 +0,0 @@
-name: CI / Documentation
-
-on:
-  push:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-docs.yml'
-      - '.github/workflows/_build-docs.yml'
-      - 'AGENTS.md'
-      - 'README.md'
-      - 'README_ZH.md'
-      - 'docs/**'
-      - 'github-pages/**'
-      - 'reme_studio/README*.md'
-      - 'reme_studio/public/og.jpg'
-      - 'typescript/README*.md'
-      - 'plugins/*/README*.md'
-      - 'benchmark/*/README*.md'
-  pull_request:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-docs.yml'
-      - '.github/workflows/_build-docs.yml'
-      - 'AGENTS.md'
-      - 'README.md'
-      - 'README_ZH.md'
-      - 'docs/**'
-      - 'github-pages/**'
-      - 'reme_studio/README*.md'
-      - 'reme_studio/public/og.jpg'
-      - 'typescript/README*.md'
-      - 'plugins/*/README*.md'
-      - 'benchmark/*/README*.md'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  documentation:
-    name: Test and build documentation
-    uses: ./.github/workflows/_build-docs.yml
-    with:
-      run_tests: true

.github/workflows/ci-packages.yml

@@ -1,40 +0,0 @@
-name: CI / Python packages
-
-on:
-  push:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-packages.yml'
-      - '.github/workflows/_build-python-packages.yml'
-      - '.github/workflows/release-python.yml'
-      - 'pyproject.toml'
-      - 'README.md'
-      - 'reme/**'
-      - 'scripts/bump_version.py'
-      - 'tests/unit/test_package_versions.py'
-      - 'LICENSE'
-  pull_request:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-packages.yml'
-      - '.github/workflows/_build-python-packages.yml'
-      - '.github/workflows/release-python.yml'
-      - 'pyproject.toml'
-      - 'README.md'
-      - 'reme/**'
-      - 'scripts/bump_version.py'
-      - 'tests/unit/test_package_versions.py'
-      - 'LICENSE'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  distributions:
-    name: Build and verify distributions
-    uses: ./.github/workflows/_build-python-packages.yml

.github/workflows/ci-python-quality.yml

@@ -1,40 +0,0 @@
-name: CI / Python quality
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  pre-commit:
-    name: Pre-commit
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Setup Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: '3.11'
-          cache: pip
-
-      - name: Update setuptools
-        run: |
-          pip install -U setuptools wheel
-
-      - name: Install
-        run: |
-          pip install -q -e reme_studio -e ".[dev,core]"
-          pip install -q --no-deps -e plugins/auto-fin -e plugins/daily_paper
-
-      - name: Pre-commit starts
-        run: pre-commit run --all-files

.github/workflows/ci-python-tests.yml

@@ -1,54 +0,0 @@
-name: CI / Python tests
-
-on:
-  push:
-    branches: [main, master, dev, develop]
-  pull_request:
-    branches: [main, master, dev, develop]
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  unit-tests:
-    name: Unit Tests - py${{ matrix.python-version }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11", "3.12", "3.13"]
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install -e reme_studio -e ".[dev,core]"
-          pip install --no-deps -e plugins/auto-fin
-          pip install -e plugins/daily_paper
-          pip install coverage
-
-      - name: Run unit tests
-        run: |
-          coverage run -m pytest tests/unit plugins/auto-fin plugins/daily_paper \
-            -v \
-            --tb=long \
-            -s \
-            --log-cli-level=WARNING
-
-      - name: Generate coverage report
-        run: coverage report -m

.github/workflows/ci-reme-studio.yml

@@ -1,90 +0,0 @@
-name: CI / ReMe Studio
-
-on:
-  push:
-    paths:
-      - "reme_studio/**"
-      - ".github/workflows/ci-reme-studio.yml"
-      - ".github/workflows/release-reme-studio.yml"
-      - "scripts/package_studio.py"
-      - "tests/unit/test_package_versions.py"
-      - "pyproject.toml"
-      - "LICENSE"
-  pull_request:
-    paths:
-      - "reme_studio/**"
-      - ".github/workflows/ci-reme-studio.yml"
-      - ".github/workflows/release-reme-studio.yml"
-      - "scripts/package_studio.py"
-      - "tests/unit/test_package_versions.py"
-      - "pyproject.toml"
-      - "LICENSE"
-
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  studio:
-    name: Studio checks
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: reme_studio
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Setup Node
-        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: "22.22.3"
-          cache: npm
-          cache-dependency-path: reme_studio/package-lock.json
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Run format check
-        run: npm run format:check
-
-      - name: Run lint
-        run: npm run lint
-
-      - name: Run tests
-        run: npm test
-
-      - name: Verify npm package
-        run: |
-          npm pack --pack-destination "${RUNNER_TEMP}"
-          tar -tzf "${RUNNER_TEMP}"/agentscope-ai-reme_studio-*.tgz | grep '^package/dist-static/index.html$'
-
-      - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: "3.11"
-
-      - name: Build and verify Python package
-        working-directory: .
-        run: |
-          python -m pip install build packaging pytest twine
-          PYTHONPATH=. python -m pytest tests/unit/test_package_versions.py -q
-          python scripts/package_studio.py
-          python -m build reme_studio --outdir dist/studio
-          python -m twine check dist/studio/*
-          STUDIO_WHEEL="$(pwd)/$(ls dist/studio/reme_studio-*.whl)"
-          python -m venv "${RUNNER_TEMP}/reme-studio-package-smoke"
-          "${RUNNER_TEMP}/reme-studio-package-smoke/bin/python" -m pip install "${STUDIO_WHEEL}"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-studio-package-smoke/bin/python" - <<'PY'
-          from reme_studio import static_dir
-
-          assert (static_dir() / "index.html").is_file()
-          PY

.github/workflows/ci-typescript.yml

@@ -1,51 +0,0 @@
-name: CI / TypeScript integrations
-
-on:
-  push:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-typescript.yml'
-      - '.github/workflows/release-typescript.yml'
-      - 'typescript/**'
-  pull_request:
-    branches: [main, master, dev, develop]
-    paths:
-      - '.github/workflows/ci-typescript.yml'
-      - '.github/workflows/release-typescript.yml'
-      - 'typescript/**'
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  package:
-    name: Type-check, test, and pack
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: typescript
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: '22.22.3'
-          cache: npm
-          cache-dependency-path: typescript/package-lock.json
-
-      - run: npm ci
-      - run: npm run format:check
-      - run: npm run lint
-      - run: npm run typecheck
-      - run: npm test
-      - run: npm run test:package
-      - name: Validate OpenClaw package contract
-        run: npx --yes clawhub@0.23.3 package validate . --json

.github/workflows/ci-windows.yml

@@ -1,51 +0,0 @@
-name: CI / Windows
-
-on:
-  push:
-    branches: [main, master, dev, develop]
-  pull_request:
-    branches: [main, master, dev, develop]
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  cli-smoke:
-    name: CLI smoke - py${{ matrix.python-version }}
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11"]
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: Install package
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install -e ".[dev,as]"
-
-      - name: Run version job
-        run: reme start config=tests/fixtures/config/version-smoke.yaml job=version
-
-      - name: Run Windows path tests
-        run: |
-          python -m pytest `
-            tests/unit/test_auto_dream.py::test_scan_day_files_includes_nested_md_and_excludes_interests `
-            tests/unit/test_auto_dream.py::test_dream_extract_matches_posix_catalog_paths `
-            tests/unit/test_read_with_neighbors.py::test_read_with_neighbors_uses_posix_nested_path `
-            -v

.github/workflows/deploy-docs.yml

@@ -1,52 +0,0 @@
-name: Deploy / Documentation
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - "github-pages/**"
-      - "docs/**"
-      - "README.md"
-      - "README_ZH.md"
-      - "reme_studio/README*.md"
-      - "reme_studio/public/og.jpg"
-      - "typescript/README*.md"
-      - "plugins/*/README*.md"
-      - "benchmark/*/README*.md"
-      - "AGENTS.md"
-      - ".github/workflows/deploy-docs.yml"
-      - ".github/workflows/_build-docs.yml"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: pages
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Build documentation
-    uses: ./.github/workflows/_build-docs.yml
-    with:
-      run_tests: true
-      upload_pages_artifact: true
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    permissions:
-      pages: write
-      id-token: write
-    steps:
-      - name: Deploy
-        id: deployment
-        uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5

.github/workflows/policy-pr-title.yml

@@ -1,40 +0,0 @@
-name: Policy / PR title
-
-on:
-  pull_request:
-    branches: [main, master, dev, develop]
-    types: [opened, edited, synchronize, reopened]
-
-permissions:
-  contents: read
-  pull-requests: read
-
-jobs:
-  check-pr-title:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check PR title format
-        uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          types: |
-            feat
-            fix
-            docs
-            ci
-            refactor
-            test
-            chore
-            perf
-            style
-            build
-            revert
-          requireScope: false
-          scopePattern: ^[a-z0-9_-]+$
-          scopePatternError: |
-            The scope must contain only lowercase letters, numbers, hyphens, and underscores.
-            Example: "feat(memory): add redis cache support"
-          validateSingleCommit: false
-          ignoreLabels: |
-            ignore-semantic-pull-request

.github/workflows/pre-commit.yml

@@ -0,0 +1,38 @@
+name: Pre-commit
+
+on: [ push, pull_request ]
+
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: True
+      matrix:
+        os: [ ubuntu-latest ]
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.10'
+    steps:
+      - uses: actions/checkout@master
+      - name: Setup Python
+        uses: actions/setup-python@master
+        with:
+          python-version: '3.10'
+      - name: Update setuptools
+        run: |
+          pip install -U setuptools wheel
+      - name: Install
+        run: |
+          pip install -q -e .[dev]
+      - name: Install pre-commit
+        run: |
+          pre-commit install
+      - name: Pre-commit starts
+        run: |
+          pre-commit run --all-files > pre-commit.log 2>&1 || true
+          cat pre-commit.log
+          if grep -q Failed pre-commit.log; then
+            echo -e "\e[41m  [**FAIL**] Please install pre-commit and format your code first. \e[0m"
+            exit 1
+          fi
+          echo -e "\e[46m  ********************************Passed******************************** \e[0m"

.github/workflows/python-publish.yml

@@ -0,0 +1,40 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish Python Package to Pypi
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel build
+    - name: Build package
+      run: python -m build
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/release-auto-fin.yml

@@ -1,157 +0,0 @@
-# 发布操作手册：
-# 1. 先将 plugins/auto-fin/pyproject.toml 中的 project.version 更新为待发布版本并合入目标分支。
-# 2. 确认插件依赖的 reme-ai 版本已经发布到 PyPI；本工作流会在构建阶段验证该依赖可下载。
-# 3. 确认 PyPI Trusted Publisher 已绑定本仓库、此工作流和 pypi environment，且 PyPI 上不存在相同版本。
-# 4. 在 GitHub 仓库的 Actions 页面选择“Release / Auto Fin plugin”，点击“Run workflow”。
-# 5. 输入与 project.version 完全一致的版本号（例如 0.1.0）后运行；版本也可以带 v 前缀。
-#
-# 推荐发布顺序：reme-ai -> reme-auto-fin -> QwenPaw 更新依赖并通过 plugins: [auto-fin] 启用。
-# 当前仅支持 workflow_dispatch 手动触发，不会因 push、tag 或 release 自动发布。
-
-name: Release / Auto Fin plugin
-
-run-name: Publish reme-auto-fin ${{ inputs.version }}
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Version from plugins/auto-fin/pyproject.toml (for example, 0.1.0)
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: publish-reme-auto-fin
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    env:
-      RELEASE_VERSION: ${{ inputs.version }}
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: '3.11'
-
-      - name: Install test and build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install build packaging pytest pytest-asyncio twine
-          python -m pip install -e ".[core]"
-          python -m pip install --no-deps -e plugins/auto-fin
-
-      - name: Validate package name and release version
-        id: package
-        run: |
-          python - "${RELEASE_VERSION}" <<'PY'
-          import os
-          import sys
-          import tomllib
-          from pathlib import Path
-
-          from packaging.requirements import Requirement
-          from packaging.version import Version
-
-          project = tomllib.loads(Path("plugins/auto-fin/pyproject.toml").read_text(encoding="utf-8"))["project"]
-          expected = Version(sys.argv[1].removeprefix("v"))
-          actual = Version(project["version"])
-          if project["name"] != "reme-auto-fin":
-              raise SystemExit(f"Expected project name 'reme-auto-fin', found {project['name']!r}")
-          if actual != expected:
-              raise SystemExit(f"Package version is {actual}, but workflow input is {expected}")
-          requirements = [requirement for requirement in project["dependencies"] if requirement.startswith("reme-ai")]
-          if len(requirements) != 1:
-              raise SystemExit(f"Expected one reme-ai dependency, found {requirements!r}")
-          reme_requirement = Requirement(requirements[0])
-          if reme_requirement.name != "reme-ai" or reme_requirement.extras:
-              raise SystemExit(f"Expected a base reme-ai dependency, found {requirements[0]!r}")
-          if Version("0.4.1.8") in reme_requirement.specifier or Version("0.4.1.9") not in reme_requirement.specifier:
-              raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {requirements[0]!r}")
-          with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
-              print(f"reme_requirement={reme_requirement}", file=output)
-          print(f"Publishing {project['name']} {actual}")
-          PY
-
-      - name: Run Auto Fin tests
-        run: python -m pytest plugins/auto-fin -q
-
-      - name: Require the plugin-enabled ReMe release on PyPI
-        env:
-          REME_REQUIREMENT: ${{ steps.package.outputs.reme_requirement }}
-        run: |
-          python -m pip download --no-deps \
-            --dest "${RUNNER_TEMP}/reme-auto-fin-base" \
-            "${REME_REQUIREMENT}"
-
-      - name: Build and check distributions
-        run: |
-          mkdir -p dist/auto-fin
-          python -m build plugins/auto-fin --outdir dist/auto-fin
-          python -m twine check dist/auto-fin/*
-
-      - name: Verify distributions and isolated installation
-        run: |
-          AUTO_FIN_WHEEL="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.whl)"
-          AUTO_FIN_SDIST="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.tar.gz)"
-          python -m zipfile -l "${AUTO_FIN_WHEEL}" | grep 'dist-info/licenses/LICENSE'
-          python -m tarfile -l "${AUTO_FIN_SDIST}" | grep '/LICENSE'
-          python -m venv "${RUNNER_TEMP}/reme-auto-fin-smoke"
-          "${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" -m pip install \
-            "agentscope[model-ollama]==2.0.7" "${AUTO_FIN_WHEEL}"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" - <<'PY'
-          from importlib.metadata import distribution
-
-          from reme.plugin_manifest import load_package_manifest
-
-          package = distribution("reme-auto-fin")
-          plugins = {entry.name: entry for entry in package.entry_points if entry.group == "reme.plugins"}
-          assert plugins["auto-fin"].value == "reme_auto_fin"
-          manifest = load_package_manifest("reme_auto_fin", plugin_name="auto-fin")
-          assert set(manifest.backends) == {
-              "auto_fin_data_step",
-              "auto_fin_topic_step",
-              "auto_fin_merge_step",
-          }
-          assert set(manifest.application_defaults["jobs"]) == {
-              "auto_fin",
-              "auto_fin_cron",
-          }
-          PY
-
-      - name: Upload distributions
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: reme-auto-fin-${{ inputs.version }}
-          path: dist/auto-fin/
-          if-no-files-found: error
-
-  publish:
-    needs: build
-    runs-on: ubuntu-latest
-    environment: pypi
-    permissions:
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Download distributions
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: reme-auto-fin-${{ inputs.version }}
-          path: dist/auto-fin
-
-      - name: Publish reme-auto-fin
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
-        with:
-          packages-dir: dist/auto-fin

.github/workflows/release-daily-paper.yml

@@ -1,157 +0,0 @@
-# Release checklist:
-# 1. Update project.version in plugins/daily_paper/pyproject.toml and merge it into the target branch.
-# 2. Publish the required reme-ai version before this plugin; the build verifies that dependency on PyPI.
-# 3. Configure PyPI Trusted Publishing for this repository/workflow and its pypi environment.
-# 4. Run "Release / Daily Paper plugin" from GitHub Actions with the exact project version (a v prefix is accepted).
-#
-# Recommended order: reme-ai -> reme-daily-paper -> downstream applications enabling plugins: [daily-paper].
-# This workflow is intentionally manual and never publishes from a push, tag, or GitHub release event.
-
-name: Release / Daily Paper plugin
-
-run-name: Publish reme-daily-paper ${{ inputs.version }}
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Version from plugins/daily_paper/pyproject.toml (for example, 0.1.0)
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: publish-reme-daily-paper
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    env:
-      RELEASE_VERSION: ${{ inputs.version }}
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: '3.11'
-
-      - name: Install test and build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install build packaging pytest pytest-asyncio twine
-          python -m pip install -e ".[core]"
-          python -m pip install -e plugins/daily_paper
-
-      - name: Validate package name, dependencies, and release version
-        id: package
-        run: |
-          python - "${RELEASE_VERSION}" <<'PY'
-          import os
-          import sys
-          import tomllib
-          from pathlib import Path
-
-          from packaging.requirements import Requirement
-          from packaging.version import Version
-
-          project = tomllib.loads(Path("plugins/daily_paper/pyproject.toml").read_text(encoding="utf-8"))["project"]
-          expected = Version(sys.argv[1].removeprefix("v"))
-          actual = Version(project["version"])
-          if project["name"] != "reme-daily-paper":
-              raise SystemExit(f"Expected project name 'reme-daily-paper', found {project['name']!r}")
-          if actual != expected:
-              raise SystemExit(f"Package version is {actual}, but workflow input is {expected}")
-          requirements = [Requirement(value) for value in project["dependencies"]]
-          reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"]
-          if len(reme_requirements) != 1 or reme_requirements[0].extras:
-              raise SystemExit(f"Expected one base reme-ai dependency, found {reme_requirements!r}")
-          if Version("0.4.1.8") in reme_requirements[0].specifier or Version("0.4.1.9") not in reme_requirements[0].specifier:
-              raise SystemExit(f"Expected reme-ai>=0.4.1.9, found {reme_requirements!r}")
-          if sum(requirement.name == "pypdf" for requirement in requirements) != 1:
-              raise SystemExit("Expected exactly one pypdf dependency")
-          with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
-              print(f"reme_requirement={reme_requirements[0]}", file=output)
-          print(f"Publishing {project['name']} {actual}")
-          PY
-
-      - name: Run Daily Paper tests
-        run: python -m pytest plugins/daily_paper -q
-
-      - name: Require the plugin-enabled ReMe release on PyPI
-        env:
-          REME_REQUIREMENT: ${{ steps.package.outputs.reme_requirement }}
-        run: |
-          python -m pip download --no-deps \
-            --dest "${RUNNER_TEMP}/reme-daily-paper-base" \
-            "${REME_REQUIREMENT}"
-
-      - name: Build and check distributions
-        run: |
-          mkdir -p dist/daily-paper
-          python -m build plugins/daily_paper --outdir dist/daily-paper
-          python -m twine check dist/daily-paper/*
-
-      - name: Verify distributions and isolated installation
-        run: |
-          DAILY_PAPER_WHEEL="$(pwd)/$(ls dist/daily-paper/reme_daily_paper-*.whl)"
-          DAILY_PAPER_SDIST="$(pwd)/$(ls dist/daily-paper/reme_daily_paper-*.tar.gz)"
-          python -m zipfile -l "${DAILY_PAPER_WHEEL}" | grep 'reme_daily_paper/plugin.yaml'
-          python -m zipfile -l "${DAILY_PAPER_WHEEL}" | grep 'reme_daily_paper/analyze.yaml'
-          python -m zipfile -l "${DAILY_PAPER_WHEEL}" | grep 'dist-info/licenses/LICENSE'
-          python -m tarfile -l "${DAILY_PAPER_SDIST}" | grep '/LICENSE'
-          python -m venv "${RUNNER_TEMP}/reme-daily-paper-smoke"
-          "${RUNNER_TEMP}/reme-daily-paper-smoke/bin/python" -m pip install \
-            "agentscope[model-ollama]==2.0.7" "${DAILY_PAPER_WHEEL}"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-daily-paper-smoke/bin/python" - <<'PY'
-          from importlib.metadata import distribution
-
-          from reme.plugin_manifest import load_package_manifest
-
-          package = distribution("reme-daily-paper")
-          plugins = {entry.name: entry for entry in package.entry_points if entry.group == "reme.plugins"}
-          assert plugins["daily-paper"].value == "reme_daily_paper"
-          manifest = load_package_manifest("reme_daily_paper", plugin_name="daily-paper")
-          assert set(manifest.backends) == {
-              "daily_paper_collect_step",
-              "daily_paper_rank_step",
-              "daily_paper_select_step",
-              "daily_paper_analyze_step",
-              "daily_paper_digest_step",
-          }
-          assert set(manifest.application_defaults["jobs"]) == {"daily_paper", "daily_paper_cron"}
-          PY
-
-      - name: Upload distributions
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: reme-daily-paper-${{ inputs.version }}
-          path: dist/daily-paper/
-          if-no-files-found: error
-
-  publish:
-    needs: build
-    runs-on: ubuntu-latest
-    environment: pypi
-    permissions:
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Download distributions
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: reme-daily-paper-${{ inputs.version }}
-          path: dist/daily-paper
-
-      - name: Publish reme-daily-paper
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
-        with:
-          packages-dir: dist/daily-paper

.github/workflows/release-python.yml

@@ -1,47 +0,0 @@
-name: Release / Python packages
-
-# Configure a PyPI Trusted Publisher for this repository, workflow, and its
-# pypi environment before running the manual release.
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Release version
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: publish-reme-ai
-  cancel-in-progress: false
-
-jobs:
-  build:
-    name: Build and verify distributions
-    uses: ./.github/workflows/_build-python-packages.yml
-    with:
-      expected_version: ${{ inputs.version }}
-      upload_artifacts: true
-
-  publish-reme:
-    needs: build
-    runs-on: ubuntu-latest
-    environment: pypi
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - name: Download ReMe distributions
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: reme-distributions
-          path: dist/reme
-
-      - name: Publish ReMe
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
-        with:
-          packages-dir: dist/reme
-          skip-existing: true

.github/workflows/release-reme-studio.yml

@@ -1,158 +0,0 @@
-# Release checklist:
-# 1. Update reme_studio/pyproject.toml, package.json, and package-lock.json to the same Studio version.
-# 2. Configure npm Trusted Publishing and PyPI Trusted Publishing with the pypi environment.
-# 3. Run this workflow manually with the exact Studio version.
-
-name: Release / ReMe Studio
-
-run-name: Publish ReMe Studio ${{ inputs.version }} (${{ inputs.npm_tag }})
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Version from the Studio Python and npm manifests
-        required: true
-        type: string
-      npm_tag:
-        description: npm distribution tag
-        required: true
-        default: latest
-        type: choice
-        options:
-          - next
-          - latest
-
-permissions:
-  contents: read
-
-concurrency:
-  group: publish-reme-studio
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    env:
-      RELEASE_VERSION: ${{ inputs.version }}
-      NPM_TAG: ${{ inputs.npm_tag }}
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: "22.22.3"
-          cache: npm
-          cache-dependency-path: reme_studio/package-lock.json
-
-      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
-        with:
-          python-version: "3.11"
-
-      - name: Validate Studio package names and version
-        run: |
-          python - <<'PY'
-          import json
-          import os
-          import tomllib
-          from pathlib import Path
-
-          studio = Path("reme_studio")
-          python_manifest = tomllib.loads((studio / "pyproject.toml").read_text(encoding="utf-8"))["project"]
-          npm_manifest = json.loads((studio / "package.json").read_text(encoding="utf-8"))
-          expected = os.environ["RELEASE_VERSION"].removeprefix("v")
-          if python_manifest["name"] != "reme_studio":
-              raise SystemExit(f"Unexpected Python package name: {python_manifest['name']}")
-          if npm_manifest["name"] != "@agentscope-ai/reme_studio":
-              raise SystemExit(f"Unexpected npm package name: {npm_manifest['name']}")
-          if python_manifest["version"] != expected or npm_manifest["version"] != expected:
-              raise SystemExit(
-                  f"Studio manifests are {python_manifest['version']} and {npm_manifest['version']}; "
-                  f"workflow input is {expected}",
-              )
-          prerelease = "-" in expected
-          if prerelease != (os.environ["NPM_TAG"] == "next"):
-              raise SystemExit("Prereleases must use next; stable releases must use latest")
-          PY
-
-      - name: Install dependencies and run checks
-        working-directory: reme_studio
-        run: |
-          npm ci
-          npm run format:check
-          npm run lint
-          npm test
-
-      - name: Build Studio distributions
-        run: |
-          python -m pip install build twine
-          mkdir -p dist/studio-python dist/studio-npm
-          npm pack ./reme_studio --pack-destination dist/studio-npm
-          python scripts/package_studio.py
-          python -m build reme_studio --outdir dist/studio-python
-          python -m twine check dist/studio-python/*
-
-      - name: Verify Studio distributions and isolated installation
-        run: |
-          STUDIO_WHEEL="$(pwd)/$(ls dist/studio-python/reme_studio-*.whl)"
-          tar -tzf dist/studio-npm/*.tgz | grep '^package/dist-static/index.html$'
-          python -m venv "${RUNNER_TEMP}/reme-studio-package-smoke"
-          "${RUNNER_TEMP}/reme-studio-package-smoke/bin/python" -m pip install "${STUDIO_WHEEL}"
-          cd "${RUNNER_TEMP}"
-          "${RUNNER_TEMP}/reme-studio-package-smoke/bin/python" - <<'PY'
-          from reme_studio import static_dir
-
-          assert (static_dir() / "index.html").is_file()
-          PY
-
-      - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: reme-studio-${{ inputs.version }}
-          path: |
-            dist/studio-python/*
-            dist/studio-npm/*
-          if-no-files-found: error
-
-  publish-python:
-    needs: build
-    runs-on: ubuntu-latest
-    environment: pypi
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: reme-studio-${{ inputs.version }}
-          path: dist
-
-      - name: Publish ReMe Studio to PyPI
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
-        with:
-          packages-dir: dist/studio-python
-          skip-existing: true
-
-  publish-npm:
-    needs: build
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      id-token: write
-    steps:
-      - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: "24"
-          registry-url: https://registry.npmjs.org
-
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: reme-studio-${{ inputs.version }}
-          path: dist
-
-      - name: Publish ReMe Studio to npm
-        env:
-          NPM_TAG: ${{ inputs.npm_tag }}
-        run: npm publish dist/studio-npm/*.tgz --access public --tag "${NPM_TAG}" --provenance

.github/workflows/release-typescript.yml

@@ -1,167 +0,0 @@
-# Release checklist:
-# 1. Update typescript/package.json and package-lock.json to the release version and merge them.
-# 2. Configure npm Trusted Publishing for agentscope-ai/ReMe and this workflow file.
-# 3. Run this workflow manually with the exact package version (an optional v prefix is accepted).
-# 4. Configure ClawHub Trusted Publishing or CLAWHUB_TOKEN before enabling ClawHub publication.
-# 5. Use the `next` tag for prereleases and `latest` only for stable releases.
-
-name: Release / TypeScript integrations
-
-run-name: Publish @agentscope-ai/reme ${{ inputs.version }} (${{ inputs.npm_tag }})
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Version from typescript/package.json (for example, 0.1.0)
-        required: true
-        type: string
-      npm_tag:
-        description: npm distribution tag
-        required: true
-        default: latest
-        type: choice
-        options:
-          - next
-          - latest
-      publish_clawhub:
-        description: Also publish the verified tarball to ClawHub
-        required: true
-        default: false
-        type: boolean
-
-permissions:
-  contents: read
-
-concurrency:
-  group: publish-agentscope-ai-reme
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.validate.outputs.version }}
-    env:
-      RELEASE_VERSION: ${{ inputs.version }}
-      NPM_TAG: ${{ inputs.npm_tag }}
-
-    steps:
-      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Set up Node
-        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: '22.22.3'
-
-      - name: Validate package name and release version
-        id: validate
-        working-directory: typescript
-        run: |
-          node --input-type=module <<'JS'
-          import { appendFileSync, readFileSync } from 'node:fs';
-
-          const manifest = JSON.parse(readFileSync('package.json', 'utf8'));
-          const expected = process.env.RELEASE_VERSION.replace(/^v/, '');
-          if (manifest.name !== '@agentscope-ai/reme') {
-            throw new Error(`Unexpected package name: ${manifest.name}`);
-          }
-          if (manifest.version !== expected) {
-            throw new Error(`package.json is ${manifest.version}, workflow input is ${expected}`);
-          }
-          const prerelease = manifest.version.includes('-');
-          const npmTag = process.env.NPM_TAG;
-          if (prerelease !== (npmTag === 'next')) {
-            throw new Error(prerelease
-              ? 'Prerelease versions must use the next npm tag'
-              : 'Stable versions must use the latest npm tag');
-          }
-          console.log(`Preparing ${manifest.name}@${manifest.version}`);
-          appendFileSync(process.env.GITHUB_OUTPUT, `version=${manifest.version}\n`);
-          JS
-
-      - name: Install dependencies
-        working-directory: typescript
-        run: npm ci
-
-      - name: Type-check and test
-        working-directory: typescript
-        run: |
-          npm run format:check
-          npm run lint
-          npm run typecheck
-          npm test
-          npm run test:package
-          npx --yes clawhub@0.23.3 package validate . --json
-
-      - name: Pack npm tarball
-        working-directory: typescript
-        run: |
-          mkdir -p "${RUNNER_TEMP}/reme-typescript-package"
-          npm pack --pack-destination "${RUNNER_TEMP}/reme-typescript-package"
-
-      - name: Upload npm tarball
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: agentscope-ai-reme-${{ inputs.version }}
-          path: ${{ runner.temp }}/reme-typescript-package/*.tgz
-          if-no-files-found: error
-
-  publish:
-    needs: build
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Set up Node for npm
-        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
-        with:
-          node-version: '24'
-          registry-url: https://registry.npmjs.org
-
-      - name: Download npm tarball
-        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-        with:
-          name: agentscope-ai-reme-${{ inputs.version }}
-          path: dist/typescript
-
-      - name: Reject an existing package version
-        env:
-          PACKAGE_VERSION: ${{ inputs.version }}
-        run: |
-          PACKAGE_VERSION="${PACKAGE_VERSION#v}"
-          if npm view "@agentscope-ai/reme@${PACKAGE_VERSION}" version >/dev/null 2>&1; then
-            echo "@agentscope-ai/reme@${PACKAGE_VERSION} already exists" >&2
-            exit 1
-          fi
-
-      - name: Publish to npm
-        env:
-          NPM_TAG: ${{ inputs.npm_tag }}
-        run: npm publish dist/typescript/*.tgz --access public --tag "${NPM_TAG}" --provenance
-
-  publish-clawhub:
-    if: ${{ inputs.publish_clawhub }}
-    needs: build
-    permissions:
-      actions: read
-      contents: read
-      id-token: write
-    uses: openclaw/clawhub/.github/workflows/package-publish.yml@87ca030c30f3cfb78ab15c8e66b5ff1469c8f9c8 # v0.23.3
-    with:
-      owner: agentscope-ai
-      family: code-plugin
-      version: ${{ needs.build.outputs.version }}
-      tags: ${{ inputs.npm_tag }}
-      source_repo: ${{ github.repository }}
-      source_commit: ${{ github.sha }}
-      source_ref: ${{ github.ref }}
-      source_path: typescript
-      package_artifact_name: agentscope-ai-reme-${{ inputs.version }}
-      wait_for_publication: true
-    secrets:
-      clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}

.github/workflows/security-codeql.yml

@@ -1,46 +0,0 @@
-name: Security / CodeQL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  schedule:
-    - cron: '0 1 * * 1'
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: read
-  security-events: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: Analyze ${{ matrix.language }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [python, javascript-typescript]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
-        with:
-          persist-credentials: false
-
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
-        with:
-          languages: ${{ matrix.language }}
-          build-mode: none
-
-      - name: Perform CodeQL analysis
-        uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4
-        with:
-          category: /language:${{ matrix.language }}

.gitignore

@@ -1,82 +1,45 @@
-# OS / editor
+.vscode
+.env*
 .DS_Store
-.idea/
-.vscode/
-.qoder/
-*.code-workspace
-
-# Local environment
-.env
-.env.*
-!.env.example
-!example.env
-.venv/
+.idea
 venv/
-env/
-private*/
-
-# Python caches / test artifacts
-__pycache__/
-*.py[cod]
-*$py.class
-.ipynb_checkpoints/
-.pytest_cache/
-.ruff_cache/
-.mypy_cache/
-.coverage
-coverage.xml
-htmlcov/
-
-# Packaging / build outputs
-build/
-dist/
-node_modules/
-*.egg-info/
-typescript/reports/
-
-# Logs / temporary files
+.ipynb_checkpoints
+.__pycache__
+__pycache__
 *.log
-nohup.out
-nohup*.out
+tmp*
+temp*
+private*
+dist/
+nohup*
+cache
 log/
-logs/
-runs/
-tmp*/
-temp*/
 .trash/
-
-# ReMe runtime data
-.reme/
-reme_workspace/
-reme_workspace_auto_fin_real_test*/
-vault/
-*.db
-*.sqlite
+runs
+logs
+rag_nodes_index.jsonl
+alfworld_data
+step_experiences/*
+build/*
+*.egg-info/*
+cookbook/appworld/data/*
+cookbook/appworld/experiments/*
+cookbook/appworld/exp_result/*
+file_vector_store/*
+cookbook/appworld/file_vector_store/*
+/.venv/
+site/*
+docs/_build/*
+test_compact_storage/*
+test_working_memory/*
+*.code-workspace
+local_vector_store/*
+reme_profile/*
+chroma_vector_store/*
+bench_results/*
+meta_memory/*
 *.sqlite3
-
-# Documentation build outputs
-docs/_build/
-site/
-
-evaluation/
-# The pi-Bench suite ships its own trace-history render config, which must
-# stay in git even though it lives under an evaluation/ directory.
-!benchmark/pibench/config/bench/evaluation/
-!benchmark/pibench/config/bench/evaluation/**
-datasets/
-
-# Claude Code skills (local only)
-.claude/skills/
-
-# Benchmark memory workspaces (created on demand by run.py via mkdir)
-benchmark/*/workspaces/
-
-# Benchmark datasets (LongMemEval via download.py, BEAM via git clone)
-benchmark/*/dataset/
-
-# Benchmark outputs (created on demand by run.py via mkdir)
-benchmark/*/results/
-
-# integration tests outputs
-tests/integration/logs/
-daily/
+**/data/*.json
+*.db
+memories/*
+.reme/*

.pre-commit-config.yaml

@@ -1,10 +1,9 @@
-exclude: ^skills/
-
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
     hooks:
       - id: check-ast
+        exclude: ^(test/|cookbook/|reme_ai/)
       - id: check-yaml
       - id: check-xml
       - id: check-toml
@@ -15,31 +14,37 @@ repos:
     rev: v4.0.0
     hooks:
       - id: add-trailing-comma
+        exclude: ^(test/|cookbook/|reme_ai/)
   - repo: https://github.com/psf/black
-    rev: 26.5.1
+    rev: 25.9.0
     hooks:
     - id: black
-      args: [--line-length=120, --target-version=py311]
+      exclude: ^(test/|cookbook/|reme_ai/)
+      args: [--line-length=120]
   - repo: https://github.com/PyCQA/flake8
     rev: 7.3.0
     hooks:
       - id: flake8
+        exclude: ^(test/|cookbook/|reme_ai/)
         args: [
           "--extend-ignore=E203",
           "--max-line-length=120"
         ]
   - repo: https://github.com/pylint-dev/pylint
-    rev: v4.0.6
+    rev: v4.0.2
     hooks:
       - id: pylint
         exclude:
             (?x)(
                 ^docs
+                | ^test/
+                | ^cookbook/
                 | pb2\.py$
                 | grpc\.py$
                 | \.demo$
                 | \.md$
                 | \.html$
+                | reme_ai/
           )
         args: [
           --disable=W0511,
@@ -78,7 +83,7 @@ repos:
           --max-module-lines=1500,
         ]
   - repo: https://github.com/regebro/pyroma
-    rev: "5.0.1"
+    rev: "5.0"
     hooks:
       - id: pyroma
         args: [--min=10, .]

AGENTS.md

@@ -1,214 +0,0 @@
-# AGENTS.md
-
-This file guides coding agents working in the ReMe repository. Keep changes small, testable, and consistent with the
-contracts expressed by the current code.
-
-## Project Principles
-
-ReMe is a local-first, file-native memory system for agents.
-
-- User-owned workspace files are the durable source of truth.
-- Indexes, catalogs, graphs, caches, and generated metadata must remain rebuildable.
-- Prefer transparent formats and predictable behavior over hidden state.
-- Preserve user control over workspace paths, configuration, and service boundaries.
-- Keep concepts focused on project intent; let code and schemas describe implementation.
-
-When convenience conflicts with these principles, favor data ownership, recoverability, and explicit behavior.
-
-## Sources of Truth
-
-Use this order when documentation and implementation disagree:
-
-1. Current code and public Pydantic schemas.
-2. Tests that describe supported behavior.
-3. CLI behavior and the built-in configuration.
-4. README files and other development documentation.
-
-Do not duplicate large implementation descriptions in documentation. Express the stable contract and link to the
-relevant module where useful. When behavior changes intentionally, update the implementation, schemas, tests, defaults,
-and concise documentation together.
-
-## Repository Map
-
-- `reme/reme.py`: CLI entry point; dispatches `start`, `find_reme`, and client calls.
-- `reme/application.py`: application assembly, dependency ordering, job execution, and lifecycle.
-- `reme/config/config_parser.py`: YAML/JSON loading, environment expansion, dot-notation parsing, and deep config
-  merging.
-- `reme/config/default.yaml`: default service, jobs, steps, and components. Other files in
-  `reme/config/` are named configuration variants.
-- `reme/schema/application_config.py`: typed application, component, and job configuration.
-- `reme/schema/`: request, response, streaming, memory, graph, and file contracts.
-- `reme/components/application_context.py`: application-wide wiring and in-memory shared state.
-- `reme/components/runtime_context.py`: request-scoped data, response, streaming queue, and stop event.
-- `reme/components/base_component.py`: component lifecycle, dependency binding, and workspace helpers.
-- `reme/components/component_registry.py`: the frozen built-in registry template and application-local registry factory.
-- `reme/components/job/`: base, stream, background, and cron job implementations.
-- `reme/components/service/`: local CLI, HTTP, and MCP service backends.
-- `reme/components/`: agent wrappers, model adapters, stores, catalogs, graphs, indexes, clients, tokenizers, and
-  outbound proxies.
-- `reme/steps/`: registered job steps grouped by common, file I/O, index, evolve, cookbook, benchmark, and transfer
-  concerns.
-- `reme/utils/`: shared utilities, including service discovery, logging, web-static resolution, session I/O, token
-  accounting, and wikilink handling.
-- `tests/unit/`: primary fast, isolated validation suite.
-- `tests/integration/`: service/model tests that may need credentials or external processes.
-- `reme_studio/`: ReMe Studio frontend source plus the independently published `reme_studio` Python package and
-  `@agentscope-ai/reme_studio` npm static distribution.
-- `typescript/`: the independently published `@agentscope-ai/reme` package, including the shared TypeScript client and
-  DeepSeek Harness and OpenClaw adapters.
-- `plugins/`: installable ReMe extensions, such as Auto Fin.
-- `integrations/`: adapters that connect ReMe to external agent hosts, such as Claude Code, DSH, and Hermes Agent.
-- `skills/`: standalone skills; `reme_memory` calls ReMe, while other skills may use separate tools or direct-file
-  conventions.
-- `benchmark/` and `cookbook/`: runnable evaluations and example workflows.
-- `docs/`: README-linked supporting pages and figures.
-
-## Development Setup
-
-ReMe requires Python 3.11 or newer. Install the editable development environment with:
-
-```bash
-pip install -e reme_studio -e ".[dev,core]"
-```
-
-Before changing behavior, inspect the adjacent implementation, schema, built-in config, and focused tests. Follow
-existing async and typing patterns unless the task explicitly requires a new contract.
-
-## Configuration and CLI Contracts
-
-- CLI syntax is `reme ACTION key=value ...`; leading `-` or `--` on arguments is accepted.
-- Nested overrides use dot notation. Values support null, booleans, numbers, JSON collections, and quoted JSON strings;
-  leading-zero numeric-looking values remain strings.
-- `config=<name-or-path>` loads a discovered config name or a `.yaml`, `.yml`, or `.json` file. With no explicit config
-  path, `default` is loaded when available.
-- Config files expand `${VAR}` and `${VAR:-default}` recursively. An undefined variable without a default is an error.
-- CLI/config overrides are deep-merged over the loaded file. Do not silently change this merge behavior or stable
-  configuration keys.
-- `ApplicationConfig` normalizes `workspace_dir` to an expanded absolute path. `session_dir`
-  must remain workspace-relative; standard transcripts live under `{session_dir}/dialog`.
-- `reme start` runs the configured service. `reme start job=<name> ...` switches to the one-shot CLI service and runs
-  the job through the normal application lifecycle.
-- Other actions use a client selected from the running service configuration when discoverable, otherwise from local
-  config. Client-selection arguments must not leak into the job payload.
-
-## Registration and Application Lifecycle
-
-Component and Step discovery is import-driven:
-
-- Implementations declare a non-`BASE` `component_type` and register with `@R.register("backend")`
-  or `R.register(Class, "backend")`.
-- Component packages must be imported through `reme/components/__init__.py`.
-- Step packages/modules must be reachable through their package `__init__.py` chain and ultimately
-  `reme/steps/__init__.py`.
-- Adding an implementation without its registration import leaves it undiscoverable at runtime. Treat implementation,
-  registration, import side effect, defaults, and tests as one change.
-
-`Application` validates config through `ApplicationContext`, creates workspace directories, instantiates the service,
-configured components, and jobs, and then manages lifecycle as follows:
-
-- Components start in topological dependency order. Missing required dependencies and cycles fail explicitly; optional
-  dependencies may resolve to `None`.
-- Jobs start after components in this order: base jobs, stream jobs, background jobs, then cron jobs.
-- Shutdown closes everything in reverse start order and then shuts down the optional thread pool.
-- If startup fails, already-started resources are closed.
-- `BaseComponent.start()` and `close()` are lock-protected and idempotent. Dependencies created by a standalone
-  `default_factory` are owned and closed by the parent component.
-
-Keep async clients, tasks, executors, and services under this lifecycle. Do not introduce an untracked long-lived
-resource.
-
-## Jobs, Steps, and State
-
-`BaseJob` resolves configured Step classes during job startup and constructs fresh Step instances for every invocation.
-Job-level kwargs are merged into each `RuntimeContext`, with call-time kwargs taking precedence. Sequential Steps in one
-invocation share the same `RuntimeContext` and `Response`.
-
-Treat Step instances as invocation-scoped:
-
-- Constructor fields and `self.kwargs` hold Step configuration and resolved dependencies. They may be cached or adjusted
-  during that one invocation, but must not be relied on across Job calls.
-- `self.context.data` holds request inputs and intermediate values shared by sequential Steps.
-- `self.context.response.answer`, `success`, and `metadata` are request-scoped output. Because the same response travels
-  through the Step chain, later Steps may consume metadata produced earlier, but it is not application-lifetime or
-  durable storage.
-- `self.app_context.metadata` holds in-memory state shared across Job/Step invocations for the life of one
-  `Application`, such as counters, tool-context state, session maps, or locks.
-- Workspace files or a dedicated Component/store hold durable state that must survive restart.
-
-Use narrow, namespaced keys in `app_context.metadata` and protect shared mutable values against concurrent access. The
-search/draft helpers intentionally mirror tool-context state into
-`self.kwargs` only when no `ApplicationContext` exists for standalone use and unit tests; do not generalize that
-compatibility fallback into persistent runtime state. If shared state becomes a stable service contract or needs
-dedicated lifecycle, locking, or persistence, promote it to a typed context field or Component.
-
-Additional Step contracts:
-
-- `Ref` dependencies resolve in this order: Step kwargs, current `RuntimeContext`, then the named application component.
-  The value is cached only on the current Step instance and cleared before each call.
-- `input_mapping` and `output_mapping` copy keys within `RuntimeContext.data`; missing sources are ignored.
-- Dispatched Steps receive the current `RuntimeContext`, so their data and response are shared.
-- Base jobs convert uncaught Step errors into `Response(success=False)`; stream jobs emit an error chunk and always a
-  terminal `DONE`; background jobs let errors reach their supervisor.
-- Background jobs are never service-exposed. MCP also skips stream jobs. Respect `enable_serve`
-  and any configured service job allowlist.
-
-## Workspace and File Safety
-
-- Application startup creates the workspace plus configured metadata, session, memory-session, resource, daily, and
-  digest directories.
-- File-operation paths are resolved against the workspace and must stay inside it. Home-relative paths are unsupported,
-  traversal escapes are rejected, and `_allowed_paths` restrictions fail closed when invalid.
-- Preserve per-path locking, encoding detection, byte limits, truncation behavior, and optimistic
-  `expected_mtime` checks when modifying file operations.
-- Do not bypass the existing file steps or stores in a way that weakens workspace containment.
-- Never write test state into the repository's `.reme/`; use `tmp_path` or another isolated workspace.
-- Do not delete or rewrite user memory to repair an index or make a test pass. Rebuild derived state from source files
-  instead.
-
-## Validation
-
-Use the narrowest useful check while iterating, then broaden it according to risk.
-
-Focused test:
-
-```bash
-pytest tests/unit/path/to/test_file.py -v
-```
-
-Main unit suite:
-
-```bash
-pytest tests/unit -v --tb=long -s --log-cli-level=WARNING
-```
-
-Repository formatting and lint checks:
-
-```bash
-pre-commit run --all-files
-```
-
-Black and Flake8 use a 120-character line limit and Python 3.11 formatting; Pylint is also run by pre-commit. If
-`reme_studio/` changes, use its Node 22.13+ scripts and run the proportionate checks from that directory, such as
-`npm run format:check`, `npm run lint`, or `npm test`.
-
-Integration tests may contact real model providers, services, or agent subprocesses and can require credentials. Do not
-run credentialed or externally mutating tests automatically; run them only when the task requires them and the necessary
-environment has been supplied or authorized. Mock network, model, and subprocess boundaries in unit tests.
-
-## Change Guardrails
-
-- Preserve unrelated user changes in a dirty working tree.
-- Make the smallest coherent change and avoid unrelated cleanup or broad refactors.
-- Do not edit generated output when the source can be changed instead. The publish workflow builds
-  `reme_studio/dist-static` and stages it under `reme_studio/src/reme_studio/static`; change `reme_studio/` source for
-  frontend work.
-- Do not silently change CLI flags, configuration keys, workspace layouts, serialized schemas, endpoint shapes,
-  streaming termination, or service interfaces. Preserve compatibility where practical and document intentional
-  migrations.
-- Do not introduce dependencies without a concrete repository-level need.
-- Do not commit `.env` files, credentials, runtime memory, logs, indexes, caches, benchmark outputs, or generated
-  Studio distributions.
-- State which validations passed and which relevant checks were not run in the final handoff.
-
-If a requirement is ambiguous, infer intent from nearby code, schemas, defaults, and tests. Ask the user only when the
-remaining choice would materially alter a public contract, user data, or an external system.

CLAUDE.md

@@ -1 +0,0 @@
-AGENTS.md

README.md

@@ -1,14 +1,16 @@
 <p align="center">
- <img src="https://raw.githubusercontent.com/agentscope-ai/ReMe/main/docs/figure/reme_logo.png" alt="ReMe Logo" width="50%">
+ <img src="docs/_static/figure/reme_logo.png" alt="ReMe Logo" width="50%">
 </p>
 
 <p align="center">
-  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.11+-blue" alt="Python Version"></a>
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.10+-blue" alt="Python Version"></a>
   <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/pypi/v/reme-ai.svg?logo=pypi" alt="PyPI Version"></a>
   <a href="https://pepy.tech/project/reme-ai/"><img src="https://img.shields.io/pypi/dm/reme-ai" alt="PyPI Downloads"></a>
   <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/commit-activity/m/agentscope-ai/ReMe?style=flat-square" alt="GitHub commit activity"></a>
+</p>
+
+<p align="center">
   <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-black" alt="License"></a>
-  <a href="https://reme.agentscope.io"><img src="https://img.shields.io/badge/docs-ReMe-blue" alt="Documentation"></a>
   <a href="./README.md"><img src="https://img.shields.io/badge/English-Click-yellow" alt="English"></a>
   <a href="./README_ZH.md"><img src="https://img.shields.io/badge/简体中文-点击查看-orange" alt="简体中文"></a>
   <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/stars/agentscope-ai/ReMe?style=social" alt="GitHub Stars"></a>
@@ -20,382 +22,658 @@
 </p>
 
 <p align="center">
-  <strong>A local-first, self-evolving personal knowledge base for AI agents.</strong><br>
+  <strong>A memory management toolkit for AI agents — Remember Me, Refine Me.</strong><br>
 </p>
 
-> Previous versions: [0.3.x](https://github.com/agentscope-ai/ReMe/tree/reme_v3) ·
-> [0.2.x](https://github.com/agentscope-ai/ReMe/tree/v0.2.0.6) ·
-> [MemoryScope](https://github.com/agentscope-ai/ReMe/tree/memoryscope_branch)
+> For the older version, please refer to the [0.2.x documentation](docs/README_0_2_x.md).
 
-## ✨ Why ReMe?
+---
 
-🧠 ReMe turns conversations and resources into readable, editable, searchable, and interconnected Markdown memory. Agents
-such as QwenPaw and DeepSeek Harness can share the same workspace to retrieve, maintain, and evolve knowledge, while
-users retain control of the durable files.
+🧠 ReMe is a memory management framework designed for **AI agents**, providing
+both [file-based](#-file-based-memory-system-remelight) and [vector-based](#-vector-based-memory-system) memory systems.
 
-- **Memory as File, File as Memory**: ReMe stores durable memory as ordinary Markdown with frontmatter and wikilinks.
-  Users and agents can inspect, edit, move, sync, and back it up with familiar tools, while indexes and generated
-  metadata remain rebuildable.
-- **Self-evolving knowledge base**: ReMe progressively turns conversations and resources into daily notes and long-term
-  knowledge, preserving sources while refining facts, preferences, procedures, and relationships over time.
-- **Recall is precise and context-aware.** BM25, optional embeddings, and wikilink expansion retrieve relevant
-  line-level passages and their relationships without loading the entire knowledge base into the agent context.
-- **One memory workspace works across agents.** Personal assistants, coding agents, and other agent runtimes can share
-  the same local workspace through native integrations, SKILL.md, CLI, HTTP, MCP, or Python APIs.
+It tackles two core problems of agent memory: **limited context window** (early information is truncated or lost in long
+conversations) and **stateless sessions** (new sessions cannot inherit history and always start from scratch).
 
-<p align="center">
-  <img src="docs/figure/design-philosophy.svg" alt="ReMe Design Philosophy" width="92%">
-</p>
+ReMe gives agents **real memory** — old conversations are automatically compacted, important information is persistently
+stored, and relevant context is automatically recalled in future interactions.
 
-## 📰 Latest Updates
+ReMe achieves state-of-the-art results on the LoCoMo and HaluMem benchmarks; see the [Experimental results](#experimental-results).
 
-- [2026.08] - Published [`@agentscope-ai/reme`](https://www.npmjs.com/package/@agentscope-ai/reme), providing native
-  ReMe memory integrations for DeepSeek Harness and OpenClaw plus a shared TypeScript HTTP client.
-- [2026.08] - Published the [ReMe blog](https://agentscope-ai.github.io/ReMe/?doc=en-reme-blog), an end-to-end introduction to its local-first memory
-  architecture, self-evolving workflows, hybrid search, proactive discovery, and benchmark results.
-- [2026.08] - [Experience-driven enhancement method](https://reme.agentscope.io/?doc=toolmemory-en) of agent tool-use execution built
-  on ReMe is available on [arXiv:2608.03403](https://arxiv.org/abs/2608.03403).
-- [2026.07] - Introduced optional plugins: [Daily Paper](https://reme.agentscope.io/?doc=daily-paper-en) for paper discovery and
-  analysis, and [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-en) for researching the latest 24 hours of topic-related CLS news
-  with local-memory search and validated historical wikilinks.
-- [2026.07] - Our
-  paper [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/)
-  has been accepted to Findings of ACL 2026.
+<details>
+<summary><b>What you can do with ReMe</b></summary>
 
-## 🚀 Quick Start
+<br>
 
-### Installation
+- **Personal assistant**: Provide long-term memory for agents like [CoPaw](https://github.com/agentscope-ai/CoPaw),
+  remembering user preferences and conversation history.
+- **Coding assistant**: Record code style preferences and project context, maintaining a consistent development
+  experience across sessions.
+- **Customer service bot**: Track user issue history and preference settings for personalized service.
+- **Task automation**: Learn success/failure patterns from historical tasks to continuously optimize execution
+  strategies.
+- **Knowledge Q&A**: Build a searchable knowledge base with semantic search and exact matching support.
+- **Multi-turn dialogue**: Automatically compress long conversations while retaining key information within limited
+  context windows.
 
-ReMe requires Python 3.11+.
+</details>
 
-Install from pip:
+---
 
-```bash
-pip install "reme-ai[core]"
+## 📁 File-based memory system (ReMeLight)
+
+> Memory as files, files as memory.
+
+Treat **memory as files** — readable, editable, and copyable.
+[CoPaw](https://github.com/agentscope-ai/CoPaw) integrates long-term memory and context management by inheriting from
+`ReMeLight`.
+
+| Traditional memory system | File-based ReMe      |
+|---------------------------|----------------------|
+| 🗄️ Database storage      | 📝 Markdown files    |
+| 🔒 Opaque                 | 👀 Always readable   |
+| ❌ Hard to modify          | ✏️ Directly editable |
+| 🚫 Hard to migrate        | 📦 Copy to migrate   |
+
+```
+working_dir/
+├── MEMORY.md              # Long-term memory: persistent info such as user preferences
+├── memory/
+│   └── YYYY-MM-DD.md      # Daily journal: automatically written after each conversation
+├── dialog/                # Raw conversation records: full dialog before compression
+│   └── YYYY-MM-DD.jsonl   # Daily conversation messages in JSONL format
+└── tool_result/           # Cache for long tool outputs (auto-managed, expired entries auto-cleaned)
+    └── <uuid>.txt
 ```
 
-Install from source:
+### Core capabilities
+
+[ReMeLight](reme/reme_light.py) is the core class of the file-based memory system. It provides full memory management
+capabilities for AI agents:
+
+<table>
+<tr><th>Category</th><th>Method</th><th>Function</th><th>Key components</th></tr>
+<tr><td rowspan="4">Context Management</td><td><code>check_context</code></td><td>📊 Check context size</td><td><a href="reme/memory/file_based/components/context_checker.py">ContextChecker</a> — checks whether context exceeds thresholds and splits messages</td></tr>
+<tr><td><code>compact_memory</code></td><td>📦 Compact history into summary</td><td><a href="reme/memory/file_based/components/compactor.py">Compactor</a> — ReActAgent that generates structured context summaries</td></tr>
+<tr><td><code>compact_tool_result</code></td><td>✂️ Compact long tool outputs</td><td><a href="reme/memory/file_based/components/tool_result_compactor.py">ToolResultCompactor</a> — truncates long tool outputs and stores them in <code>tool_result/</code> while keeping file references in messages</td></tr>
+<tr><td><code>pre_reasoning_hook</code></td><td>🔄 Pre-reasoning hook</td><td><code>compact_tool_result</code> + <code>check_context</code> + <code>compact_memory</code> + <code>summary_memory</code> (async)</td></tr>
+<tr><td rowspan="2">Long-term Memory</td><td><code>summary_memory</code></td><td>📝 Persist important memory to files</td><td><a href="reme/memory/file_based/components/summarizer.py">Summarizer</a> — ReActAgent + file tools (<code>read</code> / <code>write</code> / <code>edit</code>)</td></tr>
+<tr><td><code>memory_search</code></td><td>🔍 Semantic memory search</td><td><a href="reme/memory/file_based/tools/memory_search.py">MemorySearch</a> — hybrid retrieval with vectors + BM25</td></tr>
+<tr><td rowspan="2">Session Memory</td><td><code>get_in_memory_memory</code></td><td>💾 Create in-session memory instance</td><td>Returns ReMeInMemoryMemory with dialog_path configured for persistence</td></tr>
+<tr><td><code>await_summary_tasks</code></td><td>⏳ Wait for async summary tasks</td><td>Block until all background summary tasks complete</td></tr>
+<tr><td>-</td><td><code>start</code></td><td>🚀 Start memory system</td><td>Initialize file storage, file watcher, and embedding cache; clean up expired tool result files</td></tr>
+<tr><td>-</td><td><code>close</code></td><td>📕 Shutdown and cleanup</td><td>Clean up tool result files, stop file watcher, and persist embedding cache</td></tr>
+</table>
+
+---
+
+### 🚀 Quick start
+
+#### Installation
+
+**Install from source:**
 
 ```bash
 git clone https://github.com/agentscope-ai/ReMe.git
 cd ReMe
-pip install -e reme_studio -e ".[core]"
-cd reme_studio
-npm ci
-npm run build:static
-cd ..
+pip install -e ".[light]"
 ```
 
-The static build requires Node.js 22.13 or newer and makes Studio available from the source tree.
-
-### Start the Service
+**Update to the latest version:**
 
 ```bash
-reme start
+git pull
+pip install -e ".[light]"
 ```
 
-The default service address is `127.0.0.1:2333`. If the port is occupied, specify another port:
+#### Environment variables
 
-```bash
-reme start service.port=8181
-# reme start workspace_dir=/tmp/reme-demo service.port=8181
+`ReMeLight` uses environment variables to configure the embedding model and storage backends:
+
+| Variable             | Description                   | Example                                             |
+|----------------------|-------------------------------|-----------------------------------------------------|
+| `LLM_API_KEY`        | LLM API key                   | `sk-xxx`                                            |
+| `LLM_BASE_URL`       | LLM base URL                  | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
+| `EMBEDDING_API_KEY`  | Embedding API key (optional)  | `sk-xxx`                                            |
+| `EMBEDDING_BASE_URL` | Embedding base URL (optional) | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
+
+#### Python usage
+
+```python
+import asyncio
+
+from reme.reme_light import ReMeLight
+
+
+async def main():
+    # Initialize ReMeLight
+    reme = ReMeLight(
+        default_as_llm_config={"model_name": "qwen3.5-35b-a3b"},
+        # default_embedding_model_config={"model_name": "text-embedding-v4"},
+        default_file_store_config={"fts_enabled": True, "vector_enabled": False},
+        enable_load_env=True,
+    )
+    await reme.start()
+
+    messages = [...]  # List of conversation messages
+
+    # 1. Check context size (token counting, determine if compaction is needed)
+    messages_to_compact, messages_to_keep, is_valid = await reme.check_context(
+        messages=messages,
+        memory_compact_threshold=90000,  # Threshold to trigger compaction (tokens)
+        memory_compact_reserve=10000,  # Token count to reserve for recent messages
+    )
+
+    # 2. Compact conversation history into a structured summary
+    summary = await reme.compact_memory(
+        messages=messages,
+        previous_summary="",
+        max_input_length=128000,  # Model context window (tokens)
+        compact_ratio=0.7,  # Trigger compaction when exceeding max_input_length * 0.7
+        language="zh",  # Summary language (e.g., "zh" / "")
+    )
+
+    # 3. Compact long tool outputs (prevent tool results from blowing up context)
+    messages = await reme.compact_tool_result(messages)
+
+    # 4. Pre-reasoning hook (auto compact tool results + check context + generate summaries)
+    processed_messages, compressed_summary = await reme.pre_reasoning_hook(
+        messages=messages,
+        system_prompt="You are a helpful AI assistant.",
+        compressed_summary="",
+        max_input_length=128000,
+        compact_ratio=0.7,
+        memory_compact_reserve=10000,
+        enable_tool_result_compact=True,
+        tool_result_compact_keep_n=3,
+    )
+
+    # 5. Persist important memory to files (writes to memory/YYYY-MM-DD.md)
+    summary_result = await reme.summary_memory(
+        messages=messages,
+        language="zh",
+    )
+
+    # 6. Semantic memory search (vector + BM25 hybrid retrieval)
+    result = await reme.memory_search(query="Python version preference", max_results=5)
+
+    # 7. Create in-session memory instance (manages context for one conversation)
+    memory = reme.get_in_memory_memory()  # Auto-configures dialog_path
+    for msg in messages:
+        await memory.add(msg)
+    token_stats = await memory.estimate_tokens(max_input_length=128000)
+    print(f"Current context usage: {token_stats['context_usage_ratio']:.1f}%")
+    print(f"Message token count: {token_stats['messages_tokens']}")
+    print(f"Estimated total tokens: {token_stats['estimated_tokens']}")
+
+    # 8. Mark messages as compressed (auto-persists to dialog/YYYY-MM-DD.jsonl)
+    # await memory.mark_messages_compressed(messages_to_compact)
+
+    # Shutdown ReMeLight
+    await reme.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
-```bash
-reme version
-reme health_check
-reme help
-curl -s http://127.0.0.1:2333/version -H 'Content-Type: application/json' -d '{}'
+> 📂 Full example: [test_reme_light.py](tests/light/test_reme_light.py)
+> 📋 Sample run log: [test_reme_light_log.txt](tests/light/test_reme_light_log.txt) (223,838 tokens → 1,105 tokens, 99.5%
+> compression)
+
+### Architecture of the file-based ReMeLight memory system
+
+[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py)
+inherits
+`ReMeLight` and integrates its memory capabilities into the agent reasoning loop:
+
+```mermaid
+graph LR
+    Agent[Agent] -->|Before each reasoning step| Hook[pre_reasoning_hook]
+    Hook --> TC[compact_tool_result<br>Compact tool outputs]
+    TC --> CC[check_context<br>Token counting]
+    CC -->|Exceeds limit| CM[compact_memory<br>Generate summary]
+    CC -->|Exceeds limit| SM[summary_memory<br>Async persistence]
+    SM -->|ReAct + FileIO| Files[memory/*.md]
+    CC -->|Exceeds limit| MMC[mark_messages_compressed<br>Persist raw dialog]
+    MMC --> Dialog[dialog/*.jsonl]
+    Agent -->|Explicit call| Search[memory_search<br>Vector+BM25]
+    Agent -->|In - session| InMem[ReMeInMemoryMemory<br>Token-aware memory]
+    InMem -->|Compress/Clear| Dialog
+    Files -.->|FileWatcher| Store[(FileStore<br>Vector+FTS index)]
+    Search --> Store
 ```
 
-### 5-Minute Memory Demo
-
-With the service running, write a memory node, let ReMe index it, then retrieve it:
-
-```bash
-reme write \
-  path=digest/wiki/quick-start-demo \
-  name="Quick Start Demo" \
-  description="A first ReMe memory node" \
-  content="# Quick Start Demo
-
-ReMe stores agent memory as readable Markdown.
-
-Related: [[digest/wiki/memory-as-file.md]]"
-
-reme search query="agent memory markdown" limit=5
-reme read path=digest/wiki/quick-start-demo start_line=1 end_line=20
-```
-
-The generated file is ordinary Markdown with frontmatter:
-
-```markdown
----
-name: Quick Start Demo
-description: A first ReMe memory node
 ---
 
-# Quick Start Demo
+#### 1. `check_context` — context checking
 
-ReMe stores agent memory as readable Markdown.
+[ContextChecker](reme/memory/file_based/components/context_checker.py) uses token counting to determine whether the
+context exceeds thresholds and automatically splits messages into a "to compact" group and a "to keep" group.
 
-Related: [[digest/wiki/memory-as-file.md]]
+```mermaid
+graph LR
+    M[messages] --> H[AsMsgHandler<br>Token counting]
+    H --> C{total > threshold?}
+    C -->|No| K[Return all messages]
+    C -->|Yes| S[Keep from tail<br>reserve tokens]
+    S --> CP[messages_to_compact<br>Earlier messages]
+    S --> KP[messages_to_keep<br>Recent messages]
+    S --> V{is_valid<br>Tool calls aligned?}
 ```
 
-### ReMe Studio (Optional)
+- **Core logic**: keep `reserve` tokens from the tail; mark the rest as messages to compact.
+- **Integrity guarantee**: preserves complete user-assistant turns and tool_use/tool_result pairs without splitting
+  them.
 
-The `core` installation includes Studio. After starting ReMe, open <http://127.0.0.1:2333/> to browse, edit, and search
-the workspace. To add Studio to a base installation, use `pip install "reme-ai[web]"`. See the
-[ReMe Studio guide](https://reme.agentscope.io/?doc=studio-en) for source builds, configuration, and development.
+---
 
-### Optional Model Configuration
+#### 2. `compact_memory` — conversation compaction
 
-Configure environment variables when you want LLM-powered memory evolution or embedding retrieval. Embeddings are
-disabled by default, so the default setup does not start an embedding model or require an embedding API key.
+[Compactor](reme/memory/file_based/components/compactor.py) uses a ReActAgent to compact conversation history into a *
+*structured context summary**.
 
-```bash
-cat > .env <<'EOF'
-# Optional: used only after embedding components are explicitly enabled in the config.
-# EMBEDDING_API_KEY=sk-xxx
-# EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
-
-# Required for auto_memory, auto_resource, and auto_dream.
-LLM_API_KEY=sk-xxx
-LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
-EOF
+```mermaid
+graph LR
+    M[messages] --> H[AsMsgHandler<br>format_msgs_to_str]
+    H --> A[ReActAgent<br>reme_compactor]
+    P[previous_summary] -->|Incremental update| A
+    A --> S[Structured summary<br>Goal/Progress/Decisions...]
 ```
 
-Basic file operations, BM25 search, wikilink traversal, and reading proactive topics can run without LLM credentials.
+**Summary structure** (context checkpoints):
 
-> [!NOTE]
-> To enable embedding-based semantic retrieval, uncomment `components.as_embedding` and
-> `components.embedding_store` in [`reme/config/default.yaml`](reme/config/default.yaml), then change
-> `components.file_store.default.embedding_store` from `""` to `default`. See the
-> [memory search guide](docs/en/memory_search.md) for details.
+| Field                 | Description                                                            |
+|-----------------------|------------------------------------------------------------------------|
+| `## Goal`             | User goals                                                             |
+| `## Constraints`      | Constraints and preferences                                            |
+| `## Progress`         | Task progress                                                          |
+| `## Key Decisions`    | Key decisions                                                          |
+| `## Next Steps`       | Next step plans                                                        |
+| `## Critical Context` | Critical data such as file paths, function names, error messages, etc. |
 
-## 🤝 Use ReMe with Your Agent
+- **Incremental updates**: when `previous_summary` is provided, new conversations are merged into the existing summary.
 
-ReMe can run as a local memory service accessed through the CLI, HTTP API, or MCP server, or it can be embedded in the
-host process through its Python API. Host integrations can add memory guidance, recall, and capture to the agent
-lifecycle according to the capabilities of each runtime.
+---
 
-| Agent                          | Recommended path                                                                                                                         | Available after integration                                                                             |
-| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- |
-| **DeepSeek Harness**           | Install [`@agentscope-ai/reme`](typescript/README.md#deepseek-harness) with `dsh plugin --profile web add @agentscope-ai/reme`. | Long-term memory guidance, the `reme_search` tool, and automatic capture of completed main-agent turns. |
-| **OpenClaw**                   | Install [`@agentscope-ai/reme`](typescript/README.md#openclaw) with `openclaw plugins install @agentscope-ai/reme`.             | Native memory tools, recall before user-triggered runs, and automatic turn capture.                     |
-| **QwenPaw**                    | Embed ReMe in-process through its Python API.                                                                                            | Reuse the host lifecycle and model config while keeping memory local and file-based.                    |
-| **Claude Code**                | Start the streamable HTTP MCP service and install [the ReMe plugin](integrations/claude_code/reme).                                      | MCP recall tools, the `reme-memory` skill, and a Stop hook that records sessions automatically.         |
-| **Hermes**                     | Start the HTTP service and install [the ReMe provider](integrations/hermes_agent).                                                       | Recall before model calls and asynchronous `auto_memory` after each completed turn.                     |
-| **Codex and other CLI agents** | Install or copy the [ReMe Memory skill](skills/reme_memory/SKILL.md).                                                                    | Search, read, and write memory through the CLI; automatic capture requires host lifecycle integration.  |
+#### 3. `summary_memory` — persistent memory
 
-<p align="center"><b>Integration demos</b></p>
+[Summarizer](reme/memory/file_based/components/summarizer.py) uses a **ReAct + file tools** pattern so that the AI can
+decide what to write and where to write it.
 
-<table>
-  <tr>
-    <td align="center"></td>
-    <td width="45%" align="center"><b>Auto Memory</b></td>
-    <td width="45%" align="center"><b>Auto Dream</b></td>
-  </tr>
-  <tr>
-    <td align="center"><b>QwenPaw</b></td>
-    <td width="45%">
-      <img src="docs/figure/qwenpaw-auto-memory.gif" alt="QwenPaw Auto Memory demo" width="100%">
-    </td>
-    <td width="45%">
-      <img src="docs/figure/qwenpaw-auto-dream.gif" alt="QwenPaw Auto Dream demo" width="100%">
-    </td>
-  </tr>
-  <tr>
-    <td align="center"><b>Claude Code</b></td>
-    <td width="45%">
-      <img src="docs/figure/cc-auto-memory.gif" alt="Claude Code Auto Memory demo" width="100%">
-    </td>
-    <td width="45%">
-      <img src="docs/figure/cc-auto-dream.gif" alt="Claude Code Auto Dream demo" width="100%">
-    </td>
-  </tr>
-</table>
-
-## 🧠 How ReMe Works
-
-> Memory as File, File as Memory.
-
-ReMe treats **memory as files**, progressively processing filtered conversation source records and external resources
-from `session/` and `resource/` into `daily/`, then `digest/`. The default workspace is `.reme/` under the current
-directory; `workspace_dir=...` selects a different user-owned location.
-
-### Workspace Layout
-
-```text
-<workspace_dir>/
-├── metadata/       # Rebuildable indexes, graphs, catalogs, and caches
-├── session/        # Conversation source records and agent sessions
-│   ├── dialog/
-│   │   └── <session_id>.jsonl  # Source messages saved by auto_memory
-│   └── claude_code/
-│       └── <session_id>.jsonl  # ReMe copy used by auto_memory_cc
-├── mem_session/    # Generated agent-wrapper sessions/config, not user memory
-│   ├── agentscope/
-│   ├── claude_config/
-│   └── codex/
-├── resource/            # External raw materials
-│   ├── <resource>.<ext>  # Root-level files enter today's daily layer
-│   └── YYYY-MM-DD/
-│       └── <resource>.<ext>
-├── daily/               # Lightly processed memory: daily facts, conversation summaries, resource readings
-│   ├── YYYY-MM-DD.md
-│   └── YYYY-MM-DD/
-│       ├── <generated_name>.md  # Topic-named conversation or resource card
-│       └── interests.yaml
-└── digest/              # Long-term memory: personal facts, procedural experience, knowledge nodes
-    ├── personal/
-    │   └── {topic/event}.md
-    ├── procedure/
-    │   └── {topic/event}.md
-    └── wiki/
-        └── {topic/event}.md
+```mermaid
+graph LR
+    M[messages] --> A[ReActAgent<br>reme_summarizer]
+    A -->|read| R[Read memory/YYYY-MM-DD.md]
+    R --> T{Reason: how to merge?}
+    T -->|write| W[Overwrite]
+    T -->|edit| E[Edit in place]
+    W --> F[memory/YYYY-MM-DD.md]
+    E --> F
 ```
 
-<p align="center">
-  <img src="docs/figure/reme-overview.svg" alt="ReMe file-based memory system overview" width="92%">
-</p>
+**File tools** ([FileIO](reme/memory/file_based/tools/file_io.py)):
 
-### Memory Lifecycle
+| Tool    | Function              |
+|---------|-----------------------|
+| `read`  | Read file content     |
+| `write` | Overwrite file        |
+| `edit`  | Find-and-replace edit |
 
-ReMe follows a capture → index → consolidate → recall loop. Workspace files remain the durable source of truth;
-everything under `metadata/` is rebuildable.
+---
 
-| Capability                                  | Entry point                                     | What it does                                                                                                                                                   | Output                                                        |
-| ------------------------------------------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- |
-| [`auto_memory`](docs/en/auto_memory.md)     | Agent hook or `reme auto_memory`                | Distills useful conversation facts while preserving a filtered conversation source record.                                                                     | `session/dialog/*.jsonl`, `daily/<date>/<generated-name>.md`  |
-| [`auto_resource`](docs/en/auto_resource.md) | Resource watcher or `reme auto_resource`        | Turns files under `resource/` into source-linked, content-named daily cards.                                                                                   | `daily/<date>/<resource-card>.md`                             |
-| [`auto_index`](docs/en/memory_search.md)    | Background watcher or `reme reindex`            | Live-indexes Markdown in `daily/` and `digest/`; a full rebuild also scans `resource/` and JSONL.                                                              | Searchable chunks, BM25, wikilink graph, and optional vectors |
-| [`auto_dream`](docs/en/auto_dream.md)       | `dream_cron` or `reme auto_dream`               | By default, extracts up to five reusable units from changed files in the latest two-day window, then creates, corroborates, refines, or corrects digest nodes. | `digest/**`, `daily/<date>/interests.yaml`                    |
-| [`proactive`](docs/en/proactive.md)         | `reme proactive` before an agent decides to act | Reads topics generated by `auto_dream`; the host agent decides whether and how to mention them.                                                                | Structured topics from `daily/<date>/interests.yaml`          |
+#### 4. `compact_tool_result` — tool result compaction
 
-<table>
-  <tr>
-    <td align="center" width="50%">
-      <img src="docs/figure/memory-as-file.svg" alt="Memory as File" width="92%">
-    </td>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-memory-resource.svg" alt="Auto Memory and Resource" width="92%">
-    </td>
-  </tr>
-  <tr>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-dream-and-proactive.svg" alt="Auto Dream and Proactive" width="92%">
-    </td>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-index-and-memory-search.svg" alt="Auto Index and Memory Search" width="92%">
-    </td>
-  </tr>
-</table>
+[ToolResultCompactor](reme/memory/file_based/components/tool_result_compactor.py) addresses the problem of long tool
+outputs bloating the context.
 
-Search returns matching chunks with line ranges and bounded wikilink neighbors. Optional vector results are fused with
-BM25 through reciprocal rank fusion (RRF).
+```mermaid
+graph LR
+    M[messages] --> L{Iterate tool_result<br>len > threshold?}
+    L -->|No| K[Keep as-is]
+    L -->|Yes| T[truncate_text<br>Truncate to threshold]
+    T --> S[Write full content<br>tool_result/uuid.txt]
+    S --> R[Append file path reference<br>to message]
+    R --> C[cleanup_expired_files<br>Delete expired files]
+```
 
-> [!IMPORTANT]
->
-> `proactive` only reads and exposes interest topics produced by Auto Dream. It does not independently browse the web,
-> send notifications, or rewrite the knowledge base; the host agent decides whether and how to act on a topic.
+- **Auto cleanup**: expired files (older than `retention_days`) are deleted automatically during `start` / `close` /
+  `compact_tool_result`.
 
-## 📊 Benchmarks
+---
 
-ReMe evaluates multi-session and long-context memory with agentic search-and-read workflows. The figures below are the
-published reference runs in this repository; model, prompt, dataset, and judging details are documented with each
-benchmark.
+#### 5. `memory_search` — memory retrieval
 
-| Benchmark                                                                   | Setting      |              Sample size | Agentic score | Focus                                                              |
-| --------------------------------------------------------------------------- | ------------ | -----------------------: | ------------: | ------------------------------------------------------------------ |
-| **[LongMemEval cleaned-s](https://reme.agentscope.io/?doc=longmemeval-en)** | **Overall**  |        **500 questions** |     **89.4%** | Cross-session retrieval, knowledge updates, and temporal reasoning |
-| [BEAM](https://reme.agentscope.io/?doc=beam-en)                             | 100K context | 20 cases / 400 questions |         66.1% | Ten types of long-context memory tasks                             |
-| [BEAM](https://reme.agentscope.io/?doc=beam-en)                             | 1M context   | 35 cases / 700 questions |         65.0% | Ultra-long conversation settings                                   |
+[MemorySearch](reme/memory/file_based/tools/memory_search.py) provides **vector + BM25 hybrid retrieval**.
 
-ReMe also achieved a **0.580 PROC score across five user personas** in the repository's
-[π-Bench evaluation](https://reme.agentscope.io/?doc=pibench-en), 2.4% above NanoBot under the same test-model configuration. PROC
-measures proactive handling of hidden intent, clarification, cross-session preferences and conventions, task
-dependencies, and underspecified requests.
+```mermaid
+graph LR
+    Q[query] --> E[Embedding<br>Vectorization]
+    E --> V[vector_search<br>Semantic similarity]
+    Q --> B[BM25<br>Keyword matching]
+    V -->|" weight: 0.7 "| M[Deduplicate + weighted merge]
+    B -->|" weight: 0.3 "| M
+    M --> F[min_score filter]
+    F --> R[Top-N results]
+```
 
-## 🧩 Extensions and Plugins
+- **Fusion mechanism**: vector weight 0.7 + BM25 weight 0.3 — balancing semantic similarity and exact matches.
 
-Plugins are optional Python distributions that contribute Component, Step, or Job backends and configuration. They are
-installed separately and enabled explicitly by configuration. Daily Paper and Auto Fin are independently packaged
-plugins; see the source distributions and their documentation for [Daily Paper](plugins/daily_paper/README.md) and
-[Auto Fin](plugins/auto-fin/README.md).
+---
 
-| Plugin                                                        | Capability                                                                                                    |
-| ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- |
-| [Daily Paper](https://reme.agentscope.io/?doc=daily-paper-en) | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. |
-| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-en)       | Fetch topic-related CLS news, search ReMe history, and generate wikilink-backed Markdown reports.             |
+#### 6. `ReMeInMemoryMemory` — in-session memory
 
-See [Plugin Management](docs/en/plugin_management.md) to install, inspect, validate, enable, and uninstall ReMe plugins.
+[ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) extends AgentScope's `InMemoryMemory` to provide
+token-aware memory management and raw conversation persistence.
 
-## 📚 Documentation
+```mermaid
+graph LR
+    C[content] --> G[get_memory<br>exclude_mark=COMPRESSED]
+    G --> F[Filter out compressed messages]
+    F --> P{prepend_summary?}
+    P -->|Yes| S[Prepend previous summary]
+    S --> O[Output messages]
+    P -->|No| O
+    M[mark_messages_compressed] --> D[Persist to dialog/YYYY-MM-DD.jsonl]
+    D --> R[Remove from memory]
+```
 
-These guides cover the main user workflows and the runtime contracts implemented by the current code.
+| Function                         | Description                                              |
+|----------------------------------|----------------------------------------------------------|
+| `get_memory`                     | Filter messages by mark and auto-append summary          |
+| `estimate_tokens`                | Estimate token usage of the context                      |
+| `state_dict` / `load_state_dict` | Serialize/deserialize state (session persistence)        |
+| `mark_messages_compressed`       | Mark messages compressed and persist to dialog directory |
+| `clear_content`                  | Persist all messages before clearing memory              |
 
-| Guide                                                                     | What you will learn                                                                                 |
-| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |
-| [Quick Start](docs/en/quick_start.md)                                     | Install ReMe, start the service, and run the first file and memory operations.                      |
-| [Memory as File](docs/en/memory_as_file.md)                               | Understand workspace layers, frontmatter, wikilinks, chunks, and the file-as-source-of-truth model. |
-| [Auto Memory](docs/en/auto_memory.md)                                     | Preserve source conversations and distill reusable daily memory cards.                              |
-| [Auto Resource](docs/en/auto_resource.md)                                 | Import supported text resources and turn them into source-linked daily cards.                       |
-| [Auto Dream](docs/en/auto_dream.md) and [Auto Link](docs/en/auto_link.md) | Consolidate daily notes into evolving digest nodes and readable wikilink relationships.             |
-| [Memory Search](docs/en/memory_search.md)                                 | Use BM25, optional vectors, RRF fusion, line-range recall, and progressive link expansion.          |
-| [Proactive](docs/en/proactive.md)                                         | Read interest topics safely and integrate them into a host agent's decision flow.                   |
-| [Application Scenarios](docs/en/reme_scene.md)                            | Follow concrete financial research, coding-memory, and personal knowledge-base examples.           |
-| [Framework](docs/en/framework.md)                                         | Understand Application, Job, Step, Component, service, configuration, and lifecycle boundaries.     |
-| [TypeScript integrations](typescript/README.md)                            | Configure the shared client and native DeepSeek Harness and OpenClaw adapters.                       |
-| [ReMe Blog](https://agentscope-ai.github.io/ReMe/?doc=en-reme-blog)       | Read the product story, design rationale, examples, and benchmark summary.                          |
+**Raw conversation persistence**: When messages are compressed or cleared, they are automatically saved to
+`{dialog_path}/{date}.jsonl` with one JSON-formatted message per line.
 
-## 🛠️ Common Commands
+---
 
-Run `reme help` for the full job list. Common workspace and maintenance commands are:
+#### 7. `pre_reasoning_hook` — pre-reasoning processing
 
-| Command                                   | Purpose                                                                           |
-| ----------------------------------------- | --------------------------------------------------------------------------------- |
-| `reme status`                             | Show stateful data-component memory estimates and process RSS.                    |
-| [`reme search`](docs/en/memory_search.md) | Retrieve memory with BM25 and wikilinks by default, plus vectors when enabled.    |
-| `reme read` / `reme write` / `reme edit`  | Inspect and maintain Markdown memory files.                                       |
-| `reme traverse` / `reme graph_snapshot`   | Explore wikilink neighborhoods or the category-rooted digest graph.               |
-| `reme chat`                               | Stream a read-only, workspace-aware agent conversation. Requires LLM credentials. |
-| `reme reindex`                            | Rebuild search and wikilink indexes from existing files.                          |
+This is a unified entry point that wires all the above components together and automatically manages context before each
+reasoning step.
 
-## 🤝 Community and Contributing
+```mermaid
+graph LR
+    M[messages] --> TC[compact_tool_result<br>Compact long tool outputs]
+    TC --> CC[check_context<br>Compute remaining space]
+    CC --> D{messages_to_compact<br>Non-empty?}
+    D -->|No| K[Return original messages + summary]
+    D -->|Yes| V{is_valid?}
+    V -->|No| K
+    V -->|Yes| CM[compact_memory<br>Sync summary generation]
+    V -->|Yes| SM[add_async_summary_task<br>Async persistence]
+    CM --> R[Return messages_to_keep + new summary]
+```
 
-- **Issues, requests, and help**: Check [Open Issues](https://github.com/agentscope-ai/ReMe/issues) first. If there is no
-  related discussion, open one with the background, expected behavior, and impact scope.
-- **Code contributions**: Before making changes, read the repository's
-  [contribution guide](docs/en/contributing.md). Source, schemas, and tests are the authoritative architecture and
-  extension guide.
-- **Documentation contributions**: Update the canonical files under `docs/en/`, `docs/zh/`, or the relevant package
-  directory in this repository. The documentation site is generated from these files.
-- **Commit convention**: Conventional Commits are recommended, for example `feat(search): add link expansion option` or
-  `docs(zh): update quick start`.
-- **Pre-submit checks**: Before submitting a PR, try to run `pre-commit run --all-files` and `pytest`. If tests that
-  depend on LLMs, embeddings, or external services cannot run, explain that in the PR.
-- **Documentation**: Visit [reme.agentscope.io](https://reme.agentscope.io).
+**Execution flow**:
+
+1. `compact_tool_result` — compact long tool outputs.
+2. `check_context` — check whether the context exceeds limits.
+3. `compact_memory` — generate compact summary (sync).
+4. `summary_memory` — persist memory (async in the background).
+
+---
+
+## 🗃️ Vector-based memory system
+
+[ReMe Vector Based](reme/reme.py) is the core class for the vector-based memory system. It manages three types of
+memories:
+
+| Memory type           | Use case                                                          |
+|-----------------------|-------------------------------------------------------------------|
+| **Personal memory**   | Records user preferences and habits                               |
+| **Procedural memory** | Records task execution experience and patterns of success/failure |
+| **Tool memory**       | Records tool usage experience and parameter tuning                |
+
+### Core capabilities
+
+| Method             | Function     | Description                                                 |
+|--------------------|--------------|-------------------------------------------------------------|
+| `summarize_memory` | 🧠 Summarize | Automatically extract and store memories from conversations |
+| `retrieve_memory`  | 🔍 Retrieve  | Retrieve related memories based on a query                  |
+| `add_memory`       | ➕ Add        | Manually add memories into the vector store                 |
+| `get_memory`       | 📖 Get       | Get a single memory by ID                                   |
+| `update_memory`    | ✏️ Update    | Update existing memory content or metadata                  |
+| `delete_memory`    | 🗑️ Delete   | Delete a specific memory                                    |
+| `list_memory`      | 📋 List      | List memories with filtering and sorting                    |
+
+### Installation and environment variables
+
+Installation and environment configuration are the same as [ReMeLight](#installation).
+API keys are configured via environment variables and can be stored in a `.env` file at the project root.
+
+### Python usage
+
+```python
+import asyncio
+
+from reme import ReMe
+
+
+async def main():
+    # Initialize ReMe
+    reme = ReMe(
+        working_dir=".reme",
+        default_llm_config={
+            "backend": "openai",
+            "model_name": "qwen3.5-plus",
+        },
+        default_embedding_model_config={
+            "backend": "openai",
+            "model_name": "text-embedding-v4",
+            "dimensions": 1024,
+        },
+        default_vector_store_config={
+            "backend": "local",  # Supports local/chroma/qdrant/elasticsearch
+        },
+    )
+    await reme.start()
+
+    messages = [
+        {"role": "user", "content": "Help me write a Python script", "time_created": "2026-02-28 10:00:00"},
+        {"role": "assistant", "content": "Sure, I'll help you with that.", "time_created": "2026-02-28 10:00:05"},
+    ]
+
+    # 1. Summarize memories from conversation (automatically extract user preferences, task experience, etc.)
+    result = await reme.summarize_memory(
+        messages=messages,
+        user_name="alice",  # Personal memory
+        # task_name="code_writing",  # Procedural memory
+    )
+    print(f"Summary result: {result}")
+
+    # 2. Retrieve related memories
+    memories = await reme.retrieve_memory(
+        query="Python programming",
+        user_name="alice",
+        # task_name="code_writing",
+    )
+    print(f"Retrieved memories: {memories}")
+
+    # 3. Manually add a memory
+    memory_node = await reme.add_memory(
+        memory_content="The user prefers concise code style.",
+        user_name="alice",
+    )
+    print(f"Added memory: {memory_node}")
+    memory_id = memory_node.memory_id
+
+    # 4. Get a single memory by ID
+    fetched_memory = await reme.get_memory(memory_id=memory_id)
+    print(f"Fetched memory: {fetched_memory}")
+
+    # 5. Update memory content
+    updated_memory = await reme.update_memory(
+        memory_id=memory_id,
+        user_name="alice",
+        memory_content="The user prefers concise code with comments.",
+    )
+    print(f"Updated memory: {updated_memory}")
+
+    # 6. List all memories for the user (supports filtering and sorting)
+    all_memories = await reme.list_memory(
+        user_name="alice",
+        limit=10,
+        sort_key="time_created",
+        reverse=True,
+    )
+    print(f"User memory list: {all_memories}")
+
+    # 7. Delete a specific memory
+    await reme.delete_memory(memory_id=memory_id)
+    print(f"Deleted memory: {memory_id}")
+
+    # 8. Delete all memories (use with care)
+    # await reme.delete_all()
+
+    await reme.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Technical architecture
+
+```mermaid
+graph LR
+    User[User / Agent] --> ReMe[Vector Based ReMe]
+    ReMe --> Summarize[Summarize memories]
+    ReMe --> Retrieve[Retrieve memories]
+    ReMe --> CRUD[CRUD operations]
+    Summarize --> PersonalSum[PersonalSummarizer]
+    Summarize --> ProceduralSum[ProceduralSummarizer]
+    Summarize --> ToolSum[ToolSummarizer]
+    Retrieve --> PersonalRet[PersonalRetriever]
+    Retrieve --> ProceduralRet[ProceduralRetriever]
+    Retrieve --> ToolRet[ToolRetriever]
+    PersonalSum --> VectorStore[Vector database]
+    ProceduralSum --> VectorStore
+    ToolSum --> VectorStore
+    PersonalRet --> VectorStore
+    ProceduralRet --> VectorStore
+    ToolRet --> VectorStore
+```
+
+### Experimental results
+
+Evaluations are conducted on two benchmarks: **LoCoMo** and **HaluMem**. Experimental settings:
+
+1. **ReMe backbone**: as specified in each table.
+2. **Evaluation protocol**: LLM-as-a-Judge following MemOS — each answer is scored by GPT-4o-mini.
+
+Baseline results are reproduced from their respective papers under aligned settings where possible.
+
+### LoCoMo
+
+| Method   | Single Hop | Multi Hop | Temporal  | Open Domain | Overall   |
+|----------|------------|-----------|-----------|-------------|-----------|
+| MemoryOS | 62.43      | 56.50     | 37.18     | 40.28       | 54.70     |
+| Mem0     | 66.71      | 58.16     | 55.45     | 40.62       | 61.00     |
+| MemU     | 72.77      | 62.41     | 33.96     | 46.88       | 61.15     |
+| MemOS    | 81.45      | 69.15     | 72.27     | 60.42       | 75.87     |
+| HiMem    | 89.22      | 70.92     | 74.77     | 54.86       | 80.71     |
+| Zep      | 88.11      | 71.99     | 74.45     | 66.67       | 81.06     |
+| TiMem    | 81.43      | 62.20     | 77.63     | 52.08       | 75.30     |
+| TSM      | 84.30      | 66.67     | 71.03     | 58.33       | 76.69     |
+| MemR3    | 89.44      | 71.39     | 76.22     | 61.11       | 81.55     |
+| **ReMe** | **89.89**  | **82.98** | **83.80** | **71.88**   | **86.23** |
+
+### HaluMem
+
+| Method      | Memory Integrity | Memory Accuracy | QA Accuracy |
+|-------------|------------------|-----------------|-------------|
+| MemoBase    | 14.55            | 92.24           | 35.53       |
+| Supermemory | 41.53            | 90.32           | 54.07       |
+| Mem0        | 42.91            | 86.26           | 53.02       |
+| ProMem      | **73.80**        | 89.47           | 62.26       |
+| **ReMe**    | 67.72            | **94.06**       | **88.78**   |
+
+---
+
+## 🧪 Procedural memory paper
+
+> Our procedural (task) memory paper is available on [arXiv](https://arxiv.org/abs/2512.10696).
+
+### 🌍 [Appworld benchmark](benchmark/appworld/quickstart.md)
+
+We evaluate ReMe on the Appworld environment using Qwen3-8B (non-thinking mode):
+
+| Method   | Avg@4               | Pass@4              |
+|----------|---------------------|---------------------|
+| w/o ReMe | 0.1497              | 0.3285              |
+| w/ ReMe  | 0.1706 **(+2.09%)** | 0.3631 **(+3.46%)** |
+
+Pass@K measures the probability that at least one of K generated candidates successfully completes the task (score=1).
+The current experiments use an internal AppWorld environment, which may differ slightly from the public version.
+
+For more details on how to reproduce the experiments, see [quickstart.md](benchmark/appworld/quickstart.md).
+
+### 🔧 [BFCL-V3 benchmark](benchmark/bfcl/quickstart.md)
+
+We evaluate ReMe on the BFCL-V3 multi-turn-base task (random split 50 train / 150 val) using Qwen3-8B (thinking mode):
+
+| Method   | Avg@4               | Pass@4              |
+|----------|---------------------|---------------------|
+| w/o ReMe | 0.4033              | 0.5955              |
+| w/ ReMe  | 0.4450 **(+4.17%)** | 0.6577 **(+6.22%)** |
+
+For more details on how to reproduce the experiments, see [quickstart.md](benchmark/bfcl/quickstart.md).
+
+## ⭐ Community & support
+
+- **Star & Watch**: Starring helps more agent developers discover ReMe; Watching keeps you up to date with new releases
+  and features.
+- **Share your results**: Share how ReMe empowers your agents in Issues or Discussions — we are happy to showcase great
+  community use cases.
+- **Need a new feature?** Open a feature request; we’ll evolve ReMe together with the community.
+- **Code contributions**: All forms of contributions are welcome. Please see
+  the [contribution guide](docs/contribution.md).
+- **Acknowledgements**: We thank excellent open-source projects such as OpenClaw, Mem0, MemU, and CoPaw for their
+  inspiration and support.
 
 ### Contributors
 
-Thanks to everyone who has contributed to ReMe:
+Thanks to all who have contributed to ReMe:
 
 <a href="https://github.com/agentscope-ai/ReMe/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=agentscope-ai/ReMe" alt="Contributors" />
 </a>
 
+---
+
 ## 📄 Citation
 
 ```bibtex
-@software{ReMe2026,
-  title = {Remember me, Refine me: Memory Management Kit for Agents},
+@software{AgentscopeReMe2025,
+  title = {AgentscopeReMe: Memory Management Kit for Agents},
   author = {ReMe Team},
   url = {https://reme.agentscope.io},
-  year = {2026}
+  year = {2025}
 }
 ```
 
+---
+
 ## ⚖️ License
 
-This project is open source under the Apache License 2.0. See [LICENSE](./LICENSE) for details.
+This project is open-sourced under the Apache License 2.0. See [LICENSE](./LICENSE) for details.
+
+---
+
+## 🤔 Why ReMe?
+
+ReMe stands for **Remember Me** and **Refine Me**, symbolizing our goal to help AI agents "remember" users and "refine"
+themselves through interactions. We hope ReMe is not just a cold memory module, but a partner that truly helps agents
+understand users, accumulate experience, and continuously evolve.
+
+---
+
+## 📈 Star history
+
+[![Star History Chart](https://api.star-history.com/svg?repos=agentscope-ai/ReMe&type=Date)](https://www.star-history.com/#agentscope-ai/ReMe&Date)
+

README_ZH.md

@@ -1,14 +1,16 @@
 <p align="center">
- <img src="https://raw.githubusercontent.com/agentscope-ai/ReMe/main/docs/figure/reme_logo.png" alt="ReMe Logo" width="50%">
+ <img src="docs/_static/figure/reme_logo.png" alt="ReMe 标志" width="50%">
 </p>
 
 <p align="center">
-  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.11+-blue" alt="Python Version"></a>
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.10+-blue" alt="Python Version"></a>
   <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/pypi/v/reme-ai.svg?logo=pypi" alt="PyPI Version"></a>
   <a href="https://pepy.tech/project/reme-ai/"><img src="https://img.shields.io/pypi/dm/reme-ai" alt="PyPI Downloads"></a>
   <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/commit-activity/m/agentscope-ai/ReMe?style=flat-square" alt="GitHub commit activity"></a>
+</p>
+
+<p align="center">
   <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-black" alt="License"></a>
-  <a href="https://reme.agentscope.io"><img src="https://img.shields.io/badge/docs-ReMe-blue" alt="文档"></a>
   <a href="./README.md"><img src="https://img.shields.io/badge/English-Click-yellow" alt="English"></a>
   <a href="./README_ZH.md"><img src="https://img.shields.io/badge/简体中文-点击查看-orange" alt="简体中文"></a>
   <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/stars/agentscope-ai/ReMe?style=social" alt="GitHub Stars"></a>
@@ -20,349 +22,597 @@
 </p>
 
 <p align="center">
-  <strong>面向 AI Agent 的 local-first 自进化个人知识库。</strong><br>
+  <strong>面向智能体的记忆管理工具包，Remember Me, Refine Me.</strong><br>
 </p>
 
-> 历史版本：[0.3.x](https://github.com/agentscope-ai/ReMe/tree/reme_v3) ·
-> [0.2.x](https://github.com/agentscope-ai/ReMe/tree/v0.2.0.6) ·
-> [MemoryScope](https://github.com/agentscope-ai/ReMe/tree/memoryscope_branch)
+> 老版本请参阅 [0.2.x 版本文档](docs/README_0_2_x_ZH.md)
 
-## ✨ 为什么选择 ReMe？
+---
 
-🧠 ReMe 将对话和资料持续沉淀为可读、可编辑、可检索、相互链接的 Markdown 记忆。QwenPaw、DeepSeek Harness 等 Agent
-可以共享同一个 workspace，共同检索、维护和演化知识，而持久文件始终由用户掌控。
+🧠 ReMe 是一个专为 **AI 智能体** 打造的记忆管理框架，同时提供基于[文件系统](#-基于文件的记忆系统-remelight)
+和基于[向量库](#-基于向量库的记忆系统)的记忆系统。
 
-- **Memory as File, File as Memory**：ReMe 使用带 frontmatter 和 wikilink 的普通 Markdown 保存持久记忆。用户和 Agent
-  都可以使用熟悉的工具查看、编辑、移动、同步和备份；索引及生成的元数据均可重建。
-- **自进化知识库**：ReMe 将对话和资料逐步加工为 daily note 与长期知识，在保留来源的同时，持续提炼事实、偏好、
-  流程经验及其关系。
-- **精准召回所需上下文。** ReMe 结合 BM25、可选 embedding 和 wikilink 展开，召回带行号的相关片段及其关系，无需把整个知识库塞入
-  Agent 上下文。
-- **一个 workspace，可供不同 Agent 共同使用。** 个人助理、coding agent 和其他 Agent runtime 可以通过原生集成、SKILL.md、CLI、
-  HTTP、MCP 或 Python API 共享同一个本地记忆空间。
+它解决智能体记忆的两类核心问题：**上下文窗口有限**（长对话时早期信息被截断或丢失）、**会话无状态**（新对话无法继承历史，每次从零开始）。
 
-<p align="center">
-  <img src="docs/figure/design-philosophy.svg" alt="ReMe 设计理念" width="92%">
-</p>
+ReMe 让智能体拥有**真正的记忆力**——旧对话自动浓缩，重要信息持久保存，下次对话自动想起来。
 
-## 📰 最新动态
+在 LoCoMo 与 HaluMem 基准测试中，ReMe 取得了领先结果，详见[实验效果](#实验效果)。
 
-- [2026.08] - 发布 [`@agentscope-ai/reme`](https://www.npmjs.com/package/@agentscope-ai/reme)，提供统一 TypeScript HTTP
-  client，以及 DeepSeek Harness 和 OpenClaw 的原生 ReMe 记忆集成。
-- [2026.08] - 发布 [ReMe 博客](https://agentscope-ai.github.io/ReMe/?doc=zh-reme-blog)，系统介绍本地优先的记忆架构、自进化工作流、混合检索、
-  主动发现与评测结果。
-- [2026.08] - 基于 ReMe 的智能体工具使用
-  [经验驱动增强方法](https://reme.agentscope.io/?doc=toolmemory-zh)已发布，见
-  [arXiv:2608.03403](https://arxiv.org/abs/2608.03403)。
-- [2026.07] - 新增可选插件：[每日论文](https://reme.agentscope.io/?doc=daily-paper-zh)用于论文发现与解析，
-  [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-zh)用于研究最近 24 小时的主题相关财联社新闻，通过本地记忆搜索回顾历史材料并构建
-  wikilink。
-- [2026.07] -
-  我们的论文 [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/)
-  已被 Findings of ACL 2026 接收。
+<details>
+<summary><b>你可以用 ReMe 做什么</b></summary>
 
-## 🚀 快速开始
+<br>
 
-### 安装
+- **个人助理**：为 [CoPaw](https://github.com/agentscope-ai/CoPaw) 等智能体提供长期记忆，记住用户偏好和历史对话。
+- **编程助手**：记录代码风格偏好、项目上下文，跨会话保持一致的开发体验。
+- **客服机器人**：记录用户问题历史、偏好设置，提供个性化服务。
+- **任务自动化**：从历史任务中学习成功/失败模式，持续优化执行策略。
+- **知识问答**：构建可检索的知识库，支持语义搜索和精确匹配。
+- **多轮对话**：自动压缩长对话，在有限上下文窗口内保留关键信息。
 
-ReMe 要求 Python 3.11+。
+</details>
 
-从 pip 安装：
+---
 
-```bash
-pip install "reme-ai[core]"
+## 📁 基于文件的记忆系统 (ReMeLight)
+
+> 记忆即文件，文件即记忆
+
+将**记忆视为文件**——可读、可编辑、可复制。
+[CoPaw](https://github.com/agentscope-ai/CoPaw) 通过继承 `ReMeLight` 实现了长期记忆和上下文的管理。
+
+| 传统记忆系统    | File Based ReMe |
+|-----------|-----------------|
+| 🗄️ 数据库存储 | 📝 Markdown 文件  |
+| 🔒 不可见    | 👀 随时可读         |
+| ❌ 难修改     | ✏️ 直接编辑         |
+| 🚫 难迁移    | 📦 复制即迁移        |
+
+```
+working_dir/
+├── MEMORY.md              # 长期记忆：用户偏好等持久信息
+├── memory/
+│   └── YYYY-MM-DD.md      # 每日日记：对话结束后自动写入
+├── dialog/                # 原始对话记录：压缩前的完整对话
+│   └── YYYY-MM-DD.jsonl   # 按日期存储的对话消息（JSONL 格式）
+└── tool_result/           # 超长工具输出缓存（自动管理，超期自动清理）
+    └── <uuid>.txt
 ```
 
-从源码安装：
+### 核心能力
+
+[ReMeLight](reme/reme_light.py) 是该记忆系统的核心类，为 AI Agent 提供完整的记忆管理能力：
+
+<table>
+<tr><th>类别</th><th>方法</th><th>功能</th><th>关键组件</th></tr>
+<tr><td rowspan="4">上下文管理</td><td><code>check_context</code></td><td>📊 检查上下文大小</td><td><a href="reme/memory/file_based/components/context_checker.py">ContextChecker</a> — 检查上下文是否超出阈值并拆分 Message</td></tr>
+<tr><td><code>compact_memory</code></td><td>📦 压缩历史对话为摘要</td><td><a href="reme/memory/file_based/components/compactor.py">Compactor</a> — ReActAgent 生成结构化上下文摘要</td></tr>
+<tr><td><code>compact_tool_result</code></td><td>✂️ 压缩超长工具输出</td><td><a href="reme/memory/file_based/components/tool_result_compactor.py">ToolResultCompactor</a> — 截断超长的工具调用结果并转存到 <code>tool_result/</code>，消息中保留文件引用</td></tr>
+<tr><td><code>pre_reasoning_hook</code></td><td>🔄 推理前预处理钩子</td><td>compact_tool_result + check_context + compact_memory + summary_memory(async)</td></tr>
+<tr><td rowspan="2">长期记忆</td><td><code>summary_memory</code></td><td>📝 将重要记忆写入文件</td><td><a href="reme/memory/file_based/components/summarizer.py">Summarizer</a> — ReActAgent + 文件工具（read / write / edit）</td></tr>
+<tr><td><code>memory_search</code></td><td>🔍 语义搜索记忆</td><td><a href="reme/memory/file_based/tools/memory_search.py">MemorySearch</a> — 向量 + BM25 混合检索</td></tr>
+<tr><td rowspan="2">会话内存</td><td><code>get_in_memory_memory</code></td><td>💾 创建会话内存实例</td><td>返回 ReMeInMemoryMemory，自动配置 dialog_path 实现对话持久化</td></tr>
+<tr><td><code>await_summary_tasks</code></td><td>⏳ 等待异步摘要任务</td><td>阻塞等待所有后台摘要任务完成</td></tr>
+<tr><td>-</td><td><code>start</code></td><td>🚀 启动记忆系统</td><td>初始化文件存储、文件监控、Embedding 缓存；清理过期工具结果文件</td></tr>
+<tr><td>-</td><td><code>close</code></td><td>📕 关闭并清理</td><td>清理工具结果文件、停止文件监控、保存 Embedding 缓存</td></tr>
+</table>
+
+---
+
+### 🚀 快速开始
+
+#### 安装
+
+**从源码安装：**
 
 ```bash
 git clone https://github.com/agentscope-ai/ReMe.git
 cd ReMe
-pip install -e reme_studio -e ".[core]"
-cd reme_studio
-npm ci
-npm run build:static
-cd ..
+pip install -e ".[light]"
 ```
 
-静态构建要求 Node.js 22.13 或更高版本，并让源码安装可以直接使用 Studio。
-
-### 启动服务
+**更新到最新版本：**
 
 ```bash
-reme start
+git pull
+pip install -e ".[light]"
 ```
 
-默认服务地址是 `127.0.0.1:2333`。如果端口被占用，可以指定其他端口：
+#### 环境变量
 
-```bash
-reme start service.port=8181
-# reme start workspace_dir=/tmp/reme-demo service.port=8181
+`ReMeLight` 环境变量配置 Embedding 和存储后端
+
+| Variable             | Description             | Example                                             |
+|----------------------|-------------------------|-----------------------------------------------------|
+| `LLM_API_KEY`        | LLM API key             | `sk-xxx`                                            |
+| `LLM_BASE_URL`       | LLM base URL            | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
+| `EMBEDDING_API_KEY`  | Embedding API key (可选)  | `sk-xxx`                                            |
+| `EMBEDDING_BASE_URL` | Embedding base URL (可选) | `https://dashscope.aliyuncs.com/compatible-mode/v1` |
+
+#### Python 使用
+
+```python
+import asyncio
+
+from reme.reme_light import ReMeLight
+
+
+async def main():
+    # 初始化 ReMeLight
+    reme = ReMeLight(
+        default_as_llm_config={"model_name": "qwen3.5-35b-a3b"},
+        # default_embedding_model_config={"model_name": "text-embedding-v4"},
+        default_file_store_config={"fts_enabled": True, "vector_enabled": False},
+        enable_load_env=True,
+    )
+    await reme.start()
+
+    messages = [...]  # 对话消息列表
+
+    # 1. 检查上下文大小（Token 计数，判断是否需要压缩）
+    messages_to_compact, messages_to_keep, is_valid = await reme.check_context(
+        messages=messages,
+        memory_compact_threshold=90000,  # 触发压缩的阈值（tokens）
+        memory_compact_reserve=10000,  # 保留的近期消息 token 数
+    )
+
+    # 2. 将历史对话压缩为结构化摘要（可传入上轮摘要，实现增量更新）
+    summary = await reme.compact_memory(
+        messages=messages,
+        previous_summary="",
+        max_input_length=128000,  # 模型上下文窗口（tokens）
+        compact_ratio=0.7,  # 达到 max_input_length * 0.7 时触发压缩
+        language="zh",  # 摘要语言（zh / ""）
+    )
+
+    # 3. 压缩超长工具输出（防止工具结果撑爆上下文）
+    messages = await reme.compact_tool_result(messages)
+
+    # 4. 推理前预处理钩子（自动压缩工具结果 + 检查上下文 + 生成摘要）
+    processed_messages, compressed_summary = await reme.pre_reasoning_hook(
+        messages=messages,
+        system_prompt="你是一个有帮助的 AI 助手。",
+        compressed_summary="",
+        max_input_length=128000,
+        compact_ratio=0.7,
+        memory_compact_reserve=10000,
+        enable_tool_result_compact=True,
+        tool_result_compact_keep_n=3,
+    )
+
+    # 5. 将重要记忆写入文件（摘要写入 memory/YYYY-MM-DD.md）
+    summary_result = await reme.summary_memory(
+        messages=messages,
+        language="zh",
+    )
+
+    # 6. 语义搜索记忆（向量 + BM25 混合检索）
+    result = await reme.memory_search(query="Python 版本偏好", max_results=5)
+
+    # 7. 创建会话内存实例（管理单次对话的上下文）
+    from reme.memory.file_based.reme_in_memory_memory import ReMeInMemoryMemory
+    memory = reme.get_in_memory_memory()  # 自动配置 dialog_path
+    for msg in messages:
+        await memory.add(msg)
+    token_stats = await memory.estimate_tokens(max_input_length=128000)
+    print(f"当前上下文使用率: {token_stats['context_usage_ratio']:.1f}%")
+    print(f"消息 Token 数: {token_stats['messages_tokens']}")
+    print(f"预估总 Token 数: {token_stats['estimated_tokens']}")
+
+    # 8. 标记消息为压缩状态（自动持久化到 dialog/YYYY-MM-DD.jsonl）
+    # await memory.mark_messages_compressed(messages_to_compact)
+
+    # 关闭 ReMeLight
+    await reme.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
-```bash
-reme version
-reme health_check
-reme help
-curl -s http://127.0.0.1:2333/version -H 'Content-Type: application/json' -d '{}'
+> 📂 完整示例代码：[test_reme_light.py](tests/light/test_reme_light.py)
+> 📋 运行结果示例：[test_reme_light_log.txt](tests/light/test_reme_light_log.txt)（223,838 tokens → 1,105 tokens，压缩率99.5%）
+
+### 基于文件的 ReMeLight 记忆系统架构
+
+[CoPaw MemoryManager](https://github.com/agentscope-ai/CoPaw/blob/main/src/copaw/agents/memory/memory_manager.py) 继承
+`ReMeLight`，将记忆能力集成到 Agent 推理流程中：
+
+```mermaid
+graph LR
+    Agent[Agent] -->|每轮推理前| Hook[pre_reasoning_hook]
+    Hook --> TC[compact_tool_result<br>压缩工具输出]
+    TC --> CC[check_context<br>Token 计数]
+    CC -->|超限| CM[compact_memory<br>生成摘要]
+    CC -->|超限| SM[summary_memory<br>异步持久化]
+    SM -->|ReAct + FileIO| Files[memory/*.md]
+    CC -->|超限| MMC[mark_messages_compressed<br>持久化原始对话]
+    MMC --> Dialog[dialog/*.jsonl]
+    Agent -->|主动调用| Search[memory_search<br>向量+BM25]
+    Agent -->|会话内存| InMem[ReMeInMemoryMemory<br>Token感知内存]
+    InMem -->|压缩/清空| Dialog
+    Files -.->|FileWatcher| Store[(FileStore<br>向量+FTS索引)]
+    Search --> Store
 ```
 
-### 5 分钟记忆 Demo
-
-服务运行后，可以写入一个记忆节点，让 ReMe 索引并检索它：
-
-```bash
-reme write \
-  path=digest/wiki/quick-start-demo \
-  name="Quick Start Demo" \
-  description="第一个 ReMe 记忆节点" \
-  content="# Quick Start Demo
-
-ReMe 会把 Agent 记忆保存为可读的 Markdown。
-
-相关链接：[[digest/wiki/memory-as-file.md]]"
-
-reme search query="agent memory markdown" limit=5
-reme read path=digest/wiki/quick-start-demo start_line=1 end_line=20
-```
-
-生成的文件是普通 Markdown，并带有 frontmatter：
-
-```markdown
----
-name: Quick Start Demo
-description: 第一个 ReMe 记忆节点
 ---
 
-# Quick Start Demo
+#### 1. check_context — 上下文检查
 
-ReMe 会把 Agent 记忆保存为可读的 Markdown。
+[ContextChecker](reme/memory/file_based/components/context_checker.py) 基于 Token 计数判断上下文是否超限，自动拆分为「待压缩」和「保留」两组消息。
 
-相关链接：[[digest/wiki/memory-as-file.md]]
+```mermaid
+graph LR
+    M[messages] --> H[AsMsgHandler<br>Token 计数]
+    H --> C{total > threshold?}
+    C -->|否| K[返回全部消息]
+    C -->|是| S[从尾部向前保留<br>reserve tokens]
+    S --> CP[messages_to_compact<br>早期消息]
+    S --> KP[messages_to_keep<br>近期消息]
+    S --> V{is_valid<br>工具调用对齐?}
 ```
 
-### ReMe Studio（可选）
+- **核心逻辑**：从尾部向前保留 `reserve` tokens，超出部分标记为待压缩
+- **完整性保证**：不拆分 user-assistant 对话对，不拆分 tool_use/tool_result 配对
 
-上面的 `core` 安装已包含 Studio。启动 ReMe 后，打开 <http://127.0.0.1:2333/> 即可浏览、编辑和搜索 workspace。
-如需为基础安装单独添加 Studio，可使用 `pip install "reme-ai[web]"`。源码构建、配置和开发说明见
-[ReMe Studio 指南](https://reme.agentscope.io/?doc=studio-zh)。
+---
 
-### 可选模型配置
+#### 2. compact_memory — 对话压缩
 
-如果需要 LLM 驱动的记忆演化或 embedding 检索，可以配置环境变量。embedding 默认关闭，因此默认配置不会启动 embedding 模型，也不需要
-embedding API key。
+[Compactor](reme/memory/file_based/components/compactor.py) 使用 ReActAgent 将历史对话压缩为**结构化上下文摘要**。
 
-```bash
-cat > .env <<'EOF'
-# 可选：仅在配置中显式启用 embedding 组件后使用。
-# EMBEDDING_API_KEY=sk-xxx
-# EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
-
-# 必须：auto_memory、auto_resource 和 auto_dream 需要 LLM。
-LLM_API_KEY=sk-xxx
-LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
-EOF
+```mermaid
+graph LR
+    M[messages] --> H[AsMsgHandler<br>format_msgs_to_str]
+    H --> A[ReActAgent<br>reme_compactor]
+    P[previous_summary] -->|增量更新| A
+    A --> S[结构化摘要<br>Goal/Progress/Decisions...]
 ```
 
-基础文件读写、BM25 检索、wikilink 遍历和 proactive topics 读取可以先不配置 LLM 凭证。
+**摘要结构**（上下文检查点）：
 
-> [!NOTE]
-> 如需启用基于 embedding 的语义检索，请取消 [`reme/config/default.yaml`](reme/config/default.yaml) 中
-> `components.as_embedding` 和 `components.embedding_store` 的注释，并将
-> `components.file_store.default.embedding_store` 从 `""` 改为 `default`。完整说明见
-> [记忆检索文档](docs/zh/memory_search.md)。
+| 字段                    | 说明                 |
+|-----------------------|--------------------|
+| `## Goal`             | 用户目标               |
+| `## Constraints`      | 约束和偏好              |
+| `## Progress`         | 任务进展               |
+| `## Key Decisions`    | 关键决策               |
+| `## Next Steps`       | 下一步计划              |
+| `## Critical Context` | 文件路径、函数名、错误信息等关键数据 |
 
-## 🤝 将 ReMe 接入你的 Agent
+- **增量更新**：传入 `previous_summary` 时，自动将新对话与旧摘要合并
 
-ReMe 既可以作为本地记忆服务，通过 CLI、HTTP API 或 MCP server 接入，也可以通过 Python API 嵌入宿主进程。宿主集成可根据不同
-runtime 的能力，将记忆指引、召回和捕获接入 Agent 生命周期。
+---
 
-| Agent                      | 推荐接入方式                                                                                                                              | 接入后能力                                                            |
-| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- |
-| **DeepSeek Harness**       | 使用 `dsh plugin --profile web add @agentscope-ai/reme` 安装 [`@agentscope-ai/reme`](typescript/README_ZH.md#deepseek-harness)。 | 长期记忆指引、`reme_search` 工具，以及自动捕获已完成的主 Agent 对话。 |
-| **OpenClaw**               | 使用 `openclaw plugins install @agentscope-ai/reme` 安装 [`@agentscope-ai/reme`](typescript/README_ZH.md#openclaw)。             | 原生记忆工具、用户触发运行前召回和自动对话捕获。                      |
-| **QwenPaw**                | 通过 Python API 在进程内嵌入 ReMe。                                                                                                       | 复用宿主生命周期和模型配置，同时保持记忆本地、文件化。                |
-| **Claude Code**            | 启动 streamable HTTP MCP service，并安装 [ReMe 插件](integrations/claude_code/reme)。                                                     | MCP 召回工具、`reme-memory` skill，以及自动记录会话的 Stop hook。     |
-| **Hermes**                 | 启动 HTTP service，并安装 [ReMe provider](integrations/hermes_agent)。                                                                    | 模型调用前召回，每轮对话完成后异步执行 `auto_memory`。                |
-| **Codex 及其他 CLI Agent** | 安装或复制 [ReMe Memory skill](skills/reme_memory/SKILL.md)。                                                                             | 通过 CLI 搜索、读取和写入记忆；自动捕获需要显式接入宿主生命周期。     |
+#### 3. summary_memory — 记忆持久化
 
-<p align="center"><b>集成演示</b></p>
+[Summarizer](reme/memory/file_based/components/summarizer.py) 采用 **ReAct + 文件工具** 模式，让 AI 自主决定写什么、写到哪。
 
-<table>
-  <tr>
-    <td align="center"></td>
-    <td width="45%" align="center"><b>Auto Memory</b></td>
-    <td width="45%" align="center"><b>Auto Dream</b></td>
-  </tr>
-  <tr>
-    <td align="center"><b>QwenPaw</b></td>
-    <td width="45%">
-      <img src="docs/figure/qwenpaw-auto-memory.gif" alt="QwenPaw Auto Memory 演示" width="100%">
-    </td>
-    <td width="45%">
-      <img src="docs/figure/qwenpaw-auto-dream.gif" alt="QwenPaw Auto Dream 演示" width="100%">
-    </td>
-  </tr>
-  <tr>
-    <td align="center"><b>Claude Code</b></td>
-    <td width="45%">
-      <img src="docs/figure/cc-auto-memory.gif" alt="Claude Code Auto Memory 演示" width="100%">
-    </td>
-    <td width="45%">
-      <img src="docs/figure/cc-auto-dream.gif" alt="Claude Code Auto Dream 演示" width="100%">
-    </td>
-  </tr>
-</table>
-
-## 🧠 ReMe 如何工作
-
-> Memory as File, File as Memory.
-
-ReMe 将 **记忆视为文件**，让过滤后的对话来源记录和外部资料从 `session/`、`resource/` 渐进加工到 `daily/`，再沉淀为
-`digest/`。默认 workspace 是当前目录下的 `.reme/`；可通过 `workspace_dir=...` 选择其他由用户控制的位置。
-
-### Workspace 结构
-
-```text
-<workspace_dir>/
-├── metadata/       # 可重建的索引、图谱、catalog 和缓存
-├── session/        # 对话来源记录和 Agent session
-│   ├── dialog/
-│   │   └── <session_id>.jsonl  # auto_memory 保存的来源消息
-│   └── claude_code/
-│       └── <session_id>.jsonl  # auto_memory_cc 使用的 ReMe 副本
-├── mem_session/    # Agent wrapper 生成的 session/配置，不是用户记忆
-│   ├── agentscope/
-│   ├── claude_config/
-│   └── codex/
-├── resource/            # 外部原始材料
-│   ├── <resource>.<ext>  # 根目录文件进入当天 daily 层
-│   └── YYYY-MM-DD/
-│       └── <resource>.<ext>
-├── daily/               # 浅加工记忆：当天事实、对话摘要、资源解读
-│   ├── YYYY-MM-DD.md
-│   └── YYYY-MM-DD/
-│       ├── <generated_name>.md  # 按主题命名的对话或资源卡片
-│       └── interests.yaml
-└── digest/              # 长期记忆：个人事实、流程经验、知识节点
-    ├── personal/
-    │   └── {topic/event}.md
-    ├── procedure/
-    │   └── {topic/event}.md
-    └── wiki/
-        └── {topic/event}.md
+```mermaid
+graph LR
+    M[messages] --> A[ReActAgent<br>reme_summarizer]
+    A -->|read| R[读取 memory/YYYY-MM-DD.md]
+    R --> T{思考: 如何合并?}
+    T -->|write| W[覆盖写入]
+    T -->|edit| E[精确替换]
+    W --> F[memory/YYYY-MM-DD.md]
+    E --> F
 ```
 
-<p align="center">
-  <img src="docs/figure/reme-overview.svg" alt="ReMe 文件化记忆系统总览" width="92%">
-</p>
+**文件工具**（[FileIO](reme/memory/file_based/tools/file_io.py)）：
 
-### 记忆生命周期
+| 工具      | 功能      |
+|---------|---------|
+| `read`  | 读取文件内容  |
+| `write` | 覆盖写入文件  |
+| `edit`  | 精确匹配后替换 |
 
-ReMe 遵循 capture → index → consolidate → recall 的循环。workspace 文件是持久化的事实来源，`metadata/` 中的内容均可重建。
+---
 
-| 能力                                        | 入口                                      | 作用                                                                                         | 输出                                                         |
-| ------------------------------------------- | ----------------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------------ |
-| [`auto_memory`](docs/zh/auto_memory.md)     | Agent hook 或 `reme auto_memory`          | 提炼有长期价值的对话事实，同时保留过滤后的对话来源记录。                                     | `session/dialog/*.jsonl`、`daily/<date>/<generated-name>.md` |
-| [`auto_resource`](docs/zh/auto_resource.md) | 资源监听或 `reme auto_resource`           | 将 `resource/` 下的文件转为带来源链接、按内容命名的 daily 卡片。                             | `daily/<date>/<resource-card>.md`                            |
-| [`auto_index`](docs/zh/memory_search.md)    | 后台监听或 `reme reindex`                 | 实时索引 `daily/` 和 `digest/` 中的 Markdown；全量重建还会扫描 `resource/` 和 JSONL。        | 可检索的 chunks、BM25、wikilink 图谱和可选向量               |
-| [`auto_dream`](docs/zh/auto_dream.md)       | `dream_cron` 或 `reme auto_dream`         | 默认从最近两天内变化的文件中最多提取 5 个可复用 unit，再创建、印证、补充或修正 digest 节点。 | `digest/**`、`daily/<date>/interests.yaml`                   |
-| [`proactive`](docs/zh/proactive.md)         | Agent 决定主动行动前调用 `reme proactive` | 读取 `auto_dream` 生成的 topics；是否以及如何提醒用户由宿主 Agent 决定。                     | 来自 `daily/<date>/interests.yaml` 的结构化 topics           |
+#### 4. compact_tool_result — 工具结果压缩
 
-<table>
-  <tr>
-    <td align="center" width="50%">
-      <img src="docs/figure/memory-as-file.svg" alt="Memory as File" width="92%">
-    </td>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-memory-resource.svg" alt="Auto Memory and Resource" width="92%">
-    </td>
-  </tr>
-  <tr>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-dream-and-proactive.svg" alt="Auto Dream and Proactive" width="92%">
-    </td>
-    <td align="center" width="50%">
-      <img src="docs/figure/auto-index-and-memory-search.svg" alt="Auto Index and Memory Search" width="92%">
-    </td>
-  </tr>
-</table>
+[ToolResultCompactor](reme/memory/file_based/components/tool_result_compactor.py) 解决工具输出过长导致上下文膨胀的问题。
 
-搜索返回带行号范围的相关 chunks 和数量受限的 wikilink 邻居；可选向量结果通过 RRF 与 BM25 融合。
+```mermaid
+graph LR
+    M[messages] --> L{遍历 tool_result<br>len > threshold?}
+    L -->|否| K[保留原样]
+    L -->|是| T[truncate_text<br>截断到 threshold]
+    T --> S[完整内容写入<br>tool_result/uuid.txt]
+    S --> R[消息追加文件路径引用]
+    R --> C[cleanup_expired_files<br>清理过期文件]
+```
 
-> [!IMPORTANT]
->
-> `proactive` 只读取并暴露 Auto Dream 生成的兴趣主题，不会自行联网、发送通知或改写知识库；是否以及如何使用主题，由宿主 Agent
-> 决定。
+- **自动清理**：过期文件（超过 `retention_days`）在 `start`/`close`/`compact_tool_result` 时自动删除
 
-## 📊 评测结果
+---
 
-ReMe 通过 Agent 多轮搜索与读取的方式，评测多会话和超长上下文中的记忆能力。下表为仓库中已公开的参考实验结果；模型、prompt、数据集和评判细节见各评测文档。
+#### 5. memory_search — 记忆检索
 
-| 基准                                                                        | 设置        |            样本量 | Agentic 得分 | 主要检验内容                   |
-| --------------------------------------------------------------------------- | ----------- | ----------------: | -----------: | ------------------------------ |
-| **[LongMemEval cleaned-s](https://reme.agentscope.io/?doc=longmemeval-zh)** | **整体**    |        **500 题** |    **89.4%** | 跨会话检索、知识更新与时间推理 |
-| [BEAM](https://reme.agentscope.io/?doc=beam-zh)                             | 100K 上下文 | 20 cases / 400 题 |        66.1% | 十类长上下文记忆任务           |
-| [BEAM](https://reme.agentscope.io/?doc=beam-zh)                             | 1M 上下文   | 35 cases / 700 题 |        65.0% | 超长对话设置                   |
+[MemorySearch](reme/memory/file_based/tools/memory_search.py) 提供**向量 + BM25 混合检索**能力。
 
-在仓库的 [π-Bench 评测](https://reme.agentscope.io/?doc=pibench-zh)中，ReMe Agent 在 5 种用户角色上的平均 **PROC 得分为 0.580**
-，比相同测试模型配置的 NanoBot 高 2.4%。PROC 用于评估隐藏意图完成、针对性澄清、跨会话偏好和规范复用、跨任务依赖推断以及欠规格请求推进等主动性能力。
+```mermaid
+graph LR
+    Q[query] --> E[Embedding<br>向量化]
+    E --> V[vector_search<br>语义相似]
+    Q --> B[BM25<br>关键词匹配]
+    V -->|" weight: 0.7 "| M[去重 + 加权融合]
+    B -->|" weight: 0.3 "| M
+    M --> F[min_score 过滤]
+    F --> R[Top-N 结果]
+```
 
-## 🧩 扩展与插件
+- **融合机制**：向量权重 0.7 + BM25 权重 0.3，兼顾语义相似和精确匹配
 
-插件是可选的独立 Python distribution，可以贡献 Component、Step、Job backend 和配置，并通过配置显式启用。每日论文与 Auto Fin
-均已独立打包，源码 distribution 及说明分别见[每日论文](plugins/daily_paper/README_ZH.md)和
-[Auto Fin](plugins/auto-fin/README_ZH.md)。
+---
 
-| 插件                                                       | 能力                                                                           |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------ |
-| [每日论文](https://reme.agentscope.io/?doc=daily-paper-zh) | 发现并排序论文，使用 Agent 解读 PDF，生成文件化论文笔记和五分钟简报。          |
-| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-zh)    | 拉取主题相关财联社新闻，搜索 ReMe 历史材料并生成带 wikilink 的 Markdown 报告。 |
+#### 6. ReMeInMemoryMemory — 会话内存
 
-安装、查看、校验、启用和卸载 ReMe 插件的方法见[插件管理](docs/zh/plugin_management.md)。
+[ReMeInMemoryMemory](reme/memory/file_based/reme_in_memory_memory.py) 扩展 AgentScope 的 `InMemoryMemory`，提供 Token
+感知的内存管理和原始对话持久化能力。
 
-## 📚 文档
+```mermaid
+graph LR
+    C[content] --> G[get_memory<br>exclude_mark=COMPRESSED]
+    G --> F[排除已压缩消息]
+    F --> P{prepend_summary?}
+    P -->|是| S[头部插入 previous-summary]
+    S --> O[输出 messages]
+    P -->|否| O
+    M[mark_messages_compressed] --> D[持久化到 dialog/YYYY-MM-DD.jsonl]
+    D --> R[从内存移除]
+```
 
-下列文档覆盖主要使用流程，并以当前代码的运行时契约为准。
+| 功能                               | 说明                    |
+|----------------------------------|-----------------------|
+| `get_memory`                     | 按标记过滤，自动追加压缩摘要        |
+| `estimate_tokens`                | 估算上下文 Token 用量        |
+| `state_dict` / `load_state_dict` | 状态序列化/反序列化（会话持久化）     |
+| `mark_messages_compressed`       | 标记消息压缩并持久化到 dialog 目录 |
+| `clear_content`                  | 持久化所有消息后清空内存          |
 
-| 文档                                                                     | 主要内容                                                               |
-| ------------------------------------------------------------------------ | ---------------------------------------------------------------------- |
-| [快速开始](docs/zh/quick_start.md)                                       | 安装 ReMe、启动服务，并执行首次文件和记忆操作。                        |
-| [Memory as File](docs/zh/memory_as_file.md)                              | 理解 workspace 分层、frontmatter、wikilink、chunk 和文件事实来源模型。 |
-| [Auto Memory](docs/zh/auto_memory.md)                                    | 保留过滤后的对话来源记录，并提炼可复用的 daily 记忆卡片。              |
-| [Auto Resource](docs/zh/auto_resource.md)                                | 导入支持的文本资料，转换为可追溯来源的 daily 卡片。                    |
-| [Auto Dream](docs/zh/auto_dream.md) 与 [Auto Link](docs/zh/auto_link.md) | 将 daily 记忆整理为持续演化的 digest 节点和可读 wikilink 关系。        |
-| [记忆检索](docs/zh/memory_search.md)                                     | 使用 BM25、可选向量、RRF 融合、行号范围召回和渐进式链接扩展。          |
-| [Proactive](docs/zh/proactive.md)                                        | 安全读取兴趣主题，并将其接入宿主 Agent 的决策流程。                    |
-| [应用场景](docs/zh/reme_scene.md)                                        | 查看金融研究、研发记忆和个人知识库的完整使用示例。                     |
-| [框架说明](docs/zh/framework.md)                                         | 理解 Application、Job、Step、Component、service、配置和生命周期边界。  |
-| [TypeScript 集成](typescript/README_ZH.md)                               | 配置统一 client，以及 DeepSeek Harness 和 OpenClaw 原生适配器。        |
-| [ReMe 博客](https://agentscope-ai.github.io/ReMe/?doc=zh-reme-blog)      | 了解完整产品故事、设计动机、使用示例和评测摘要。                       |
+**原始对话持久化**：当消息被压缩或清空时，自动保存到 `{dialog_path}/{date}.jsonl`，每行一条 JSON 格式的消息记录。
 
-## 🛠️ 常用命令
+---
 
-运行 `reme help` 可查看完整 job 列表。常用 workspace 与维护命令如下：
+#### 7. pre_reasoning_hook — 推理前预处理
 
-| 命令                                      | 作用                                                          |
-| ----------------------------------------- | ------------------------------------------------------------- |
-| `reme status`                             | 查看有状态数据组件的内存估算及进程 RSS。                      |
-| [`reme search`](docs/zh/memory_search.md) | 默认使用 BM25 和 wikilink 检索，启用后增加向量检索。          |
-| `reme read` / `reme write` / `reme edit`  | 检查和维护 Markdown 记忆文件。                                |
-| `reme traverse` / `reme graph_snapshot`   | 浏览 wikilink 邻域或按类别组织的 digest 图。                  |
-| `reme chat`                               | 与可感知 workspace 的只读 Agent 进行流式对话；需要 LLM 凭证。 |
-| `reme reindex`                            | 基于已有文件重建检索和 wikilink 索引。                        |
+整合上述组件的统一入口，在每轮推理前自动管理上下文。
 
-## 🤝 社区与贡献
+```mermaid
+graph LR
+    M[messages] --> TC[compact_tool_result<br>压缩超长工具输出]
+    TC --> CC[check_context<br>计算剩余空间]
+    CC --> D{messages_to_compact<br>非空?}
+    D -->|否| K[返回原消息 + 原摘要]
+    D -->|是| V{is_valid?}
+    V -->|否| K
+    V -->|是| CM[compact_memory<br>同步生成摘要]
+    V -->|是| SM[add_async_summary_task<br>异步持久化]
+    CM --> R[返回 messages_to_keep + 新摘要]
+```
 
-- **问题反馈、需求与帮助**：请先查看 [Open Issues](https://github.com/agentscope-ai/ReMe/issues)；如无相关讨论，可新建 Issue
-  说明背景、目标行为和影响范围。
-- **代码贡献**：改动前建议阅读仓库内的[贡献指南](docs/zh/contributing.md)。架构与扩展方式以源码、schema 和测试为准。
-- **文档贡献**：请直接更新本仓库 `docs/en/`、`docs/zh/` 或对应 package 目录中的规范源文件；文档站点会从这些文件生成。
-- **提交规范**：建议使用 Conventional Commits，例如 `feat(search): add link expansion option`、
-  `docs(zh): update quick start`。
-- **提交前检查**：提交 PR 前请尽量运行 `pre-commit run --all-files` 和 `pytest`；如有依赖 LLM、embedding 或外部服务的测试无法运行，请在
-  PR 中说明。
-- **项目文档**：访问 [reme.agentscope.io](https://reme.agentscope.io)。
+**执行流程**：
+
+1. `compact_tool_result` — 压缩超长工具输出
+2. `check_context` — 检查上下文是否超限
+3. `compact_memory` — 生成压缩摘要（同步）
+4. `summary_memory` — 持久化记忆（异步后台）
+
+---
+
+## 🗃️ 基于向量库的记忆系统
+
+[ReMe Vector Based](reme/reme.py) 是基于向量库的记忆系统核心类，支持三种记忆类型的统一管理：
+
+| 记忆类型         | 用途               |
+|--------------|------------------|
+| **个人记忆**     | 记录用户偏好、习惯        |
+| **任务/程序性记忆** | 记录任务执行经验、成功/失败模式 |
+| **工具记忆**     | 记录工具使用经验、参数优化    |
+
+### 核心能力
+
+| 方法                 | 功能       | 说明             |
+|--------------------|----------|----------------|
+| `summarize_memory` | 🧠 记忆总结  | 从对话中自动提取并存储记忆  |
+| `retrieve_memory`  | 🔍 记忆检索  | 根据查询检索相关记忆     |
+| `add_memory`       | ➕ 添加记忆   | 手动添加记忆到向量库     |
+| `get_memory`       | 📖 获取记忆  | 通过 ID 获取单条记忆   |
+| `update_memory`    | ✏️ 更新记忆  | 更新已有记忆的内容或元数据  |
+| `delete_memory`    | 🗑️ 删除记忆 | 删除指定记忆         |
+| `list_memory`      | 📋 列出记忆  | 列出某类记忆，支持过滤和排序 |
+
+### 安装与环境变量
+
+安装和环境变量配置与 [ReMeLight 一致](#安装)，通过环境变量设置 API 密钥，可写在项目根目录的 `.env` 文件中。
+
+
+### Python 使用
+
+```python
+import asyncio
+
+from reme import ReMe
+
+
+async def main():
+    # 初始化 ReMe
+    reme = ReMe(
+        working_dir=".reme",
+        default_llm_config={
+            "backend": "openai",
+            "model_name": "qwen3.5-plus",
+        },
+        default_embedding_model_config={
+            "backend": "openai",
+            "model_name": "text-embedding-v4",
+            "dimensions": 1024,
+        },
+        default_vector_store_config={
+            "backend": "local",  # 支持 local/chroma/qdrant/elasticsearch
+        },
+    )
+    await reme.start()
+
+    messages = [
+        {"role": "user", "content": "帮我写一个 Python 脚本", "time_created": "2026-02-28 10:00:00"},
+        {"role": "assistant", "content": "好的，我来帮你写", "time_created": "2026-02-28 10:00:05"},
+    ]
+
+    # 1. 从对话中总结记忆（自动提取用户偏好、任务经验等）
+    result = await reme.summarize_memory(
+        messages=messages,
+        user_name="alice",  # 个人记忆
+        # task_name="code_writing",  # 任务记忆
+    )
+    print(f"总结结果: {result}")
+
+    # 2. 检索相关记忆
+    memories = await reme.retrieve_memory(
+        query="Python 编程",
+        user_name="alice",
+        # task_name="code_writing",
+    )
+    print(f"检索结果: {memories}")
+
+    # 3. 手动添加记忆
+    memory_node = await reme.add_memory(
+        memory_content="用户喜欢简洁的代码风格",
+        user_name="alice",
+    )
+    print(f"添加的记忆: {memory_node}")
+    memory_id = memory_node.memory_id
+
+    # 4. 通过 ID 获取单条记忆
+    fetched_memory = await reme.get_memory(memory_id=memory_id)
+    print(f"获取的记忆: {fetched_memory}")
+
+    # 5. 更新记忆内容
+    updated_memory = await reme.update_memory(
+        memory_id=memory_id,
+        user_name="alice",
+        memory_content="用户喜欢简洁且带注释的代码风格",
+    )
+    print(f"更新后的记忆: {updated_memory}")
+
+    # 6. 列出用户的所有记忆（支持过滤和排序）
+    all_memories = await reme.list_memory(
+        user_name="alice",
+        limit=10,
+        sort_key="time_created",
+        reverse=True,
+    )
+    print(f"用户记忆列表: {all_memories}")
+
+    # 7. 删除指定记忆
+    await reme.delete_memory(memory_id=memory_id)
+    print(f"已删除记忆: {memory_id}")
+
+    # 8. 删除所有记忆（谨慎使用）
+    # await reme.delete_all()
+
+    await reme.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### 技术架构
+
+```mermaid
+graph LR
+    User[用户 / Agent] --> ReMe[Vector Based ReMe]
+    ReMe --> Summarize[记忆总结]
+    ReMe --> Retrieve[记忆检索]
+    ReMe --> CRUD[增删改查]
+    Summarize --> PersonalSum[PersonalSummarizer]
+    Summarize --> ProceduralSum[ProceduralSummarizer]
+    Summarize --> ToolSum[ToolSummarizer]
+    Retrieve --> PersonalRet[PersonalRetriever]
+    Retrieve --> ProceduralRet[ProceduralRetriever]
+    Retrieve --> ToolRet[ToolRetriever]
+    PersonalSum --> VectorStore[向量数据库]
+    ProceduralSum --> VectorStore
+    ToolSum --> VectorStore
+    PersonalRet --> VectorStore
+    ProceduralRet --> VectorStore
+    ToolRet --> VectorStore
+```
+
+### 实验效果
+
+本实验部分在 LoCoMo和HaluMem 两个数据集上进行评测，实验设置如下：
+
+1. **ReMe 使用模型**：如各表 backbone 列所示。
+2. **评估使用模型**：采用 LLM-as-a-Judge 协议（参照 MemOS）——每条回答由 GPT-4o-mini 裁判模型打分。
+
+实验设置尽量与各基线论文保持一致，以复用其公开结果。
+
+### LoCoMo
+
+| Method   | Single Hop | Multi Hop | Temporal  | Open Domain | Overall   |
+|----------|------------|-----------|-----------|-------------|-----------|
+| MemoryOS | 62.43      | 56.50     | 37.18     | 40.28       | 54.70     |
+| Mem0     | 66.71      | 58.16     | 55.45     | 40.62       | 61.00     |
+| MemU     | 72.77      | 62.41     | 33.96     | 46.88       | 61.15     |
+| MemOS    | 81.45      | 69.15     | 72.27     | 60.42       | 75.87     |
+| HiMem    | 89.22      | 70.92     | 74.77     | 54.86       | 80.71     |
+| Zep      | 88.11      | 71.99     | 74.45     | 66.67       | 81.06     |
+| TiMem    | 81.43      | 62.20     | 77.63     | 52.08       | 75.30     |
+| TSM      | 84.30      | 66.67     | 71.03     | 58.33       | 76.69     |
+| MemR3    | 89.44      | 71.39     | 76.22     | 61.11       | 81.55     |
+| **ReMe** | **89.89**  | **82.98** | **83.80** | **71.88**   | **86.23** |
+
+### HaluMem
+
+| Method      | Memory Integrity | Memory Accuracy | QA Accuracy |
+|-------------|------------------|-----------------|-------------|
+| MemoBase    | 14.55            | 92.24           | 35.53       |
+| Supermemory | 41.53            | 90.32           | 54.07       |
+| Mem0        | 42.91            | 86.26           | 53.02       |
+| ProMem      | **73.80**        | 89.47           | 62.26       |
+| **ReMe**    | 67.72            | **94.06**       | **88.78**   |
+
+---
+
+## 🧪 程序化记忆论文
+
+> 我们的程序性（任务）记忆论文已在 [arXiv](https://arxiv.org/abs/2512.10696) 发布
+
+### 🌍 [Appworld 实验](benchmark/appworld/quickstart.md)
+
+我们在 Appworld 环境上使用 Qwen3-8B（非思考模式）进行评测：
+
+| 方法      | Avg@4               | Pass@4              |
+|---------|---------------------|---------------------|
+| 无 ReMe  | 0.1497              | 0.3285              |
+| 使用 ReMe | 0.1706 **(+2.09%)** | 0.3631 **(+3.46%)** |
+
+Pass@K 衡量在生成 K 个候选中，至少一个成功完成任务（score=1）的概率。
+当前实验使用的是内部 AppWorld 环境，可能与对外版本存在轻微差异。
+
+关于如何复现实验的更多细节，见 [quickstart.md](benchmark/appworld/quickstart.md)
+
+### 🔧 [BFCL-V3 实验](benchmark/bfcl/quickstart.md)
+
+我们在 BFCL-V3 multi-turn-base 任务（随机划分 50 train / 150 val）上，使用 Qwen3-8B（思考模式）进行评测：
+
+| 方法      | Avg@4               | Pass@4              |
+|---------|---------------------|---------------------|
+| 无 ReMe  | 0.4033              | 0.5955              |
+| 使用 ReMe | 0.4450 **(+4.17%)** | 0.6577 **(+6.22%)** |
+
+关于如何复现实验的更多细节，见 [quickstart.md](benchmark/bfcl/quickstart.md)
+
+## ⭐ 社区与支持
+
+- **Star 与 Watch**：Star 可让更多智能体开发者发现 ReMe；Watch 可助你第一时间获知新版本与特性。
+- **分享你的成果**：在 Issue 或 Discussion 中分享 ReMe 为你的智能体解锁了什么——我们非常乐意展示社区的优秀案例。
+- **需要新功能？** 提交 Feature Request，我们将与社区一起完善。
+- **代码贡献**：欢迎任何形式的代码贡献，请参阅 [贡献指南](docs/contribution.md)。
+- **致谢**：感谢 OpenClaw、Mem0、MemU、CoPaw 等优秀的开源项目，为项目带来诸多启发与帮助。
 
 ### 贡献者
 
@@ -372,17 +622,34 @@ ReMe 通过 Agent 多轮搜索与读取的方式，评测多会话和超长上
   <img src="https://contrib.rocks/image?repo=agentscope-ai/ReMe" alt="贡献者" />
 </a>
 
+---
+
 ## 📄 引用
 
 ```bibtex
-@software{ReMe2026,
-  title = {Remember me, Refine me: Memory Management Kit for Agents},
+@software{AgentscopeReMe2025,
+  title = {AgentscopeReMe: Memory Management Kit for Agents},
   author = {ReMe Team},
   url = {https://reme.agentscope.io},
-  year = {2026}
+  year = {2025}
 }
 ```
 
+---
+
 ## ⚖️ 许可证
 
 本项目基于 Apache License 2.0 开源，详情参见 [LICENSE](./LICENSE) 文件。
+
+---
+
+## 🤔 为什么叫 ReMe？
+
+ReMe 是 **Remember Me** 和 **Refine Me** 的缩写，寓意让 AI 智能体「记住我」并在交互中「精进自我」。我们希望 ReMe
+不只是一个冷冰冰的记忆模块，而是能让智能体真正理解用户、积累经验、持续进化的伙伴。
+
+---
+
+## 📈 Star 历史
+
+[![Star History Chart](https://api.star-history.com/svg?repos=agentscope-ai/ReMe&type=Date)](https://www.star-history.com/#agentscope-ai/ReMe&Date)

benchmark/appworld/appworld_react_agent.py

@@ -0,0 +1,360 @@
+# flake8: noqa: E402, E501
+# pylint: disable=E0611
+"""A minimal ReAct Agent for AppWorld tasks."""
+import os
+import re
+import time
+import json
+import datetime
+from typing import List, Any
+
+
+import ray
+import requests
+from tqdm import tqdm
+from loguru import logger
+from openai import OpenAI
+from jinja2 import Template
+from dotenv import load_dotenv
+
+from prompt import NEW_PROMPT_TEMPLATE
+from appworld import AppWorld, load_task_ids
+
+os.environ["APPWORLD_ROOT"] = "."
+
+load_dotenv("../../.env")
+
+
+@ray.remote
+class AppworldReactAgent:
+    """A minimal ReAct Agent for AppWorld tasks."""
+
+    def __init__(
+        self,
+        index: int,
+        task_ids: List[str],
+        experiment_name: str,
+        model_name: str = "qwen3-8b",
+        temperature: float = 0.9,
+        max_interactions: int = 30,
+        max_response_size: int = 129024,
+        num_trials: int = 1,
+        use_memory: bool = False,
+        memory_base_url: str = "http://0.0.0.0:8002/",
+        use_memory_addition: bool = False,
+        use_memory_deletion: bool = False,
+        delete_freq: int = 10,
+        freq_threshold: int = 5,
+        utility_threshold: float = 0.5,
+    ):
+
+        self.index: int = index
+        self.task_ids: List[str] = task_ids
+        self.experiment_name: str = experiment_name
+        self.model_name: str = model_name
+        self.temperature: float = temperature
+        self.max_interactions: int = max_interactions
+        self.max_response_size: int = max_response_size
+        self.num_trials: int = num_trials
+        self.use_memory: bool = use_memory
+        self.use_memory_addition: bool = use_memory_addition if use_memory else False
+        self.use_memory_deletion: bool = use_memory_deletion if use_memory else False
+        self.delete_freq: int = delete_freq
+        self.freq_threshold: int = freq_threshold
+        self.utility_threshold: float = utility_threshold
+
+        self.llm_client = OpenAI()
+        self.memory_base_url: str = memory_base_url
+
+        self.history: List[List[List[dict]]] = [[] for _ in range(num_trials)]
+        self.retrieved_memory_list: List[List[List[Any]]] = [[] for _ in range(num_trials)]
+
+        for run_id in range(num_trials):
+            for _ in range(len(task_ids)):
+                self.retrieved_memory_list[run_id].append([])
+                self.history[run_id].append([])
+
+    def call_llm(self, messages: list) -> str:
+        """Call the LLM to generate a response to the messages."""
+        for i in range(100):
+            try:
+                response = self.llm_client.chat.completions.create(
+                    model=self.model_name,
+                    messages=messages,
+                    temperature=self.temperature,
+                    extra_body={"enable_thinking": False},
+                    seed=0,
+                )
+
+                return response.choices[0].message.content
+
+            except Exception as e:
+                logger.exception(f"encounter error with {e.args}")
+                time.sleep(1 + i * 10)
+
+        return "call llm error"
+
+    def prompt_messages(self, run_id, task_index, previous_memories: None, world: AppWorld):
+        """Prompt the messages to the LLM."""
+        app_descriptions = json.dumps(
+            [{"name": k, "description": v} for (k, v) in world.task.app_descriptions.items()],
+            indent=1,
+        )
+        dictionary = {"supervisor": world.task.supervisor, "app_descriptions": app_descriptions}
+        sys_prompt = Template(NEW_PROMPT_TEMPLATE.lstrip()).render(dictionary)
+        query = world.task.instruction
+        if self.use_memory:
+            if len(previous_memories) == 0:
+                response = self.get_memory(world.task.instruction)
+                if response and "memory_list" in response["metadata"]:
+                    self.retrieved_memory_list[run_id][task_index] = response["metadata"]["memory_list"]
+                    task_memory = re.sub(r"\bMemory\s*(\d+)\s*[:]", r"Experience \1:", response["answer"])
+                    logger.info(f"loaded task_memory: {task_memory}")
+                    query = (
+                        "Task:\n"
+                        + query
+                        + "\n\nSome Related Experience to help you to complete the task:\n"
+                        + task_memory
+                    )
+            else:
+                formatted_memories = []
+                for i, memory in enumerate(previous_memories, 1):
+                    condition = memory["when_to_use"]
+                    memory_content = memory["content"]
+                    memory_text = f"Experience {i}:\n When to use: {condition}\n Content: {memory_content}\n"
+                    formatted_memories.append(memory_text)
+                query = (
+                    "Task:\n"
+                    + query
+                    + "\n\nSome Related Experience to help you to complete the task:\n"
+                    + "\n".join(formatted_memories)
+                )
+        messages = [
+            {"role": "system", "content": sys_prompt},
+            {"role": "user", "content": query},
+        ]
+        self.history[run_id][task_index] = messages
+
+    @staticmethod
+    def get_reward(world) -> float:
+        """Get the reward for the Appworld world."""
+        tracker = world.evaluate()
+        num_passes = len(tracker.passes)
+        num_failures = len(tracker.failures)
+        return num_passes / (num_passes + num_failures)
+
+    def extract_code_and_fix_content(
+        self,
+        text: str,
+        ignore_multiple_calls=True,
+    ) -> tuple[str, str]:
+        """Extract the code and fix the content."""
+        full_code_regex = r"```python\n(.*?)```"
+        partial_code_regex = r".*```python\n(.*)"
+
+        original_text = text
+        output_code = ""
+        match_end = 0
+        # Handle multiple calls
+        for re_match in re.finditer(full_code_regex, original_text, flags=re.DOTALL):
+            code = re_match.group(1).strip()
+            if ignore_multiple_calls:
+                text = original_text[: re_match.end()]
+                return code, text
+            output_code += code + "\n"
+            match_end = re_match.end()
+        # check for partial code match at end (no terminating ```)  following the last match
+        partial_match = re.match(
+            partial_code_regex,
+            original_text[match_end:],
+            flags=re.DOTALL,
+        )
+        if partial_match:
+            output_code += partial_match.group(1).strip()
+            # terminated due to stop condition. Add stop condition to output.
+            if not text.endswith("\n"):
+                text = text + "\n"
+            text = text + "```"
+        if len(output_code) == 0:
+            return text, text
+        else:
+            return output_code, text
+
+    def execute(self):
+        """Execute the Appworld tasks."""
+        result = []
+        counter = 0
+        for task_index, task_id in enumerate(tqdm(self.task_ids, desc=f"run_index={self.index}")):
+            t_result = None
+            previous_memories = []
+            # Run each task num_trials times
+            for run_id in range(self.num_trials):
+                start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                with AppWorld(task_id=task_id, experiment_name=f"{self.experiment_name}_run_{run_id}") as world:
+                    before_score = self.get_reward(world)
+                    for i in range(self.max_interactions):
+                        if i == 0:
+                            self.prompt_messages(
+                                run_id=run_id,
+                                task_index=task_index,
+                                previous_memories=previous_memories,
+                                world=world,
+                            )
+                        code_msg = self.call_llm(self.history[run_id][task_index])
+                        code, _ = self.extract_code_and_fix_content(code_msg)
+                        self.history[run_id][task_index].append({"role": "assistant", "content": code})
+
+                        output = world.execute(code)
+                        # if len(output) > self.max_response_size:
+                        #     # logger.warning(f"output exceed max size={len(output)}")
+                        #     output = output[: self.max_response_size]
+                        self.history[run_id][task_index].append(
+                            {"role": "user", "content": "Output:\n```\n" + output + "```\n\n"},
+                        )
+
+                        if world.task_completed():
+                            break
+
+                    after_score = self.get_reward(world)
+                    uplift_score = after_score - before_score
+
+                    if self.use_memory:
+                        if self.use_memory_addition:
+                            new_traj_list = [
+                                self.get_traj_from_task_history(task_id, self.history[run_id][task_index], after_score),
+                            ]
+                            previous_memories = self.summary_memory(new_traj_list)
+                            if after_score == 1:
+                                self.add_memory(previous_memories)
+
+                        # update the freq & utility attributes of retrieved memories
+                        update_utility: bool = after_score == 1
+                        self.update_memory_information(self.retrieved_memory_list[run_id][task_index], update_utility)
+
+                    counter += 1
+                    if self.use_memory_deletion:  # and counter % self.delete_freq == 0:
+                        self.delete_memory()
+
+                    t_result = {
+                        "task_id": world.task_id,
+                        "run_id": run_id,
+                        "experiment_name": self.experiment_name,
+                        "task_completed": world.task_completed(),
+                        "before_score": before_score,
+                        "after_score": after_score,
+                        "uplift_score": uplift_score,
+                        "task_history": self.history[run_id][task_index],
+                        "task_start_time": start_time,
+                    }
+                    if after_score == 1:
+                        break
+            result.append(t_result)
+
+        return result
+
+    def handle_api_response(self, response: requests.Response):
+        """Handle API response with proper error checking"""
+        if response.status_code != 200:
+            print(f"Error: {response.status_code}")
+            print(response.text)
+            return None
+
+        return response.json()
+
+    def get_memory(self, query: str):
+        """Retrieve relevant task memories based on a query"""
+        response = requests.post(
+            url=f"{self.memory_base_url}retrieve_task_memory",
+            json={
+                "query": query,
+                "enable_llm_rerank": False,
+                "enable_score_filter": False,
+                "top_k": 5,
+                "enable_llm_rewrite": False,
+            },
+        )
+
+        result = self.handle_api_response(response)
+        if not result:
+            return None
+
+        logger.info(f"query: {query}, response: {result}")
+        return result
+
+    def get_traj_from_task_history(self, task_id: str, task_history: list, reward: float):
+        """Get the trajectory from the task history."""
+        pattern = r"\n\nSome Related Experience to help you to complete the task:.*"
+        task_history[1]["content"] = re.sub(pattern, "", task_history[1]["content"], flags=re.DOTALL)
+        return {
+            "task_id": task_id,
+            "messages": task_history,
+            "score": reward,
+        }
+
+    def summary_memory(self, trajectories):
+        """Generate a summary of conversation messages and create task memories"""
+
+        response = requests.post(
+            url=f"{self.memory_base_url}summary_task_memory",
+            json={
+                "trajectories": trajectories,
+                "success_threshold": 1.0,
+                "enable_soft_comparison": True,
+                "validation_threshold": 0.5,
+            },
+        )
+
+        result = self.handle_api_response(response)
+        if not result:
+            return []
+
+        # Extract memory list from response
+        memory_list = result.get("metadata", {}).get("memory_list", [])
+        print(f"Task memory list created: {len(memory_list)} memories")
+        return memory_list
+
+    def add_memory(self, memory_list):
+        """Add the memory to the memory pool."""
+        response = requests.post(
+            url=f"{self.memory_base_url}add_task_memory",
+            json={
+                "memory_list": memory_list,
+            },
+        )
+        response.raise_for_status()
+
+    def update_memory_information(self, memory_list, update_utility: bool = False):
+        """Update the memory information."""
+        response = requests.post(
+            url=f"{self.memory_base_url}record_task_memory",
+            json={
+                "memory_list": memory_list,
+                "update_utility": update_utility,
+            },
+        )
+        response.raise_for_status()
+        logger.info(response.json())
+
+    def delete_memory(self):
+        """Delete the memory from the memory pool."""
+        response = requests.post(
+            url=f"{self.memory_base_url}delete_task_memory",
+            json={
+                "freq_threshold": self.freq_threshold,
+                "utility_threshold": self.utility_threshold,
+            },
+        )
+        response.raise_for_status()
+
+
+def main():
+    """Main function to run the Appworld React Agent."""
+    dataset_name = "train"
+    task_ids = load_task_ids(dataset_name)
+    agent = AppworldReactAgent(index=0, task_ids=task_ids[0:1], experiment_name=dataset_name, num_trials=1)
+    result = agent.execute()
+    logger.info(f"result={json.dumps(result)}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/appworld/prompt.py

@@ -0,0 +1,660 @@
+# flake8: noqa: E402, E501
+# pylint: disable=C0114,C0301
+# This is a basic prompt template containing all the necessary onboarding information to solve AppWorld tasks. It explains the role of the agent and the supervisor, how to explore the API documentation, how to operate the interactive coding environment and call APIs via a simple task, and provides key instructions and disclaimers.
+
+# You can adapt it as needed by your agent. You can also choose to bypass API docs app and build your own API retrieval, e.g., for FullCodeRefl, IPFunCall, etc, we asked an LLM to predict relevant APIs separately and put its documentation directly in the prompt.
+PROMPT_TEMPLATE = """
+USER:
+I am your supervisor and you are a super intelligent AI Assistant whose job is to achieve my day-to-day tasks completely autonomously.
+
+To do this, you will need to interact with app/s (e.g., spotify, venmo, etc) using their associated APIs on my behalf. For this you will undertake a *multi-step conversation* using a python REPL environment. That is, you will write the python code and the environment will execute it and show you the result, based on which, you will write python code for the next step and so on, until you've achieved the goal. This environment will let you interact with app/s using their associated APIs on my behalf.
+
+Here are three key APIs that you need to know to get more information
+
+# To get a list of apps that are available to you.
+print(apis.api_docs.show_app_descriptions())
+
+# To get the list of apis under any app listed above, e.g. supervisor
+print(apis.api_docs.show_api_descriptions(app_name='supervisor'))
+
+# To get the specification of a particular api, e.g. supervisor app's show_account_passwords
+print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords'))
+
+Each code execution will produce an output that you can use in subsequent calls. Using these APIs, you can now generate code, that the environment will execute, to solve the task.
+
+For example, consider the task:
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+
+Task:
+
+What is the password for my Spotify account?
+
+ASSISTANT:
+# Okay. Lets first find which apps are available to get the password by looking at the app descriptions.
+print(apis.api_docs.show_app_descriptions())
+
+USER:
+[
+  {
+    "name": "api_docs",
+    "description": "An app to search and explore API documentation."
+  },
+  {
+    "name": "supervisor",
+    "description": "An app to access supervisor's personal information, account credentials, addresses, payment cards, and manage the assigned task."
+  },
+  ...
+  {
+    "name": "spotify",
+    "description": "A music streaming app to stream songs and manage song, album and playlist libraries."
+  },
+  {
+    "name": "venmo",
+    "description": "A social payment app to send, receive and request money to and from others."
+  },
+  ...
+]
+
+
+ASSISTANT:
+# Looks like the supervisor app could help me with that. Lets see what apis are available under this app.
+print(apis.api_docs.show_api_descriptions(app_name='supervisor'))
+
+
+USER:
+[
+  ...
+  "show_account_passwords : Show your supervisor's account passwords."
+  ...
+]
+
+
+ASSISTANT:
+# I can use `show_account_passwords` to get the passwords. Let me see its detailed specification to understand its arguments and output structure.
+print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords'))
+
+USER:
+{
+  'app_name': 'supervisor',
+  'api_name': 'show_account_passwords',
+  'path': '/account_passwords',
+  'method': 'GET',
+  'description': "Show your supervisor's app account passwords.",
+  'parameters': [],
+  'response_schemas': {
+    'success': [{'account_name': 'string', 'password': 'string'}],
+    'failure': {'message': 'string'}
+  }
+}
+
+ASSISTANT:
+# Okay, it requires no arguments. So I can just call it directly.
+print(apis.supervisor.show_account_passwords())
+
+USER:
+[
+  {
+    "account_name": "spotify",
+    "password": "dummy_spotify_pass"
+  },
+  {
+    "account_name": "file_system",
+    "password": "dummy_fs_pass"
+  },
+  ...
+]
+
+
+ASSISTANT:
+# So the Spotify password is an entry in the `passwords` list with the account_name=spotify.
+spotify_password = [account_password["account_name"] == "spotify" for account_password in passwords][0]["password"]
+print(spotify_password)
+
+
+USER:
+dummy_spotify_pass
+
+ASSISTANT:
+# When the task is completed, I need to call apis.supervisor.complete_task(). If there is an answer, I need to pass it as an argument `answer`. I will pass the spotify_password as an answer.
+apis.supervisor.complete_task(answer=spotify_password)
+
+
+USER:
+Marked the active task complete.
+
+
+----------------------------------------------
+
+USER:
+**Key instructions and disclaimers**:
+
+1. The email addresses, access tokens and variables (e.g. spotify_password) in the example above were only for demonstration. Obtain the correct information by calling relevant APIs yourself.
+2. Only generate valid code blocks, i.e., do not put them in ```...``` or add any extra formatting. Any thoughts should be put as code comments.
+3. You can use the variables from the previous code blocks in the subsequent code blocks.
+4. Write small chunks of code and only one chunk of code in every step. Make sure everything is working correctly before making any irreversible change.
+5. The provided Python environment has access to its standard library. But modules and functions that have a risk of affecting the underlying OS, file system or process are disabled. You will get an error if do call them.
+6. Any reference to a file system in the task instructions means the file system *app*, operable via given APIs, and not the actual file system the code is running on. So do not write code making calls to os-level modules and functions.
+7. To interact with apps, only use the provided APIs, and not the corresponding Python packages. E.g., do NOT use `spotipy` for Spotify. Remember, the environment only has the standard library.
+8. The provided API documentation has both the input arguments and the output JSON schemas. All calls to APIs and parsing its outputs must be as per this documentation.
+9. For APIs that return results in "pages", make sure to consider all pages.
+10. To obtain current date or time, use Python functions like `datetime.now()` or obtain it from the phone app. Do not rely on your existing knowledge of what the current date or time is.
+11. For all temporal requests, use proper time boundaries, e.g., if I ask for something that happened yesterday, make sure to consider the time between 00:00:00 and 23:59:59. All requests are concerning a single, default (no) time zone.
+12. Any reference to my friends, family or any other person or relation refers to the people in my phone's contacts list.
+13. All my personal information, and information about my app account credentials, physical addresses and owned payment cards are stored in the "supervisor" app. You can access them via the APIs provided by the supervisor app.
+14. Once you have completed the task, call `apis.supervisor.complete_task()`. If the task asks for some information, return it as the answer argument, i.e. call `apis.supervisor.complete_task(answer=<answer>)`. For tasks that do not require an answer, just skip the answer argument or pass it as None.
+15. The answers, when given, should be just entity or number, not full sentences, e.g., `answer=10` for "How many songs are in the Spotify queue?". When an answer is a number, it should be in numbers, not in words, e.g., "10" and not "ten".
+16. You can also pass `status="fail"` in the complete_task API if you are sure you cannot solve it and want to exit.
+17. You must make all decisions completely autonomously and not ask for any clarifications or confirmations from me or anyone else.
+
+USER:
+Using these APIs, now generate code to solve the actual task:
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+
+Task:
+
+{{ instruction }}
+"""
+
+PROMPT_TEMPLATE_WITH_EXPERIENCE = """
+USER:
+I am your supervisor and you are a super intelligent AI Assistant whose job is to achieve my day-to-day tasks completely autonomously.
+
+To do this, you will need to interact with app/s (e.g., spotify, venmo, etc) using their associated APIs on my behalf. For this you will undertake a *multi-step conversation* using a python REPL environment. That is, you will write the python code and the environment will execute it and show you the result, based on which, you will write python code for the next step and so on, until you've achieved the goal. This environment will let you interact with app/s using their associated APIs on my behalf.
+
+Here are three key APIs that you need to know to get more information
+
+# To get a list of apps that are available to you.
+print(apis.api_docs.show_app_descriptions())
+
+# To get the list of apis under any app listed above, e.g. supervisor
+print(apis.api_docs.show_api_descriptions(app_name='supervisor'))
+
+# To get the specification of a particular api, e.g. supervisor app's show_account_passwords
+print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords'))
+
+Each code execution will produce an output that you can use in subsequent calls. Using these APIs, you can now generate code, that the environment will execute, to solve the task.
+
+For example, consider the task:
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+
+Task:
+
+What is the password for my Spotify account?
+
+ASSISTANT:
+# Okay. Lets first find which apps are available to get the password by looking at the app descriptions.
+print(apis.api_docs.show_app_descriptions())
+
+USER:
+[
+  {
+    "name": "api_docs",
+    "description": "An app to search and explore API documentation."
+  },
+  {
+    "name": "supervisor",
+    "description": "An app to access supervisor's personal information, account credentials, addresses, payment cards, and manage the assigned task."
+  },
+  ...
+  {
+    "name": "spotify",
+    "description": "A music streaming app to stream songs and manage song, album and playlist libraries."
+  },
+  {
+    "name": "venmo",
+    "description": "A social payment app to send, receive and request money to and from others."
+  },
+  ...
+]
+
+
+ASSISTANT:
+# Looks like the supervisor app could help me with that. Lets see what apis are available under this app.
+print(apis.api_docs.show_api_descriptions(app_name='supervisor'))
+
+
+USER:
+[
+  ...
+  "show_account_passwords : Show your supervisor's account passwords."
+  ...
+]
+
+
+ASSISTANT:
+# I can use `show_account_passwords` to get the passwords. Let me see its detailed specification to understand its arguments and output structure.
+print(apis.api_docs.show_api_doc(app_name='supervisor', api_name='show_account_passwords'))
+
+USER:
+{
+  'app_name': 'supervisor',
+  'api_name': 'show_account_passwords',
+  'path': '/account_passwords',
+  'method': 'GET',
+  'description': "Show your supervisor's app account passwords.",
+  'parameters': [],
+  'response_schemas': {
+    'success': [{'account_name': 'string', 'password': 'string'}],
+    'failure': {'message': 'string'}
+  }
+}
+
+ASSISTANT:
+# Okay, it requires no arguments. So I can just call it directly.
+print(apis.supervisor.show_account_passwords())
+
+USER:
+[
+  {
+    "account_name": "spotify",
+    "password": "dummy_spotify_pass"
+  },
+  {
+    "account_name": "file_system",
+    "password": "dummy_fs_pass"
+  },
+  ...
+]
+
+
+ASSISTANT:
+# So the Spotify password is an entry in the `passwords` list with the account_name=spotify.
+spotify_password = [account_password["account_name"] == "spotify" for account_password in passwords][0]["password"]
+print(spotify_password)
+
+
+USER:
+dummy_spotify_pass
+
+ASSISTANT:
+# When the task is completed, I need to call apis.supervisor.complete_task(). If there is an answer, I need to pass it as an argument `answer`. I will pass the spotify_password as an answer.
+apis.supervisor.complete_task(answer=spotify_password)
+
+
+USER:
+Marked the active task complete.
+
+
+----------------------------------------------
+
+USER:
+**Key instructions and disclaimers**:
+
+1. The email addresses, access tokens and variables (e.g. spotify_password) in the example above were only for demonstration. Obtain the correct information by calling relevant APIs yourself.
+2. Only generate valid code blocks, i.e., do not put them in ```...``` or add any extra formatting. Any thoughts should be put as code comments.
+3. You can use the variables from the previous code blocks in the subsequent code blocks.
+4. Write small chunks of code and only one chunk of code in every step. Make sure everything is working correctly before making any irreversible change.
+5. The provided Python environment has access to its standard library. But modules and functions that have a risk of affecting the underlying OS, file system or process are disabled. You will get an error if do call them.
+6. Any reference to a file system in the task instructions means the file system *app*, operable via given APIs, and not the actual file system the code is running on. So do not write code making calls to os-level modules and functions.
+7. To interact with apps, only use the provided APIs, and not the corresponding Python packages. E.g., do NOT use `spotipy` for Spotify. Remember, the environment only has the standard library.
+8. The provided API documentation has both the input arguments and the output JSON schemas. All calls to APIs and parsing its outputs must be as per this documentation.
+9. For APIs that return results in "pages", make sure to consider all pages.
+10. To obtain current date or time, use Python functions like `datetime.now()` or obtain it from the phone app. Do not rely on your existing knowledge of what the current date or time is.
+11. For all temporal requests, use proper time boundaries, e.g., if I ask for something that happened yesterday, make sure to consider the time between 00:00:00 and 23:59:59. All requests are concerning a single, default (no) time zone.
+12. Any reference to my friends, family or any other person or relation refers to the people in my phone's contacts list.
+13. All my personal information, and information about my app account credentials, physical addresses and owned payment cards are stored in the "supervisor" app. You can access them via the APIs provided by the supervisor app.
+14. Once you have completed the task, call `apis.supervisor.complete_task()`. If the task asks for some information, return it as the answer argument, i.e. call `apis.supervisor.complete_task(answer=<answer>)`. For tasks that do not require an answer, just skip the answer argument or pass it as None.
+15. The answers, when given, should be just entity or number, not full sentences, e.g., `answer=10` for "How many songs are in the Spotify queue?". When an answer is a number, it should be in numbers, not in words, e.g., "10" and not "ten".
+16. You can also pass `status="fail"` in the complete_task API if you are sure you cannot solve it and want to exit.
+17. You must make all decisions completely autonomously and not ask for any clarifications or confirmations from me or anyone else.
+18. Some Related Experience to help you to complete the task:
+{{experience}}
+
+USER:
+Using these APIs, now generate code to solve the actual task:
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+
+Task:
+
+{{ instruction }}
+"""
+
+NEW_PROMPT_TEMPLATE = """
+USER:
+I am your supervisor and you are a super intelligent AI Assistant whose job is to achieve my day-to-day tasks completely autonomously.
+
+To do this, you will need to interact with app/s (e.g., spotify, venmo etc) using their associated APIs on my behalf. For this you will undertake a *multi-step conversation* using a python REPL environment. That is, you will write the python code and the environment will execute it and show you the result, based on which, you will write python code for the next step and so on, until you've achieved the goal. This environment will let you interact with app/s using their associated APIs on my behalf.
+
+Here are three key APIs that you need to know to get more information
+
+# To get a list of apps that are available to you.
+
+```python
+print(apis.api_docs.show_app_descriptions())
+```
+
+# To get the list of apis under any app listed above, e.g. spotify
+
+```python
+print(apis.api_docs.show_api_descriptions(app_name='spotify'))
+```
+
+# To get the specification of a particular api, e.g. spotify app's login api
+
+```python
+print(apis.api_docs.show_api_doc(app_name='spotify', api_name='login'))
+```
+
+Each code execution will produce an output that you can use in subsequent calls. Using these APIs, you can now generate code, that I will execute, to solve the task. Let's start with the task
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+Task: How many playlists do I have in Spotify?
+
+ASSISTANT:
+Okay. Lets first find which APIs are available to use in Spotify.
+Code:
+```python
+print(apis.api_docs.show_api_descriptions(app_name='spotify'))
+```
+
+USER:
+Output:
+```
+[
+ ...
+ "login : Login to your account.",
+ "logout : Logout from your account.",
+ ...
+]
+```
+
+
+ASSISTANT:
+Okay. Looks like I can use the `login` api. Lets find its specifications.
+
+Code:
+```python
+print(apis.api_docs.show_api_doc(app_name='spotify', api_name='login'))
+```
+
+
+USER:
+Output:
+```
+{
+ "app_name": "spotify",
+ "api_name": "login",
+ "path": "/auth/token",
+ "method": "POST",
+ "description": "Login to your account.",
+ "parameters": [
+  {
+   "name": "username",
+   "type": "string",
+   "required": true,
+   "description": "Your account email.",
+   "default": null,
+   "constraints": []
+  },
+  {
+   "name": "password",
+   "type": "string",
+   "required": true,
+   "description": "Your account password.",
+   "default": null,
+   "constraints": []
+  }
+ ],
+ "response_schemas": {
+  "success": {
+   "token_type": "string",
+   "access_token": "string"
+  },
+  "failure": {
+   "message": "string"
+  }
+ }
+}
+```
+
+
+ASSISTANT:
+I need the supervisor's username and password. Lets see if any app can help me get that.
+
+Code:
+```python
+print(apis.api_docs.show_app_descriptions())
+```
+
+
+USER:
+Output:
+```
+{{ app_descriptions }}
+```
+
+
+ASSISTANT:
+Looks like the supervisor app could help me with that.
+
+Code:
+```python
+print(apis.api_docs.show_api_descriptions(app_name='supervisor'))
+```
+
+
+USER:
+Output:
+```
+[
+  ...
+  "show_account_passwords : Show your supervisor's account passwords."
+  ...
+]
+```
+
+
+ASSISTANT:
+I can use `show_account_passwords` to get the passwords. So I will call that and save it in a variable for future reference.
+Code:
+```python
+passwords=apis.supervisor.show_account_passwords()
+print(passwords)
+```
+
+
+USER:
+Output:
+```
+[
+  {
+    "account_name": "spotify",
+    "password": "dummy_spotify_pass"
+  },
+  {
+    "account_name": "file_system",
+    "password": "dummy_fs_pass"
+  }
+]
+```
+
+
+ASSISTANT:
+So the Spotify password is an entry in the `passwords` list with the account_name=spotify. I can use it to login now with the provided email address: {{ supervisor.email }}.
+
+Code:
+ ```python
+spotify_password = [account_password["account_name"] == "spotify" for account_password in passwords][0]["password"]
+login_result = apis.spotify.login(username='{{ supervisor.email }}', password=spotify_password)
+print(login_result)
+```
+
+
+USER:
+Output:
+```
+{
+ "token_type": "Bearer",
+ "access_token": "fake_access_token"
+}
+```
+
+
+ASSISTANT:
+First, let's check the available APIs for the Spotify app.
+Code:
+```python
+print(apis.api_docs.show_api_descriptions(app_name='spotify'))
+```
+
+
+USER:
+Output:
+```
+[
+...
+{
+  "name": "show_playlist_library",
+  "description": "Get a list of playlists in the user's playlist library."
+},
+{
+  "name": "show_playlist",
+  "description": "Get detailed information about a specific playlist. You can view your own playlists or others' playlists if they are public."
+ },
+...
+]
+```
+
+
+ASSISTANT:
+The `show_playlist_library` API seems to be the one we need. Let's check its specifications.
+Code:
+```python
+print(apis.api_docs.show_api_doc(app_name='spotify', api_name='show_playlist_library'))
+```
+
+
+USER:
+Output:
+```
+{
+ "app_name": "spotify",
+ "api_name": "show_playlist_library",
+ "path": "/private_playlists",
+ "method": "GET",
+ "description": "Get a list of playlists in the user's playlist library.",
+ "parameters": [
+  {
+   "name": "access_token",
+   "type": "string",
+   "required": true,
+   "description": "Access token obtained from spotify app login.",
+   "default": null,
+   "constraints": []
+  },
+  {
+   "name": "page_index",
+   "type": "integer",
+   "required": false,
+   "description": "The index of the page to retrieve.",
+   "default": 0,
+   "constraints": [
+    "value >= 0.0"
+   ]
+  },
+  {
+   "name": "page_limit",
+   "type": "integer",
+   "required": false,
+   "description": "The maximum number of results to return per page.",
+   "default": 5,
+   "constraints": [
+    "value >= 1.0, <= 20.0"
+   ]
+  },
+  {
+   "name": "is_public",
+   "type": "boolean",
+   "required": false,
+   "description": "Whether to show public playlists or private playlists.",
+   "default": null,
+   "constraints": []
+  }
+ ],
+ "response_schema": [
+  {
+   "title": "string",
+   "created_at": "2019-01-01T00:00:00",
+   "is_public": true,
+   "rating": 0.0,
+   "like_count": 1,
+   "owner_email": "user@example.com",
+   "playlist_id": 1,
+   "song_ids": [
+    1
+   ]
+  }
+ ]
+}
+```
+
+
+ASSISTANT:
+I need to page through all the playlists to get the list of playlists and save it in `playlists`.
+Code:
+```python
+page_index = 0
+playlists = []
+while page_index < 10:
+  playlist_page = apis.spotify.show_playlist_library(access_token=spotify_access_token, page_index=page_index)
+  if playlist_page:
+    playlists.extend(playlist_page)
+    page_index += 1
+  else:
+    break
+num_playlists = len(playlists)
+print(num_playlists)
+
+```
+
+USER:
+Output:
+```
+23
+```
+
+
+ASSISTANT:
+Now that the task is completed, I can call apis.supervisor.complete_task(). Since this task has an answer to be returned, I will pass that as an argument.
+
+Code:
+```python
+apis.supervisor.complete_task(answer=num_playlists)
+```
+
+
+USER:
+Output:
+Marked the active task complete.
+
+
+----------------------------------------------
+
+USER:
+**Key instructions**:
+(1) Make sure to end code blocks with ``` followed by a newline(\n).
+
+(2) Remember you can use the variables in your code in subsequent code blocks.
+
+(3) Remember that the email addresses, access tokens and variables (e.g. spotify_password) in the example above are not valid anymore.
+
+(4) You can use the "supervisor" app to get information about my accounts and use the "phone" app to get information about friends and family.
+
+(5) Always look at API specifications (using apis.api_docs.show_api_doc) before calling an API.
+
+(6) Write small chunks of code and only one chunk of code in every step. Make sure everything is working correctly before making any irreversible change.
+
+(7) Many APIs return items in "pages". Make sure to run through all the pages by looping over `page_index`.
+
+(8) Once you have completed the task, make sure to call apis.supervisor.complete_task(). If the task asked for some information, return it as the answer argument, i.e. call apis.supervisor.complete_task(answer=<answer>). Many tasks do not require an answer, so in those cases, just call apis.supervisor.complete_task() i.e. do not pass any argument.
+
+USER:
+Using these APIs, now generate code to solve the actual task:
+
+My name is: {{ supervisor.first_name }} {{ supervisor.last_name }}. My personal email is {{ supervisor.email }} and phone number is {{ supervisor.phone_number }}.
+
+"""

benchmark/appworld/quickstart.md

@@ -0,0 +1,128 @@
+# AppWorld
+Experiment Quick Start Guide
+
+This guide helps you quickly set up and run AppWorld experiments with ReMe integration.
+
+## Env Setup
+
+### 1. Clone the Repository
+
+```bash
+git clone https://github.com/agentscope-ai/ReMe.git
+cd ReMe/benchmark/appworld
+```
+
+### 2. Appworld Environment Setup
+
+Create a new conda environment with Python 3.12:
+
+```bash
+conda create -p ./appworld-env python==3.12
+conda activate ./appworld-env
+```
+
+Install required Python packages:
+
+```bash
+pip install -r requirements.txt
+```
+
+Install AppWorld and download the dataset:
+
+```bash
+pip install appworld
+appworld install
+appworld download data
+```
+
+**Note**: The AppWorld data will be saved in the current directory.
+
+### 3. Start ReMe Service
+
+Install ReMe (if not already installed)
+If you haven't installed the ReMe environment yet, follow these steps:
+```bash
+# Go back to the project root
+cd ../..
+
+# Create ReMe environment
+conda create -p ./reme-env python==3.12
+conda activate ./reme-env
+
+# Install ReMe
+pip install .
+```
+
+Launch the ReMe service to enable memory library functionality:
+
+```bash
+reme2 \
+  backend=http \
+  http.port=8002 \
+  llms.default.model_name=qwen3-8b \
+  embedding_models.default.model_name=text-embedding-v4 \
+  vector_stores.default.backend=es \
+  vector_stores.default.collection_name=appworld \
+  vector_stores.default.hosts=http://xx.yy.zz.mm:nn
+```
+
+### 4. Common Issues
+
+**AppWorld data not found**: Ensure `appworld download data` completed successfully
+
+**pydantic version issue**:  AppWorld depends on an older version of pydantic, which is why a separate environment is needed. If you encounter issues running the experiments, try `pip install appworld` to override the dependencies.
+
+
+
+## Run Experiments
+
+### 1. Test: With Memory vs Without Memory
+
+Run the main experiment script to compare performance with and without memory:
+
+```bash
+python run_appworld.py
+```
+
+**What this does:**
+- Runs AppWorld tasks on the test-normal set
+- Compares agent performance with ReMe memory (`use_memory=True`) vs without memory
+- Uses multiple workers for parallel processing
+- Runs each task multiple times for statistical significance
+- Results are automatically saved to `./exp_result/` directory
+
+**Configuration options in `run_appworld.py`:**
+- `max_workers`: Number of parallel workers (default: 16)
+- `num_runs`: Number of times each task is repeated (default: 4)
+- `batch_size`: Number of concurrent tasks per batch (default: 8)
+- `num_trials`: Maximum number of self-reflections, failure-aware reflection mechanism is triggered when num_trials>1 (default: 1)
+- `model_name`: Task execution model (default: "qwen3-8b")
+- `use_memory`: Whether to use ReMe memory library (default: True)
+- `use_memory_addition`: Whether to enable selective addition (default: False)
+- `use_memory_deletion`: Whether to enable utility-based deletion (default: False)
+
+### 2. View Experiment Results
+
+After running experiments, analyze the statistical results:
+
+```bash
+python run_exp_statistic.py
+```
+
+**What this script does:**
+- Processes all result files in `./exp_result/`
+- Calculates best@k, pass@k metrics for different k values
+- Generates a summary table showing performance comparisons
+- Saves results to `experiment_summary.csv`
+
+**Metrics explained:**
+- `best@k`: Takes groups of k runs per task, finds the maximum score in each group, then averages these maximums
+- `pass@k`: Takes groups of k runs per task, measures the probability that at least one out of k independent task runs is successful.
+- Higher k values show potential performance, lower k values show consistency
+- In our AppWorld experiments, we report Task Goal Completion (TGC) metric, which measures percentage of tasks for which the agent passes all evaluation tests.
+
+**Output Files**
+
+- `./exp_result/*.jsonl`: Raw experiment results for each configuration
+- `./exp_result/experiment_summary.csv`: Statistical summary table
+- Console output: Real-time progress and summary statistics

benchmark/appworld/requirements.txt

@@ -0,0 +1,7 @@
+fastapi
+uvicorn
+uuid
+jinja2
+loguru
+openai
+pandas

benchmark/appworld/run_appworld.py

@@ -0,0 +1,197 @@
+# pylint: disable=E0611
+"""Run the Appworld React Agent."""
+
+import os
+import json
+import time
+from pathlib import Path
+
+import ray
+import requests
+from loguru import logger
+from dotenv import load_dotenv
+from appworld import load_task_ids
+from appworld_react_agent import AppworldReactAgent
+
+os.environ["APPWORLD_ROOT"] = "."
+
+load_dotenv("../../.env")
+
+
+def run_agent(
+    run_index: int,
+    max_workers: int,
+    model_name: str,
+    dataset_name: str,
+    experiment_suffix: str,
+    num_trials: int = 1,
+    use_memory: bool = False,
+    memory_base_url: str = "http://0.0.0.0:8002/",
+    use_memory_addition: bool = False,
+    use_memory_deletion: bool = False,
+    delete_freq: int = 10,
+    freq_threshold: int = 5,
+    utility_threshold: float = 0.5,
+    batch_size: int = 4,
+):
+    """Run the Appworld React Agent."""
+    experiment_name = dataset_name + "_" + experiment_suffix
+    path: Path = Path(f"./exp_result/{model_name}")
+    path.mkdir(parents=True, exist_ok=True)
+
+    task_ids = load_task_ids(dataset_name)
+
+    result: list = []
+
+    def dump_file():
+        with open(path / f"{experiment_name}.jsonl", "a", encoding="utf-8") as f:
+            for x in result:
+                f.write(json.dumps(x) + "\n")
+
+    if max_workers > 1:
+        # Process tasks in batches
+        total_tasks = len(task_ids)
+        num_batches = (total_tasks + batch_size - 1) // batch_size  # Ceiling division
+
+        logger.info(f"Total tasks: {total_tasks}, Batch size: {batch_size}, Number of batches: {num_batches}")
+
+        for batch_idx in range(num_batches):
+            # Initialize Ray for this batch
+            start_idx = batch_idx * batch_size
+            end_idx = min(start_idx + batch_size, total_tasks)
+            batch_task_ids = task_ids[start_idx:end_idx]
+
+            logger.info(f"Starting batch {batch_idx + 1}/{num_batches} with {len(batch_task_ids)} tasks")
+
+            # Initialize Ray with the number of CPUs needed for this batch
+            ray.init(num_cpus=len(batch_task_ids))
+
+            future_list: list = []
+            for i, task_id in enumerate(batch_task_ids):
+                actor = AppworldReactAgent.remote(
+                    index=start_idx + i,
+                    model_name=model_name,
+                    task_ids=[task_id],
+                    experiment_name=experiment_name,
+                    num_trials=num_trials,
+                    use_memory=use_memory,
+                    memory_base_url=memory_base_url,
+                    use_memory_addition=use_memory_addition,
+                    use_memory_deletion=use_memory_deletion,
+                    delete_freq=delete_freq,
+                    freq_threshold=freq_threshold,
+                    utility_threshold=utility_threshold,
+                )
+                future = actor.execute.remote()
+                future_list.append(future)
+                time.sleep(1)
+
+            logger.info(f"Batch {batch_idx + 1} submit complete, waiting for results...")
+
+            # Collect results from this batch
+            for i, (task_id, future) in enumerate(zip(batch_task_ids, future_list)):
+                try:
+                    t_result = ray.get(future)
+                    if t_result:
+                        if isinstance(t_result, list):
+                            result.extend(t_result)
+                        else:
+                            result.append(t_result)
+                except Exception:
+                    logger.exception(f"run ray error with task_id={task_id}")
+
+                logger.info(f"Batch {batch_idx + 1}: task {i + 1}/{len(batch_task_ids)} complete")
+
+            # Shutdown Ray to free resources before next batch
+            ray.shutdown()
+            logger.info(f"Batch {batch_idx + 1}/{num_batches} complete, Ray resources released")
+
+            # Optional: small delay between batches
+            if batch_idx < num_batches - 1:
+                time.sleep(2)
+
+        dump_file()
+
+    else:
+        agent = AppworldReactAgent(
+            index=run_index,
+            model_name=model_name,
+            task_ids=task_ids,
+            experiment_name=experiment_name,
+            num_trials=num_trials,
+            use_memory=use_memory,
+            memory_base_url=memory_base_url,
+            use_memory_addition=use_memory_addition,
+            use_memory_deletion=use_memory_deletion,
+            delete_freq=delete_freq,
+            freq_threshold=freq_threshold,
+            utility_threshold=utility_threshold,
+        )
+        result = agent.execute()
+
+        dump_file()
+
+
+def handle_api_response(response: requests.Response):
+    """Handle API response with proper error checking"""
+    if response.status_code != 200:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+        return None
+
+    return response.json()
+
+
+def load_memory(path: str = "docs/library", api_url: str = "http://0.0.0.0:8002/"):
+    """Load memories from disk into the vector store"""
+    response = requests.post(
+        url=f"{api_url}load_memory",
+        json={
+            "load_file_path": path,
+            "clear_existing": True,
+        },
+    )
+
+    result = handle_api_response(response)
+    if result:
+        print(f"Memory loaded from {path}")
+
+
+def main():
+    """Main function to run the Appworld React Agent."""
+    max_workers = 16
+    batch_size = 8
+
+    num_runs = 4  # Number of runs
+    num_trials = 1  # for self-reflection
+    model_name = "qwen3-8b"
+    use_memory = True
+    use_memory_addition = False
+    use_memory_deletion = False
+    memory_base_url = "http://0.0.0.0:8002/"
+
+    if use_memory:
+        load_file_path = "docs/library/paper_data/task/appworld_qwen3_8b.jsonl"
+        load_memory(load_file_path, memory_base_url)
+
+    for i in range(num_runs):
+        run_agent(
+            run_index=i,
+            max_workers=max_workers,
+            model_name=model_name,
+            dataset_name="test_normal",
+            experiment_suffix="with-fixed-memory",
+            num_trials=num_trials,
+            use_memory=use_memory,
+            memory_base_url=memory_base_url,
+            use_memory_addition=use_memory_addition,
+            use_memory_deletion=use_memory_deletion,
+            delete_freq=5,
+            freq_threshold=5,
+            utility_threshold=0.5,
+            batch_size=batch_size,
+        )
+
+
+if __name__ == "__main__":
+    main()

benchmark/appworld/run_exp_statistic.py

@@ -0,0 +1,164 @@
+"""Run the experiment statistic."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import pandas as pd
+from loguru import logger
+
+
+def calculate_best_at_k(scores: list, k: int) -> float:
+    """
+    Calculate best@k
+    Divide scores into groups of size k, take the maximum value in each group,
+    then average these maximum values
+
+    Args:
+        scores: List of after_score values for all runs of a task
+        k: Group size
+
+    Returns:
+        best@k value
+    """
+    if len(scores) % k != 0:
+        raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
+
+    group_maxs = []
+    for i in range(0, len(scores), k):
+        group = scores[i : i + k]
+        group_maxs.append(max(group))
+
+    return sum(group_maxs) / len(group_maxs)
+
+
+def calculate_pass_at_k(scores: list, k: int) -> float:
+    """Calculate pass@k."""
+    if len(scores) % k != 0:
+        raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
+
+    group_maxs = []
+    for i in range(0, len(scores), k):
+        group = scores[i : i + k]
+        is_pass = 1.0 if max(group) >= 1.0 else 0.0
+        group_maxs.append(is_pass)
+
+    return sum(group_maxs) / len(group_maxs)
+
+
+def get_possible_k_values(total_runs: int) -> list:
+    """
+    Get all possible k values (factors of total_runs)
+
+    Args:
+        total_runs: Total number of runs
+
+    Returns:
+        List of k values in descending order
+    """
+    k_values = []
+    for k in range(1, total_runs + 1):
+        if total_runs % k == 0:
+            k_values.append(k)
+    return sorted(k_values, reverse=True)  # Sort from large to small
+
+
+def run_exp_statistic():
+    """Run the experiment statistic."""
+    path: Path = Path("./exp_result/qwen3-8b")
+
+    # Store results for all experiments
+    all_results = {}
+
+    for file in path.glob("*.jsonl"):  # [f for f in path.glob("*.jsonl") if not f.stem[-1].isdigit()]
+        # Group results by task_id
+        task_results = defaultdict(list)
+
+        with open(file, "r", encoding="utf-8") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                data = json.loads(line)
+
+                if isinstance(data, list):
+                    for part_data in data:
+                        task_id = part_data["task_id"]
+                        after_score = part_data["after_score"]
+                        task_results[task_id].append(after_score)
+                else:
+                    task_id = data["task_id"]
+                    after_score = data["after_score"]
+                    task_results[task_id].append(after_score)
+
+        if not task_results:
+            logger.warning(f"No valid data found in file {file}")
+            continue
+
+        # Check if each task has consistent number of runs
+        run_counts = [len(scores) for scores in task_results.values()]
+        if len(set(run_counts)) > 1:
+            logger.warning(f"Inconsistent number of runs for different tasks in file {file}: {set(run_counts)}")
+            continue
+
+        num_runs = run_counts[0]
+        logger.info(f"File {file}: {len(task_results)} tasks, {num_runs} runs per task")
+
+        # Get all possible k values
+        k_values = get_possible_k_values(num_runs)
+        logger.info(f"Calculable best@k values: {k_values}")
+
+        # Calculate various best@k values
+        file_results = {"file": file.name}
+
+        for k in k_values:
+            best_at_k_scores = []
+            pass_at_k_scores = []
+            for task_id, scores in task_results.items():
+                try:
+                    best_k_score = calculate_best_at_k(scores, k)
+                    pass_at_k_score = calculate_pass_at_k(scores, k)
+                    pass_at_k_scores.append(pass_at_k_score)
+                    best_at_k_scores.append(best_k_score)
+                except ValueError as e:
+                    logger.error(f"Error calculating best@{k} for task {task_id}: {e}")
+                    continue
+
+            if best_at_k_scores:
+                avg_best_at_k = sum(best_at_k_scores) / len(best_at_k_scores)
+                file_results[f"best@{k}"] = avg_best_at_k
+                logger.info(f"file={file.name} best@{k}={avg_best_at_k:.4f}")
+
+            if pass_at_k_scores:
+                avg_pass_at_k = sum(pass_at_k_scores) / len(pass_at_k_scores)
+                file_results[f"pass@{k}"] = avg_pass_at_k
+                logger.info(f"file={file.name} pass@{k}={avg_pass_at_k:.4f}")
+
+        all_results[file.name] = file_results
+
+    # Create and display table
+    if all_results:
+        df = pd.DataFrame(list(all_results.values()))
+        df = df.set_index("file")
+
+        # Sort columns by the number in column name (best@8, best@4, best@2, best@1)
+        pass_columns = [col for col in df.columns if col.startswith("pass@")]
+        # best_columns = [col for col in df.columns]
+        pass_columns.sort(key=lambda x: x, reverse=False)
+        df = df[pass_columns]
+
+        print("\n" + "=" * 80)
+        print("Experiment Results Summary Table")
+        print("=" * 80)
+        print(df.round(4))
+        print("=" * 80)
+
+        # Save table to CSV
+        output_path = path / "experiment_summary.csv"
+        df.to_csv(output_path)
+        logger.info(f"Results table saved to: {output_path}")
+    else:
+        logger.warning("No valid experiment results found")
+
+
+if __name__ == "__main__":
+    run_exp_statistic()

benchmark/beam/README.md

@@ -1,124 +0,0 @@
-[中文版 / Chinese version](./README_ZH.md)
-
-# BEAM Benchmark
-
-BEAM is a benchmark for **memory capability over long-context chat cases**. Each
-case contains a very long chat history split into batches; ReMe converts each
-batch into a session, ingests them in chronological order, then answers probing
-questions via an agentic (ReAct) mode. Answers are scored with BEAM's
-rubric-based `answer_judge` job, which produces both a graded score and a binary
-verdict, and per-type averages are reported.
-
-BEAM ships dataset variants by chat size — `100K` / `500K` / `1M` / `10M` — so
-memory systems can be stressed at different context lengths. Question types
-include abstention, contradiction resolution, event ordering, information
-extraction, instruction following, knowledge update, multi-session reasoning,
-preference following, summarization, and temporal reasoning.
-
-> For the shared setup (dependencies, credentials, log conventions) see the
-> [top-level benchmark README](../README.md).
-
-## 1. Get the Dataset
-
-BEAM is a public repository, cloned into `benchmark/beam/dataset/`:
-
-```bash
-mkdir -p benchmark/beam/dataset
-cd benchmark/beam/dataset
-git clone https://github.com/mohammadtavakoli78/BEAM.git
-```
-
-After cloning, `benchmark/beam/dataset/BEAM/` should contain `chats/`, `src/`,
-`topics/` and other subdirectories.
-
-## 2. Run
-
-From the repository root:
-
-```bash
-python benchmark/beam/run.py
-python benchmark/beam/run.py --config benchmark/beam/config.yaml
-python benchmark/beam/run.py -q                        # quiet
-python benchmark/beam/run.py --eval_only               # reuse existing workspaces, query + judge only
-```
-
-## 3. Pipeline
-
-1. For each case, load `chat.json` and convert each batch into a ReMe session.
-2. Ingest sessions in chronological order into an isolated workspace, then `digest_update`.
-3. Answer each probing question via agentic (ReAct) mode.
-4. Score answers with BEAM's rubric-based `answer_judge` job and print per-type averages.
-
-## 4. Key config — `benchmark/beam/config.yaml`
-
-| Key | Meaning |
-| --- | --- |
-| `dataset.beam_root` | BEAM dataset root (`benchmark/beam/dataset/BEAM`). |
-| `dataset.chat_size` | Variant to run: `100K` / `500K` / `1M` / `10M`. |
-| `dataset.case_ids` | Specific cases (e.g. `["1","2"]`); empty = all cases. |
-| `dataset.start_index` / `num_items` | Case pagination (`num_items` `0` = all). |
-| `dataset.workspace_root` | Per-case workspace root (`benchmark/beam/workspaces/beam`). |
-| `evaluation.num_workers` | `0` = auto, `1` = sequential, `>1` = parallel. |
-| `reme.config` | ReMe config used (`beam.yaml`). |
-| `output.dir` | Results directory (`benchmark/beam/results`). |
-
-## 5. Outputs
-
-Results are JSON files written to `output.dir` as
-`results_<chat_size>_<timestamp>.json`, with a per-type score summary also
-printed to the console. Logging conventions are shared across benchmarks — see
-the [top-level README](../README.md#outputs--logs).
-
-## 6. Reference Results
-
-> The results below use the longmemeval-version prompt.
-
-### 100K
-
-agentscope==2.0.4.post1, conda reme env, 20 workers, eval-only (reusing prebuilt memory)
-(2026-08-05, 20 cases / 400 Qs, total 46.0 min)
-
-| Type | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|---|
-| abstention | 0.550 | 0.550 | 96,031 | 1,070 | 97,101 | 4.58 |
-| contradiction_resolution | 0.438 | 0.412 | 32,263 | 872 | 33,135 | 2.48 |
-| event_ordering | 0.501 | 0.423 | 140,195 | 5,163 | 145,358 | 4.70 |
-| information_extraction | 0.873 | 0.832 | 50,245 | 883 | 51,128 | 3.15 |
-| instruction_following | 0.750 | 0.725 | 37,986 | 848 | 38,834 | 2.67 |
-| knowledge_update | 0.688 | 0.675 | 31,198 | 651 | 31,849 | 2.27 |
-| multi_session_reasoning | 0.626 | 0.584 | 85,038 | 4,563 | 89,601 | 4.28 |
-| preference_following | 0.925 | 0.912 | 34,281 | 989 | 35,270 | 2.50 |
-| summarization | 0.623 | 0.461 | 89,657 | 2,056 | 91,713 | 4.12 |
-| temporal_reasoning | 0.637 | 0.625 | 34,563 | 1,049 | 35,612 | 2.52 |
-| **OVERALL** | **0.661** | **0.620** | **63,146** | **1,814** | **64,960** | **3.33** |
-
-Memory Construction average token consumption (default agent, full build over 20 cases):
-
-| Agent | input tok/case | output tok/case | total tok/case |
-|---|---|---|---|
-| default | 2,172,316 | 136,697 | 2,309,013 |
-
-### 1M
-
-agentscope==2.0.4.post1, conda reme env, 20 workers, full memory build
-(2026-08-05, 35 cases / 700 Qs, total 459.2 min)
-
-| Type | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|---|
-| abstention | 0.429 | 0.429 | 118,707 | 1,178 | 119,886 | 4.20 |
-| contradiction_resolution | 0.391 | 0.364 | 49,787 | 810 | 50,597 | 2.50 |
-| event_ordering | 0.558 | 0.456 | 201,514 | 3,889 | 205,403 | 4.79 |
-| information_extraction | 0.809 | 0.772 | 78,950 | 894 | 79,844 | 3.00 |
-| instruction_following | 0.852 | 0.832 | 55,757 | 924 | 56,681 | 2.81 |
-| knowledge_update | 0.779 | 0.771 | 45,981 | 665 | 46,646 | 2.37 |
-| multi_session_reasoning | 0.658 | 0.612 | 138,133 | 2,873 | 141,006 | 4.40 |
-| preference_following | 0.798 | 0.777 | 51,796 | 920 | 52,716 | 2.53 |
-| summarization | 0.693 | 0.537 | 158,794 | 2,905 | 161,700 | 4.44 |
-| temporal_reasoning | 0.536 | 0.536 | 100,176 | 3,148 | 103,324 | 3.90 |
-| **OVERALL** | **0.650** | **0.609** | **99,959** | **1,821** | **101,780** | **3.49** |
-
-Memory Construction average token consumption (default agent, full build over 35 cases):
-
-| Agent | input tok/case | output tok/case | total tok/case |
-|---|---|---|---|
-| default | 31,943,817 | 1,417,061 | 33,360,878 |

benchmark/beam/README_ZH.md

@@ -1,119 +0,0 @@
-# BEAM 评测
-
-[English version](./README.md)
-
-BEAM 是一个面向**长上下文对话场景**的记忆能力评测基准。每个 case 包含一段被切分为多个
-batch 的超长对话；ReMe 将每个 batch 转换为一个会话，按时间顺序摄入后，以 agentic（ReAct）
-模式回答探测问题。答案由 BEAM 基于 rubric 的 `answer_judge` 任务打分，同时给出分级分数与二元
-判定，并输出各类型平均分。
-
-BEAM 按对话规模提供多种数据变体 —— `100K` / `500K` / `1M` / `10M`，可在不同上下文长度下
-压测记忆系统。题型包括 abstention（拒答）、contradiction resolution（矛盾消解）、event
-ordering（事件排序）、information extraction（信息抽取）、instruction following（指令遵循）、
-knowledge update（知识更新）、multi-session reasoning（多会话推理）、preference following
-（偏好遵循）、summarization（摘要）与 temporal reasoning（时间推理）。
-
-> 公共设置（依赖、凭据、日志约定）见[总评测说明](../README_ZH.md)。
-
-## 1. 获取数据集
-
-BEAM 是公开仓库，clone 到 `benchmark/beam/dataset/` 下：
-
-```bash
-mkdir -p benchmark/beam/dataset
-cd benchmark/beam/dataset
-git clone https://github.com/mohammadtavakoli78/BEAM.git
-```
-
-clone 完成后，`benchmark/beam/dataset/BEAM/` 目录下应包含 `chats/`、`src/`、`topics/` 等子目录。
-
-## 2. 运行
-
-在仓库根目录执行：
-
-```bash
-python benchmark/beam/run.py
-python benchmark/beam/run.py --config benchmark/beam/config.yaml
-python benchmark/beam/run.py -q                        # 安静模式
-python benchmark/beam/run.py --eval_only               # 复用已有工作区，仅执行查询 + 评判
-```
-
-## 3. 流程
-
-1. 为每个 case 加载 `chat.json`，将每个 batch 转换为一个 ReMe 会话。
-2. 按时间顺序将会话摄入独立工作区，随后执行 `digest_update`。
-3. 以 agentic（ReAct）模式回答每个探测问题。
-4. 通过 BEAM 基于 rubric 的 `answer_judge` 任务打分，并输出各类型平均分。
-
-## 4. 关键配置 —— `benchmark/beam/config.yaml`
-
-| 配置项 | 含义 |
-| --- | --- |
-| `dataset.beam_root` | BEAM 数据集根目录（`benchmark/beam/dataset/BEAM`）。 |
-| `dataset.chat_size` | 运行的变体：`100K` / `500K` / `1M` / `10M`。 |
-| `dataset.case_ids` | 指定 case（如 `["1","2"]`），空表示全部。 |
-| `dataset.start_index` / `num_items` | case 分页（`num_items` 为 `0` 表示全部）。 |
-| `dataset.workspace_root` | case 工作区根目录（`benchmark/beam/workspaces/beam`）。 |
-| `evaluation.num_workers` | `0` = 自动，`1` = 串行，`>1` = 并行。 |
-| `reme.config` | 使用的 ReMe 配置（`beam.yaml`）。 |
-| `output.dir` | 结果目录（`benchmark/beam/results`）。 |
-
-## 5. 输出
-
-结果以 JSON 文件写入 `output.dir`，文件名为 `results_<chat_size>_<timestamp>.json`，
-同时控制台会打印含各类型分数的汇总。日志约定在各基准间通用，见
-[总说明](../README_ZH.md#输出与日志)。
-
-## 6. 参考结果
-
-> 以下结果使用 longmemeval 版本的 prompt。
-
-### 100K
-
-agentscope==2.0.4.post1，conda reme 环境，20 并发，eval-only（复用已构建 memory）
-（2026-08-05，20 cases / 400 Qs，总耗时 46.0 min）
-
-| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|---|
-| abstention | 0.550 | 0.550 | 96,031 | 1,070 | 97,101 | 4.58 |
-| contradiction_resolution | 0.438 | 0.412 | 32,263 | 872 | 33,135 | 2.48 |
-| event_ordering | 0.501 | 0.423 | 140,195 | 5,163 | 145,358 | 4.70 |
-| information_extraction | 0.873 | 0.832 | 50,245 | 883 | 51,128 | 3.15 |
-| instruction_following | 0.750 | 0.725 | 37,986 | 848 | 38,834 | 2.67 |
-| knowledge_update | 0.688 | 0.675 | 31,198 | 651 | 31,849 | 2.27 |
-| multi_session_reasoning | 0.626 | 0.584 | 85,038 | 4,563 | 89,601 | 4.28 |
-| preference_following | 0.925 | 0.912 | 34,281 | 989 | 35,270 | 2.50 |
-| summarization | 0.623 | 0.461 | 89,657 | 2,056 | 91,713 | 4.12 |
-| temporal_reasoning | 0.637 | 0.625 | 34,563 | 1,049 | 35,612 | 2.52 |
-| **OVERALL** | **0.661** | **0.620** | **63,146** | **1,814** | **64,960** | **3.33** |
-
-Memory Construction 平均 token 消耗（default agent，20 cases 全量构建）：
-
-| Agent | input tok/case | output tok/case | total tok/case |
-|---|---|---|---|
-| default | 2,172,316 | 136,697 | 2,309,013 |
-
-### 1M
-
-agentscope==2.0.4.post1，conda reme 环境，20 并发，全量构建 memory
-（2026-08-05，35 cases / 700 Qs，总耗时 459.2 min）
-
-| 题型 | Agentic | Binary | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|---|
-| abstention | 0.429 | 0.429 | 118,707 | 1,178 | 119,886 | 4.20 |
-| contradiction_resolution | 0.391 | 0.364 | 49,787 | 810 | 50,597 | 2.50 |
-| event_ordering | 0.558 | 0.456 | 201,514 | 3,889 | 205,403 | 4.79 |
-| information_extraction | 0.809 | 0.772 | 78,950 | 894 | 79,844 | 3.00 |
-| instruction_following | 0.852 | 0.832 | 55,757 | 924 | 56,681 | 2.81 |
-| knowledge_update | 0.779 | 0.771 | 45,981 | 665 | 46,646 | 2.37 |
-| multi_session_reasoning | 0.658 | 0.612 | 138,133 | 2,873 | 141,006 | 4.40 |
-| preference_following | 0.798 | 0.777 | 51,796 | 920 | 52,716 | 2.53 |
-| summarization | 0.693 | 0.537 | 158,794 | 2,905 | 161,700 | 4.44 |
-| temporal_reasoning | 0.536 | 0.536 | 100,176 | 3,148 | 103,324 | 3.90 |
-| **OVERALL** | **0.650** | **0.609** | **99,959** | **1,821** | **101,780** | **3.49** |
-
-Memory Construction 平均 token 消耗（default agent，35 cases 全量构建）：
-
-| Agent | input tok/case | output tok/case | total tok/case |
-|---|---|---|---|
-| default | 31,943,817 | 1,417,061 | 33,360,878 |

benchmark/beam/config.yaml

@@ -1,24 +0,0 @@
-# BEAM evaluation configuration
-# This file controls what/how to evaluate.
-
-dataset:
-  beam_root: "benchmark/beam/dataset/BEAM"   # BEAM dataset root
-  chat_size: "1M"                        # 100K | 500K | 1M | 10M (dataset variant)
-  case_ids: []                          # empty = all cases; or ["1", "2", "3"]
-  start_index: 0                        # first case index (for pagination)
-  num_items: 0                          # 0 = all cases; >0 = limit
-  workspace_root: "benchmark/beam/workspaces/beam"  # workspace root for case workspaces
-
-evaluation:
-  num_workers: 20                       # 0 = auto; 1 = sequential; >1 = parallel (per-case)
-  compress_session: false               # true = compress session chunks in search_v2 (query-aware); false = no compression
-
-reme:
-  config: "beam.yaml"                   # reme config (in reme/config/)
-
-output:
-  dir: "benchmark/beam/results"
-  log_dir: "logs"                       # log directory (relative to project root)
-  log_prefix: "beam"                    # benchmark name used in log filenames
-  log_to_console: true
-  log_to_file: true

benchmark/beam/kill.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-# 杀死指定进程及其所有子进程
-# Usage: bash kill.sh <PID>
-
-if [ -z "$1" ]; then
-    echo "Usage: bash kill.sh <PID>"
-    echo "  杀死指定进程及其所有子进程"
-    exit 1
-fi
-
-PID=$1
-
-# 检查进程是否存在
-if ! kill -0 "$PID" 2>/dev/null; then
-    echo "进程 $PID 不存在"
-    exit 1
-fi
-
-# 递归收集所有子进程（包括子进程的子进程）
-collect_children() {
-    local parent=$1
-    local children
-    children=$(ps -o pid= --ppid "$parent" 2>/dev/null | tr -d ' ')
-    for child in $children; do
-        collect_children "$child"
-    done
-    echo "$parent"
-}
-
-# 收集进程树（子进程在前，父进程在后，保证先杀子再杀父）
-PROCESS_TREE=$(collect_children "$PID")
-TOTAL=$(echo "$PROCESS_TREE" | wc -l | tr -d ' ')
-
-echo "进程树（共 $TOTAL 个进程）:"
-while read -r p; do
-    cmd=$(ps -o args= -p "$p" 2>/dev/null | head -c 80)
-    printf "  PID=%-8s %s\n" "$p" "$cmd"
-done <<< "$PROCESS_TREE"
-
-# 先 SIGTERM 优雅终止
-echo ""
-echo "发送 SIGTERM..."
-while read -r p; do
-    kill "$p" 2>/dev/null
-done <<< "$PROCESS_TREE"
-
-# 等待最多 5 秒
-for i in $(seq 1 5); do
-    alive=false
-    while read -r p; do
-        if kill -0 "$p" 2>/dev/null; then
-            alive=true
-        fi
-    done <<< "$PROCESS_TREE"
-    if [ "$alive" = false ]; then
-        break
-    fi
-    sleep 1
-done
-
-# 检查是否还有残留，强制 SIGKILL
-remaining=false
-while read -r p; do
-    if kill -0 "$p" 2>/dev/null; then
-        remaining=true
-    fi
-done <<< "$PROCESS_TREE"
-
-if [ "$remaining" = true ]; then
-    echo "部分进程未响应，发送 SIGKILL..."
-    while read -r p; do
-        kill -9 "$p" 2>/dev/null
-    done <<< "$PROCESS_TREE"
-fi
-
-echo "已终止进程树（根 PID=$PID，共 $TOTAL 个进程）"

benchmark/beam/run.py

@@ -1,891 +0,0 @@
-"""BEAM evaluation runner for ReMe.
-
-Evaluates ReMe's memory capability using the BEAM dataset.
-Each case gets an isolated workspace; chat.json batches are ingested as
-sessions in chronological order; finally probing questions are answered
-via an agentic (ReAct) approach, then
-judged by BEAM's rubric-based LLM-as-judge.
-
-Usage:
-    python benchmark/beam/run.py
-    python benchmark/beam/run.py --config benchmark/beam/config.yaml
-    python benchmark/beam/run.py -q                          # quiet: only eval-level logs
-    python benchmark/beam/run.py --log-level WARNING         # reduce eval runner logs
-    python benchmark/beam/run.py --reme-log-level WARNING    # reduce reme internal logs
-    python benchmark/beam/run.py --eval_only                 # query+judge only, reuse existing workspace
-"""
-
-import json
-import logging
-import os
-import re
-import shutil
-import time
-import threading
-from datetime import datetime
-from pathlib import Path
-
-import yaml
-from dotenv import load_dotenv
-
-# Load .env from project root
-_PROJECT_ROOT = Path(__file__).parent.parent.parent
-load_dotenv(_PROJECT_ROOT / ".env")
-
-# Workspace root — read from config.yaml (dataset.workspace_root)
-_WORKSPACE_ROOT_DEFAULT = "benchmark/beam/workspaces/beam"
-
-# ---------------------------------------------------------------------------
-# Logging
-# ---------------------------------------------------------------------------
-_DEFAULT_LOG_FORMAT = "%(asctime)s | %(levelname)s | %(message)s"
-
-logging.basicConfig(level=logging.INFO, format=_DEFAULT_LOG_FORMAT)
-logger = logging.getLogger("beam")
-
-# Noisy library loggers silenced by default
-_NOISY_LOGGERS = [
-    "httpx",
-    "httpcore",
-    "openai",
-    "uvicorn",
-    "multipart",
-    "asyncio",
-    "watchfiles",
-    "filelock",
-]
-
-
-def setup_logging(
-    log_level: str,
-    reme_log_level: str,
-    log_dir: str | None = None,
-):
-    """Configure logging for the eval runner and reme internals.
-
-    Args:
-        log_level: Level for the eval runner logger (DEBUG/INFO/WARNING/ERROR).
-        reme_log_level: Level for reme's internal loguru logger.
-        log_dir: Per-run log directory (absolute path). None = no file logging.
-    """
-    numeric = getattr(logging, log_level.upper(), logging.INFO)
-    # Eval runner logger
-    logging.getLogger().setLevel(numeric)
-    logger.setLevel(numeric)
-
-    # Suppress noisy library loggers when above DEBUG
-    if numeric > logging.DEBUG:
-        for name in _NOISY_LOGGERS:
-            lib_logger = logging.getLogger(name)
-            lib_logger.setLevel(max(numeric, logging.WARNING))
-
-    # Add file handler for eval runner if log_dir is specified
-    if log_dir:
-        os.makedirs(log_dir, exist_ok=True)
-        log_filepath = os.path.join(log_dir, "runner.log")
-        file_handler = logging.FileHandler(log_filepath, encoding="utf-8")
-        file_handler.setLevel(numeric)
-        file_handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FORMAT))
-        logging.getLogger().addHandler(file_handler)
-        logger.info(f"Eval runner log file: {log_filepath}")
-
-    # Reme internal logger (loguru) — will be applied per-worker via _configure_worker
-    os.environ["REME_LOG_LEVEL"] = reme_log_level.upper()
-    if log_dir:
-        os.environ["REME_LOG_DIR"] = log_dir
-
-
-def _configure_worker(
-    log_level: str,
-    reme_log_level: str,
-    log_dir: str | None = None,
-):
-    """Set up logging inside a multiprocessing worker process.
-
-    Must be called at the top of each worker because child processes inherit
-    parent state but loguru sinks are NOT shared across fork/spawn.
-    """
-    numeric = getattr(logging, log_level.upper(), logging.INFO)
-    logging.basicConfig(level=numeric, format=_DEFAULT_LOG_FORMAT, force=True)
-    logging.getLogger("beam").setLevel(numeric)
-    if numeric > logging.DEBUG:
-        for name in _NOISY_LOGGERS:
-            logging.getLogger(name).setLevel(max(numeric, logging.WARNING))
-
-    # Add file handler for eval runner in worker process
-    if log_dir:
-        os.makedirs(log_dir, exist_ok=True)
-        pid = os.getpid()
-        log_filepath = os.path.join(log_dir, f"worker-{pid}.log")
-        file_handler = logging.FileHandler(log_filepath, encoding="utf-8")
-        file_handler.setLevel(numeric)
-        file_handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FORMAT))
-        logging.getLogger().addHandler(file_handler)
-
-    # Re-initialize loguru for reme internals at the desired level
-    from reme.utils import get_logger
-
-    reme_log_dir = log_dir or "logs"
-    get_logger(log_dir=reme_log_dir, level=reme_log_level.upper(), force_init=True)
-
-
-# ---------------------------------------------------------------------------
-# Config loading
-# ---------------------------------------------------------------------------
-def load_eval_config(config_path: str | None = None) -> dict:
-    """Load evaluation config yaml with env-var expansion."""
-    if config_path is None:
-        config_path = str(Path(__file__).parent / "config.yaml")
-    with open(config_path, encoding="utf-8") as f:
-        raw = f.read()
-
-    # Expand ${VAR} and ${VAR:-default}
-    def _expand(m):
-        expr = m.group(1)
-        if ":-" in expr:
-            key, default = expr.split(":-", 1)
-            return os.environ.get(key, default)
-        return os.environ.get(expr, "")
-
-    raw = re.sub(r"\$\{([^}]+)\}", _expand, raw)
-    return yaml.safe_load(raw)
-
-
-# ---------------------------------------------------------------------------
-# BEAM data loading
-# ---------------------------------------------------------------------------
-def parse_beam_time_anchor(time_str: str) -> datetime:
-    """Parse BEAM time_anchor format: 'March-15-2024' -> datetime."""
-    for fmt in ("%B-%d-%Y", "%b-%d-%Y"):
-        try:
-            return datetime.strptime(time_str, fmt)
-        except ValueError:
-            continue
-    raise ValueError(f"Cannot parse time_anchor: {time_str!r}")
-
-
-def load_beam_chat(chat_path: Path, chat_size: str, case_id: str) -> list[dict]:
-    """Load BEAM chat.json and convert to ReMe session format.
-
-    Each batch becomes one session with all its turns flattened.
-    Each turn resolves its own time_anchor independently; turns without
-    an explicit time_anchor inherit from the most recent preceding turn.
-    Returns list of sessions, each with:
-      - session_id: str
-      - date: str (YYYY-MM-DD)  — derived from the *first* turn's time
-      - messages: list[dict] with name, role, content, created_at
-    """
-    with open(chat_path, encoding="utf-8") as f:
-        batches = json.load(f)
-
-    sessions = []
-    for batch in batches:
-        batch_num = batch["batch_number"]
-
-        # Resolve batch-level fallback (used when no turn has a time_anchor)
-        batch_anchor = batch.get("time_anchor")
-        if not batch_anchor:
-            batch_anchor = "January-1-2024"
-
-        # Flatten all turns, resolving time_anchor per turn
-        messages = []
-        prev_dt = None  # carries forward from previous turn
-        first_dt = None  # for session-level date
-
-        for turn in batch["turns"]:
-            # Find this turn's own time_anchor from its messages
-            turn_anchor = None
-            for msg in turn:
-                if msg.get("time_anchor"):
-                    turn_anchor = msg["time_anchor"]
-                    break
-
-            if turn_anchor:
-                dt = parse_beam_time_anchor(turn_anchor)
-            elif prev_dt is not None:
-                dt = prev_dt  # inherit from previous turn
-            else:
-                dt = parse_beam_time_anchor(batch_anchor)
-
-            if first_dt is None:
-                first_dt = dt
-            prev_dt = dt
-
-            for msg in turn:
-                role = msg["role"]
-                messages.append(
-                    {
-                        "name": role,
-                        "role": role,
-                        "content": msg["content"],
-                        "created_at": dt.strftime("%Y-%m-%dT%H:%M:%S"),
-                    },
-                )
-
-        sessions.append(
-            {
-                "session_id": f"beam_{chat_size}_{case_id}_batch{batch_num}",
-                "date": first_dt.strftime("%Y-%m-%d"),
-                "messages": messages,
-            },
-        )
-
-    return sessions
-
-
-def get_available_cases(beam_root: Path, chat_size: str) -> list[str]:
-    """Return sorted list of case IDs for a given chat size."""
-    chats_dir = beam_root / "chats" / chat_size
-    if not chats_dir.exists():
-        return []
-    return sorted(
-        [d.name for d in chats_dir.iterdir() if d.is_dir()],
-        key=int,
-    )
-
-
-# ---------------------------------------------------------------------------
-# Answer generation
-# ---------------------------------------------------------------------------
-async def answer_question_agentic(app, question: str, compress_session: bool = False) -> tuple[str, dict]:
-    """Answer a probing question using ReMe's agentic_answer job.
-
-    Returns (answer, metadata)
-    """
-    from reme.utils.evaluation_interface import track_agent_token_usage, track_job_counts
-
-    with (
-        track_job_counts(["search"], app.context) as tool_counts,
-        track_agent_token_usage(
-            ["bench"],
-            app.context,
-        ) as token_usages,
-    ):
-        query_resp = await app.run_job(
-            "agentic_answer",
-            query=question,
-            compress_session=compress_session,
-        )
-    answer = (query_resp.answer or "").strip()
-
-    return answer, {
-        "mode": "agentic",
-        "tool_counts": tool_counts,
-        "token_usage": token_usages["bench"],
-    }
-
-
-# ---------------------------------------------------------------------------
-# BEAM rubric-based LLM-as-Judge
-# ---------------------------------------------------------------------------
-async def judge_answer(
-    app,
-    question: str,
-    llm_response: str,
-    rubric: list[str],
-    question_type: str = "",
-) -> dict:
-    """Judge an answer via the answer_judge job (beam_rubric_judge_step)."""
-    judge_resp = await app.run_job(
-        "answer_judge",
-        llm_response=llm_response,
-        rubric=rubric,
-        probing_question=question,
-        question_type=question_type,
-    )
-    result = {
-        "llm_judge_score": (judge_resp.metadata or {}).get("llm_judge_score", 0.0),
-        "llm_judge_responses": (judge_resp.metadata or {}).get("llm_judge_responses", []),
-    }
-    # Include event_ordering extra metrics if present
-    eo = (judge_resp.metadata or {}).get("event_ordering")
-    if eo:
-        result["event_ordering"] = eo
-    return result
-
-
-# ---------------------------------------------------------------------------
-# Main evaluation pipeline
-# ---------------------------------------------------------------------------
-async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False) -> dict:
-    """Evaluate a single BEAM case end-to-end.
-
-    Args:
-        eval_config: The evaluation configuration dict.
-        case_id: The case directory name (e.g. "1").
-        eval_only: If True, skip ingestion and only run query+judge
-            using the existing workspace.
-
-    Returns:
-        A results dict with all questions, answers, and judgments.
-    """
-    from reme import Application
-    from reme.config import resolve_app_config
-
-    dataset_cfg = eval_config["dataset"]
-    chat_size = dataset_cfg["chat_size"]
-    compress_session = bool(eval_config["evaluation"].get("compress_session", False))
-    beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/beam/dataset/BEAM")
-    chat_path = beam_root / "chats" / chat_size / case_id / "chat.json"
-    probing_questions_path = beam_root / "chats" / chat_size / case_id / "probing_questions" / "probing_questions.json"
-
-    if not chat_path.exists():
-        raise FileNotFoundError(f"Chat file not found: {chat_path}")
-    if not probing_questions_path.exists():
-        raise FileNotFoundError(f"Probing questions not found: {probing_questions_path}")
-
-    logger.info(
-        "[Case %s] size=%s%s",
-        case_id,
-        chat_size,
-        " [eval_only]" if eval_only else "",
-    )
-
-    # Workspace setup
-    workspace_root = _PROJECT_ROOT / dataset_cfg.get("workspace_root", _WORKSPACE_ROOT_DEFAULT)
-    case_dir = workspace_root / f"{chat_size}_{case_id}"
-    workspace_dir = str(case_dir / ".reme")
-
-    if eval_only:
-        if not case_dir.exists() or not Path(workspace_dir).exists():
-            raise FileNotFoundError(
-                f"[Case {case_id}] eval_only: workspace not found at {case_dir}. "
-                f"Run without --eval_only first to build the workspace.",
-            )
-    else:
-        if case_dir.exists():
-            shutil.rmtree(case_dir)
-            logger.info(f"[Case {case_id}] Cleaned existing workspace: {case_dir}")
-        else:
-            logger.info(f"[Case {case_id}] Workspace not found, creating: {case_dir}")
-        case_dir.mkdir(parents=True, exist_ok=True)
-
-    # Pre-initialize ReMe's loguru logger with the correct log_dir
-    output_cfg = eval_config.get("output", {})
-    if output_cfg.get("log_to_file", False):
-        reme_log_dir = os.environ.get("REME_LOG_DIR")
-        if reme_log_dir:
-            from reme.utils import get_logger
-
-            get_logger(
-                log_dir=reme_log_dir,
-                level=os.environ.get("REME_LOG_LEVEL", "INFO"),
-                log_to_console=output_cfg.get("log_to_console", True),
-                log_to_file=True,
-                force_init=True,
-            )
-
-    cfg = resolve_app_config(
-        config=eval_config["reme"]["config"],
-        workspace_dir=workspace_dir,
-        log_to_console=output_cfg.get("log_to_console", True),
-        log_to_file=output_cfg.get("log_to_file", False),
-        enable_logo=False,
-    )
-
-    app = Application(**cfg)
-    await app.start()
-
-    from reme.utils.evaluation_interface import check_agent_token_usage  # noqa: E402
-
-    _MEM_AGENT_NAMES = ("default", "bench")
-    sessions_ingested = 0
-    memory_token_usage: dict[str, dict[str, int | None]] = {}
-    try:
-        if not eval_only:
-            # ── Phase 1: Ingest sessions (with token tracking) ─────────
-            sessions = load_beam_chat(chat_path, chat_size, case_id)
-            logger.info(f"[Case {case_id}] Loaded {len(sessions)} sessions from chat.json")
-
-            # Snapshot token counters before memory construction
-            mem_token_start = {name: check_agent_token_usage(name, app.context) for name in _MEM_AGENT_NAMES}
-
-            for i, session in enumerate(sessions):
-                logger.info(
-                    f"[Case {case_id}] Ingesting session {i+1}/{len(sessions)}: "
-                    f"id={session['session_id']} date={session['date']} "
-                    f"msgs={len(session['messages'])}",
-                )
-                resp = await app.run_job(
-                    "auto_memory",
-                    messages=session["messages"],
-                    session_id=session["session_id"],
-                    date=session["date"],
-                )
-                if not resp.success:
-                    logger.warning(f"[Case {case_id}] auto_memory failed: {resp.answer}")
-                else:
-                    logger.info(
-                        f"[Case {case_id}] auto_memory success: " f"{resp.answer[:100] if resp.answer else ''}",
-                    )
-                await app.run_job("index_update")
-                sessions_ingested += 1
-
-            # Final digest update
-            logger.info(f"[Case {case_id}] Running digest_update...")
-            await app.run_job("digest_update")
-            logger.info(f"[Case {case_id}] Ingestion complete.")
-
-            # Compute memory construction token deltas
-            for name in _MEM_AGENT_NAMES:
-                end_usage = check_agent_token_usage(name, app.context)
-                delta: dict[str, int | None] = {}
-                for metric in _TOKEN_USAGE_METRICS:
-                    current = end_usage[metric]
-                    start = mem_token_start[name][metric]
-                    delta[metric] = None if current is None else current - (start or 0)
-                memory_token_usage[name] = delta
-            logger.info(f"[Case {case_id}] Memory construction token usage: {memory_token_usage}")
-
-        # ── Phase 2: Answer + Judge probing questions ───────────────
-        with open(probing_questions_path, encoding="utf-8") as f:
-            probing_questions = json.load(f)
-
-        total_questions = sum(len(v) for v in probing_questions.values())
-        logger.info(f"[Case {case_id}] Total probing questions: {total_questions}")
-
-        all_question_results = []
-        q_idx = 0
-
-        for q_type in probing_questions:
-            logger.info(
-                f"[Case {case_id}] Question type: {q_type} " f"({len(probing_questions[q_type])} questions)",
-            )
-
-            for i, q in enumerate(probing_questions[q_type]):
-                q_idx += 1
-                question = q["question"]
-                rubric = q.get("rubric", [])
-                logger.info(
-                    f"[Case {case_id}] [{q_idx}/{total_questions}] " f"{q_type} Q{i+1}: {question[:100]}...",
-                )
-
-                q_result = {
-                    "question_type": q_type,
-                    "question_index": i,
-                    "question": question,
-                    "rubric": rubric,
-                }
-
-                # Agentic answer
-                try:
-                    agentic_answer, agentic_meta = await answer_question_agentic(
-                        app,
-                        question,
-                        compress_session=compress_session,
-                    )
-                except Exception as e:
-                    logger.error(f"[Case {case_id}] Agentic answer failed: {e}")
-                    agentic_answer = f"(error: {e})"
-                    agentic_meta = {"error": str(e)}
-
-                if not agentic_answer:
-                    agentic_answer = "(no answer generated)"
-                logger.info(f"[Case {case_id}] Agentic answer: {agentic_answer[:200]}...")
-                logger.info(
-                    f"[Case {case_id}] Agentic tool calls: {agentic_meta.get('tool_counts', {})}",
-                )
-                logger.info(f"[Case {case_id}] Bench token usage: {agentic_meta.get('token_usage', {})}")
-
-                # Judge agentic answer
-                logger.info(f"[Case {case_id}] Judging agentic ({q_type})...")
-                agentic_judgment = await judge_answer(
-                    app,
-                    question,
-                    agentic_answer,
-                    rubric,
-                    question_type=q_type,
-                )
-                logger.info(
-                    f"[Case {case_id}] Agentic score: " f"{agentic_judgment['llm_judge_score']:.3f}",
-                )
-
-                q_result["agentic_response"] = agentic_answer
-                q_result["agentic_judgment"] = agentic_judgment
-                q_result["agentic_metadata"] = agentic_meta
-
-                all_question_results.append(q_result)
-
-    finally:
-        await app.close()
-
-    return {
-        "case_id": case_id,
-        "chat_size": chat_size,
-        "sessions_ingested": sessions_ingested,
-        "total_questions": len(all_question_results),
-        "questions": all_question_results,
-        "memory_token_usage": memory_token_usage,
-    }
-
-
-# ---------------------------------------------------------------------------
-# Worker: runs a single case in its own process with its own event loop
-# ---------------------------------------------------------------------------
-def _evaluate_case_worker(task_input: tuple) -> dict:
-    """Worker function for multiprocessing. Each process gets its own event loop."""
-    eval_config, case_id, log_level, reme_log_level, eval_only, log_dir = task_input
-    import asyncio  # pylint: disable=import-outside-toplevel
-
-    _configure_worker(log_level, reme_log_level, log_dir=log_dir)
-
-    # Suppress httpx GC noise
-    logging.getLogger("asyncio").setLevel(logging.CRITICAL)
-
-    return asyncio.run(evaluate_case(eval_config, case_id, eval_only=eval_only))
-
-
-def _indexed_worker(indexed_input: tuple) -> tuple:
-    """Module-level wrapper for imap_unordered with index tracking."""
-    idx, task_input = indexed_input
-    return idx, _evaluate_case_worker(task_input)
-
-
-def _resolve_num_workers(configured: int) -> int:
-    """Resolve num_workers: 0=auto (cpu_count-2, min 1), 1=sequential, >1=parallel."""
-    if configured == 0:
-        return max(1, (os.cpu_count() or 4) - 2)
-    return max(1, configured)
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-def main(  # pylint: disable=too-many-statements
-    config_path: str | None = None,
-    log_level: str = "INFO",
-    reme_log_level: str = "INFO",
-    eval_only: bool = False,
-):
-    """Run the BEAM evaluation pipeline.
-
-    Args:
-        config_path: Path to the YAML config file.
-        log_level: Log level for the eval runner.
-        reme_log_level: Log level for reme internal logs.
-        eval_only: If True, skip ingestion and only run query+judge using
-            existing workspaces.
-    """
-    from multiprocessing import Pool  # pylint: disable=import-outside-toplevel
-
-    # Load config BEFORE logging setup so log_dir is available
-    eval_config = load_eval_config(config_path)
-
-    # Resolve per-run log directory from config
-    output_cfg = eval_config.get("output", {})
-    log_dir_abs = None
-    if output_cfg.get("log_to_file", False):
-        log_dir_raw = output_cfg.get("log_dir", "logs")
-        log_prefix = output_cfg.get("log_prefix", "beam")
-        run_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        log_dir_abs = str(_PROJECT_ROOT / log_dir_raw / f"{log_prefix}_{run_ts}")
-
-    setup_logging(log_level, reme_log_level, log_dir=log_dir_abs)
-    dataset_cfg = eval_config["dataset"]
-    chat_size = dataset_cfg["chat_size"]
-    beam_root = _PROJECT_ROOT / dataset_cfg.get("beam_root", "benchmark/beam/dataset/BEAM")
-
-    # Determine which cases to run
-    case_ids = dataset_cfg.get("case_ids") or []
-    if not case_ids:
-        case_ids = get_available_cases(beam_root, chat_size)
-
-    # Pagination
-    start = dataset_cfg.get("start_index", 0)
-    num_items = dataset_cfg.get("num_items", 0)
-    if num_items > 0:
-        case_ids = case_ids[start : start + num_items]
-    elif start > 0:
-        case_ids = case_ids[start:]
-
-    if not case_ids:
-        logger.error(f"No cases found for chat_size={chat_size}")
-        return
-
-    logger.info(
-        "Evaluating %d case(s) for chat_size=%s: %s%s",
-        len(case_ids),
-        chat_size,
-        case_ids,
-        " [eval_only: query+judge only]" if eval_only else "",
-    )
-
-    # Resolve parallelism
-    num_workers = _resolve_num_workers(eval_config["evaluation"].get("num_workers", 1))
-    logger.info(f"Using {num_workers} worker(s)")
-
-    # Create output directory
-    output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/beam/results")
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Create workspace root directory
-    workspace_root = _PROJECT_ROOT / dataset_cfg.get("workspace_root", _WORKSPACE_ROOT_DEFAULT)
-    workspace_root.mkdir(parents=True, exist_ok=True)
-
-    # Pre-check: verify all workspaces exist in eval_only mode
-    if eval_only:
-        missing_cases = []
-        for case_id in case_ids:
-            case_dir = workspace_root / f"{chat_size}_{case_id}"
-            if not case_dir.exists() or not (case_dir / ".reme").exists():
-                missing_cases.append(case_id)
-        if missing_cases:
-            preview = missing_cases[:10]
-            suffix = "..." if len(missing_cases) > 10 else ""
-            raise FileNotFoundError(
-                f"eval_only: {len(missing_cases)} workspace(s) not found under {workspace_root}. "
-                f"Missing cases: {preview}{suffix}. "
-                f"Run without --eval_only first to build the workspaces.",
-            )
-
-    # Build task args
-    task_args = [(eval_config, case_id, log_level, reme_log_level, eval_only, log_dir_abs) for case_id in case_ids]
-
-    # Progress tracking
-    total_items = len(task_args)
-    completed_count = [0]
-    start_time = time.time()
-    progress_lock = threading.Lock()
-
-    def _print_progress(prefix: str = "PROGRESS"):
-        elapsed = time.time() - start_time
-        elapsed_min = elapsed / 60
-        done = completed_count[0]
-        pct = 100.0 * done / total_items if total_items else 0
-        eta_str = "N/A"
-        if done > 0:
-            eta_sec = elapsed / done * (total_items - done)
-            eta_str = f"{eta_sec/60:.1f}min"
-        print(
-            f"[{prefix}] {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} | "
-            f"{done}/{total_items} ({pct:.1f}%) completed | "
-            f"elapsed={elapsed_min:.1f}min | ETA={eta_str}",
-            flush=True,
-        )
-
-    def _progress_timer():
-        """Background thread: print progress every 10 minutes."""
-        while not _timer_stop.is_set():
-            _timer_stop.wait(600)
-            if not _timer_stop.is_set():
-                with progress_lock:
-                    _print_progress()
-
-    _timer_stop = threading.Event()
-    timer_thread = threading.Thread(target=_progress_timer, daemon=True)
-    timer_thread.start()
-
-    # Run evaluation
-    if num_workers == 1:
-        results = []
-        for task_input in task_args:
-            result = _evaluate_case_worker(task_input)
-            results.append(result)
-            with progress_lock:
-                completed_count[0] += 1
-    else:
-        results = [None] * total_items
-        indexed_args = list(enumerate(task_args))
-
-        with Pool(processes=num_workers) as pool:
-            for idx, result in pool.imap_unordered(_indexed_worker, indexed_args):
-                results[idx] = result
-                with progress_lock:
-                    completed_count[0] += 1
-
-    # Stop progress timer
-    _timer_stop.set()
-    timer_thread.join(timeout=2)
-
-    # Save results
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    output_file = output_dir / f"results_{chat_size}_{timestamp}.json"
-    with open(output_file, "w", encoding="utf-8") as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-    logger.info(f"Results saved to {output_file}")
-
-    # Final progress
-    _print_progress("FINAL")
-
-    # Print concise summary
-    print("\n" + "=" * 70)
-    print(f"  BEAM EVALUATION RESULTS  |  size={chat_size}  cases={len(results)}")
-    print("=" * 70)
-
-    # Per-type stats (agentic only)
-    type_scores: dict[str, list[float]] = {}
-    type_binary_scores: dict[str, list[float]] = {}
-    all_scores: list[float] = []
-    all_binary_scores: list[float] = []
-    all_tool_call_totals: list[int] = []
-    all_token_usages: list[dict[str, int | None]] = []
-    all_memory_token_usages: list[dict[str, dict[str, int | None]]] = []
-
-    for case_result in results:
-        if "error" in case_result:
-            continue
-        mem_usage = case_result.get("memory_token_usage", {})
-        if mem_usage:
-            all_memory_token_usages.append(mem_usage)
-        for q in case_result.get("questions", []):
-            judgment = q.get("agentic_judgment", {})
-            score = judgment.get("llm_judge_score", 0.0)
-            # Binary: convert each rubric item score to 0/1, then average
-            judge_responses = judgment.get("llm_judge_responses", [])
-            if judge_responses:
-                binary_scores_per_item = [1.0 if r.get("score", 0) >= 1.0 else 0.0 for r in judge_responses]
-                binary_score = sum(binary_scores_per_item) / len(binary_scores_per_item)
-            else:
-                binary_score = 1.0 if score > 0.99 else 0.0
-            qtype = q["question_type"]
-            if qtype not in type_scores:
-                type_scores[qtype] = []
-                type_binary_scores[qtype] = []
-            type_scores[qtype].append(score)
-            type_binary_scores[qtype].append(binary_score)
-            all_scores.append(score)
-            all_binary_scores.append(binary_score)
-            metadata = q.get("agentic_metadata", {})
-            all_tool_call_totals.append(sum(metadata.get("tool_counts", {}).values()))
-            all_token_usages.append(metadata.get("token_usage", {}))
-
-    # Memory construction token usage summary
-    if all_memory_token_usages:
-        print("\n  ── Memory Construction Token Usage ──")
-        for agent_name in ("default", "bench"):
-            for metric in _TOKEN_USAGE_METRICS:
-                values = [
-                    usage[agent_name][metric]
-                    for usage in all_memory_token_usages
-                    if usage.get(agent_name, {}).get(metric) is not None
-                ]
-                if values:
-                    total = sum(values)
-                    mean, std = _mean_and_std(values)
-                    print(
-                        f"    {agent_name}/{metric}: total={total} mean={mean:.2f} std={std:.2f} ({len(values)} cases)",
-                    )
-                else:
-                    print(f"    {agent_name}/{metric}: unavailable")
-        print()
-
-    print("\n  ── AGENTIC ──")
-    if all_scores:
-        for qtype in sorted(type_scores.keys()):
-            scores = type_scores[qtype]
-            avg = sum(scores) / len(scores) if scores else 0
-            bin_scores = type_binary_scores[qtype]
-            bin_avg = sum(bin_scores) / len(bin_scores) if bin_scores else 0
-            print(f"    {qtype:<40s}: {avg:.3f}  binary={bin_avg:.3f}  ({len(scores)} Qs)")
-        overall = sum(all_scores) / len(all_scores) if all_scores else 0
-        binary_overall = sum(all_binary_scores) / len(all_binary_scores) if all_binary_scores else 0
-        print(f"    {'-'*38}")
-        print(f"    {'OVERALL':<40s}: {overall:.3f}  binary={binary_overall:.3f}  ({len(all_scores)} Qs)")
-        tool_call_mean, tool_call_std = _mean_and_std(all_tool_call_totals)
-        print(f"    Tool calls/query: mean={tool_call_mean:.2f} std={tool_call_std:.2f}")
-        print("    Bench reported tokens/query:")
-        for metric in _TOKEN_USAGE_METRICS:
-            values = [usage[metric] for usage in all_token_usages if usage.get(metric) is not None]
-            if values:
-                mean, std = _mean_and_std(values)
-                print(f"      {metric}: mean={mean:.2f} std={std:.2f}")
-            else:
-                print(f"      {metric}: unavailable")
-    else:
-        print("    (no results)")
-
-    # Per-case summary
-    print("\n  ── Per-Case Summary ──")
-    for case_result in results:
-        case_id = case_result["case_id"]
-        if "error" in case_result:
-            print(f"    Case {case_id}: ERROR — {case_result['error']}")
-            continue
-        n_qs = case_result.get("total_questions", 0)
-        n_sessions = case_result.get("sessions_ingested", 0)
-        mem_usage = case_result.get("memory_token_usage", {})
-        parts = [f"Case {case_id}: {n_sessions} sessions, {n_qs} questions"]
-        # Append memory construction total tokens if available
-        for agent_name in ("default", "bench"):
-            agent_usage = mem_usage.get(agent_name, {})
-            total = agent_usage.get("total_tokens")
-            if total is not None:
-                parts.append(f"mem_{agent_name}_tokens={total}")
-        questions = case_result.get("questions", [])
-        scores = [q.get("agentic_judgment", {}).get("llm_judge_score", 0.0) for q in questions]
-        if scores:
-            avg = sum(scores) / len(scores)
-            # Binary: 0/1 per rubric item, average per question, then across questions
-            bin_scores = []
-            for q in questions:
-                judge_responses = q.get("agentic_judgment", {}).get("llm_judge_responses", [])
-                if judge_responses:
-                    item_bins = [1.0 if r.get("score", 0) >= 1.0 else 0.0 for r in judge_responses]
-                    bin_scores.append(sum(item_bins) / len(item_bins))
-                else:
-                    s = q.get("agentic_judgment", {}).get("llm_judge_score", 0.0)
-                    bin_scores.append(1.0 if s > 0.99 else 0.0)
-            bin_avg = sum(bin_scores) / len(bin_scores)
-            parts.append(f"agentic={avg:.3f} binary={bin_avg:.3f}")
-        print(f"    {' | '.join(parts)}")
-
-    print("=" * 70)
-    total_elapsed = time.time() - start_time
-    print(f"\n  Total time: {total_elapsed/60:.1f} min")
-    print("\n" + "=" * 70)
-    print("  [DONE] BEAM EVALUATION COMPLETED SUCCESSFULLY")
-    print("=" * 70 + "\n")
-
-
-_TOKEN_USAGE_METRICS = (
-    "input_tokens",
-    "output_tokens",
-    "total_tokens",
-)
-
-
-def _mean_and_std(values: list[int]) -> tuple[float, float]:
-    """Return population mean and standard deviation for one per-question metric."""
-    if not values:
-        return 0.0, 0.0
-    mean = sum(values) / len(values)
-    return mean, (sum((value - mean) ** 2 for value in values) / len(values)) ** 0.5
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser(description="BEAM evaluation runner")
-    parser.add_argument("--config", type=str, default=None, help="Path to config.yaml")
-    parser.add_argument(
-        "--log-level",
-        type=str,
-        default="INFO",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        help="Log level for the eval runner (default: INFO)",
-    )
-    parser.add_argument(
-        "--reme-log-level",
-        type=str,
-        default="INFO",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        help="Log level for reme internal logs — loguru (default: INFO)",
-    )
-    parser.add_argument(
-        "-q",
-        "--quiet",
-        action="store_true",
-        help="Shortcut for --log-level WARNING --reme-log-level WARNING",
-    )
-    parser.add_argument(
-        "--eval_only",
-        action="store_true",
-        help="Skip ingestion. Reuse existing workspaces and only run query+judge.",
-    )
-    args = parser.parse_args()
-
-    if args.quiet:
-        args.log_level = "WARNING"
-        args.reme_log_level = "WARNING"
-
-    main(args.config, args.log_level, args.reme_log_level, eval_only=args.eval_only)

benchmark/bfcl/bfcl_agent.py

@@ -0,0 +1,726 @@
+# flake8: noqa: E402
+# pylint: disable=too-many-return-statements
+"""A minimal ReAct Agent for BFCL-v3(multi-turn) tasks."""
+
+import re
+import os
+import time
+import json
+import warnings
+import tempfile
+import datetime
+from pathlib import Path
+from typing import Dict, List, Any
+
+import ray
+import requests
+from tqdm import tqdm
+from loguru import logger
+from openai import OpenAI
+from dotenv import load_dotenv
+
+from bfcl_utils import (
+    load_test_case,
+    handle_user_turn,
+    handle_tool_calls,
+    extract_tool_schema,
+    extract_single_turn_response,
+    extract_multi_turn_responses,
+    capture_and_print_score_files,
+    create_error_response,
+)
+from bfcl_eval.model_handler.api_inference.qwen import QwenAPIHandler
+from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import (
+    is_empty_execute_response,
+)
+from bfcl_eval.eval_checker.eval_runner import (
+    multi_turn_runner,
+    ast_file_runner,
+)
+from bfcl_eval.eval_checker.eval_runner_helper import record_cost_latency
+from bfcl_eval.utils import (
+    is_multi_turn,
+    is_relevance_or_irrelevance,
+    find_file_with_suffix,
+    load_file,
+)
+
+os.environ["BFCL_DATA_PATH"] = "data/multiturn_data_base_val.jsonl"
+os.environ["BFCL_ANSWER_PATH"] = "data/possible_answer"
+load_dotenv("../../.env")
+
+
+@ray.remote
+class BFCLAgent:
+    """A minimal ReAct Agent for BFCL-v3(multi-turn) tasks."""
+
+    def __init__(
+        self,
+        index: int,
+        task_ids: List[str],
+        experiment_name: str,
+        data_path: str = os.getenv("BFCL_DATA_PATH"),
+        answer_path: Path = Path(os.getenv("BFCL_ANSWER_PATH")),
+        model_name: str = "qwen3-8b",
+        temperature: float = 0.9,
+        max_interactions: int = 30,
+        max_response_size: int = 2000,
+        num_trials: int = 1,
+        enable_thinking: bool = False,
+        use_memory: bool = False,
+        use_memory_addition: bool = False,
+        use_memory_deletion: bool = False,
+        delete_freq: int = 10,
+        freq_threshold: int = 5,
+        utility_threshold: float = 0.5,
+        memory_base_url: str = "http://0.0.0.0:8002/",
+    ):
+
+        self.index: int = index
+        self.task_ids: List[str] = task_ids
+        self.categories: List[str] = [task_id.rsplit("_", 1)[0] if "_" in task_id else task_id for task_id in task_ids]
+        self.experiment_name: str = experiment_name
+        self.data_path: str = data_path
+        self.answer_path: Path = answer_path
+        self.model_name: str = model_name
+        self.temperature: float = temperature
+        self.max_interactions: int = max_interactions
+        self.max_response_size: int = max_response_size
+        self.num_trials: int = num_trials
+        self.enable_thinking: bool = enable_thinking
+        self.use_memory: bool = use_memory
+        self.use_memory_addition: bool = use_memory_addition if use_memory else False
+        self.use_memory_deletion: bool = use_memory_deletion if use_memory else False
+        self.delete_freq: int = delete_freq
+        self.freq_threshold: int = freq_threshold
+        self.utility_threshold: float = utility_threshold
+        self.memory_base_url: str = memory_base_url
+
+        self.llm_client = OpenAI()
+
+        self.history: List[List[List[dict]]] = [[] for _ in range(num_trials)]
+        self.retrieved_memory_list: List[List[List[Any]]] = [[] for _ in range(num_trials)]
+        self.test_entry: List[List[Dict[str, Any]]] = [[] for _ in range(num_trials)]
+        self.original_test_entry: List[List[Dict[str, Any]]] = [[] for _ in range(num_trials)]
+        self.tool_schema: List[List[List[dict]]] = [[] for _ in range(num_trials)]
+        self.current_turn = [[0 for _ in range(len(task_ids))] for _ in range(num_trials)]
+
+        for run_id in range(num_trials):
+            for task_index in range(len(task_ids)):
+                self.init_state(run_id, task_index)
+
+    def init_state(self, run_id, i) -> Dict[str, Any]:
+        """Initialize the state of the agent."""
+        self.test_entry[run_id].append(load_test_case(self.data_path, self.task_ids[i]))
+        self.original_test_entry[run_id].append(self.test_entry[run_id][i].get("extra", {}))
+        self.tool_schema[run_id].append(extract_tool_schema(self.test_entry[run_id][i].get("tools", [{}])))
+
+        msg = self.test_entry[run_id][i].get("messages", [])
+        self.history[run_id].append(msg)
+        self.retrieved_memory_list[run_id].append([])
+        self.current_turn[run_id][i] = 1
+
+    def update_task_history_with_memory(self, run_id, task_index, previous_memories: None):
+        """Update the task history with memory."""
+        query = self.history[run_id][task_index][0]["content"]
+        if len(previous_memories) == 0:
+            response = self.get_memory(query)
+            if response and "memory_list" in response["metadata"]:
+                self.retrieved_memory_list[run_id][task_index] = response["metadata"]["memory_list"]
+                task_memory = re.sub(r"\bMemory\s*(\d+)\s*[:]", r"Experience \1 :", response["answer"])
+                logger.info(f"loaded task_memory: {task_memory}")
+                self.history[run_id][task_index][0] = self.get_query_with_memory(query, task_memory)
+        else:
+            formatted_memories = []
+            for i, memory in enumerate(previous_memories, 1):
+                condition = memory["when_to_use"]
+                memory_content = memory["content"]
+                memory_text = f"Experience {i} :\n When to use: {condition}\n Content: {memory_content}\n"
+                formatted_memories.append(memory_text)
+            self.history[run_id][task_index][0] = self.get_query_with_memory(query, "\n".join(formatted_memories))
+
+    def get_query_with_memory(self, query: str, memory: str):
+        """Get the query with memory."""
+        return {
+            "role": "user",
+            "content": "Task:\n" + query + "\n\nSome Related Experience to help you to complete the task:\n" + memory,
+        }
+
+    def get_query_without_experience(self, query: str):
+        """Get the query without experience."""
+        if "\n\nSome Related Experience" in query:
+            query = query.split("\n\nSome Related Experience")[0].split("Task:\n")[-1]
+        return query
+
+    def get_traj_from_task_history(self, task_id: str, task_history: list, reward: float):
+        """Get the trajectory from the task history."""
+        return {
+            "task_id": task_id,
+            "messages": task_history,
+            "score": reward,
+        }
+
+    def handle_api_response(self, response: requests.Response):
+        """Handle API response with proper error checking"""
+        if response.status_code != 200:
+            print(f"Error: {response.status_code}")
+            print(response.text)
+            return None
+
+        return response.json()
+
+    def get_memory(self, query: str):
+        """Retrieve relevant task memories based on a query"""
+        response = requests.post(
+            url=f"{self.memory_base_url}retrieve_task_memory",
+            json={
+                "query": query,
+                "enable_llm_rerank": False,
+                "enable_score_filter": False,
+                "top_k": 5,
+                "enable_llm_rewrite": False,
+            },
+        )
+
+        result = self.handle_api_response(response)
+        if not result:
+            return None
+
+        logger.info(f"query: {query}, response: {result}")
+        return result
+
+    def summary_memory(self, trajectories):
+        """Generate a summary of conversation messages and create task memories"""
+        response = requests.post(
+            url=f"{self.memory_base_url}summary_task_memory",
+            json={
+                "trajectories": trajectories,
+                "success_threshold": 1.0,
+                "enable_soft_comparison": True,
+                "validation_threshold": 0.5,
+            },
+        )
+
+        result = self.handle_api_response(response)
+        if not result:
+            return []
+
+        # Extract memory list from response
+        memory_list = result.get("metadata", {}).get("memory_list", [])
+        logger.info(f"add new memories: {memory_list}")
+        return memory_list
+
+    def add_memory(self, memory_list):
+        """Add the memory to the memory pool."""
+        response = requests.post(
+            url=f"{self.memory_base_url}add_task_memory",
+            json={
+                "memory_list": memory_list,
+            },
+        )
+        response.raise_for_status()
+
+    def update_memory_information(self, memory_list, update_utility: bool = False):
+        """Update the memory information."""
+        response = requests.post(
+            url=f"{self.memory_base_url}record_task_memory",
+            json={
+                "memory_list": memory_list,
+                "update_utility": update_utility,
+            },
+        )
+        response.raise_for_status()
+        logger.info(response.json())
+
+    def delete_memory(self):
+        """Delete the memory from the memory pool."""
+        response = requests.post(
+            url=f"{self.memory_base_url}delete_task_memory",
+            json={
+                "freq_threshold": self.freq_threshold,
+                "utility_threshold": self.utility_threshold,
+            },
+        )
+        response.raise_for_status()
+
+    def call_llm(self, messages: list, tool_schemas: list[dict]) -> str:
+        """Call the LLM."""
+        for i in range(100):
+            try:
+                response = self.llm_client.chat.completions.create(
+                    model=self.model_name,
+                    messages=messages,
+                    tools=tool_schemas,
+                    temperature=self.temperature,
+                    seed=0,
+                    extra_body={"enable_thinking": self.enable_thinking},
+                    stream=self.enable_thinking,
+                    parallel_tool_calls=True,
+                )
+                if not self.enable_thinking:
+                    out_msg = response.choices[0].message
+                    return out_msg.model_dump(exclude_unset=True, exclude_none=True)
+                else:
+                    reasoning_content = ""  # Complete reasoning process
+                    answer_content = ""  # Define complete response
+                    tool_info = []  # Store tool invocation information
+                    is_answering = (
+                        False  # Determine whether the reasoning process has finished and response has started
+                    )
+
+                    for chunk in response:
+                        if not chunk.choices:
+                            # Handle usage information
+                            continue
+
+                        delta = chunk.choices[0].delta
+                        # Handle AI's thought process (chain reasoning)
+                        if hasattr(delta, "reasoning_content") and delta.reasoning_content is not None:
+                            reasoning_content += delta.reasoning_content
+                        # Handle final response content
+                        else:
+                            if not is_answering:  # Print title when entering the response phase for the first time
+                                is_answering = True
+                            if delta.content is not None:
+                                answer_content += delta.content
+
+                            # Handle tool invocation information (support parallel tool calls)
+                            if delta.tool_calls is not None:
+                                for tool_call in delta.tool_calls:
+                                    index = tool_call.index  # Tool call index, used for parallel calls
+
+                                    # Dynamically expand tool information storage list
+                                    while len(tool_info) <= index:
+                                        tool_info.append(
+                                            {
+                                                "id": "",
+                                                "type": "function",
+                                                "index": index,
+                                                "function": {"name": "", "arguments": ""},
+                                            },
+                                        )
+
+                                    # Collect tool call ID (used for subsequent function calls)
+                                    if tool_call.id:
+                                        tool_info[index]["id"] += tool_call.id
+
+                                    # Collect function name (used for subsequent routing to specific functions)
+                                    if tool_call.function and tool_call.function.name:
+                                        tool_info[index]["function"]["name"] += tool_call.function.name
+
+                                    # Collect function parameters (in JSON string format, need subsequent parsing)
+                                    if tool_call.function and tool_call.function.arguments:
+                                        tool_info[index]["function"]["arguments"] += tool_call.function.arguments
+                    msg = {
+                        "role": "assistant",
+                        "content": answer_content,
+                        "reasoning_content": reasoning_content,
+                    }
+                    if tool_info:
+                        msg["tool_calls"] = tool_info
+                    return msg
+            except Exception as e:
+                logger.exception(f"encounter error with {e.args}")
+                time.sleep(1 + i * 10)
+
+        return "call llm error"
+
+    def env_step(self, run_id: int, index: int, messages: str) -> str:
+        """
+        Process one step in the conversation.
+        Both single turn and multi turn are supported.
+
+        Args:
+            messages: List of conversation messages, with the last one being assistant response
+            test_entry: Test entry containing initial_config, involved_classes, question etc.
+            **kwargs: Additional arguments for compatibility
+
+        Returns:
+            Dict containing next message and tools if applicable
+        """
+        try:
+            if not messages:
+                return handle_user_turn(self.original_test_entry[run_id][index], self.current_turn[run_id][index])
+
+            if messages[-1]["role"] != "assistant":
+                return create_error_response(
+                    "Last message must be from assistant",
+                )
+
+            if "tool_calls" in messages[-1] and len(messages[-1]["tool_calls"]) > 0:
+                try:
+                    tool_calls = messages[-1]["tool_calls"]
+                    decoded_calls = self._convert_tool_calls_to_execution_format(
+                        tool_calls,
+                    )
+                    # decoded_calls:[function(param=xxx)]
+                    print(f"decoded_calls: {decoded_calls}")
+                    if is_empty_execute_response(decoded_calls):
+                        warnings.warn(
+                            f"is_empty_execute_response: {is_empty_execute_response(decoded_calls)}",
+                        )
+                        return handle_user_turn(
+                            self.original_test_entry[run_id][index],
+                            self.current_turn[run_id][index],
+                        )
+                    return handle_tool_calls(
+                        tool_calls,
+                        decoded_calls,
+                        self.original_test_entry[run_id][index],
+                        self.current_turn[run_id][index],
+                    )
+                except Exception as e:
+                    warnings.warn(f"Errors during tool invocation: {str(e)}")
+                    return handle_user_turn(self.original_test_entry[run_id][index], self.current_turn[run_id][index])
+            else:
+                return handle_user_turn(self.original_test_entry[run_id][index], self.current_turn[run_id][index])
+
+        except Exception as e:
+            return create_error_response(f"Failed to process request: {str(e)}")
+
+    def _convert_tool_calls_to_execution_format(
+        self,
+        tool_calls: List[Dict[str, Any]],
+    ) -> List[str]:
+        """
+        Convert OpenAI format tool calls to execution format.
+
+        Args:
+            tool_calls: List of tool calls in OpenAI format
+
+        Returns:
+            List of function calls in string format
+        """
+        execution_list = []
+
+        for tool_call in tool_calls:
+            function = tool_call.get("function", {})
+            function_name = function.get("name", "")
+
+            try:
+                arguments = function.get("arguments", "{}")
+                if isinstance(arguments, str):
+                    args_dict = json.loads(arguments)
+                else:
+                    args_dict = arguments
+
+                args_str = ", ".join([f"{k}={repr(v)}" for k, v in args_dict.items()])
+                execution_list.append(f"{function_name}({args_str})")
+
+            except Exception:
+                execution_list.append(f"{function_name}()")
+
+        return execution_list
+
+    def get_reward(self, run_id, index) -> float:
+        """Get the reward."""
+        try:
+            if not self.history[run_id][index] or not self.original_test_entry[run_id][index]:
+                return 0.0
+
+            model_name = "env_handler"
+            handler = QwenAPIHandler(
+                model_name,
+                temperature=1.0,
+            )  # FIXME: magic number
+
+            model_result_data = self._convert_conversation_to_eval_format(run_id, index)
+
+            prompt_data = [self.original_test_entry[run_id][index]]
+
+            state = {"leaderboard_table": {}}
+            record_cost_latency(
+                state["leaderboard_table"],
+                model_name,
+                [model_result_data],
+            )
+
+            if is_relevance_or_irrelevance(self.categories[index]):
+                accuracy, _ = self._eval_relevance_test(
+                    handler,
+                    model_result_data,
+                    prompt_data,
+                    model_name,
+                    self.category,
+                )
+            else:
+                # Find the corresponding possible answer file
+
+                possible_answer_file = find_file_with_suffix(
+                    self.answer_path,
+                    self.categories[index],
+                )
+                possible_answer = load_file(possible_answer_file, sort_by_id=True)
+                possible_answer = [item for item in possible_answer if item["id"] == self.task_ids[index]]
+                if is_multi_turn(self.categories[index]):
+                    accuracy, _ = self._eval_multi_turn_test(
+                        handler,
+                        model_result_data,
+                        prompt_data,
+                        possible_answer,
+                        model_name,
+                        self.categories[index],
+                    )
+                else:
+                    accuracy, _ = self._eval_single_turn_test(
+                        handler,
+                        model_result_data,
+                        prompt_data,
+                        possible_answer,
+                        model_name,
+                        self.categories[index],
+                    )
+            print(f"model_result_data: {model_result_data}")
+            if possible_answer:
+                print(f"possible_answer: {possible_answer}")
+            else:
+                print("possible_answer: None")
+
+            return accuracy
+
+        except Exception:
+            import traceback
+
+            traceback.print_exc()
+            return 0
+
+    def _convert_conversation_to_eval_format(self, run_id, index) -> Dict[str, Any]:
+        """
+        Convert conversation history to evaluation format.
+
+        Args:
+            conversation_result: Result from run_conversation
+            original_test_entry: Original test entry data
+
+        Returns:
+            Data in format expected by multi_turn_runner or other runners
+        """
+        if is_multi_turn(self.categories[index]):
+            turns_data = extract_multi_turn_responses(self.history[run_id][index])
+        else:
+            turns_data = extract_single_turn_response(self.history[run_id][index])
+
+        model_result_data = {
+            "id": self.task_ids[index],
+            "result": turns_data,
+            "latency": 0,
+            "input_token_count": 0,
+            "output_token_count": 0,
+        }
+
+        return model_result_data
+
+    def _eval_multi_turn_test(
+        self,
+        handler,
+        model_result_data,
+        prompt_data,
+        possible_answer,
+        model_name,
+        test_category,
+    ):
+        """
+        Evaluate multi-turn test.
+
+        Args:
+            handler: Model handler instance
+            model_result_data: Model result data
+            prompt_data: Prompt data
+            possible_answer: Possible answer data
+            model_name: Name of the model
+            test_category: Category of the test
+
+        Returns:
+            Tuple of (accuracy, total_count)
+        """
+        with tempfile.TemporaryDirectory() as temp_dir:
+            score_dir = Path(temp_dir)
+            accuracy, total_count = multi_turn_runner(
+                handler=handler,
+                model_result=[model_result_data],
+                prompt=prompt_data,
+                possible_answer=possible_answer,
+                model_name=model_name,
+                test_category=test_category,
+                score_dir=score_dir,
+            )
+            capture_and_print_score_files(
+                score_dir,
+                model_name,
+                test_category,
+                "multi_turn",
+            )
+            return accuracy, total_count
+
+    def _eval_single_turn_test(
+        self,
+        handler,
+        model_result_data,
+        prompt_data,
+        possible_answer,
+        model_name,
+        test_category,
+    ):
+        """
+        Evaluate single-turn AST test.
+
+        Args:
+            handler: Model handler instance
+            model_result_data: Model result data
+            prompt_data: Prompt data
+            possible_answer: Possible answer data
+            model_name: Name of the model
+            test_category: Category of the test
+
+        Returns:
+            Tuple of (accuracy, total_count)
+        """
+        language = "Python"
+        if "java" in test_category.lower():
+            language = "Java"
+        elif "js" in test_category.lower() or "javascript" in test_category.lower():
+            language = "JavaScript"
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            score_dir = Path(temp_dir)
+            accuracy, total_count = ast_file_runner(
+                handler=handler,
+                model_result=[model_result_data],
+                prompt=prompt_data,
+                possible_answer=possible_answer,
+                language=language,
+                test_category=test_category,
+                model_name=model_name,
+                score_dir=score_dir,
+            )
+            capture_and_print_score_files(
+                score_dir,
+                model_name,
+                test_category,
+                "single_turn",
+            )
+            return accuracy, total_count
+
+    def execute(self):
+        """Execute the agent."""
+        result = []
+        counter = 0
+        for task_index, task_id in enumerate(tqdm(self.task_ids, desc=f"ray_index={self.index}")):
+            t_result = None
+            previous_memories = []
+            for run_id in range(self.num_trials):
+                try:
+                    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    for i in range(self.max_interactions):
+                        if self.use_memory and i == 0:
+                            self.update_task_history_with_memory(run_id, task_index, previous_memories)
+                        llm_output = self.call_llm(
+                            self.history[run_id][task_index],
+                            self.tool_schema[run_id][task_index],
+                        )
+                        self.history[run_id][task_index].append(llm_output)
+
+                        env_output = self.env_step(run_id, task_index, self.history[run_id][task_index])
+                        # Possible env_output returns after environment interaction:
+                        # 1. Triggers a query with available tools list:
+                        #    {"messages": [{"role": "user", "content": user_query}], "tools": tools}
+                        # 2. Returns tool invocation result: {"messages":
+                        #    [{"role": "tool", "content": {<exec_results>}, 'tool_call_id': 'chatcmpl-tool-xxx'}]}
+                        #    <exec_results>: when success, returns result dicts, e.g., {"travel_cost_list": [x]},
+                        #    when error, returns error message,
+                        #    e.g., {"error": "cd: temporary: No such directory. You cannot use path ..."}
+                        # 3. Conversation completion:
+                        #    {"messages": [{"role": "env", "content": "[CONVERSATION_COMPLETED]"}]}
+                        # 4. Program error: {"messages": [{"role": "env", "content": f"[ERROR] {error_message}"}]}
+
+                        # tool_list update
+                        if "tools" in env_output:
+                            self.tool_schema[run_id][task_index] = extract_tool_schema(env_output["tools"])
+
+                        new_tool_calls = []
+                        new_tool_call_ids = []
+                        next_user_msg = ""
+                        for idx, msg in enumerate(env_output.get("messages", [])):
+                            if msg["role"] == "tool" and len(msg["content"]) > 0:
+                                new_tool_calls.append(msg.get("content", ""))
+                                new_tool_call_ids.append(msg.get("tool_call_id", ""))
+                            elif msg["role"] == "user":
+                                next_user_msg = msg.get("content", "")
+                                self.current_turn[run_id][task_index] += 1
+                            else:  # for env role messages
+                                next_user_msg = msg.get("content", "")
+
+                        if new_tool_calls:
+                            for idx, call in enumerate(new_tool_calls):
+                                self.history[run_id][task_index].append(
+                                    {"role": "tool", "content": str(call), "tool_call_id": new_tool_call_ids[idx]},
+                                )
+                        else:
+                            self.history[run_id][task_index].append({"role": "user", "content": next_user_msg})
+
+                        logger.info(f"index={self.index} task_id={task_id} iteration={i}")
+
+                        if self.task_completed(run_id, task_index):
+                            break
+
+                    reward = self.get_reward(run_id, task_index)
+                    if self.use_memory:
+                        if self.use_memory_addition:
+                            new_traj_list = [
+                                self.get_traj_from_task_history(task_id, self.history[run_id][task_index], reward),
+                            ]
+                            previous_memories = self.summary_memory(new_traj_list)
+                            if reward == 1:
+                                self.add_memory(previous_memories)
+
+                        # update the freq & utility attributes of retrieved memories
+                        update_utility: bool = reward == 1
+                        self.update_memory_information(self.retrieved_memory_list[run_id][task_index], update_utility)
+
+                    counter += 1
+                    if self.use_memory_deletion and counter % self.delete_freq == 0:
+                        self.delete_memory()
+
+                    t_result = {
+                        "run_id": run_id,
+                        "task_id": self.task_ids[task_index],
+                        "experiment_name": self.experiment_name,
+                        "task_completed": self.task_completed(run_id, task_index),
+                        "reward": reward,
+                        "task_history": self.history[run_id][task_index],
+                        "task_start_time": start_time,
+                    }
+                    if reward == 1:
+                        break
+
+                except Exception as e:
+                    logger.exception(f"encounter error with {e.args}")
+            result.append(t_result)
+        return result
+
+    def task_completed(self, run_id, index):
+        """
+        Check if task is completed.
+
+        Returns:
+            True if task is completed, False otherwise
+        """
+        return self.history[run_id][index][-1]["content"] == "[CONVERSATION_COMPLETED]"
+
+
+def main():
+    """Main function to run the BFCLAgent."""
+    with open(os.getenv("BFCL_DATA_PATH"), "r", encoding="utf-8") as f:
+        task_ids = [json.loads(l)["id"] for l in f]
+    dataset_name = "dev"
+    agent = BFCLAgent(
+        index=0,
+        task_ids=[task_ids[0]],
+        experiment_name=f"qwen3_8b_{dataset_name}",
+    )
+    result = agent.execute()
+    logger.info(f"result={json.dumps(result)}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/bfcl/bfcl_utils.py

@@ -0,0 +1,399 @@
+"""Utils for evaluation on BFCL tasks"""
+
+import json
+from pathlib import Path
+from typing import Dict, List, Any
+
+from bfcl_eval.constants.default_prompts import (
+    DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+)
+from bfcl_eval.constants.type_mappings import GORILLA_TO_OPENAPI
+from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import (
+    execute_multi_turn_func_call,
+)
+from bfcl_eval.model_handler.model_style import ModelStyle
+from bfcl_eval.model_handler.utils import (
+    convert_to_tool,
+    default_decode_execute_prompting,
+    func_doc_language_specific_pre_processing,
+)
+
+
+def load_test_case(data_path: str, test_id: str | None) -> Dict[str, Any]:
+    """
+    load test cases by id
+    """
+    if not Path(data_path).exists():
+        raise FileNotFoundError(f"BFCL data file '{data_path}' not found")
+
+    if test_id is None:
+        raise ValueError("task_id is required")
+
+    with open(data_path, "r", encoding="utf-8") as f:
+        if str(test_id).isdigit():  # pylint: disable=R1720
+            idx = int(test_id)
+            for line_no, line in enumerate(f):
+                if line_no == idx:
+                    return json.loads(line)
+            raise ValueError(f"Test case index {idx} not found in {data_path}")
+        else:
+            for line in f:
+                data = json.loads(line)
+                if data.get("id") == test_id:
+                    return data
+            raise ValueError(f"Test case id '{test_id}' not found in {data_path}")
+
+
+def handle_user_turn(
+    test_entry: Dict[str, Any],
+    current_turn: int,
+) -> Dict[str, Any]:
+    """
+    Handle user turn by returning appropriate content from test_entry["question"].
+    For non-first turns, processes user query and tools.
+
+    Args:
+        test_entry: Test entry containing conversation data
+        current_turn: Current turn number
+
+    Returns:
+        Response containing next user message and tools
+    """
+    try:
+        current_turn_message = []
+        tools = compile_tools(test_entry)
+        questions = test_entry.get("question", [])
+        holdout_function = test_entry.get("holdout_function", {})
+
+        if str(current_turn) in holdout_function:
+            test_entry["function"].extend(holdout_function[str(current_turn)])
+            tools = compile_tools(test_entry)
+            assert len(questions[current_turn]) == 0, "Holdout turn should not have user message."
+            current_turn_message = [
+                {
+                    "role": "user",
+                    "content": DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+                },
+            ]
+            return create_user_response(current_turn_message, tools)
+        if current_turn >= len(questions):
+            return create_completion_response()
+
+        current_turn_message = questions[current_turn]
+
+        return create_user_response(current_turn_message, tools)
+
+    except Exception as e:
+        return create_error_response(f"Failed to process user message: {str(e)}")
+
+
+def handle_tool_calls(  # pylint: disable=W0613
+    tool_calls: List[Dict[str, Any]],
+    decoded_calls: list[str],
+    test_entry: Dict[str, Any],
+    current_turn: int,
+) -> Dict[str, Any]:
+    """
+    Handle tool calls from assistant.
+
+    Args:
+        tool_calls: List of tool calls in OpenAI format
+        decoded_calls: List of decoded function calls
+        test_entry: Test entry containing environment data
+        current_turn: Current turn number
+
+    Returns:
+        Response containing tool execution results
+    """
+    execution_results, _ = execute_multi_turn_func_call(
+        func_call_list=decoded_calls,
+        initial_config=test_entry["initial_config"],
+        involved_classes=test_entry["involved_classes"],
+        model_name="env_handler",
+        test_entry_id=test_entry["id"],
+        long_context=("long_context" in test_entry["id"] or "composite" in test_entry["id"]),
+        is_evaL_run=False,
+    )
+    # print('execution_results in handler_tool_calls:', execution_results)
+
+    return create_tool_response(tool_calls, execution_results)
+
+
+def compile_tools(test_entry: dict) -> list:
+    """
+    Compile functions into tools format.
+
+    Args:
+        test_entry: Test entry containing functions
+
+    Returns:
+        List of tools in OpenAI format
+    """
+    functions: list = test_entry["function"]
+    test_category: str = test_entry["id"].rsplit("_", 1)[0]
+
+    functions = func_doc_language_specific_pre_processing(functions, test_category)
+    tools = convert_to_tool(functions, GORILLA_TO_OPENAPI, ModelStyle.OpenAI_Completions)
+
+    return tools
+
+
+def create_tool_response(
+    tool_calls: List[Dict[str, Any]],
+    execution_results: List[str],
+) -> Dict[str, Any]:
+    """
+    Create response for tool calls.
+
+    Args:
+        tool_calls: List of tool calls
+        execution_results: List of execution results
+
+    Returns:
+        Response containing tool execution results
+    """
+    tool_messages = []
+    for i, (tool_call, result) in enumerate(zip(tool_calls, execution_results)):
+        tool_messages.append(
+            {
+                "role": "tool",
+                "content": result,
+                "tool_call_id": tool_call.get("id", f"call_{i}"),
+            },
+        )
+
+    return {"messages": tool_messages}
+
+
+def create_user_response(
+    question_turn: List[Dict[str, Any]],
+    tools: List[Dict[str, Any]],
+) -> Dict[str, Any]:
+    """
+    Create response containing user message.
+
+    Args:
+        question_turn: List of messages for current turn
+        tools: List of available tools
+
+    Returns:
+        Response containing user message and tools
+    """
+    user_content = ""
+    for msg in question_turn:
+        if msg["role"] == "user":
+            user_content = msg["content"]
+            break
+
+    return {"messages": [{"role": "user", "content": user_content}], "tools": tools}
+
+
+def create_completion_response() -> Dict[str, Any]:
+    """
+    Create response indicating conversation completion.
+
+    Returns:
+        Response with completion message
+    """
+    return {"messages": [{"role": "env", "content": "[CONVERSATION_COMPLETED]"}]}
+
+
+def create_error_response(error_message: str) -> Dict[str, Any]:
+    """
+    Create response for error conditions.
+
+    Args:
+        error_message: Error message to include
+
+    Returns:
+        Response containing error message
+    """
+    return {"messages": [{"role": "env", "content": f"[ERROR] {error_message}"}]}
+
+
+def decode_execute(result):
+    """
+    Decode execute results for compatibility with evaluation framework.
+
+    Args:
+        result: Result to decode
+
+    Returns:
+        List of decoded function calls
+    """
+    return default_decode_execute_prompting(result)
+
+
+def extract_single_turn_response(messages: List[Dict[str, Any]]) -> str:
+    """
+    Extract single-turn response from conversation messages.
+
+    Args:
+        messages: List of conversation messages
+
+    Returns:
+        String representation of the response
+    """
+    for message in reversed(messages):
+        if message["role"] == "assistant":
+            if "tool_calls" in message and message["tool_calls"]:
+                formatted_calls = []
+                for tool_call in message["tool_calls"]:
+                    formatted_call = format_single_tool_call_for_eval(
+                        tool_call,
+                    )
+                    if formatted_call:
+                        formatted_calls.append(formatted_call)
+                return "\n".join(formatted_calls) if formatted_calls else ""
+            elif message.get("content"):
+                return message["content"]
+
+    return ""
+
+
+def extract_multi_turn_responses(
+    messages: List[Dict[str, Any]],
+) -> List[List[str]]:
+    """
+    Extract multi-turn responses from conversation messages.
+
+    Args:
+        messages: List of conversation messages
+
+    Returns:
+        List of turns, each turn is a list of function call strings
+    """
+    turns_data = []
+    current_turn_responses = []
+
+    i = 0
+    while i < len(messages):
+        message = messages[i]
+
+        if message["role"] == "user":
+            if current_turn_responses:
+                turns_data.append(current_turn_responses)
+                current_turn_responses = []
+
+            i += 1
+            while i < len(messages) and messages[i]["role"] == "assistant":
+                assistant_msg = messages[i]
+
+                if "tool_calls" in assistant_msg and assistant_msg["tool_calls"]:
+                    for tool_call in assistant_msg["tool_calls"]:
+                        formatted_call = format_single_tool_call_for_eval(
+                            tool_call,
+                        )
+                        if formatted_call:
+                            current_turn_responses.append(formatted_call)
+
+                i += 1
+
+                while i < len(messages) and messages[i]["role"] == "tool":
+                    i += 1
+        else:
+            i += 1
+
+    if current_turn_responses:
+        turns_data.append(current_turn_responses)
+
+    return turns_data
+
+
+def format_single_tool_call_for_eval(tool_call: Dict[str, Any]) -> str:
+    """
+    Format a single tool call into string representation for evaluation.
+
+    Args:
+        tool_call: Single tool call in OpenAI format
+
+    Returns:
+        Formatted string representation
+    """
+    function = tool_call.get("function", {})
+    function_name = function.get("name", "")
+
+    try:
+        arguments = function.get("arguments", "{}")
+        if isinstance(arguments, str):
+            args_dict = json.loads(arguments)
+        else:
+            args_dict = arguments
+
+        args_str = ", ".join([f"{k}={repr(v)}" for k, v in args_dict.items()])
+        return f"{function_name}({args_str})"
+
+    except Exception:
+        return f"{function_name}()"
+
+
+def capture_and_print_score_files(
+    score_dir: Path,
+    model_name: str,
+    test_category: str,
+    eval_type: str,
+):
+    """
+    Capture and print contents of score files written to score_dir.
+
+    Args:
+        score_dir: Directory containing score files
+        model_name: Name of the model
+        test_category: Category of the test
+        eval_type: Type of evaluation (relevance/multi_turn/single_turn)
+    """
+    try:
+        print(f"\n=== {eval_type.upper()} Evaluation Result Files ===")
+        print(f"Model: {model_name}")
+        print(f"Test Category: {test_category}")
+        print(f"Evaluation Type: {eval_type}")
+
+        for file_path in score_dir.rglob("*"):
+            if file_path.is_file():
+                relative_path = file_path.relative_to(score_dir)
+                print(f"\n--- File: {relative_path} ---")
+
+                try:
+                    with open(file_path, "r", encoding="utf-8") as f:
+                        content = f.read()
+
+                    if (
+                        file_path.suffix == ".json"
+                        or content.strip().startswith("{")
+                        or content.strip().startswith("[")
+                    ):
+                        try:
+                            lines = content.strip().split("\n")
+                            formatted_lines = []
+                            for line in lines:
+                                if line.strip():
+                                    parsed = json.loads(line)
+                                    formatted_lines.append(
+                                        json.dumps(
+                                            parsed,
+                                            ensure_ascii=False,
+                                            indent=2,
+                                        ),
+                                    )
+                            content = "\n".join(formatted_lines)
+                        except json.JSONDecodeError:
+                            pass
+
+                    print(content)
+
+                except UnicodeDecodeError:
+                    print(f"[Binary file, size: {file_path.stat().st_size} bytes]")
+                except Exception as e:
+                    print(f"[Error reading file: {str(e)}]")
+
+        print(f"=== {eval_type.upper()} Evaluation Result Files End ===\n")
+
+    except Exception as e:
+        print(f"Error capturing evaluation result files: {str(e)}")
+
+
+def extract_tool_schema(tools):
+    """Reformat tool schema"""
+    for i in range(len(tools)):  # pylint: disable=C0200
+        tools[i]["function"].pop("response")
+    return tools

benchmark/bfcl/default_ids.py

@@ -0,0 +1,206 @@
+# pylint: disable=C0114
+DEFAULT_TRAIN_IDS: set[str] = {
+    "multi_turn_base_102",
+    "multi_turn_base_107",
+    "multi_turn_base_110",
+    "multi_turn_base_114",
+    "multi_turn_base_115",
+    "multi_turn_base_118",
+    "multi_turn_base_122",
+    "multi_turn_base_123",
+    "multi_turn_base_128",
+    "multi_turn_base_13",
+    "multi_turn_base_130",
+    "multi_turn_base_132",
+    "multi_turn_base_133",
+    "multi_turn_base_143",
+    "multi_turn_base_144",
+    "multi_turn_base_146",
+    "multi_turn_base_15",
+    "multi_turn_base_158",
+    "multi_turn_base_169",
+    "multi_turn_base_17",
+    "multi_turn_base_172",
+    "multi_turn_base_176",
+    "multi_turn_base_182",
+    "multi_turn_base_187",
+    "multi_turn_base_197",
+    "multi_turn_base_199",
+    "multi_turn_base_22",
+    "multi_turn_base_23",
+    "multi_turn_base_24",
+    "multi_turn_base_36",
+    "multi_turn_base_40",
+    "multi_turn_base_44",
+    "multi_turn_base_47",
+    "multi_turn_base_48",
+    "multi_turn_base_5",
+    "multi_turn_base_51",
+    "multi_turn_base_59",
+    "multi_turn_base_63",
+    "multi_turn_base_65",
+    "multi_turn_base_66",
+    "multi_turn_base_67",
+    "multi_turn_base_68",
+    "multi_turn_base_70",
+    "multi_turn_base_75",
+    "multi_turn_base_77",
+    "multi_turn_base_78",
+    "multi_turn_base_79",
+    "multi_turn_base_81",
+    "multi_turn_base_83",
+    "multi_turn_base_93",
+}
+
+DEFAULT_VAL_IDS: set[str] = {
+    "multi_turn_base_0",
+    "multi_turn_base_1",
+    "multi_turn_base_10",
+    "multi_turn_base_100",
+    "multi_turn_base_101",
+    "multi_turn_base_103",
+    "multi_turn_base_104",
+    "multi_turn_base_105",
+    "multi_turn_base_106",
+    "multi_turn_base_108",
+    "multi_turn_base_109",
+    "multi_turn_base_11",
+    "multi_turn_base_111",
+    "multi_turn_base_112",
+    "multi_turn_base_113",
+    "multi_turn_base_116",
+    "multi_turn_base_117",
+    "multi_turn_base_119",
+    "multi_turn_base_12",
+    "multi_turn_base_120",
+    "multi_turn_base_121",
+    "multi_turn_base_124",
+    "multi_turn_base_125",
+    "multi_turn_base_126",
+    "multi_turn_base_127",
+    "multi_turn_base_129",
+    "multi_turn_base_131",
+    "multi_turn_base_134",
+    "multi_turn_base_135",
+    "multi_turn_base_136",
+    "multi_turn_base_137",
+    "multi_turn_base_138",
+    "multi_turn_base_139",
+    "multi_turn_base_14",
+    "multi_turn_base_140",
+    "multi_turn_base_141",
+    "multi_turn_base_142",
+    "multi_turn_base_145",
+    "multi_turn_base_147",
+    "multi_turn_base_148",
+    "multi_turn_base_149",
+    "multi_turn_base_150",
+    "multi_turn_base_151",
+    "multi_turn_base_152",
+    "multi_turn_base_153",
+    "multi_turn_base_154",
+    "multi_turn_base_155",
+    "multi_turn_base_156",
+    "multi_turn_base_157",
+    "multi_turn_base_159",
+    "multi_turn_base_16",
+    "multi_turn_base_160",
+    "multi_turn_base_161",
+    "multi_turn_base_162",
+    "multi_turn_base_163",
+    "multi_turn_base_164",
+    "multi_turn_base_165",
+    "multi_turn_base_166",
+    "multi_turn_base_167",
+    "multi_turn_base_168",
+    "multi_turn_base_170",
+    "multi_turn_base_171",
+    "multi_turn_base_173",
+    "multi_turn_base_174",
+    "multi_turn_base_175",
+    "multi_turn_base_177",
+    "multi_turn_base_178",
+    "multi_turn_base_179",
+    "multi_turn_base_18",
+    "multi_turn_base_180",
+    "multi_turn_base_181",
+    "multi_turn_base_183",
+    "multi_turn_base_184",
+    "multi_turn_base_185",
+    "multi_turn_base_186",
+    "multi_turn_base_188",
+    "multi_turn_base_189",
+    "multi_turn_base_19",
+    "multi_turn_base_190",
+    "multi_turn_base_191",
+    "multi_turn_base_192",
+    "multi_turn_base_193",
+    "multi_turn_base_194",
+    "multi_turn_base_195",
+    "multi_turn_base_196",
+    "multi_turn_base_198",
+    "multi_turn_base_2",
+    "multi_turn_base_20",
+    "multi_turn_base_21",
+    "multi_turn_base_25",
+    "multi_turn_base_26",
+    "multi_turn_base_27",
+    "multi_turn_base_28",
+    "multi_turn_base_29",
+    "multi_turn_base_3",
+    "multi_turn_base_30",
+    "multi_turn_base_31",
+    "multi_turn_base_32",
+    "multi_turn_base_33",
+    "multi_turn_base_34",
+    "multi_turn_base_35",
+    "multi_turn_base_37",
+    "multi_turn_base_38",
+    "multi_turn_base_39",
+    "multi_turn_base_4",
+    "multi_turn_base_41",
+    "multi_turn_base_42",
+    "multi_turn_base_43",
+    "multi_turn_base_45",
+    "multi_turn_base_46",
+    "multi_turn_base_49",
+    "multi_turn_base_50",
+    "multi_turn_base_52",
+    "multi_turn_base_53",
+    "multi_turn_base_54",
+    "multi_turn_base_55",
+    "multi_turn_base_56",
+    "multi_turn_base_57",
+    "multi_turn_base_58",
+    "multi_turn_base_6",
+    "multi_turn_base_60",
+    "multi_turn_base_61",
+    "multi_turn_base_62",
+    "multi_turn_base_64",
+    "multi_turn_base_69",
+    "multi_turn_base_7",
+    "multi_turn_base_71",
+    "multi_turn_base_72",
+    "multi_turn_base_73",
+    "multi_turn_base_74",
+    "multi_turn_base_76",
+    "multi_turn_base_8",
+    "multi_turn_base_80",
+    "multi_turn_base_82",
+    "multi_turn_base_84",
+    "multi_turn_base_85",
+    "multi_turn_base_86",
+    "multi_turn_base_87",
+    "multi_turn_base_88",
+    "multi_turn_base_89",
+    "multi_turn_base_9",
+    "multi_turn_base_90",
+    "multi_turn_base_91",
+    "multi_turn_base_92",
+    "multi_turn_base_94",
+    "multi_turn_base_95",
+    "multi_turn_base_96",
+    "multi_turn_base_97",
+    "multi_turn_base_98",
+    "multi_turn_base_99",
+}

benchmark/bfcl/init_task_memory_pool.py

@@ -0,0 +1,235 @@
+# pylint: disable=W0621,W1514
+"""Init task memory pool"""
+import argparse
+import json
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import List, Dict, Any
+
+import requests
+
+
+def load_task_case(data_path: str, task_id: str | None) -> Dict[str, Any]:
+    """
+    load training cases by id
+    """
+    if not Path(data_path).exists():
+        raise FileNotFoundError(f"BFCL data file '{data_path}' not found")
+
+    if task_id is None:
+        raise ValueError("task_id is required")
+
+    with open(data_path, "r", encoding="utf-8") as f:
+        if str(task_id).isdigit():  # pylint: disable=R1720
+            idx = int(task_id)
+            for line_no, line in enumerate(f):
+                if line_no == idx:
+                    return json.loads(line)
+            raise ValueError(f"Task case index {idx} not found in {data_path}")
+        else:
+            for line in f:
+                data = json.loads(line)
+                if data.get("id") == task_id:
+                    return data
+            raise ValueError(f"Task case id '{task_id}' not found in {data_path}")
+
+
+def get_tool_prompt(tools):
+    """Construct prompt with provided tools"""
+    tool_prompt = (
+        "\n\n# Tools\n\nYou may call one or more functions to assist with the user query."
+        "\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>"
+    )
+    for tool in tools:
+        tool_prompt += "\n" + json.dumps(tool)
+    tool_prompt += (
+        "\n</tools>\n\nFor each function call, return a json object with function name"
+        " and arguments within <tool_call></tool_call> XML tags:"
+        '\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>'
+    )
+    return tool_prompt
+
+
+def group_trajectories_by_task_id(jsonl_entries: List[Dict[str, Any]]) -> List[List[Any]]:
+    """
+    group trajectories by task_id
+
+    Args:
+        jsonl_entries: JSONL entry list
+
+    Returns:
+        List[List[Any]]: trajectory list grouped by task_id
+    """
+    grouped = defaultdict(list)
+
+    for entry in jsonl_entries:
+        task_id = entry.get("task_id", "")
+        taks_case = load_task_case("data/multiturn_data_base.jsonl", task_id)
+        tools = taks_case.get("tools", [{}])
+        from bfcl_utils import extract_tool_schema
+
+        tool_schema = extract_tool_schema(tools)
+        entry["task_history"][0]["content"] += get_tool_prompt(tool_schema)
+        grouped[task_id].append(entry)
+
+    # retain only the two with the highest and lowest rewards
+    filtered_groups = []
+    for _, trajectories in grouped.items():
+        if len(trajectories) == 1:
+            # when only one trajectory, retain it
+            filtered_groups.append(trajectories)
+        elif len(trajectories) == 2:
+            # when there are two trajectories, retain them
+            filtered_groups.append(trajectories)
+        else:
+            # when there are more than two trajectories, choose the two with the highest and lowest rewards
+            trajectories.sort(key=lambda t: t["reward"])
+            min_reward_traj = trajectories[0]  # highest reward
+            max_reward_traj = trajectories[-1]  # lowest reward
+            filtered_groups.append([min_reward_traj, max_reward_traj])
+
+    return filtered_groups
+
+
+def post_to_summarizer(trajectories: List[Any], service_url: str) -> Dict[str, Any]:
+    """
+    post trajectories to summarizer service
+
+    Args:
+        trajectories: trajectory list
+        service_url: summarizer service URL
+
+    Returns:
+        response json
+    """
+    trajectory_dicts = [
+        {
+            "task_id": traj["task_id"],
+            "messages": traj["task_history"],
+            "score": traj["reward"],
+        }
+        for traj in trajectories
+    ]
+
+    request_data = {
+        "trajectories": trajectory_dicts,
+        "success_threshold": 1.0,
+        "enable_soft_comparison": True,
+        "validation_threshold": 0.5,
+    }
+
+    try:
+        response = requests.post(f"{service_url}/summary_task_memory", json=request_data)
+        response.raise_for_status()
+        return response.json()
+    except Exception as e:
+        return {"error": str(e), "trajectories_count": len(trajectories)}
+
+
+def process_trajectories_with_threads(
+    grouped_trajectories: List[List[Any]],
+    service_url: str,
+    n_threads: int = 4,
+) -> List[Dict[str, Any]]:
+    """
+    use threads to process trajectories
+
+    Args:
+        grouped_trajectories: group trajectory list by task_id
+        service_url: memory summarizer service URL
+        n_threads: number of threads
+
+    Returns:
+        all results
+    """
+    results = []
+
+    with ThreadPoolExecutor(max_workers=n_threads) as executor:
+        future_to_group = {
+            executor.submit(post_to_summarizer, group, service_url): i for i, group in enumerate(grouped_trajectories)
+        }
+
+        for future in as_completed(future_to_group):
+            group_index = future_to_group[future]
+            try:
+                result = future.result()
+                result["group_index"] = group_index
+                result["group_size"] = len(grouped_trajectories[group_index])
+                results.append(result)
+                if "memory_list" in result["metadata"]:
+                    print(f'✅ Group {group_index} processed: {result["metadata"].get("memory_list", 0)}')
+                    memory_list = result["metadata"].get("memory_list", [])
+                    response = requests.post(url=f"{service_url}/add_task_memory", json={"memory_list": memory_list})
+                    response.raise_for_status()
+                else:
+                    print(f"❌ Group {group_index} processed: error")
+            except Exception as e:
+                error_result = {
+                    "group_index": group_index,
+                    "group_size": len(grouped_trajectories[group_index]),
+                    "error": str(e),
+                }
+                results.append(error_result)
+                print(f"❌ Group {group_index} failed: {e}")
+
+    return results
+
+
+def main():
+    """Main function to convert JSONL to memories using ReMe service."""
+    parser = argparse.ArgumentParser(description="Convert JSONL to memories using ReMe service")
+    parser.add_argument("--jsonl_file", type=str, required=True, help="Path to the JSONL file")
+    parser.add_argument("--service_url", type=str, default="http://localhost:8002", help="ReMe service URL")
+    parser.add_argument("--output_file", type=str, help="Output file to save results (optional)")
+    parser.add_argument("--n_threads", type=int, default=4, help="Number of threads for processing")
+
+    args = parser.parse_args()
+
+    print(f"Processing JSONL file: {args.jsonl_file}")
+    print(f"Service URL: {args.service_url}")
+    print(f"Threads: {args.n_threads}")
+
+    with open(args.jsonl_file, "r") as f:
+        data = [json.loads(line) for line in f]
+    print(f"Loaded {len(data)} entries from JSONL file")
+
+    grouped_trajectories = group_trajectories_by_task_id(data)
+    print(f"Total groups: {len(grouped_trajectories)}")
+
+    results = process_trajectories_with_threads(
+        grouped_trajectories,
+        args.service_url,
+        n_threads=args.n_threads,
+    )
+
+    print(f"Processed {len(results)} groups")
+
+    success_count = sum(1 for r in results if "error" not in r)
+    error_count = len(results) - success_count
+    total_memories = sum(len(r["metadata"].get("memory_list", [])) for r in results if "memory_list" in r["metadata"])
+
+    print(f"✅ Success: {success_count}")
+    print(f"❌ Errors: {error_count}")
+    print(f"📊 Total task memories created: {total_memories}")
+
+    if args.output_file:
+        try:
+            summary = {
+                "jsonl_file": args.jsonl_file,
+                "total_groups": len(grouped_trajectories),
+                "success_count": success_count,
+                "error_count": error_count,
+                "total_task_memories": total_memories,
+                "results": results,
+            }
+
+            with open(args.output_file, "w") as f:
+                json.dump(summary, f, indent=2)
+            print(f"Results saved to: {args.output_file}")
+        except Exception as e:
+            print(f"Error saving results: {e}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/bfcl/preprocess.py

@@ -0,0 +1,73 @@
+# pylint: disable=W0621
+"""Preprocess multi-turn test cases"""
+
+import json
+
+
+from pathlib import Path
+from bfcl_eval.model_handler.model_style import ModelStyle
+from bfcl_eval.eval_checker.eval_runner_helper import load_file
+from bfcl_eval.constants.type_mappings import GORILLA_TO_OPENAPI
+from bfcl_eval.constants.eval_config import MULTI_TURN_FUNC_DOC_PATH
+from bfcl_eval.constants.category_mapping import MULTI_TURN_FUNC_DOC_FILE_MAPPING
+from bfcl_eval.model_handler.utils import (
+    convert_to_tool,
+    func_doc_language_specific_pre_processing,
+)
+
+
+def process_multi_turn_test_case(file_path, output_path):
+    """
+    Multi-turn test cases don't have the function doc in the prompt. We need to add them here.
+    """
+    test_cases = []
+    with open(output_path, "w", encoding="utf-8") as outf:
+        with open(file_path, encoding="utf-8") as f:
+            file = f.readlines()
+            for line in file:
+                entry = json.loads(line)
+                if "multi_turn" not in entry["id"]:
+                    continue
+                test_category: str = entry["id"].rsplit("_", 1)[0]
+                involved_classes = entry["involved_classes"]
+                entry["function"] = []
+                for func_collection in involved_classes:
+                    # func_doc is a list of dict
+                    func_doc = load_file(
+                        MULTI_TURN_FUNC_DOC_PATH / MULTI_TURN_FUNC_DOC_FILE_MAPPING[func_collection],
+                    )
+                    entry["function"].extend(func_doc)
+
+                # Handle Miss Func category; we need to remove the holdout function doc
+                if "missed_function" in entry:
+                    for turn_index, missed_func_names in entry["missed_function"].items():
+                        entry["missed_function"][turn_index] = []
+                        for missed_func_name in missed_func_names:
+                            for i, func_doc in enumerate(entry["function"]):
+                                if func_doc["name"] == missed_func_name:
+                                    # Add the missed function doc to the missed_function list
+                                    entry["missed_function"][turn_index].append(func_doc)
+                                    # Remove it from the function list
+                                    entry["function"].pop(i)
+                                    break
+
+                functions = func_doc_language_specific_pre_processing(entry["function"], test_category)
+                tools = convert_to_tool(functions, GORILLA_TO_OPENAPI, ModelStyle.OpenAI_Completions)
+
+                test_cases.append(
+                    {
+                        "id": entry["id"],
+                        "messages": entry["question"][0],
+                        "tools": tools,
+                        "extra": entry,
+                    },
+                )
+                outf.write(json.dumps(test_cases[-1], ensure_ascii=False) + "\n")
+
+    return test_cases
+
+
+if __name__ == "__main__":
+    file_path = Path("./gorilla/berkeley-function-call-leaderboard/bfcl_eval/data/BFCL_v3_multi_turn_base.json")
+    output_path = "data/multiturn_data_base.jsonl"
+    preprocessed_test_cases = process_multi_turn_test_case(file_path, output_path)

benchmark/bfcl/quickstart.md

@@ -0,0 +1,129 @@
+# BFCL
+Experiment Quick Start Guide
+
+This guide helps you quickly set up and run BFCL experiments with ReMe integration.
+
+## Env Setup
+
+### 1. BFCL installation
+
+#### Clone the repository
+```bash
+cd ReMe/benchmark/bfcl
+git clone https://github.com/ShishirPatil/gorilla.git
+cd gorilla
+git checkout ea13468
+```
+
+#### Change directory to the `berkeley-function-call-leaderboard`
+```bash
+cd berkeley-function-call-leaderboard
+```
+
+#### Install the package in editable mode
+```bash
+pip install -e .
+cd ../..
+pip install -r requirements.txt
+```
+
+#### Move the dataset to the data folder under bfcl
+```bash
+cp -r gorilla/berkeley-function-call-leaderboard/bfcl_eval/data ./
+```
+
+#### Preprocess the data to get the suitable data format
+```bash
+python preprocess.py
+```
+
+**Note**: The original BFCL data is designed as a benchmark dataset and does not have a train/validation split, you can use ``split_into_trainval.py`` to split data into train and validation sets.
+
+```bash
+python split_into_trainval.py --input ./data/multiturn_data_base.jsonl  --train ./data/multiturn_data_base_train.jsonl --val ./data/multiturn_data_base_val.jsonl
+```
+
+### 2. Start ReMe Service
+
+After collecting trajectories, Launch the ReMe service (make sure you have installed ReMe environment, if not please follow the steps in the [ReMe Installation Guide](https://github.com/agentscope-ai/ReMe/blob/main/doc/README.md) to install):
+
+```bash
+reme2 \
+  backend=http \
+  http.port=8002 \
+  llms.default.model_name=qwen3-8b \
+  embedding_models.default.model_name=text-embedding-v4 \
+  vector_stores.default.backend=local \
+  vector_stores.default.collection_name=bfcl
+```
+
+<details>
+<summary>Option: init the task memory pool from scratch</summary>
+
+- First, collect agent trajectories on training data set without task memory:
+
+  ```bash
+  # important: num_runs = 8, use_memory = False, experiment_suffix="wo-memory", data_path="data/multiturn_data_base_train.jsonl"
+  python run_bfcl.py
+  ```
+
+- Second, using ReMe to construct the initial task memory pool:
+  ```bash
+  python init_task_memory_pool.py --jsonl_file ./exp_result/qwen3-8b/with_think/bfcl-multi-turn-base_wo-memory.jsonl
+  ```
+
+  > Parameters:
+  > `jsonl_file`: Path to the collloaded trajectories
+  > `service_url`: ReMe service URL (default: `http://localhost:8002`)
+  > `n_threads`: Number of threads for processing
+  > `output_file`: Output file to save results (optional)
+
+  Now you have inited the task memory pool using `local` backend. Then, run the following `curl` command to dump the memory library:
+  ```bash
+  curl -X POST "http://0.0.0.0:8002/dump_memory" \
+    -H "Content-Type: application/json" \
+    -d '{
+      "dump_file_path": "./library/bfcl.jsonl",
+    }'
+  ```
+
+- Next time, you can import this previously exported task memory data to populate the new started workspace with existing knowledge:
+  ```bash
+  curl -X POST "http://0.0.0.0:8002/load_memory" \
+    -H "Content-Type: application/json" \
+    -d '{
+      "load_file_path": "./library/bfcl.jsonl",
+      "clear_existing": true
+    }'
+  ```
+</details>
+
+### 3. Run Experiments on Validation Set
+
+Run you can compare agent performance on the validation set with task memory (`use_memory=True`) and without task memory:
+
+```bash
+# remember to change the configuration options, e.g., `data_path=./data/multiturn_data_base_val.jsonl`
+python run_bfcl.py
+```
+
+**Note**:
+- `max_workers`: Number of parallel workers
+- `num_runs`: Number of times each task is repeated
+- `model_name`: LLM model name
+- `enable_thinking`: Control the model's thinking mode
+- `data_path`: Path to the training dataset (default: `./data/multiturn_data_base_val.jsonl`)
+- `answer_path`: Path to the possible answer, which are used to evaluate the model's output function (default: `./data/possible_answer`)
+- Results are automatically saved to `./exp_result/{model_name}/{no_think/with_think}` directory
+
+After running experiments, analyze the statistical results:
+
+```bash
+python run_exp_statistic.py
+```
+
+**What this script does:**
+- Processes all result files in `./exp_result/`
+- Calculates best@k&pass@k metrics for different k values
+- Generates a summary table showing performance comparisons
+- Saves results to `experiment_summary.csv`

benchmark/bfcl/requirements.txt

@@ -0,0 +1,6 @@
+jinja2
+loguru
+openai
+ray
+pandas
+soundfile

benchmark/bfcl/run_bfcl.py

@@ -0,0 +1,151 @@
+"""Run evaluation on BFCL-V3-Multi-Turn-Base dataset."""
+
+import time
+import json
+from pathlib import Path
+
+import ray
+import requests
+from loguru import logger
+from dotenv import load_dotenv
+from bfcl_agent import BFCLAgent
+
+load_dotenv("../../.env")
+
+
+def run_agent(
+    max_workers: int,
+    dataset_name: str,
+    experiment_suffix: str,
+    model_name: str = "qwen3-8b",
+    enable_thinking: bool = False,
+    data_path: str = "data/multiturn_data_base_val.jsonl",
+    answer_path: Path = Path("data/possible_answer"),
+    num_trials: int = 1,
+    use_memory: bool = False,
+    memory_base_url: str = "http://0.0.0.0:8002/",
+    use_memory_addition: bool = True,
+    use_memory_deletion: bool = False,
+    delete_freq: int = 10,
+    freq_threshold: int = 5,
+    utility_threshold: float = 0.5,
+):
+    """Run the agent"""
+    experiment_name = dataset_name + "_" + experiment_suffix
+    path: Path = Path(
+        f"./exp_result/{model_name}/with_think" if enable_thinking else f"./exp_result/{model_name}/no_think",
+    )
+    path.mkdir(parents=True, exist_ok=True)
+
+    with open(data_path, "r", encoding="utf-8") as f:
+        task_ids = [json.loads(line)["id"] for line in f]
+
+    result: list = []
+
+    def dump_file():
+        with open(path / f"{experiment_name}.jsonl", "a", encoding="utf-8") as f:
+            for x in result:
+                f.write(json.dumps(x) + "\n")
+
+    future_list: list = []
+    for i in range(max_workers):
+        actor = BFCLAgent.remote(
+            index=i,
+            model_name=model_name,
+            task_ids=task_ids[i::max_workers],
+            experiment_name=experiment_name,
+            data_path=data_path,
+            answer_path=answer_path,
+            num_trials=num_trials,
+            use_memory=use_memory,
+            memory_base_url=memory_base_url,
+            use_memory_addition=use_memory_addition,
+            use_memory_deletion=use_memory_deletion,
+            delete_freq=delete_freq,
+            freq_threshold=freq_threshold,
+            utility_threshold=utility_threshold,
+            enable_thinking=enable_thinking,
+        )
+        future = actor.execute.remote()
+        future_list.append(future)
+        time.sleep(1)
+    logger.info("submit complete")
+
+    for i, future in enumerate(future_list):
+        t_result = ray.get(future)
+        if t_result:
+            if isinstance(t_result, list):
+                result.extend(t_result)
+            else:
+                result.append(t_result)
+
+        logger.info(f"{i + 1}/{len(task_ids)} complete")
+    dump_file()
+
+
+def handle_api_response(response: requests.Response):
+    """Handle API response with proper error checking"""
+    if response.status_code != 200:
+        print(f"Error: {response.status_code}")
+        print(response.text)
+        return None
+
+    return response.json()
+
+
+def load_memory(path: str = "docs/library", api_url: str = "http://0.0.0.0:8002/"):
+    """Load memories from disk into the vector store"""
+    response = requests.post(
+        url=f"{api_url}load_memory",
+        json={
+            "load_file_path": path,
+            "clear_existing": True,
+        },
+    )
+
+    result = handle_api_response(response)
+    if result:
+        print(f"Memory loaded from {path}")
+
+
+def main():
+    """Main function"""
+    max_workers = 4
+    if max_workers > 1:
+        ray.init(num_cpus=max_workers)
+
+    num_runs = 4
+    num_trials = 1
+    model_name = "qwen3-8b"
+    enable_thinking = True
+    use_memory = True
+    use_memory_addition = False
+    use_memory_deletion = False
+    memory_base_url = "http://0.0.0.0:8003/"
+
+    if use_memory:
+        load_file_path = "docs/library/paper_data/task/bfcl_qwen3_8b.jsonl"
+        load_memory(load_file_path, memory_base_url)
+
+    for _ in range(num_runs):
+        run_agent(
+            max_workers=max_workers,
+            model_name=model_name,
+            dataset_name="bfcl-multi-turn-base-val",
+            experiment_suffix="w-fixed-memory",
+            data_path="data/multiturn_data_base_val.jsonl",
+            answer_path=Path("data/possible_answer"),
+            enable_thinking=enable_thinking,
+            num_trials=num_trials,
+            use_memory=use_memory,
+            memory_base_url=memory_base_url,
+            use_memory_addition=use_memory_addition,
+            use_memory_deletion=use_memory_deletion,
+            delete_freq=5,
+            freq_threshold=5,
+            utility_threshold=0.5,
+        )
+
+
+if __name__ == "__main__":
+    main()

benchmark/bfcl/run_exp_statistic.py

@@ -0,0 +1,163 @@
+"""Run the experiment statistic."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+
+import pandas as pd
+from loguru import logger
+
+
+def calculate_best_at_k(scores: list, k: int) -> float:
+    """
+    Calculate best@k
+    Divide scores into groups of size k, take the maximum value in each group,
+    then average these maximum values
+
+    Args:
+        scores: List of after_score values for all runs of a task
+        k: Group size
+
+    Returns:
+        best@k value
+    """
+    if len(scores) % k != 0:
+        raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
+
+    group_maxs = []
+    for i in range(0, len(scores), k):
+        group = scores[i : i + k]
+        group_maxs.append(max(group))
+
+    return sum(group_maxs) / len(group_maxs)
+
+
+def calculate_pass_at_k(scores: list, k: int) -> float:
+    """Calculate pass@k."""
+    if len(scores) % k != 0:
+        raise ValueError(f"Length of scores ({len(scores)}) must be divisible by k ({k})")
+
+    group_maxs = []
+    for i in range(0, len(scores), k):
+        group = scores[i : i + k]
+        is_pass = 1.0 if max(group) >= 1.0 else 0.0
+        group_maxs.append(is_pass)
+
+    return sum(group_maxs) / len(group_maxs)
+
+
+def get_possible_k_values(total_runs: int) -> list:
+    """
+    Get all possible k values (factors of total_runs)
+
+    Args:
+        total_runs: Total number of runs
+
+    Returns:
+        List of k values in descending order
+    """
+    k_values = []
+    for k in range(1, total_runs + 1):
+        if total_runs % k == 0:
+            k_values.append(k)
+    return sorted(k_values, reverse=True)  # Sort from large to small
+
+
+def run_exp_statistic():
+    """Run the experiment statistic."""
+    path: Path = Path("./exp_result/qwen3-8b/with_think")
+
+    # Store results for all experiments
+    all_results = {}
+    for file in path.glob("*.jsonl"):
+        # Group results by task_id
+        task_results = defaultdict(list)
+        print(file)
+        with open(file, "r", encoding="utf-8") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                data = json.loads(line)
+
+                if isinstance(data, list):
+                    for part_data in data:
+                        task_id = part_data["task_id"]
+                        after_score = part_data["reward"]
+                        task_results[task_id].append(after_score)
+                else:
+                    task_id = data["task_id"]
+                    after_score = data["reward"]
+                    task_results[task_id].append(after_score)
+
+        if not task_results:
+            logger.warning(f"No valid data found in file {file}")
+            continue
+
+        # Check if each task has consistent number of runs
+        run_counts = [len(scores) for scores in task_results.values()]
+        if len(set(run_counts)) > 1:
+            logger.warning(f"Inconsistent number of runs for different tasks in file {file}: {set(run_counts)}")
+            continue
+
+        num_runs = run_counts[0]
+        logger.info(f"File {file}: {len(task_results)} tasks, {num_runs} runs per task")
+
+        # Get all possible k values
+        k_values = get_possible_k_values(num_runs)
+        logger.info(f"Calculable best@k values: {k_values}")
+
+        # Calculate various best@k values
+        file_results = {"file": file.name}
+
+        for k in k_values:
+            best_at_k_scores = []
+            pass_at_k_scores = []
+            for task_id, scores in task_results.items():
+                try:
+                    best_k_score = calculate_best_at_k(scores, k)
+                    pass_at_k_score = calculate_pass_at_k(scores, k)
+                    pass_at_k_scores.append(pass_at_k_score)
+                    best_at_k_scores.append(best_k_score)
+                except ValueError as e:
+                    logger.error(f"Error calculating best@{k} for task {task_id}: {e}")
+                    continue
+
+            if best_at_k_scores:
+                avg_best_at_k = sum(best_at_k_scores) / len(best_at_k_scores)
+                file_results[f"best@{k}"] = avg_best_at_k
+                logger.info(f"file={file.name} best@{k}={avg_best_at_k:.4f}")
+
+            if pass_at_k_scores:
+                avg_pass_at_k = sum(pass_at_k_scores) / len(pass_at_k_scores)
+                file_results[f"pass@{k}"] = avg_pass_at_k
+                logger.info(f"file={file.name} pass@{k}={avg_pass_at_k:.4f}")
+
+        all_results[file.name] = file_results
+
+    # Create and display table
+    if all_results:
+        df = pd.DataFrame(list(all_results.values()))
+        df = df.set_index("file")
+
+        # Sort columns by the number in column name (best@8, best@4, best@2, best@1)
+        # best_columns = [col for col in df.columns if col.startswith('best@')]
+        best_columns = list(df.columns)
+        best_columns.sort(key=lambda x: x, reverse=False)
+        df = df[best_columns]
+
+        print("\n" + "=" * 80)
+        print("Experiment Results Summary Table")
+        print("=" * 80)
+        print(df.round(4))
+        print("=" * 80)
+
+        # Save table to CSV
+        output_path = path / "experiment_summary.csv"
+        df.to_csv(output_path)
+        logger.info(f"Results table saved to: {output_path}")
+    else:
+        logger.warning("No valid experiment results found")
+
+
+if __name__ == "__main__":
+    run_exp_statistic()

benchmark/bfcl/split_into_trainval.py

@@ -0,0 +1,69 @@
+"""Split the JSONL file into train and validation sets."""
+
+import argparse
+import json
+import random
+
+from default_ids import DEFAULT_TRAIN_IDS, DEFAULT_VAL_IDS
+
+
+def split_jsonl(
+    input_file: str,
+    train_file: str,
+    val_file: str,
+    ratio: float = 0.75,
+    random_split: bool = False,
+) -> None:
+    """Split the JSONL file into train and validation sets."""
+    with open(input_file, "r", encoding="utf-8") as f:
+        data = [json.loads(line) for line in f]
+
+    if random_split:
+        random.shuffle(data)
+        split_idx = int(len(data) * ratio)
+        train_data = data[:split_idx]
+        val_data = data[split_idx:]
+    else:
+        train_data = []
+        val_data = []
+        unknown_ids: list[str] = []
+        for obj in data:
+            if "id" not in obj:
+                raise ValueError(f"Missing 'id' field in input file: {input_file}")
+            obj_id = str(obj["id"])
+            if obj_id in DEFAULT_TRAIN_IDS:
+                train_data.append(obj)
+            elif obj_id in DEFAULT_VAL_IDS:
+                val_data.append(obj)
+            else:
+                unknown_ids.append(obj_id)
+
+        if len(train_data) + len(val_data) != len(data):
+            missing = len(data) - (len(train_data) + len(val_data))
+            examples = ", ".join(unknown_ids) if unknown_ids else "(none)"
+            raise ValueError(
+                f"{missing} samples in {input_file} not found in train_ref/val_ref id sets. Examples: {examples}",
+            )
+
+    with open(train_file, "w", encoding="utf-8") as f:
+        for item in train_data:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+    with open(val_file, "w", encoding="utf-8") as f:
+        for item in val_data:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Split JSONL file into train and validation sets.")
+    parser.add_argument("--input", required=True, help="Path to input JSONL file")
+    parser.add_argument("--train", required=True, help="Path to output train file")
+    parser.add_argument("--val", required=True, help="Path to output validation file")
+    parser.add_argument("--ratio", type=float, default=0.5, help="Train ratio (default: 0.8)")
+    parser.add_argument(
+        "--random",
+        action="store_true",
+        help="Whether to randomly split input into train/val. "
+        "If false, split strictly by default train/val id sets (see default_ids.py).",
+    )
+    args = parser.parse_args()
+    split_jsonl(args.input, args.train, args.val, args.ratio, args.random)

benchmark/halumem/cat_correct_scripts.sh

@@ -0,0 +1 @@
+cat bench_results/reme/Martin\ Mark/session* | grep '"result_type": "' | awk -F'"' '{total++; if($4=="Correct") count++} END {printf "Correct Rate: %.2f%% (%d/%d)\n", (count/total)*100, count, total}'

benchmark/halumem/eval_reme.yaml

@@ -0,0 +1,547 @@
+TEMPLATE_MEMOS: |
+  Memories for user {user_id}:
+    {memories}
+
+PROMPT_MEMZERO_JSON: |
+  # CONTEXT:
+  {context}
+
+  # CONTEXT PRIORITY:
+  When the context contains information from multiple sources, follow this strict priority order:
+  1. **Historical Dialogue** (highest priority) - Direct conversation content
+  2. **Extracted Memories** (medium priority) - Summarized memory points
+  3. **User Profile** (lowest priority) - General user information
+
+  # Question:
+  {question}
+
+  # OUTPUT FORMAT:
+  Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
+  Please provide your response in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "reasoning content",
+    "answer": "Provide a detailed answer"
+  }}
+  ```
+
+PROMPT_MEMZERO_JSON2: |
+  # CONTEXT:
+  {context}
+
+  # CONTEXT PRIORITY:
+  When the context contains information from multiple sources, follow this strict priority order:
+  1. **Historical Dialogue** (highest priority) - Direct conversation content
+  2. **Extracted Memories** (medium priority) - Summarized memory points
+  3. **User Profile** (lowest priority) - General user information
+
+  # Question:
+  {question}
+
+  # OUTPUT FORMAT:
+  Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
+  Please provide your response in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "reasoning content",
+    "answer": "Provide a detailed answer"
+  }}
+  ```
+
+PROMPT_MEMZERO: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories from both speakers
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.),
+     calculate the actual date based on the memory timestamp. For example, if a memory from
+     4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example,
+     convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
+     timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories from both speakers. Do not confuse character
+     names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+PROMPT_ZEP: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.),
+     calculate the actual date based on the memory timestamp. For example, if a memory from
+     4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example,
+     convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
+     timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories. Do not confuse character
+     names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  Context:
+
+  {context}
+
+  Question: {question}
+  Answer:
+
+PROMPT_MEMOS: |
+  You are a knowledgeable and helpful AI assistant.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories. Synthesize information across different entries if needed to form a complete answer.
+  2. Pay close attention to the timestamps to determine the answer. If memories contain contradictory information, the **most recent memory** is the source of truth.
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories.
+  4. Your answer must be grounded in the memories. However, you may use general world knowledge to interpret or complete information found within a memory (e.g., identifying a landmark mentioned by description).
+  5. If the question involves time references (like "last year", "two months ago", etc.), you **must** calculate the actual date based on the memory's timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years in your final answer.
+  7. Do not confuse character names mentioned in memories with the actual users who created them.
+  8. The answer must be brief (under 5-6 words) and direct, with no extra description.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question.
+  2. Synthesize findings from multiple memories if a single entry is insufficient.
+  3. Examine timestamps and content carefully, looking for explicit dates, times, locations, or events.
+  4. If the answer requires calculation (e.g., converting relative time references), perform the calculation.
+  5. Formulate a precise, concise answer based on the evidence from the memories (and allowed world knowledge).
+  6. Double-check that your answer directly addresses the question asked and adheres to all instructions.
+  7. Ensure your final answer is specific and avoids vague time references.
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+PROMPT_MEMOBASE: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories from both speakers
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.), calculate the actual date based on the memory timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example, convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories from both speakers. Do not confuse character names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+
+EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY: |
+  You are a strict **"Memory Integrity" evaluator**.
+  Your core task is to assess whether an AI memory system has **missed any key memory points** after processing a conversation. This evaluation measures the system's **memory integrity**, i.e., its ability to resist **amnesia** or **omission**.
+
+  # Evaluation Context & Data:
+
+  1. **Extracted Memories:**
+     These are all the memory items actually extracted by the memory system.
+     {memories}
+
+  2. **Expected Memory Point:**
+     The key memory point that *should* have been extracted.
+     {expected_memory_point}
+
+  # Evaluation Instructions:
+
+  1. For each **Expected Memory Point**, search within the **Extracted Memories** list for corresponding or related information. Ignore unrelated items.
+  2. Based on the following scoring rubric, rate how well the memory system captured the **Expected Memory Point** and provide a detailed explanation.
+
+  # Scoring Rubric:
+
+  * **2:** Fully covered or implied.
+    One or more items in "Extracted Memories" fully cover or logically imply all information in the "Expected Memory Point."
+
+  * **1:** Partially covered or mentioned.
+    Some information in "Extracted Memories" mentions part of the "Expected Memory Point," but key information is missing, inaccurate, or slightly incorrect.
+
+  * **0:** Not mentioned or incorrect.
+    "Extracted Memories" contains no mention of the "Expected Memory Point," or the corresponding information is entirely wrong.
+
+  # Scoring Notes:
+
+  * For **compound Expected Memory Points** (with multiple elements such as person/event/time/location/preference, etc.):
+
+    * All elements correct → **2 points**
+    * Some elements correct / uncertain → **1 point**
+    * Key elements missing or wrong → **0 points**
+
+  * Semantic matching is acceptable; exact wording is **not** required.
+
+  * If "Extracted Memories" contains **conflicting information**, assign the **best possible coverage score** and mention the conflict in your reasoning.
+
+  * Extra or stylistically different memories do **not** reduce the score; only the coverage of the **Expected Memory Point** matters.
+
+  * For uncertain wording ("might," "probably," "tends to," etc.):
+
+    * If the Expected Memory Point is a definite statement, usually assign **1 point**.
+
+  * If critical fields (e.g., time, entity name, relationship) are partly wrong but others match → **1 point**.
+
+    * If all key fields are wrong or missing → **0 points**.
+
+  # Output Format:
+
+  Please output your result in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "Provide a concise justification for the score",
+    "score": "2|1|0"
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_MEMORY_ACCURACY: |
+  You are a **Dialogue Memory Accuracy Evaluator.** Your task is to evaluate the **accuracy** of a memory extracted by an AI memory system, based on three given inputs: the dialogue content, the *target (gold)* memory points (the correct annotated memories), and the *candidate* memory to be evaluated. The goal is to output a **structured evaluation result**.
+
+  # Input Content
+
+  * **Dialogue:**
+    {dialogue}
+
+  * **Golden Memories (Target Memory Points):**
+    The correct memory points pre-annotated for this dialogue in the evaluation dataset.
+    {golden_memories}
+
+  * **Candidate Memory:**
+    The memory extracted by the system to be evaluated.
+    {candidate_memory}
+
+  # Evaluation Principles and Definitions
+
+  ### 1) Support / Entailment
+
+  * An **information point** (atomic fact) in the candidate memory is considered *supported* if it can be directly stated or semantically entailed (via synonym, paraphrase, or equivalent expression) by the *Dialogue* or *Golden Memories*.
+  * Only the given dialogue and golden memories can be used for judgment — **no external knowledge** or assumptions are allowed.
+    Any information not appearing in or inferable from these two sources is considered *unsupported*.
+  * Pay careful attention to **negation**, **quantities**, **time**, and **subjects**.
+    If the candidate statement contradicts the dialogue or golden memories, it is considered a **conflict**.
+
+  ### 2) Memory Accuracy Score (integer: 0 / 1 / 2)
+
+  * **2 points:** Every information point in the candidate memory is supported by the dialogue or golden memories, with **no contradictions or hallucinations**.
+  * **1 point:** The candidate memory is *partially correct* (at least one supported information point) but also includes *unsupported* or *contradictory* content.
+  * **0 points:** The candidate memory is **entirely unsupported or contradictory** to the sources (i.e., a "hallucinated memory").
+
+  > Note:
+  > * If a candidate memory contains multiple information points, **any unsupported or contradictory element** prevents a full score (2).
+  > * If both supported and unsupported/conflicting content appear, assign a score of **1**.
+
+  ### 3) Inclusion in Golden Memories (Boolean field-level judgment)
+
+  **Definition:**
+
+  * **Atomic information point:** the smallest factual unit in the candidate memory (e.g., *name = Li Si*, *age = 25*, *location = Beijing*, *preference = coffee*, *budget ≤ 2000*, *meeting_time = Wednesday 10:00*, *tool = Zoom*, etc.).
+  * **Field / Slot:** the semantic dimension of an information point (e.g., *name*, *age*, *residence*, *food preference*, *budget*, *meeting time*, *meeting tool*, etc.).
+
+  **Judgment Rules (independent of correctness):**
+
+  * **true:**
+    Every atomic information point in the candidate memory has a corresponding **field** in the golden memories (allowing for synonyms, paraphrases, or equivalent expressions; ignore value, polarity, or quantity differences).
+
+    * Note: A single field in the gold list may match multiple candidate points (e.g., multiple "drink preference" facts can be covered by one "drink preference" field in gold).
+  * **false:**
+    If **any** atomic information point's field in the candidate memory cannot be found in the golden memories, mark as *false*.
+
+  **Important Notes:**
+
+  * Field matching is restricted to fields that are **explicitly present or semantically recognizable** in the golden memories — no external knowledge may be used to expand the field set.
+  * Differences in **values** (e.g., "Zhang San" vs. "Li Si"), **polarity** (like/dislike), or **exact number/time** do **not** affect this Boolean judgment.
+
+  # Evaluation Procedure
+
+  For each candidate memory:
+
+  1. **Decompose** it into atomic information points (e.g., name, number, location, preference).
+  2. For each information point, **search** the dialogue and golden memories for supporting or contradictory evidence.
+  3. Assign the **accuracy_score** (0 / 1 / 2) according to the rules above.
+  4. Determine **is_included_in_golden_memories (true/false)**:
+
+     * Identify each information point's field;
+     * If *all* fields exist in the golden memories, mark as *true*; otherwise, *false*.
+  5. Provide a **concise Chinese explanation** in `"reason"`, citing key evidence (short excerpts allowed), and clearly state any unsupported or contradictory parts if applicable.
+
+  # Output Format (strictly required)
+
+  Output **only one JSON object**, with the following three fields:
+
+  * `"accuracy_score"`: `"0"` or `"1"` or `"2"`
+  * `"is_included_in_golden_memories"`: `"true"` or `"false"`
+  * `"reason"`: `"brief explanation in Chinese"`
+
+  Do **not** include any other text, explanation, or fields.
+  Do **not** include the candidate memory text inside the JSON.
+
+  Please output **only** the following JSON (in a code block):
+
+  ```json
+  {{
+    "reason": "Brief explanation in Chinese"
+    "accuracy_score": "2 | 1 | 0",
+    "is_included_in_golden_memories": "true | false",
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_UPDATE_MEMORY: |
+  Your task is to **evaluate the update accuracy** of an AI memory system.
+  Based on the information provided below, determine whether the system-generated **“Generated Memories”** correctly **includes** the **Target Memory for Update**.
+
+  # Background Information
+
+  The following information is provided for evaluation:
+
+  1. **Generated Memories:**
+     This is the list of memory points generated by the system after the current dialogue.
+     {memories}
+
+  2. **Target Memory for Update:**
+     This is the correct, updated version of the memory point that should have been produced — the one we focus on in this evaluation.
+     {updated_memory}
+
+  3. **Original Memory Content:**
+     This is the original version of the target memory before the update.
+     {original_memory}
+
+  # Evaluation Criteria
+
+  Please make your judgment **strictly based on the content update of the “Target Memory for Update.”**
+  Use the following categories:
+
+  ### Correct Update
+
+  * **Generated Memories** **contains all information points** from the “Target Memory for Update,” accurately and completely reflecting the intended update.
+  * **Key fields** (e.g., date, time, values, proper nouns, etc.) must match exactly.
+  * The **original memory** is effectively replaced or marked as outdated.
+  * Synonymous or slightly rephrased expressions are acceptable.
+
+  ### Hallucinated Update
+
+  * **Factual error:** The **Generated Memories** includes a new memory related to the “Target Memory for Update,” but its content contains factual mistakes or contradictions compared to the correct update.
+
+  ### Omitted Update
+
+  * **Completely omitted:** The **Generated Memories** contains no new memory related to the “Target Memory for Update.”
+  * **Partially omitted:** A related new memory was generated in **Generated Memories**, but it **misses key information** that should have been included.
+
+  ### Other
+
+  Used for update failures that do **not clearly fall** into the above categories of “Hallucination” or “Omission.”
+
+  # Output Requirements
+
+  Please return your evaluation strictly in the following JSON format and provide a concise explanation.
+
+  ```json
+  {{
+    "reason": "Briefly explain your reasoning here and why it fits this category.",
+    "evaluation_result": "Correct | Hallucination | Omission | Other"
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_QUESTION: |
+  You are an **evaluation expert for AI memory system question answering**.
+  Based **only** on the provided **“Question”**, **“Reference Answer”**, and **“Key Memory Points”** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **“Memory System Response.”** Classify it as one of **“Correct”**, **“Hallucination”**, or **“Omission.”** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
+
+  # Evaluation Criteria
+
+  ## Answer Type Classification
+
+  ### 1. Correct
+
+  * The “Memory System Response” accurately answers the “Question,” and its content is **semantically equivalent** to the “Reference Answer.”
+  * It contains **no contradictions** with the “Key Memory Points” or “Reference Answer.”
+  * It introduces **no unsupported details** beyond the “Key Memory Points” that could alter the conclusion.
+  * Synonyms, paraphrasing, and reasonable summarization are acceptable.
+
+  ### 2. Hallucination
+
+  * The “Memory System Response” includes information or facts that **contradict or are inconsistent** with the “Reference Answer” or the “Key Memory Points.”
+  * When the “Reference Answer” is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
+  * Extra irrelevant information that does **not change** the conclusion is **not** considered hallucination by itself; however, if it **changes or misleads** the conclusion, or **contradicts** the “Key Memory Points,” it should be judged as a **Hallucination**.
+
+  ### 3. Omission
+
+  * The response is **incomplete** compared to the “Reference Answer.”
+  * It explicitly states “don’t know,” “can’t remember,” or “no related memory,” even though relevant information exists in the “Key Memory Points.”
+  * For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
+
+  ## Priority Rules (Conflict Handling)
+
+  * If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
+  * If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
+  * Only when the meaning is **fully equivalent** to the reference answer should it be classified as **Correct**.
+
+  ## Detailed Guidelines and Tolerance
+
+  * Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
+  * For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
+  * If the reference answer is *“unknown / cannot be determined”* and the system provides a definite fact, that is a **Hallucination**.
+    If the system also answers *“unknown”* (without guessing), it may be **Correct**.
+  * The evaluation must rely **only** on the *Reference Answer*, *Key Memory Points*, and *System Response* — no external context, world knowledge, or speculative reasoning is allowed.
+
+  # Information for Evaluation
+
+  * **Question:**
+    {question}
+
+  * **Reference Answer:**
+    {reference_answer}
+
+  * **Key Memory Points:**
+    {key_memory_points}
+
+  * **Memory System Response:**
+    {response}
+
+  # Output Requirements
+
+  Please provide your evaluation result **strictly** in the JSON format below.
+  Do **not** add any extra explanation or comments outside the JSON block.
+
+  ```json
+  {{
+    "reasoning": "Provide a concise and traceable evaluation rationale: first compare the system’s response with the Key Memory Points (which were correctly used, which were missing, and whether there was any fabrication/contradiction), then assess its consistency with the Reference Answer, and finally state the classification basis.",
+    "evaluation_result": "Correct | Hallucination | Omission"
+  }}
+  ```
+
+
+EVALUATION_PROMPT_FOR_QUESTION2: |
+  You are an **evaluation expert for AI memory system question answering**.
+
+  Based **only** on the provided **"Question"**, **"Reference Answer"**, and **"Key Memory Points"** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **"Memory System Response."** Classify it as one of **"Correct"**, **"Hallucination"**, or **"Omission."** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
+
+  # Evaluation Criteria
+
+  ## Answer Type Classification
+
+  ### 1. Correct
+
+  * The "Memory System Response" accurately answers the "Question," and its content is **semantically equivalent** to the "Reference Answer."
+  * It contains **no contradictions** with the "Key Memory Points" or "Reference Answer."
+  * **Extra details not present in the Key Memory Points are allowed and should not be penalized**, as long as they:
+    - Do not contradict the Key Memory Points or Reference Answer
+    - Do not change or mislead the core conclusion
+    - Are reasonable additional context that the memory system may have retained from the conversation
+  * The memory system may have stored additional information beyond the Key Memory Points. Such extra information should be treated as **supplementary context** rather than hallucination, provided it does not conflict with the core answer.
+  * Synonyms, paraphrasing, and reasonable summarization are acceptable.
+
+  ### 2. Hallucination
+
+  * The "Memory System Response" includes information or facts that **contradict or are inconsistent** with the "Reference Answer" or the "Key Memory Points."
+  * The response provides information that **directly contradicts** known facts from the Key Memory Points.
+  * When the "Reference Answer" is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
+  * **Important:** Extra information that is NOT in Key Memory Points is **NOT automatically a hallucination**. Only classify as hallucination if the extra information:
+    - Directly contradicts the Key Memory Points or Reference Answer
+    - Changes or misleads the core conclusion in a way that makes the answer incorrect
+    - Provides a definitive answer when the Reference Answer indicates uncertainty
+
+  ### 3. Omission
+
+  * The response is **incomplete** compared to the "Reference Answer."
+  * It explicitly states "don't know," "can't remember," or "no related memory," even though relevant information exists in the "Key Memory Points."
+  * For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
+
+  ## Priority Rules (Conflict Handling)
+
+  * If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
+  * If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
+  * If the core answer is correct and complete, classify as **Correct** even if there are extra details not in Key Memory Points (as long as they don't contradict or mislead).
+
+  ## Detailed Guidelines and Tolerance
+
+  * Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
+  * For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
+  * If the reference answer is *"unknown / cannot be determined"* and the system provides a definite fact, that is a **Hallucination**.
+    If the system also answers *"unknown"* (without guessing), it may be **Correct**.
+  * **Focus on evaluating whether the core answer to the question is correct**, not whether the response is limited to only the Key Memory Points.
+  * Extra contextual information (e.g., additional preferences, related details) should be viewed as enrichment, not as errors, unless they contradict or mislead.
+
+  # Information for Evaluation
+
+  * **Question:**
+    {question}
+
+  * **Reference Answer:**
+    {reference_answer}
+
+  * **Key Memory Points:**
+    {key_memory_points}
+
+  * **Memory System Response:**
+    {response}
+
+  # Output Requirements
+
+  Please provide your evaluation result **strictly** in the JSON format below.
+  Do **not** add any extra explanation or comments outside the JSON block.
+
+  ```json
+  {{
+    "reasoning": "Provide a concise and traceable evaluation rationale: first verify that the system's response correctly includes all required elements from the Reference Answer, then check if any information contradicts the Key Memory Points or Reference Answer. Extra details not in Key Memory Points should be noted but not penalized unless they contradict or mislead. Finally state the classification basis.",
+    "evaluation_result": "Correct | Hallucination | Omission"
+  }}
+  ```
+  """

benchmark/halumem/eval_scripts.sh

@@ -0,0 +1,5 @@
+clear && python benchmark/halumem/eval_reme.py \
+        --data_path /Users/yuli/workspace/HaluMem/data/HaluMem-Medium.jsonl \
+        --reme_model_name qwen3.5-plus \
+        --batch_size 10000 \
+        --algo_version default

benchmark/locomo/eval_reme.yaml

@@ -0,0 +1,180 @@
+TEMPLATE_MEMOS: |
+  Memories for user {user_id}:
+    {memories}
+
+PROMPT_MEMZERO_JSON: |
+  # CONTEXT:
+  {context}
+
+  # CONTEXT PRIORITY:
+  When the context contains information from multiple sources, follow this strict priority order:
+  1. **Historical Dialogue** (highest priority) - Direct conversation content
+  2. **Extracted Memories** (medium priority) - Summarized memory points
+  3. **User Profile** (lowest priority) - General user information
+
+  # Question:
+  {question}
+
+   # INSTRUCTIONS:
+      1. Carefully analyze all provided memories (facts and entities)
+      2. Pay special attention to the timestamps (event_time) to determine when events occurred
+      3. If the question asks about a specific event or fact, look for direct evidence in the memories
+      4. If the memories contain contradictory information, prioritize the most recent memory
+      5. Always convert relative time references to specific dates, months, or years
+      6. Be as specific as possible when talking about people, places, and events
+      7. Timestamps in memories represent the time the event was mentioned in a message, not the actual time the event occurred
+
+
+  # OUTPUT FORMAT:
+  Please provide your response in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "reasoning content",
+    "answer": "Provide a detailed answer"
+  }}
+  ```
+
+SYSTEM_PROMPT: |
+  You are an expert grader that determines if answers to questions match a gold standard answer
+
+USER_PROMPT: |
+  Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given the following data:
+    (1) a question (posed by one user to another user),
+    (2) a 'gold' (ground truth) answer,
+    (3) a generated answer
+  which you will score as CORRECT/WRONG.
+
+  The point of the question is to ask about something one user should know about the other user based on their prior conversations.
+  The gold answer will usually be a concise and short answer that includes the referenced topic, for example:
+  Question: Do you remember what I got the last time I went to Hawaii?
+  Gold answer: A shell necklace
+  The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.
+
+  For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.
+
+  Now it's time for the real question:
+  Question: {question}
+  Gold answer: {golden_answer}
+  Generated answer: {generated_answer}
+
+  First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.
+  Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.
+
+  Just return the label CORRECT or WRONG in a json format with the key as "label".
+
+user_message_summary_1: |
+  You are a Memory Agent responsible for managing {memory_type} memories about {memory_target}.
+
+  ## Latest Conversation
+  Format: round<index> [<timestamp>] <role/name>: <content>
+  {context}
+
+  ## Task
+  ### Step 1: Create Memory Draft
+  Use `add_draft_and_retrieve_similar_memory` to create a memory draft list based on the latest conversation.
+  - For each memory draft, fill in the required parameters:
+    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
+    * `memory_content`: concise memory content extracted from the conversation
+  - Use actual names from the conversation (e.g., "Bob likes apples") instead of generic references (e.g., "user likes apples")
+  - Extract all important information comprehensively—do not miss critical details, but avoid any fabrications or unfounded assumptions
+  - The tool will retrieve similar historical memories via vector search to help you in Step 2
+
+  ### Step 2: Add Memories
+  Review each memory draft from Step 1 and compare it with the retrieved historical memories, then use `add_memory` to manage all memories in one call:
+
+  - For each new memory, fill in the required parameters:
+    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
+    * `memory_content`: memory content
+  - Add memories when:
+    * The draft contains new information not present in historical memories
+
+
+  **General Guidelines:**
+  - **Skip** drafts if their content is already fully covered by historical memories (avoid redundancy)
+  - You can add memories in a single `add_memory` tool call
+
+user_message_summary_2: |
+  You are a Profile Agent responsible for managing profiles about {memory_target}.
+
+  ## Latest Conversation
+  Format: round<index> [<timestamp>] <role/name>: <content>
+  {context}
+
+  ## Current Profiles
+  {profiles}
+
+  ## Task
+  Analyze the Latest Conversation and use `update_profiles` to manage profiles (both updates and additions in one call):
+
+  **For profiles_to_update** (updating existing profiles):
+  - For each profile to update, fill in the required parameters:
+    * `profile_id`: ID of the profile to update (from Current Profiles)
+    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
+    * `profile_key`: profile key or category (e.g., 'name', 'age', 'occupation')
+    * `profile_value`: updated profile value, please be concise. (e.g., 'John Smith')
+
+  **For profiles_to_add** (adding new profiles):
+  - For each new profile, fill in the required parameters:
+    * `message_time`: timestamp from the conversation (e.g., '2020-01-01 00:00:00')
+    * `profile_key`: profile key or category (e.g., 'name', 'age', 'occupation')
+    * `profile_value`: profile value (e.g., 'John Smith')
+  - Add profiles when:
+    * The information represents a new distinct profile not present in Current Profiles
+    * The profile key doesn't exist in Current Profiles
+    * The information cannot be merged into existing profiles
+
+  **General Guidelines:**
+  - Extract all important information comprehensively—do not miss critical details, but avoid any fabrications or unfounded assumptions
+  - You can update and add profiles in a single tool call
+
+user_message_retrieve: |
+  You are a Memory Retrieval Agent specialized in retrieving {memory_type} memories about {memory_target}.
+
+  ## User Profile
+  {user_profile}
+
+  ## User Question
+  {context}
+
+  ## Multi-Phase Retrieval Strategy
+  Follow these phases sequentially to gather comprehensive information:
+
+  ### Phase 1: Semantic Search (No Time Filter)
+  **Tool**: `retrieve_memory` (without time constraints)
+  **Objective**: Cast a wide net to find potentially relevant memories
+  **Approach**:
+  - Execute 3-5 diverse search queries using different formulations:
+    * Original question verbatim
+    * Rephrased variations (different wording, synonyms)
+    * Entity-focused queries (extract and search specific names, places, events)
+    * Keyword-based searches (core concepts, topics)
+    * Related context queries (broader themes)
+  - Review all results before proceeding to next phase
+
+  ### Phase 2: Deep Dive into History
+  **Tool**: `read_history`
+  **When to use**: After exhausting retrieval attempts OR when specific conversation context is needed
+  **Important Constraints**:
+  - Each history is very long and resource-intensive to read
+  - **Maximum limit: Read no more than 3 histories total**
+  - Only use this phase when absolutely necessary for answering the question
+  **Approach**:
+  - Extract `history_id` from retrieved memory references
+  - Prioritize the most relevant or recent histories
+  - Can read multiple histories at once by passing multiple history_ids
+  - Be selective: choose only the top 1-3 most promising histories
+  - Use this to understand the full conversation surrounding a memory
+
+  ## Response Guidelines
+  - Base your answer EXCLUSIVELY on user profile, retrieved memories, and history data
+  - Never infer, assume, or hallucinate information
+  - Always cite sources with timestamps: `[timestamp] Memory content`
+  - Present conflicting information transparently with respective timestamps
+  - If you find sufficient information to answer the user's question, you may output directly without exhausting all search phases
+  - Exhaust all search strategies before concluding information doesn't exist
+
+  ### Output any tangentially related findings, Format:
+  [timestamp] [memory/profile/history] [relevant content1]
+  [timestamp] [memory/profile/history] [relevant content2]
+

benchmark/longmemeval/README.md

@@ -1,96 +0,0 @@
-[中文版 / Chinese version](./README_ZH.md)
-
-# LongMemEval Benchmark
-
-LongMemEval is a benchmark for **long-term memory over multi-session chat
-histories**. Each item provides a chronologically ordered set of chat sessions
-between a user and an assistant, followed by a probing question whose answer is
-only recoverable by reasoning over the user-owned memory. ReMe ingests the
-sessions into an isolated per-item workspace, answers the question via an
-agentic (ReAct) mode, and scores the answer with an LLM-as-judge.
-
-Question types include single-session (user / assistant / preference),
-multi-session reasoning, knowledge update, and temporal reasoning.
-
-> For the shared setup (dependencies, credentials, log conventions) see the
-> [top-level benchmark README](../README.md).
-
-## 1. Get the Dataset
-
-ReMe uses only the **cleaned-S** split, hosted on HuggingFace:
-[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2).
-The download script fetches it via the hf-mirror.com mirror; to use a different
-mirror, modify `BASE_URL` in [`download.py`](./download.py).
-
-```bash
-cd benchmark/longmemeval
-python download.py            # saves dataset/longmemeval_s_reme_cleaned.json; skips if already present
-```
-
-Ground truth is embedded in the data file.
-
-## 2. Run
-
-From the repository root:
-
-```bash
-python benchmark/longmemeval/run.py
-python benchmark/longmemeval/run.py --config benchmark/longmemeval/config.yaml
-python benchmark/longmemeval/run.py -q                        # quiet: only eval-level logs
-python benchmark/longmemeval/run.py --log-level WARNING       # reduce eval runner logs
-python benchmark/longmemeval/run.py --reme-log-level WARNING  # reduce reme internal logs
-python benchmark/longmemeval/run.py --eval_only               # reuse existing workspaces, query + judge only
-```
-
-## 3. Pipeline
-
-1. Load the dataset (ground truth is embedded in the data file).
-2. For each item, create an isolated workspace and ingest sessions in chronological order.
-3. Trigger `auto_dream` when consecutive sessions cross the configured hour (default 23:00).
-4. Answer each question via agentic (ReAct) mode.
-5. Judge the answer (binary yes/no) with the `answer_judge` job and print per-type accuracy.
-
-## 4. Key config — `benchmark/longmemeval/config.yaml`
-
-| Key | Meaning |
-| --- | --- |
-| `dataset.path` | Dataset file to evaluate (e.g. `longmemeval_s_reme_cleaned.json`); ground truth is included. |
-| `dataset.start_index` / `num_items` | Slice of items to evaluate. |
-| `dataset.question_types` | Filter by question type; empty = all. |
-| `dataset.workspace_root` | Per-item workspace root (`benchmark/longmemeval/workspaces/longmemeval-s`). |
-| `evaluation.num_workers` | `0` = auto (cpu-2), `1` = sequential, `>1` = parallel. |
-| `evaluation.filter_future_sessions` | Only ingest sessions with timestamp ≤ `question_date`. |
-| `reme.config` | ReMe config used (`lme.yaml`). |
-| `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | Dream triggering behavior. |
-| `output.dir` | Results directory (`benchmark/longmemeval/results`). |
-
-## 5. Outputs
-
-Results are JSON files written to `output.dir` as `results_<timestamp>.json`,
-with a per-type accuracy summary also printed to the console. Logging
-conventions are shared across benchmarks — see the
-[top-level README](../README.md#outputs--logs).
-
-## 6. Reference Results
-
-### cleaned-s
-
-**Basic settings**
-
-1. Modified auto-memory prompt, auto-dream disabled.
-2. All sessions in reme-memory are strictly earlier than the question time.
-
-**Results**
-
-agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only (reusing prebuilt memory)
-(2026-08-06, 500 items, total 10.0 min)
-
-| Type | Agentic | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|
-| knowledge-update | 0.910 | 31,581 | 589 | 32,169 | 2.90 |
-| multi-session | 0.842 | 52,837 | 1,474 | 54,311 | 4.21 |
-| single-session-assistant | 1.000 | 15,596 | 279 | 15,875 | 1.89 |
-| single-session-preference | 0.633 | 36,802 | 818 | 37,620 | 3.60 |
-| single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 |
-| temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 |
-| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** |

benchmark/longmemeval/README_ZH.md

@@ -1,90 +0,0 @@
-# LongMemEval 评测
-
-[English version](./README.md)
-
-LongMemEval 是一个面向**多轮多会话历史的长期记忆能力**的评测基准。每个条目提供一组按时间
-顺序排列的用户与助手之间的会话，以及一个只能通过推理用户自有记忆才能回答的探测问题。ReMe
-将会话摄入按条目隔离的工作区，以 agentic（ReAct）模式回答问题，最后由 LLM-as-judge 打分。
-
-题型包括单会话（user / assistant / preference）、多会话推理、知识更新与时间推理等。
-
-> 公共设置（依赖、凭据、日志约定）见[总评测说明](../README_ZH.md)。
-
-## 1. 获取数据集
-
-ReMe 仅使用 **cleaned-S** 版本，数据托管在 HuggingFace：
-[agentscope-ai/ReMe_longmemeval_clean_s_v2](https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2)。
-下载脚本经 hf-mirror.com 镜像源获取，如需更换源请修改 [`download.py`](./download.py) 中的
-`BASE_URL`。
-
-```bash
-cd benchmark/longmemeval
-python download.py            # 保存为 dataset/longmemeval_s_reme_cleaned.json，已存在则自动跳过
-```
-
-ground truth 已内嵌在数据文件中。
-
-## 2. 运行
-
-在仓库根目录执行：
-
-```bash
-python benchmark/longmemeval/run.py
-python benchmark/longmemeval/run.py --config benchmark/longmemeval/config.yaml
-python benchmark/longmemeval/run.py -q                        # 安静模式：仅评测级日志
-python benchmark/longmemeval/run.py --log-level WARNING       # 降低评测 runner 日志
-python benchmark/longmemeval/run.py --reme-log-level WARNING  # 降低 reme 内部日志
-python benchmark/longmemeval/run.py --eval_only               # 复用已有工作区，仅执行查询 + 评判
-```
-
-## 3. 流程
-
-1. 加载数据集（ground truth 已内嵌在数据文件中）。
-2. 为每个条目创建独立工作区，按时间顺序摄入会话。
-3. 当相邻会话跨越配置的时刻（默认 23:00）时触发 `auto_dream`。
-4. 以 agentic（ReAct）模式回答每个问题。
-5. 通过 `answer_judge` 任务对答案做二元（yes/no）评判，并输出各类型准确率。
-
-## 4. 关键配置 —— `benchmark/longmemeval/config.yaml`
-
-| 配置项 | 含义 |
-| --- | --- |
-| `dataset.path` | 待评测的数据集文件（如 `longmemeval_s_reme_cleaned.json`），已包含 ground truth。 |
-| `dataset.start_index` / `num_items` | 评测条目的切片范围。 |
-| `dataset.question_types` | 按问题类型过滤，空表示全部。 |
-| `dataset.workspace_root` | 条目工作区根目录（`benchmark/longmemeval/workspaces/longmemeval-s`）。 |
-| `evaluation.num_workers` | `0` = 自动（cpu-2），`1` = 串行，`>1` = 并行。 |
-| `evaluation.filter_future_sessions` | 仅摄入时间戳 ≤ `question_date` 的会话。 |
-| `reme.config` | 使用的 ReMe 配置（`lme.yaml`）。 |
-| `reme.dream_trigger_hour` / `dream_scan_days` / `dream_max_units` | dream 触发行为。 |
-| `output.dir` | 结果目录（`benchmark/longmemeval/results`）。 |
-
-## 5. 输出
-
-结果以 JSON 文件写入 `output.dir`，文件名为 `results_<timestamp>.json`，
-同时控制台会打印含各类型准确率的汇总。日志约定在各基准间通用，见
-[总说明](../README_ZH.md#输出与日志)。
-
-## 6. 参考结果
-
-### cleaned-s
-
-**基础设置**
-
-1. 使用修改后的 auto-memory prompt，关闭 auto-dream 机制
-2. reme-memory 中的全部 session 的时间一定早于 question 的时间
-
-**结果**
-
-agentscope==2.0.4.post1, conda reme env, 32 workers, eval-only（复用预构建记忆）
-（2026-08-06，500 题，总计 10.0 min）
-
-| 类型 | Agentic | input tok/q | output tok/q | total tok/q | tool calls/q |
-|---|---|---|---|---|---|
-| knowledge-update | 0.910 | 31,581 | 589 | 32,169 | 2.90 |
-| multi-session | 0.842 | 52,837 | 1,474 | 54,311 | 4.21 |
-| single-session-assistant | 1.000 | 15,596 | 279 | 15,875 | 1.89 |
-| single-session-preference | 0.633 | 36,802 | 818 | 37,620 | 3.60 |
-| single-session-user | 0.986 | 27,433 | 359 | 27,792 | 2.60 |
-| temporal-reasoning | 0.902 | 62,674 | 985 | 63,659 | 4.97 |
-| **OVERALL** | **0.894** | **43,448** | **876** | **44,324** | **3.69** |

benchmark/longmemeval/compute_stats.py

@@ -0,0 +1,346 @@
+"""
+LongMemEval Evaluation Statistics Analyzer
+
+Computes detailed statistics from evaluation results including:
+- Overall accuracy
+- Accuracy by question type
+- Timing statistics (summary, retrieval)
+- Memory extraction statistics
+
+Usage:
+    python bench/longmemeval/compute_stats.py \
+        --results_dir bench/longmemeval/bench_results/longmemeval_reme
+"""
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+
+def load_results(results_dir: str) -> list[dict]:
+    """Load all question result files from the directory.
+
+    Args:
+        results_dir: Path to the results directory
+
+    Returns:
+        List of result dictionaries
+    """
+    results_path = Path(results_dir)
+    results = []
+
+    # Load individual question files
+    question_files = sorted(results_path.glob("question_*.json"))
+
+    for file_path in question_files:
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                result = json.load(f)
+                results.append(result)
+        except Exception as e:
+            print(f"⚠️ Error loading {file_path}: {e}")
+
+    return results
+
+
+def compute_accuracy_stats(results: list[dict]) -> dict[str, Any]:
+    """Compute overall and per-type accuracy statistics.
+
+    Args:
+        results: List of result dictionaries
+
+    Returns:
+        Dictionary with accuracy statistics
+    """
+    total = len(results)
+    correct = 0
+    incorrect = 0
+    error = 0
+
+    # Per question type statistics
+    type_stats = defaultdict(lambda: {"total": 0, "correct": 0, "incorrect": 0, "error": 0})
+
+    for r in results:
+        qtype = r.get("question_type", "unknown")
+        judgment = r.get("judgment", {})
+        is_correct = judgment.get("is_correct")
+
+        type_stats[qtype]["total"] += 1
+
+        if is_correct is True:
+            correct += 1
+            type_stats[qtype]["correct"] += 1
+        elif is_correct is False:
+            incorrect += 1
+            type_stats[qtype]["incorrect"] += 1
+        else:
+            error += 1
+            type_stats[qtype]["error"] += 1
+
+    # Compute accuracies
+    overall = {
+        "total": total,
+        "correct": correct,
+        "incorrect": incorrect,
+        "error": error,
+        "accuracy": correct / total if total > 0 else 0,
+        "accuracy_valid": correct / (correct + incorrect) if (correct + incorrect) > 0 else 0,
+    }
+
+    by_type = {}
+    for qtype, stats in type_stats.items():
+        valid = stats["correct"] + stats["incorrect"]
+        by_type[qtype] = {
+            **stats,
+            "accuracy": stats["correct"] / stats["total"] if stats["total"] > 0 else 0,
+            "accuracy_valid": stats["correct"] / valid if valid > 0 else 0,
+        }
+
+    return {
+        "overall": overall,
+        "by_question_type": by_type,
+    }
+
+
+def compute_timing_stats(results: list[dict]) -> dict[str, Any]:
+    """Compute timing statistics.
+
+    Args:
+        results: List of result dictionaries
+
+    Returns:
+        Dictionary with timing statistics
+    """
+    summary_times = []
+    retrieve_times = []
+
+    for r in results:
+        summary_ms = r.get("summary_duration_ms", 0)
+        retrieve_ms = r.get("retrieve_duration_ms", 0)
+
+        if summary_ms > 0:
+            summary_times.append(summary_ms)
+        if retrieve_ms > 0:
+            retrieve_times.append(retrieve_ms)
+
+    def compute_stats(times: list[float]) -> dict:
+        if not times:
+            return {"count": 0, "total_ms": 0, "avg_ms": 0, "min_ms": 0, "max_ms": 0}
+
+        return {
+            "count": len(times),
+            "total_ms": sum(times),
+            "total_min": sum(times) / 1000 / 60,
+            "avg_ms": sum(times) / len(times),
+            "min_ms": min(times),
+            "max_ms": max(times),
+        }
+
+    return {
+        "summary": compute_stats(summary_times),
+        "retrieve": compute_stats(retrieve_times),
+        "total_time_min": (sum(summary_times) + sum(retrieve_times)) / 1000 / 60,
+    }
+
+
+def compute_memory_stats(results: list[dict]) -> dict[str, Any]:
+    """Compute memory extraction statistics.
+
+    Args:
+        results: List of result dictionaries
+
+    Returns:
+        Dictionary with memory statistics
+    """
+    memory_counts = []
+    session_counts = []
+
+    for r in results:
+        memories = r.get("extracted_memories", [])
+        num_sessions = r.get("num_sessions", 0)
+
+        memory_counts.append(len(memories))
+        session_counts.append(num_sessions)
+
+    def compute_stats(counts: list[int]) -> dict:
+        if not counts:
+            return {"count": 0, "total": 0, "avg": 0, "min": 0, "max": 0}
+
+        return {
+            "count": len(counts),
+            "total": sum(counts),
+            "avg": sum(counts) / len(counts),
+            "min": min(counts),
+            "max": max(counts),
+        }
+
+    return {
+        "memories_per_question": compute_stats(memory_counts),
+        "sessions_per_question": compute_stats(session_counts),
+    }
+
+
+def print_report(
+    accuracy_stats: dict,
+    timing_stats: dict,
+    memory_stats: dict,
+    results_dir: str,
+):
+    """Print formatted statistics report.
+
+    Args:
+        accuracy_stats: Accuracy statistics
+        timing_stats: Timing statistics
+        memory_stats: Memory statistics
+        results_dir: Path to results directory
+    """
+    print("\n" + "=" * 80)
+    print("LONGMEMEVAL EVALUATION STATISTICS")
+    print(f"Results Directory: {results_dir}")
+    print("=" * 80)
+
+    # Overall accuracy
+    overall = accuracy_stats["overall"]
+    print("\n📊 Overall Accuracy:")
+    print(f"  Total Questions:    {overall['total']}")
+    print(f"  ✅ Correct:         {overall['correct']} ({100 * overall['accuracy']:.2f}%)")
+    print(
+        f"  ❌ Incorrect:       {overall['incorrect']} "
+        f"({100 * overall['incorrect'] / overall['total'] if overall['total'] > 0 else 0:.2f}%)",
+    )
+    if overall["error"] > 0:
+        print(f"  ⚠️ Error:           {overall['error']} ({100 * overall['error'] / overall['total']:.2f}%)")
+    print(f"  Accuracy (valid):   {100 * overall['accuracy_valid']:.2f}%")
+
+    # Accuracy by question type
+    print("\n📊 Accuracy by Question Type:")
+    print("-" * 60)
+    print(f"{'Question Type':<30} {'Correct':<10} {'Total':<10} {'Accuracy':<10}")
+    print("-" * 60)
+
+    by_type = accuracy_stats["by_question_type"]
+    for qtype in sorted(by_type.keys()):
+        stats = by_type[qtype]
+        print(f"{qtype:<30} {stats['correct']:<10} {stats['total']:<10} {100 * stats['accuracy']:.2f}%")
+
+    print("-" * 60)
+
+    # Timing statistics
+    print("\n⏱️ Timing Statistics:")
+    summary = timing_stats["summary"]
+    retrieve = timing_stats["retrieve"]
+
+    print("  Memory Summarization:")
+    print(f"    Total Time:  {summary['total_min']:.2f} min")
+    print(f"    Avg per Q:   {summary['avg_ms']:.0f} ms")
+    print(f"    Min/Max:     {summary['min_ms']:.0f} / {summary['max_ms']:.0f} ms")
+
+    print("  Memory Retrieval:")
+    print(f"    Total Time:  {retrieve['total_min']:.2f} min")
+    print(f"    Avg per Q:   {retrieve['avg_ms']:.0f} ms")
+    print(f"    Min/Max:     {retrieve['min_ms']:.0f} / {retrieve['max_ms']:.0f} ms")
+
+    print(f"  Total Time:    {timing_stats['total_time_min']:.2f} min")
+
+    # Memory statistics
+    print("\n📝 Memory Statistics:")
+    mem = memory_stats["memories_per_question"]
+    sess = memory_stats["sessions_per_question"]
+
+    print("  Extracted Memories per Question:")
+    print(f"    Total:   {mem['total']}")
+    print(f"    Average: {mem['avg']:.1f}")
+    print(f"    Min/Max: {mem['min']} / {mem['max']}")
+
+    print("  Sessions per Question:")
+    print(f"    Average: {sess['avg']:.1f}")
+    print(f"    Min/Max: {sess['min']} / {sess['max']}")
+
+    print("\n" + "=" * 80)
+
+
+def save_statistics(
+    accuracy_stats: dict,
+    timing_stats: dict,
+    memory_stats: dict,
+    output_file: str,
+):
+    """Save statistics to JSON file.
+
+    Args:
+        accuracy_stats: Accuracy statistics
+        timing_stats: Timing statistics
+        memory_stats: Memory statistics
+        output_file: Path to output file
+    """
+    stats = {
+        "accuracy": accuracy_stats,
+        "timing": timing_stats,
+        "memory": memory_stats,
+    }
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=4, ensure_ascii=False)
+
+    print(f"\n📁 Statistics saved to: {output_file}")
+
+
+def main(results_dir: str, output_file: str = None):
+    """Main function to compute and display statistics.
+
+    Args:
+        results_dir: Path to results directory
+        output_file: Optional path to save statistics JSON
+    """
+    print(f"\nLoading results from: {results_dir}")
+
+    results = load_results(results_dir)
+
+    if not results:
+        print("❌ No results found!")
+        return
+
+    print(f"Loaded {len(results)} question results")
+
+    # Compute statistics
+    accuracy_stats = compute_accuracy_stats(results)
+    timing_stats = compute_timing_stats(results)
+    memory_stats = compute_memory_stats(results)
+
+    # Print report
+    print_report(accuracy_stats, timing_stats, memory_stats, results_dir)
+
+    # Save to file if specified
+    if output_file:
+        save_statistics(accuracy_stats, timing_stats, memory_stats, output_file)
+    else:
+        # Default output file in results directory
+        default_output = Path(results_dir) / "statistics.json"
+        save_statistics(accuracy_stats, timing_stats, memory_stats, str(default_output))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Compute statistics from LongMemEval evaluation results",
+    )
+    parser.add_argument(
+        "--results_dir",
+        type=str,
+        default="bench_results/longmemeval_reme",
+        help="Path to results directory containing question_*.json files",
+    )
+    parser.add_argument(
+        "--output_file",
+        type=str,
+        default=None,
+        help="Path to save statistics JSON (default: <results_dir>/statistics.json)",
+    )
+
+    args = parser.parse_args()
+
+    main(
+        results_dir=args.results_dir,
+        output_file=args.output_file,
+    )

benchmark/longmemeval/config.yaml

@@ -1,33 +0,0 @@
-# LongMemEval evaluation configuration
-# This file controls what/how to evaluate.
-
-dataset:
-  path: "benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json"
-  start_index: 0             # first item index
-  num_items: 500            # how many items to evaluate (starting from start_index)
-  max_sessions: 0            # 0 = all sessions; >0 = limit sessions per item for testing
-  question_types: []         # filter by question_type; empty list = no filtering (all types)
-  workspace_root: "benchmark/longmemeval/workspaces/longmemeval-s"  # workspace root for item workspaces
-
-evaluation:
-  # LLM-as-judge uses the 'judge' as_llm component defined in lme.yaml
-  # Model and credentials are configured there (reading from .env)
-  # Judgment is always binary (yes/no) — defined in lme/llm_judge.yaml
-  num_workers: 32            # 0 = auto (cpu_count - 2, min 1); 1 = sequential; >1 = parallel
-  filter_future_sessions: true  # true = only ingest sessions with timestamp <= question_date
-  compress_session: false   # true = compress session chunks in search_v2 (query-aware); false = no compression
-
-reme:
-  config: "lme.yaml" # reme config to use (in reme/config/)
-  # Dream trigger: when gap between consecutive sessions crosses this hour (23:00)
-  dream_trigger_hour: 23
-  # Dream scan_days for each trigger
-  dream_scan_days: 2
-  dream_max_units: 5
-
-output:
-  dir: "benchmark/longmemeval/results"
-  log_dir: "logs"                # log directory (relative to project root)
-  log_prefix: "longmemeval"      # benchmark name used in log filenames
-  log_to_console: true
-  log_to_file: true

benchmark/longmemeval/download.py

@@ -1,67 +0,0 @@
-"""Download the LongMemEval cleaned-S dataset used by ReMe.
-
-Source: https://huggingface.co/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2
-(downloaded via the hf-mirror.com mirror for reliability).
-
-The file ``longmemeval_s_reme_cleaned.json`` is saved under ``dataset/`` next to this
-script using the same name as on the remote (``benchmark/longmemeval/config.yaml``
-points to it).
-
-Usage:
-    python download.py           # download cleaned-S (skip if it already exists)
-"""
-
-import os
-import sys
-import urllib.request
-
-BASE_URL = "https://hf-mirror.com/datasets/agentscope-ai/ReMe_longmemeval_clean_s_v2/resolve/main"
-TARGET_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dataset")
-
-# Files to download (saved with the same name as on the remote).
-FILES = [
-    "longmemeval_s_reme_cleaned.json",
-]
-
-
-def download_file(filename: str):
-    """Download a single file from the mirror to the target directory."""
-    url = f"{BASE_URL}/{filename}"
-    dest = os.path.join(TARGET_DIR, filename)
-
-    if os.path.exists(dest):
-        size = os.path.getsize(dest)
-        print(f"  [skip] {filename} already exists ({size / 1024 / 1024:.1f} MB)")
-        return
-
-    print(f"  [downloading] {filename} ...")
-    try:
-        urllib.request.urlretrieve(url, dest, reporthook=_progress)
-        size = os.path.getsize(dest)
-        print(f"\n  [done] {filename} ({size / 1024 / 1024:.1f} MB)")
-    except Exception as e:
-        print(f"\n  [error] {filename}: {e}")
-        if os.path.exists(dest):
-            os.remove(dest)
-        sys.exit(1)
-
-
-def _progress(block_num, block_size, total_size):
-    downloaded = block_num * block_size
-    if total_size > 0:
-        pct = min(100, downloaded * 100 / total_size)
-        mb = downloaded / 1024 / 1024
-        total_mb = total_size / 1024 / 1024
-        sys.stdout.write(f"\r    {mb:.1f}/{total_mb:.1f} MB ({pct:.1f}%)")
-    else:
-        mb = downloaded / 1024 / 1024
-        sys.stdout.write(f"\r    {mb:.1f} MB downloaded")
-    sys.stdout.flush()
-
-
-if __name__ == "__main__":
-    os.makedirs(TARGET_DIR, exist_ok=True)
-    print(f"Downloading LongMemEval cleaned-S dataset to: {TARGET_DIR}\n")
-    for fname in FILES:
-        download_file(fname)
-    print("\nAll files downloaded successfully!")

benchmark/longmemeval/eval_longmemeval_reme_retrieve.py

@@ -0,0 +1,921 @@
+"""
+LongMemEval Benchmark Evaluator for ReMe - Retrieve Only
+
+A simplified evaluation pipeline that only runs the retrieve and judge phases:
+1. Loads LongMemEval benchmark data
+2. Skips memory summarization (assumes memories are already in vector store)
+3. Uses questions to query memory and generate answers
+4. Uses LLM to judge answer correctness
+5. Generates comprehensive metrics
+
+This is useful for debugging/tuning the retrieve phase without re-running summary.
+
+Usage:
+    python benchmark/longmemeval/eval_longmemeval_reme_retrieve.py \
+        --data_path dataset/longmemeval/longmemeval_s_cleaned.json \
+        --top_k 20 --start_index 0 --end_index 10
+"""
+
+import asyncio
+import json
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from loguru import logger
+
+from reme.reme import ReMe
+
+
+# ==================== Configuration ====================
+
+
+@dataclass
+class RetrieveEvalConfig:
+    """Evaluation configuration parameters for retrieve-only mode."""
+
+    data_path: str
+    top_k: int = 10
+    start_index: int = 0
+    end_index: Optional[int] = None
+    max_concurrency: int = 1
+    output_dir: str = "cache/bench_results/longmemeval_reme_retrieve"
+    reme_model_name: str = "qwen-flash"
+    eval_model_name: str = "qwen3-max"
+    algo_version: str = "v1"
+    samples_per_type: int = -1  # Number of samples per question type, -1 for all
+    enable_thinking_params: bool = False
+    # Optional: path to previous summary results to reload memories
+    summary_results_dir: Optional[str] = None
+
+
+# ==================== Answer Judge Prompts ====================
+
+
+def get_anscheck_prompt(task: str, question: str, answer: str, response: str, abstention: bool = False) -> str:
+    """Generate the answer checking prompt based on question type.
+
+    Args:
+        task: Question type, e.g. 'single-session-user', 'multi-session', 'temporal-reasoning'
+        question: The question content
+        answer: The reference answer
+        response: The model's response
+        abstention: Whether this is an unanswerable question
+
+    Returns:
+        Prompt for judging answer correctness
+    """
+    if not abstention:
+        if task in ["single-session-user", "single-session-assistant", "multi-session"]:
+            template = (
+                "I will give you a question, a correct answer, and a response from a model. Please answer yes i"
+                "f the response contains the correct answer. Otherwise, answer no. If the response is equival"
+                "ent to the correct answer or contains all the intermediate steps to get the correct answer, "
+                "you should also answer yes. If the response only contains a subset of the information required"
+                " by the answer, answer no. \n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs"
+                " the model response correct? Answer yes or no only."
+            )
+            prompt = template.format(question, answer, response)
+        elif task == "temporal-reasoning":
+            template = (
+                "I will give you a question, a correct answer, and a response from a model. Please answer yes"
+                " if the response contains the correct answer. Otherwise, answer no. If the response is equiv"
+                "alent to the correct answer or contains all the intermediate steps to get the correct answer"
+                ", you should also answer yes. If the response only contains a subset of the information requ"
+                "ired by the answer, answer no. In addition, do not penalize off-by-one errors for the numbe"
+                "r of days. If the question asks for the number of days/weeks/months, etc., and the model ma"
+                "kes off-by-one errors (e.g., predicting 19 days when the answer is 18), the model's respon"
+                "se is still correct. \n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs th"
+                "e model response correct? Answer yes or no only."
+            )
+            prompt = template.format(question, answer, response)
+        elif task == "knowledge-update":
+            template = (
+                "I will give you a question, a correct answer, and a response from a model. Please answer yes "
+                "if the response contains the correct answer. Otherwise, answer no. If the response contains "
+                "some previous information along with an updated answer, the response should be considered "
+                "as correct as long as the updated answer is the required answer.\n\nQuestion: {}\n\nCorrec"
+                "t Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only."
+            )
+            prompt = template.format(question, answer, response)
+        elif task == "single-session-preference":
+            template = (
+                "I will give you a question, a rubric for desired personalized response, and a response from a"
+                " model. Please answer yes if the response satisfies the desired response. Otherwise, answer"
+                " no. The model does not need to reflect all the points in the rubric. The response is corr"
+                "ect as long as it recalls and utilizes the user's personal information correctly.\n\nQuest"
+                "ion: {}\n\nRubric: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes o"
+                "r no only."
+            )
+            prompt = template.format(question, answer, response)
+        else:
+            # Default template
+            template = (
+                "I will give you a question, a correct answer, and a response from a model. Please answer y"
+                "es if the response contains the correct answer. Otherwise, answer no. If the response is "
+                "equivalent to the correct answer or contains all the intermediate steps to get the correc"
+                "t answer, you should also answer yes. If the response only contains a subset of the infor"
+                "mation required by the answer, answer no. \n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel"
+                " Response: {}\n\nIs the model response correct? Answer yes or no only."
+            )
+            prompt = template.format(question, answer, response)
+
+    else:
+        template = (
+            "I will give you an unanswerable question, an explanation, and a response from a model. Please "
+            "answer yes if the model correctly identifies the question as unanswerable. The model could say "
+            "that the information is incomplete, or some other information is given but the asked informati"
+            "on is not.\n\nQuestion: {}\n\nExplanation: {}\n\nModel Response: {}\n\nDoes the model correct"
+            "ly identify the question as unanswerable? Answer yes or no only."
+        )
+        prompt = template.format(question, answer, response)
+    return prompt
+
+
+# ==================== Utilities ====================
+
+
+class DataLoader:
+    """Handles loading and parsing of LongMemEval data."""
+
+    @staticmethod
+    def load_json(file_path: str) -> list[dict]:
+        """Load all entries from a JSON file."""
+        with open(file_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    @staticmethod
+    def filter_by_type(data: list[dict], samples_per_type: int = -1) -> list[tuple[int, dict]]:
+        """Filter data by question type with specified number of samples per type.
+
+        Args:
+            data: List of question entries
+            samples_per_type: Number of samples per type, -1 for all
+
+        Returns:
+            List of tuples (original_index, entry) for selected samples
+        """
+        if samples_per_type == -1:
+            # Return all with original indices
+            return list(enumerate(data))
+
+        # Group by question type
+        type_groups: dict[str, list[tuple[int, dict]]] = {}
+        for i, entry in enumerate(data):
+            qtype = entry.get("question_type", "unknown")
+            if qtype not in type_groups:
+                type_groups[qtype] = []
+            type_groups[qtype].append((i, entry))
+
+        # Select samples from each type
+        selected = []
+        for qtype, entries in type_groups.items():
+            count = min(samples_per_type, len(entries))
+            selected.extend(entries[:count])
+            logger.info(f"  {qtype}: selected {count}/{len(entries)} samples")
+
+        # Sort by original index to maintain order
+        selected.sort(key=lambda x: x[0])
+        return selected
+
+
+class FileManager:
+    """Manages file I/O operations."""
+
+    def __init__(self, base_dir: str):
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+    def save_question_result(self, idx: int, question_id: str, data: dict):
+        """Save result for a single question."""
+        file_path = self.base_dir / f"question_{idx:04d}_{question_id}.json"
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=4, ensure_ascii=False)
+        logger.info(f"✅ Saved question result to {file_path}")
+
+    def load_question_result(self, idx: int, question_id: str) -> Optional[dict]:
+        """Load result for a single question if exists."""
+        file_path = self.base_dir / f"question_{idx:04d}_{question_id}.json"
+        if not file_path.exists():
+            return None
+        with open(file_path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def save_summary(self, results: list[dict]):
+        """Save summary of all results."""
+        file_path = self.base_dir / "summary.json"
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(results, f, indent=4, ensure_ascii=False)
+        logger.info(f"✅ Saved summary to {file_path}")
+
+
+# ==================== Evaluation Functions ====================
+
+
+async def answer_question_with_memories(
+    reme: ReMe,
+    question: str,
+    memories: str,
+    user_id: str = None,
+    model_name: str = "qwen3-max",
+):
+    """
+    Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template.
+
+    Args:
+        reme: ReMe instance with default_llm and prompt_handler
+        question: The question to answer
+        memories: The retrieved memories (formatted as context)
+        user_id: Optional user ID for context formatting
+        model_name: Model name to use for LLM request
+
+    Returns:
+        dict with 'reasoning' and 'answer' fields
+    """
+    # Format context with memories
+    if user_id:
+        context = reme.prompt_handler.prompt_format(
+            "TEMPLATE_MEMOS",
+            user_id=user_id,
+            memories=memories,
+        )
+    else:
+        context = f"Memories:\n{memories}"
+
+    # Use PROMPT_MEMZERO_JSON template for structured JSON response
+    prompt = reme.prompt_handler.prompt_format(
+        "PROMPT_MEMZERO_JSON",
+        context=context,
+        question=question,
+    )
+
+    result = await reme.get_llm(name=model_name).simple_request_for_json(
+        prompt=prompt,
+        model_name=None,
+    )
+
+    return result
+
+
+# ==================== Memory Operations ====================
+
+
+class RetrieveProcessor:
+    """Handles ReMe memory retrieve operations only."""
+
+    def __init__(
+        self,
+        reme: ReMe,
+        reme_model_name: str = "qwen-flash",
+        eval_model_name: str = "qwen3-max",
+        algo_version: str = "v1",
+        enable_thinking_params: bool = False,
+    ):
+        self.reme = reme
+        self.reme_model_name = reme_model_name
+        self.eval_model_name = eval_model_name
+        self.algo_version = algo_version
+        self.enable_thinking_params = enable_thinking_params
+
+    async def search_memory(
+        self,
+        query: str,
+        user_id: str,
+        top_k: int = 20,
+    ) -> tuple[dict, list, float]:
+        """
+        Search memory using ReMe and return structured answer with reasoning.
+
+        Returns:
+            tuple: (answer_dict, agent_messages, duration_ms)
+                answer_dict contains: {"reasoning": str, "answer": str, "memories": str}
+        """
+        start = time.time()
+
+        # Retrieve memories from ReMe using new API
+        result = await self.reme.retrieve_memory(
+            llm_config_name="qwen3-max",
+            query=query,
+            retrieve_top_k=top_k,
+            user_name=user_id,
+            version=self.algo_version,
+            return_dict=True,
+            enable_time_filter=True,
+            enable_thinking_params=True,
+        )
+
+        # Extract memories from response
+        memories = result["answer"]
+        agent_messages = [x.simple_dump(enable_argument_dict=True) for x in result["messages"]]
+        retrieved_nodes = [x.model_dump(exclude_none=True) for x in result["retrieved_nodes"]]
+
+        # Use LLM to generate structured answer from memories
+        answer_result = await answer_question_with_memories(
+            reme=self.reme,
+            question=query,
+            memories=memories,
+            user_id=user_id,
+            model_name=self.eval_model_name,
+        )
+
+        # Add original memories to the result
+        answer_result["memories"] = memories
+        answer_result["retrieved_nodes"] = retrieved_nodes
+
+        duration_ms = (time.time() - start) * 1000
+        return answer_result, agent_messages, duration_ms
+
+
+# ==================== Answer Judge ====================
+
+
+class LongMemEvalJudge:
+    """LongMemEval answer judge using LLM."""
+
+    def __init__(self, reme: ReMe, model: str = "qwen3-max"):
+        self.reme = reme
+        self.model = model
+
+    async def judge_answer(
+        self,
+        question_type: str,
+        question: str,
+        answer: str,
+        response: str,
+        abstention: bool = False,
+    ) -> dict:
+        """
+        Judge if the model's response is correct.
+
+        Returns:
+            dict with is_correct, llm_response, and judge_prompt
+        """
+        prompt = get_anscheck_prompt(question_type, question, answer, response, abstention)
+
+        try:
+            llm_response = await self.reme.get_llm("default").simple_request(
+                prompt=prompt,
+                model_name=self.model,
+            )
+            llm_response_lower = llm_response.strip().lower()
+            is_correct = llm_response_lower.startswith("yes")
+
+            return {
+                "is_correct": is_correct,
+                "llm_response": llm_response,
+                "judge_prompt": prompt,
+            }
+        except Exception as e:
+            return {
+                "is_correct": None,
+                "error": str(e),
+                "judge_prompt": prompt,
+            }
+
+
+# ==================== Metrics ====================
+
+
+class MetricsAggregator:
+    """Aggregates evaluation metrics for LongMemEval."""
+
+    @staticmethod
+    def compute_metrics(results: list[dict]) -> dict[str, Any]:
+        """Compute overall and per-type metrics."""
+        total = len(results)
+        correct = sum(1 for r in results if r.get("judgment", {}).get("is_correct") is True)
+        incorrect = sum(1 for r in results if r.get("judgment", {}).get("is_correct") is False)
+        error = total - correct - incorrect
+
+        metrics = {
+            "total": total,
+            "correct": correct,
+            "incorrect": incorrect,
+            "error": error,
+            "accuracy": correct / total if total > 0 else 0,
+            "accuracy_valid": correct / (correct + incorrect) if (correct + incorrect) > 0 else 0,
+        }
+
+        # Per question type statistics
+        type_stats = {}
+        for r in results:
+            qtype = r.get("question_type", "unknown")
+            if qtype not in type_stats:
+                type_stats[qtype] = {"total": 0, "correct": 0, "incorrect": 0}
+            type_stats[qtype]["total"] += 1
+            if r.get("judgment", {}).get("is_correct") is True:
+                type_stats[qtype]["correct"] += 1
+            elif r.get("judgment", {}).get("is_correct") is False:
+                type_stats[qtype]["incorrect"] += 1
+
+        metrics["by_question_type"] = {
+            qtype: {
+                **stats,
+                "accuracy": stats["correct"] / stats["total"] if stats["total"] > 0 else 0,
+                "accuracy_valid": (
+                    stats["correct"] / (stats["correct"] + stats["incorrect"])
+                    if (stats["correct"] + stats["incorrect"]) > 0
+                    else 0
+                ),
+            }
+            for qtype, stats in type_stats.items()
+        }
+
+        return metrics
+
+    @staticmethod
+    def compute_timing_stats(results: list[dict]) -> dict[str, Any]:
+        """Compute timing statistics."""
+        retrieve_times = []
+
+        for r in results:
+            retrieve_ms = r.get("retrieve_duration_ms", 0)
+            if retrieve_ms > 0:
+                retrieve_times.append(retrieve_ms)
+
+        def compute_stats(times: list[float]) -> dict:
+            if not times:
+                return {"count": 0, "total_ms": 0, "avg_ms": 0, "min_ms": 0, "max_ms": 0}
+
+            return {
+                "count": len(times),
+                "total_ms": sum(times),
+                "total_min": sum(times) / 1000 / 60,
+                "avg_ms": sum(times) / len(times),
+                "min_ms": min(times),
+                "max_ms": max(times),
+            }
+
+        return {
+            "retrieve": compute_stats(retrieve_times),
+            "total_time_min": sum(retrieve_times) / 1000 / 60,
+        }
+
+
+# ==================== Main Pipeline ====================
+
+
+class LongMemEvalRetrieveEvaluator:
+    """Retrieve-only evaluator for LongMemEval benchmark using ReMe."""
+
+    def __init__(self, config: RetrieveEvalConfig):
+        self.config = config
+        self.reme = ReMe(
+            default_llm_config={
+                "model_name": self.config.reme_model_name,
+            },
+            llms={
+                "qwen-plus-think": {
+                    "backend": "openai",
+                    "model_name": "qwen-plus",
+                    "extra_body": {
+                        "enable_thinking": True,
+                    },
+                },
+                "qwen3-max-think": {
+                    "backend": "openai",
+                    "model_name": "qwen3-max",
+                    "extra_body": {
+                        "enable_thinking": True,
+                    },
+                },
+                "qwen3-max": {
+                    "backend": "openai",
+                    "model_name": "qwen3-max",
+                    "extra_body": {
+                        "enable_thinking": False,
+                    },
+                },
+            },
+        )
+
+        # Load evaluation prompts into ReMe's prompt handler
+        prompts_yaml_path = Path(__file__).parent / "eval_reme.yaml"
+        self.reme.prompt_handler.load_prompt_by_file(prompts_yaml_path)
+
+        self.file_manager = FileManager(config.output_dir)
+        self.retrieve_processor = RetrieveProcessor(
+            self.reme,
+            config.reme_model_name,
+            config.eval_model_name,
+            config.algo_version,
+            config.enable_thinking_params,
+        )
+        self.judge = LongMemEvalJudge(self.reme, config.eval_model_name)
+        self.data_loader = DataLoader()
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self.reme.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit with cleanup."""
+        await self.reme.close()
+        return False
+
+    async def process_question_entry(self, entry: dict, idx: int) -> dict:
+        """Process a single question entry (retrieve + judge only).
+
+        Args:
+            entry: A question entry from LongMemEval dataset
+            idx: Index of the question
+
+        Returns:
+            Result dictionary
+        """
+        question_id = entry["question_id"]
+        question = entry["question"]
+        answer = entry["answer"]
+        question_type = entry["question_type"]
+        question_date = entry.get("question_date", "")
+        haystack_dates = entry["haystack_dates"]
+        haystack_session_ids = entry["haystack_session_ids"]
+        haystack_sessions = entry["haystack_sessions"]
+
+        # Use question_id as user_id for isolation (same as full eval)
+        user_id = f"longmemeval_{question_id}"
+
+        logger.info(f"\n{'='*60}")
+        logger.info(f"Question ID: {question_id}")
+        logger.info(f"Question Type: {question_type}")
+        logger.info(f"Question: {question}")
+        logger.info(f"Question_date: {question_date}")
+        logger.info(f"Answer: {answer}")
+        logger.info(f"Number of sessions: {len(haystack_sessions)}")
+        logger.info(f"{'='*60}")
+
+        # Skip summary phase - directly search memory and answer question
+        logger.info("  Retrieving and answering question using ReMe...")
+        answer_dict, retrieve_messages, retrieve_duration_ms = await self.retrieve_processor.search_memory(
+            query=f"[Question_date: {question_date} | Question_type: {question_type}] " + question,
+            user_id=user_id,
+            top_k=self.config.top_k,
+        )
+
+        # Extract answer and reasoning from the structured response
+        model_response = answer_dict.get("answer", "")
+        model_reasoning = answer_dict.get("reasoning", "")
+        retrieved_memories = answer_dict.get("memories", "")
+        retrieved_nodes = answer_dict.get("retrieved_nodes", [])
+
+        # Judge answer correctness
+        logger.info("  Judging answer correctness...")
+        judgment = await self.judge.judge_answer(
+            question_type=question_type,
+            question=question,
+            answer=answer,
+            response=model_response,
+        )
+
+        is_correct = judgment.get("is_correct")
+        logger.info(
+            f"  → Answer judgment: {'Correct' if is_correct else 'Incorrect' if is_correct is False else 'Error'}",
+        )
+
+        result = {
+            "question_id": question_id,
+            "question_type": question_type,
+            "question": question,
+            "answer": answer,
+            "question_date": question_date,
+            "haystack_dates": haystack_dates,
+            "haystack_session_ids": haystack_session_ids,
+            "num_sessions": len(haystack_sessions),
+            "model_response": model_response,
+            "model_reasoning": model_reasoning,
+            "retrieved_memories": retrieved_memories,
+            "retrieved_nodes": retrieved_nodes,
+            "judgment": judgment,
+            "retrieve_duration_ms": retrieve_duration_ms,
+            "retrieve_messages": retrieve_messages,
+        }
+
+        # Save individual result
+        self.file_manager.save_question_result(idx, question_id, result)
+
+        logger.info(f"  Question {question_id} - Completed")
+
+        return result
+
+    async def run_evaluation(self):
+        """Run the retrieve-only evaluation pipeline with parallel processing."""
+        start_time = time.time()
+
+        # NOTE: Do NOT clear vector store - we assume memories are already there from previous summary run
+
+        # Load dataset
+        logger.info(f"Loading dataset from: {self.config.data_path}")
+        all_data = self.data_loader.load_json(self.config.data_path)
+        logger.info(f"Total questions in dataset: {len(all_data)}")
+
+        # Filter by question type
+        logger.info(f"Filtering by type (samples_per_type={self.config.samples_per_type}):")
+        filtered_data = self.data_loader.filter_by_type(all_data, self.config.samples_per_type)
+        logger.info(f"Selected {len(filtered_data)} questions after filtering")
+
+        # Apply start_index and end_index on filtered data
+        end_index = self.config.end_index or len(filtered_data)
+        start_index = self.config.start_index
+        end_index = min(end_index, len(filtered_data))
+
+        # Get the slice we want to process
+        data_to_process = filtered_data[start_index:end_index]
+        total_questions = len(data_to_process)
+
+        logger.info(f"Processing {total_questions} questions (index {start_index} to {end_index - 1})")
+
+        print("\n" + "=" * 80)
+        print("LONGMEMEVAL EVALUATION - REME (RETRIEVE ONLY)")
+        print(f"Samples per type: {self.config.samples_per_type} (-1 = all)")
+        print(f"Questions to process: {total_questions} | Top-K: {self.config.top_k}")
+        print(f"Max Concurrency: {self.config.max_concurrency}")
+        print(f"ReMe Model: {self.config.reme_model_name} | Eval Model: {self.config.eval_model_name}")
+        print(f"Algo Version: {self.config.algo_version}")
+        print("⚠️  NOTE: Assumes memories are already in vector store from previous summary run")
+        print("=" * 80 + "\n")
+
+        # Use semaphore to control concurrency
+        semaphore = asyncio.Semaphore(self.config.max_concurrency)
+
+        async def process_with_semaphore(idx: int, original_idx: int, entry: dict) -> Optional[dict]:
+            """Process a question with semaphore for concurrency control."""
+            async with semaphore:
+                question_id = entry["question_id"]
+
+                # Check cache first (use original index for cache file naming)
+                cached_result = self.file_manager.load_question_result(original_idx, question_id)
+                if cached_result:
+                    print(f"⚡ [{idx}/{total_questions}] Skipping question {original_idx} (cached)")
+                    return cached_result
+
+                print(f"\n{'#'*60}")
+                print(f"### [{idx}/{total_questions}] Processing Question {original_idx} ###")
+                print(f"{'#'*60}")
+
+                try:
+                    result = await self.process_question_entry(entry, original_idx)
+                    print(f"✅ [{idx}/{total_questions}] Completed question {original_idx}")
+                    return result
+                except Exception as e:
+                    logger.error(f"❌ Error processing question {original_idx}: {e}")
+                    import traceback
+
+                    traceback.print_exc()
+                    return {
+                        "question_id": question_id,
+                        "error": str(e),
+                        "question_type": entry.get("question_type", "unknown"),
+                        "question": entry.get("question", ""),
+                        "answer": entry.get("answer", ""),
+                        "judgment": {"is_correct": None, "error": str(e)},
+                    }
+
+        # Create all tasks from filtered data (each item is a tuple of (original_idx, entry))
+        tasks = [
+            process_with_semaphore(idx + 1, original_idx, entry)
+            for idx, (original_idx, entry) in enumerate(data_to_process)
+        ]
+
+        # Execute in parallel with controlled concurrency
+        all_results = await asyncio.gather(*tasks, return_exceptions=False)
+
+        # Filter out None results if any
+        all_results = [r for r in all_results if r is not None]
+
+        # Save summary
+        self.file_manager.save_summary(all_results)
+
+        elapsed = time.time() - start_time
+        print(f"\n✅ Processing completed in {elapsed:.2f}s")
+        if total_questions > 0:
+            print(f"   Average time per question: {elapsed / total_questions:.2f}s")
+
+        # Compute and report metrics
+        self._report_metrics(all_results)
+
+        return all_results
+
+    def _report_metrics(self, results: list[dict]):
+        """Report evaluation metrics."""
+        metrics = MetricsAggregator.compute_metrics(results)
+        timing_stats = MetricsAggregator.compute_timing_stats(results)
+
+        print("\n" + "=" * 80)
+        print("EVALUATION SUMMARY - LONGMEMEVAL - REME (RETRIEVE ONLY)")
+        print("=" * 80 + "\n")
+
+        print("📊 Overall Results:")
+        print(f"  ✅ Correct: {metrics['correct']}/{metrics['total']} ({100*metrics['accuracy']:.2f}%)")
+        print(
+            f"  ❌ Incorrect: {metrics['incorrect']}/{metrics['total']}"
+            f" ({100*metrics['incorrect']/metrics['total'] if metrics['total'] > 0 else 0:.2f}%)",
+        )
+        if metrics["error"] > 0:
+            print(f"  ⚠️ Error: {metrics['error']}/{metrics['total']} ({100*metrics['error']/metrics['total']:.2f}%)")
+        print(f"  Accuracy (valid): {100*metrics['accuracy_valid']:.2f}%")
+
+        print("\n📊 Accuracy by Question Type:")
+        print("-" * 60)
+        print(f"{'Question Type':<30} {'Correct':<10} {'Total':<10} {'Accuracy':<10}")
+        print("-" * 60)
+        for qtype in sorted(metrics["by_question_type"].keys()):
+            stats = metrics["by_question_type"][qtype]
+            print(f"{qtype:<30} {stats['correct']:<10} {stats['total']:<10} {100*stats['accuracy']:.2f}%")
+        print("-" * 60)
+
+        print("\n⏱️ Timing Statistics (Retrieve Only):")
+        retrieve = timing_stats["retrieve"]
+        print("  Memory Retrieval:")
+        print(f"    Total Time:  {retrieve['total_min']:.2f} min")
+        print(f"    Avg per Q:   {retrieve['avg_ms']:.0f} ms")
+        print(f"  Total Time:    {timing_stats['total_time_min']:.2f} min")
+
+        # Save metrics
+        final_results = {
+            "accuracy": metrics,
+            "timing": timing_stats,
+        }
+        metrics_file = self.file_manager.base_dir / "eval_statistics.json"
+        with open(metrics_file, "w", encoding="utf-8") as f:
+            json.dump(final_results, f, indent=4, ensure_ascii=False)
+        print(f"\n📁 Statistics saved to: {metrics_file}")
+
+        print("\n" + "=" * 80)
+
+
+# ==================== Entry Point ====================
+
+
+async def main_async(
+    data_path: str,
+    top_k: int = 20,
+    start_index: int = 0,
+    end_index: Optional[int] = None,
+    max_concurrency: int = 1,
+    output_dir: str = "bench_results/longmemeval_reme_retrieve",
+    reme_model_name: str = "qwen-flash",
+    eval_model_name: str = "qwen3-max",
+    algo_version: str = "v1",
+    samples_per_type: int = -1,
+    enable_thinking_params: bool = False,
+    summary_results_dir: Optional[str] = None,
+):
+    """Main async entry point for LongMemEval retrieve-only evaluation with proper resource cleanup."""
+    config = RetrieveEvalConfig(
+        data_path=data_path,
+        top_k=top_k,
+        start_index=start_index,
+        end_index=end_index,
+        max_concurrency=max_concurrency,
+        output_dir=output_dir,
+        reme_model_name=reme_model_name,
+        eval_model_name=eval_model_name,
+        algo_version=algo_version,
+        samples_per_type=samples_per_type,
+        enable_thinking_params=enable_thinking_params,
+        summary_results_dir=summary_results_dir,
+    )
+
+    # Use async context manager for automatic cleanup
+    async with LongMemEvalRetrieveEvaluator(config) as evaluator:
+        await evaluator.run_evaluation()
+
+
+def main(
+    data_path: str,
+    top_k: int = 20,
+    start_index: int = 0,
+    end_index: Optional[int] = None,
+    max_concurrency: int = 1,
+    output_dir: str = "bench_results/longmemeval_reme_retrieve",
+    reme_model_name: str = "qwen-flash",
+    eval_model_name: str = "qwen3-max",
+    algo_version: str = "v1",
+    samples_per_type: int = -1,
+    enable_thinking_params: bool = False,
+    summary_results_dir: Optional[str] = None,
+):
+    """Main entry point for LongMemEval retrieve-only evaluation."""
+    asyncio.run(
+        main_async(
+            data_path=data_path,
+            top_k=top_k,
+            start_index=start_index,
+            end_index=end_index,
+            max_concurrency=max_concurrency,
+            output_dir=output_dir,
+            reme_model_name=reme_model_name,
+            eval_model_name=eval_model_name,
+            algo_version=algo_version,
+            samples_per_type=samples_per_type,
+            enable_thinking_params=enable_thinking_params,
+            summary_results_dir=summary_results_dir,
+        ),
+    )
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Evaluate ReMe on LongMemEval benchmark (Retrieve Phase Only)",
+    )
+    parser.add_argument(
+        "--data_path",
+        type=str,
+        # default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_s_cleaned.json",
+        default="/Users/zhouwk/PycharmProjects/MemAgent/dataset/longmemeval/longmemeval_oracle.json",
+        help="Path to LongMemEval JSON file",
+    )
+    parser.add_argument(
+        "--top_k",
+        type=int,
+        default=10,
+        help="Number of memories to retrieve (default: 10)",
+    )
+    parser.add_argument(
+        "--start_index",
+        type=int,
+        default=0,
+        help="Start index for processing questions (default: 0)",
+    )
+    parser.add_argument(
+        "--end_index",
+        type=int,
+        default=None,
+        help="End index for processing questions (default: None, process all)",
+    )
+    parser.add_argument(
+        "--max_concurrency",
+        type=int,
+        default=8,
+        help="Maximum concurrent question processing (default: 1)",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="bench_results/longmemeval_reme_retrieve_gpt4",
+        help="Output directory for results",
+    )
+    parser.add_argument(
+        "--reme_model_name",
+        type=str,
+        default="gpt-4o-mini-2024-07-18",
+        help="Model name for ReMe operations (default: gpt-4o-mini-2024-07-18)",
+    )
+    parser.add_argument(
+        "--eval_model_name",
+        type=str,
+        default="gpt-4o-mini-2024-07-18",
+        help="Model name for evaluation/judgment (default: gpt-4o-mini-2024-07-18)",
+    )
+    parser.add_argument(
+        "--algo_version",
+        type=str,
+        default="longmemeval",
+        help="Algorithm version for retrieval (default: longmemeval)",
+    )
+    parser.add_argument(
+        "--samples_per_type",
+        type=int,
+        default=4,
+        help="Number of samples per question type, -1 for all (default: 4)",
+    )
+    parser.add_argument(
+        "--enable_thinking_params",
+        action="store_true",
+        default=False,
+        help="Enable thinking parameters for retrieval (default: False)",
+    )
+    parser.add_argument(
+        "--summary_results_dir",
+        type=str,
+        default="/Users/zhouwk/PycharmProjects/ReMe/benchmark/longmemeval/bench_results",
+        help="Optional: path to previous summary results directory (for reference)",
+    )
+    parser.add_argument(
+        "--no_cache",
+        action="store_true",
+        default=False,
+        help="Ignore cached results and re-run all questions (default: False)",
+    )
+
+    args = parser.parse_args()
+    print(f"args={args}!")
+
+    main(
+        data_path=args.data_path,
+        top_k=args.top_k,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        max_concurrency=args.max_concurrency,
+        output_dir=args.output_dir,
+        reme_model_name=args.reme_model_name,
+        eval_model_name=args.eval_model_name,
+        algo_version=args.algo_version,
+        samples_per_type=args.samples_per_type,
+        enable_thinking_params=args.enable_thinking_params,
+        summary_results_dir=args.summary_results_dir,
+    )

benchmark/longmemeval/eval_reme.yaml

@@ -0,0 +1,548 @@
+TEMPLATE_MEMOS: |
+  Memories for user {user_id}:
+    {memories}
+
+PROMPT_MEMZERO_JSON: |
+  # CONTEXT:
+  {context}
+
+  # CONTEXT PRIORITY:
+  When the context contains information from multiple sources, follow this strict priority order:
+  1. **Historical Dialogue** (highest priority) - Direct conversation content
+  2. **Extracted Memories** (medium priority) - Summarized memory points
+  3. **User Profile** (lowest priority) - General user information
+
+  # Question:
+  {question}
+
+  # OUTPUT FORMAT:
+  Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
+  Please provide your response in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "reasoning content",
+    "answer": "Provide a detailed answer"
+  }}
+  ```
+
+PROMPT_MEMZERO_JSON2: |
+  # CONTEXT:
+  {context}
+
+  # CONTEXT PRIORITY:
+  When the context contains information from multiple sources, follow this strict priority order:
+  1. **Historical Dialogue** (highest priority) - Direct conversation content
+  2. **Extracted Memories** (medium priority) - Summarized memory points
+  3. **User Profile** (lowest priority) - General user information
+
+  # Question:
+  {question}
+
+  # OUTPUT FORMAT:
+  Do not hallucinate; strictly answer the user's question based on the content of the CONTEXT.
+  Please provide your response in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "reasoning content",
+    "answer": "Provide a detailed answer"
+  }}
+  ```
+
+PROMPT_MEMZERO: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories from both speakers
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.),
+     calculate the actual date based on the memory timestamp. For example, if a memory from
+     4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example,
+     convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
+     timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories from both speakers. Do not confuse character
+     names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+PROMPT_ZEP: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.),
+     calculate the actual date based on the memory timestamp. For example, if a memory from
+     4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example,
+     convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory
+     timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories. Do not confuse character
+     names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  Context:
+
+  {context}
+
+  Question: {question}
+  Answer:
+
+PROMPT_MEMOS: |
+  You are a knowledgeable and helpful AI assistant.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories. Synthesize information across different entries if needed to form a complete answer.
+  2. Pay close attention to the timestamps to determine the answer. If memories contain contradictory information, the **most recent memory** is the source of truth.
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories.
+  4. Your answer must be grounded in the memories. However, you may use general world knowledge to interpret or complete information found within a memory (e.g., identifying a landmark mentioned by description).
+  5. If the question involves time references (like "last year", "two months ago", etc.), you **must** calculate the actual date based on the memory's timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years in your final answer.
+  7. Do not confuse character names mentioned in memories with the actual users who created them.
+  8. The answer must be brief (under 5-6 words) and direct, with no extra description.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question.
+  2. Synthesize findings from multiple memories if a single entry is insufficient.
+  3. Examine timestamps and content carefully, looking for explicit dates, times, locations, or events.
+  4. If the answer requires calculation (e.g., converting relative time references), perform the calculation.
+  5. Formulate a precise, concise answer based on the evidence from the memories (and allowed world knowledge).
+  6. Double-check that your answer directly addresses the question asked and adheres to all instructions.
+  7. Ensure your final answer is specific and avoids vague time references.
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+PROMPT_MEMOBASE: |
+  You are an intelligent memory assistant tasked with retrieving accurate information from conversation memories.
+
+  # CONTEXT:
+  You have access to memories from two speakers in a conversation. These memories contain
+  timestamped information that may be relevant to answering the question.
+
+  # INSTRUCTIONS:
+  1. Carefully analyze all provided memories from both speakers
+  2. Pay special attention to the timestamps to determine the answer
+  3. If the question asks about a specific event or fact, look for direct evidence in the memories
+  4. If the memories contain contradictory information, prioritize the most recent memory
+  5. If there is a question about time references (like "last year", "two months ago", etc.), calculate the actual date based on the memory timestamp. For example, if a memory from 4 May 2022 mentions "went to India last year," then the trip occurred in 2021.
+  6. Always convert relative time references to specific dates, months, or years. For example, convert "last year" to "2022" or "two months ago" to "March 2023" based on the memory timestamp. Ignore the reference while answering the question.
+  7. Focus only on the content of the memories from both speakers. Do not confuse character names mentioned in memories with the actual users who created those memories.
+  8. The answer should be less than 5-6 words.
+
+  # APPROACH (Think step by step):
+  1. First, examine all memories that contain information related to the question
+  2. Examine the timestamps and content of these memories carefully
+  3. Look for explicit mentions of dates, times, locations, or events that answer the question
+  4. If the answer requires calculation (e.g., converting relative time references), show your work
+  5. Formulate a precise, concise answer based solely on the evidence in the memories
+  6. Double-check that your answer directly addresses the question asked
+  7. Ensure your final answer is specific and avoids vague time references
+
+  {context}
+
+  Question: {question}
+
+  Answer:
+
+
+EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY: |
+  You are a strict **"Memory Integrity" evaluator**.
+  Your core task is to assess whether an AI memory system has **missed any key memory points** after processing a conversation. This evaluation measures the system’s **memory integrity**, i.e., its ability to resist **amnesia** or **omission**.
+
+  # Evaluation Context & Data:
+
+  1. **Extracted Memories:**
+     These are all the memory items actually extracted by the memory system.
+     {memories}
+
+  2. **Expected Memory Point:**
+     The key memory point that *should* have been extracted.
+     {expected_memory_point}
+
+  # Evaluation Instructions:
+
+  1. For each **Expected Memory Point**, search within the **Extracted Memories** list for corresponding or related information. Ignore unrelated items.
+  2. Based on the following scoring rubric, rate how well the memory system captured the **Expected Memory Point** and provide a detailed explanation.
+
+  # Scoring Rubric:
+
+  * **2:** Fully covered or implied.
+    One or more items in “Extracted Memories” fully cover or logically imply all information in the “Expected Memory Point.”
+
+  * **1:** Partially covered or mentioned.
+    Some information in “Extracted Memories” mentions part of the “Expected Memory Point,” but key information is missing, inaccurate, or slightly incorrect.
+
+  * **0:** Not mentioned or incorrect.
+    “Extracted Memories” contains no mention of the “Expected Memory Point,” or the corresponding information is entirely wrong.
+
+  # Scoring Notes:
+
+  * For **compound Expected Memory Points** (with multiple elements such as person/event/time/location/preference, etc.):
+
+    * All elements correct → **2 points**
+    * Some elements correct / uncertain → **1 point**
+    * Key elements missing or wrong → **0 points**
+
+  * Semantic matching is acceptable; exact wording is **not** required.
+
+  * If “Extracted Memories” contains **conflicting information**, assign the **best possible coverage score** and mention the conflict in your reasoning.
+
+  * Extra or stylistically different memories do **not** reduce the score; only the coverage of the **Expected Memory Point** matters.
+
+  * For uncertain wording (“might,” “probably,” “tends to,” etc.):
+
+    * If the Expected Memory Point is a definite statement, usually assign **1 point**.
+
+  * If critical fields (e.g., time, entity name, relationship) are partly wrong but others match → **1 point**.
+
+    * If all key fields are wrong or missing → **0 points**.
+
+  # Output Format:
+
+  Please output your result in the following JSON format:
+
+  ```json
+  {{
+    "reasoning": "Provide a concise justification for the score",
+    "score": "2|1|0"
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_MEMORY_ACCURACY: |
+  You are a **Dialogue Memory Accuracy Evaluator.** Your task is to evaluate the **accuracy** of a memory extracted by an AI memory system, based on three given inputs: the dialogue content, the *target (gold)* memory points (the correct annotated memories), and the *candidate* memory to be evaluated. The goal is to output a **structured evaluation result**.
+
+  # Input Content
+
+  * **Dialogue:**
+    {dialogue}
+
+  * **Golden Memories (Target Memory Points):**
+    The correct memory points pre-annotated for this dialogue in the evaluation dataset.
+    {golden_memories}
+
+  * **Candidate Memory:**
+    The memory extracted by the system to be evaluated.
+    {candidate_memory}
+
+  # Evaluation Principles and Definitions
+
+  ### 1) Support / Entailment
+
+  * An **information point** (atomic fact) in the candidate memory is considered *supported* if it can be directly stated or semantically entailed (via synonym, paraphrase, or equivalent expression) by the *Dialogue* or *Golden Memories*.
+  * Only the given dialogue and golden memories can be used for judgment — **no external knowledge** or assumptions are allowed.
+    Any information not appearing in or inferable from these two sources is considered *unsupported*.
+  * Pay careful attention to **negation**, **quantities**, **time**, and **subjects**.
+    If the candidate statement contradicts the dialogue or golden memories, it is considered a **conflict**.
+
+  ### 2) Memory Accuracy Score (integer: 0 / 1 / 2)
+
+  * **2 points:** Every information point in the candidate memory is supported by the dialogue or golden memories, with **no contradictions or hallucinations**.
+  * **1 point:** The candidate memory is *partially correct* (at least one supported information point) but also includes *unsupported* or *contradictory* content.
+  * **0 points:** The candidate memory is **entirely unsupported or contradictory** to the sources (i.e., a “hallucinated memory”).
+
+  > Note:
+  >
+  > * If a candidate memory contains multiple information points, **any unsupported or contradictory element** prevents a full score (2).
+  > * If both supported and unsupported/conflicting content appear, assign a score of **1**.
+
+  ### 3) Inclusion in Golden Memories (Boolean field-level judgment)
+
+  **Definition:**
+
+  * **Atomic information point:** the smallest factual unit in the candidate memory (e.g., *name = Li Si*, *age = 25*, *location = Beijing*, *preference = coffee*, *budget ≤ 2000*, *meeting_time = Wednesday 10:00*, *tool = Zoom*, etc.).
+  * **Field / Slot:** the semantic dimension of an information point (e.g., *name*, *age*, *residence*, *food preference*, *budget*, *meeting time*, *meeting tool*, etc.).
+
+  **Judgment Rules (independent of correctness):**
+
+  * **true:**
+    Every atomic information point in the candidate memory has a corresponding **field** in the golden memories (allowing for synonyms, paraphrases, or equivalent expressions; ignore value, polarity, or quantity differences).
+
+    * Note: A single field in the gold list may match multiple candidate points (e.g., multiple “drink preference” facts can be covered by one “drink preference” field in gold).
+  * **false:**
+    If **any** atomic information point’s field in the candidate memory cannot be found in the golden memories, mark as *false*.
+
+  **Important Notes:**
+
+  * Field matching is restricted to fields that are **explicitly present or semantically recognizable** in the golden memories — no external knowledge may be used to expand the field set.
+  * Differences in **values** (e.g., “Zhang San” vs. “Li Si”), **polarity** (like/dislike), or **exact number/time** do **not** affect this Boolean judgment.
+
+  # Evaluation Procedure
+
+  For each candidate memory:
+
+  1. **Decompose** it into atomic information points (e.g., name, number, location, preference).
+  2. For each information point, **search** the dialogue and golden memories for supporting or contradictory evidence.
+  3. Assign the **accuracy_score** (0 / 1 / 2) according to the rules above.
+  4. Determine **is_included_in_golden_memories (true/false)**:
+
+     * Identify each information point’s field;
+     * If *all* fields exist in the golden memories, mark as *true*; otherwise, *false*.
+  5. Provide a **concise Chinese explanation** in `"reason"`, citing key evidence (short excerpts allowed), and clearly state any unsupported or contradictory parts if applicable.
+
+  # Output Format (strictly required)
+
+  Output **only one JSON object**, with the following three fields:
+
+  * `"accuracy_score"`: `"0"` or `"1"` or `"2"`
+  * `"is_included_in_golden_memories"`: `"true"` or `"false"`
+  * `"reason"`: `"brief explanation in Chinese"`
+
+  Do **not** include any other text, explanation, or fields.
+  Do **not** include the candidate memory text inside the JSON.
+
+  Please output **only** the following JSON (in a code block):
+
+  ```json
+  {{
+    "accuracy_score": "2 | 1 | 0",
+    "is_included_in_golden_memories": "true | false",
+    "reason": "Brief explanation in Chinese"
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_UPDATE_MEMORY: |
+  Your task is to **evaluate the update accuracy** of an AI memory system.
+  Based on the information provided below, determine whether the system-generated **“Generated Memories”** correctly **includes** the **Target Memory for Update**.
+
+  # Background Information
+
+  The following information is provided for evaluation:
+
+  1. **Generated Memories:**
+     This is the list of memory points generated by the system after the current dialogue.
+     {memories}
+
+  2. **Target Memory for Update:**
+     This is the correct, updated version of the memory point that should have been produced — the one we focus on in this evaluation.
+     {updated_memory}
+
+  3. **Original Memory Content:**
+     This is the original version of the target memory before the update.
+     {original_memory}
+
+  # Evaluation Criteria
+
+  Please make your judgment **strictly based on the content update of the “Target Memory for Update.”**
+  Use the following categories:
+
+  ### Correct Update
+
+  * **Generated Memories** **contains all information points** from the “Target Memory for Update,” accurately and completely reflecting the intended update.
+  * **Key fields** (e.g., date, time, values, proper nouns, etc.) must match exactly.
+  * The **original memory** is effectively replaced or marked as outdated.
+  * Synonymous or slightly rephrased expressions are acceptable.
+
+  ### Hallucinated Update
+
+  * **Factual error:** The **Generated Memories** includes a new memory related to the “Target Memory for Update,” but its content contains factual mistakes or contradictions compared to the correct update.
+
+  ### Omitted Update
+
+  * **Completely omitted:** The **Generated Memories** contains no new memory related to the “Target Memory for Update.”
+  * **Partially omitted:** A related new memory was generated in **Generated Memories**, but it **misses key information** that should have been included.
+
+  ### Other
+
+  Used for update failures that do **not clearly fall** into the above categories of “Hallucination” or “Omission.”
+
+  # Output Requirements
+
+  Please return your evaluation strictly in the following JSON format and provide a concise explanation.
+
+  ```json
+  {{
+    "reason": "Briefly explain your reasoning here and why it fits this category.",
+    "evaluation_result": "Correct | Hallucination | Omission | Other"
+  }}
+  ```
+
+EVALUATION_PROMPT_FOR_QUESTION: |
+  You are an **evaluation expert for AI memory system question answering**.
+  Based **only** on the provided **“Question”**, **“Reference Answer”**, and **“Key Memory Points”** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **“Memory System Response.”** Classify it as one of **“Correct”**, **“Hallucination”**, or **“Omission.”** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
+
+  # Evaluation Criteria
+
+  ## Answer Type Classification
+
+  ### 1. Correct
+
+  * The “Memory System Response” accurately answers the “Question,” and its content is **semantically equivalent** to the “Reference Answer.”
+  * It contains **no contradictions** with the “Key Memory Points” or “Reference Answer.”
+  * It introduces **no unsupported details** beyond the “Key Memory Points” that could alter the conclusion.
+  * Synonyms, paraphrasing, and reasonable summarization are acceptable.
+
+  ### 2. Hallucination
+
+  * The “Memory System Response” includes information or facts that **contradict or are inconsistent** with the “Reference Answer” or the “Key Memory Points.”
+  * When the “Reference Answer” is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
+  * Extra irrelevant information that does **not change** the conclusion is **not** considered hallucination by itself; however, if it **changes or misleads** the conclusion, or **contradicts** the “Key Memory Points,” it should be judged as a **Hallucination**.
+
+  ### 3. Omission
+
+  * The response is **incomplete** compared to the “Reference Answer.”
+  * It explicitly states “don’t know,” “can’t remember,” or “no related memory,” even though relevant information exists in the “Key Memory Points.”
+  * For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
+
+  ## Priority Rules (Conflict Handling)
+
+  * If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
+  * If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
+  * Only when the meaning is **fully equivalent** to the reference answer should it be classified as **Correct**.
+
+  ## Detailed Guidelines and Tolerance
+
+  * Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
+  * For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
+  * If the reference answer is *“unknown / cannot be determined”* and the system provides a definite fact, that is a **Hallucination**.
+    If the system also answers *“unknown”* (without guessing), it may be **Correct**.
+  * The evaluation must rely **only** on the *Reference Answer*, *Key Memory Points*, and *System Response* — no external context, world knowledge, or speculative reasoning is allowed.
+
+  # Information for Evaluation
+
+  * **Question:**
+    {question}
+
+  * **Reference Answer:**
+    {reference_answer}
+
+  * **Key Memory Points:**
+    {key_memory_points}
+
+  * **Memory System Response:**
+    {response}
+
+  # Output Requirements
+
+  Please provide your evaluation result **strictly** in the JSON format below.
+  Do **not** add any extra explanation or comments outside the JSON block.
+
+  ```json
+  {{
+    "reasoning": "Provide a concise and traceable evaluation rationale: first compare the system’s response with the Key Memory Points (which were correctly used, which were missing, and whether there was any fabrication/contradiction), then assess its consistency with the Reference Answer, and finally state the classification basis.",
+    "evaluation_result": "Correct | Hallucination | Omission"
+  }}
+  ```
+
+
+EVALUATION_PROMPT_FOR_QUESTION2: |
+  You are an **evaluation expert for AI memory system question answering**.
+
+  Based **only** on the provided **"Question"**, **"Reference Answer"**, and **"Key Memory Points"** (the essential facts needed to derive the reference answer), strictly evaluate the **accuracy** of the **"Memory System Response."** Classify it as one of **"Correct"**, **"Hallucination"**, or **"Omission."** Do **not** use any external knowledge or subjective inference. Finally, output your judgment **strictly** in the specified JSON format.
+
+  # Evaluation Criteria
+
+  ## Answer Type Classification
+
+  ### 1. Correct
+
+  * The "Memory System Response" accurately answers the "Question," and its content is **semantically equivalent** to the "Reference Answer."
+  * It contains **no contradictions** with the "Key Memory Points" or "Reference Answer."
+  * **Extra details not present in the Key Memory Points are allowed and should not be penalized**, as long as they:
+    - Do not contradict the Key Memory Points or Reference Answer
+    - Do not change or mislead the core conclusion
+    - Are reasonable additional context that the memory system may have retained from the conversation
+  * The memory system may have stored additional information beyond the Key Memory Points. Such extra information should be treated as **supplementary context** rather than hallucination, provided it does not conflict with the core answer.
+  * Synonyms, paraphrasing, and reasonable summarization are acceptable.
+
+  ### 2. Hallucination
+
+  * The "Memory System Response" includes information or facts that **contradict or are inconsistent** with the "Reference Answer" or the "Key Memory Points."
+  * The response provides information that **directly contradicts** known facts from the Key Memory Points.
+  * When the "Reference Answer" is labeled as *unknown/uncertain*, yet the response provides a specific verifiable fact or conclusion.
+  * **Important:** Extra information that is NOT in Key Memory Points is **NOT automatically a hallucination**. Only classify as hallucination if the extra information:
+    - Directly contradicts the Key Memory Points or Reference Answer
+    - Changes or misleads the core conclusion in a way that makes the answer incorrect
+    - Provides a definitive answer when the Reference Answer indicates uncertainty
+
+  ### 3. Omission
+
+  * The response is **incomplete** compared to the "Reference Answer."
+  * It explicitly states "don't know," "can't remember," or "no related memory," even though relevant information exists in the "Key Memory Points."
+  * For multi-element questions, **all elements must be correct and present**; omission of **any** element is considered an **Omission**.
+
+  ## Priority Rules (Conflict Handling)
+
+  * If the response contains **both missing necessary information** and **fabricated/contradictory information**, classify it as **Hallucination**.
+  * If there is **no fabrication/contradiction** but some necessary information is missing, classify it as **Omission**.
+  * If the core answer is correct and complete, classify as **Correct** even if there are extra details not in Key Memory Points (as long as they don't contradict or mislead).
+
+  ## Detailed Guidelines and Tolerance
+
+  * Equivalent expressions of numbers, times, and units are acceptable, but the **numerical values themselves must not differ**.
+  * For multi-element questions, **all elements must be complete and accurate**; missing any element counts as **Omission**.
+  * If the reference answer is *"unknown / cannot be determined"* and the system provides a definite fact, that is a **Hallucination**.
+    If the system also answers *"unknown"* (without guessing), it may be **Correct**.
+  * **Focus on evaluating whether the core answer to the question is correct**, not whether the response is limited to only the Key Memory Points.
+  * Extra contextual information (e.g., additional preferences, related details) should be viewed as enrichment, not as errors, unless they contradict or mislead.
+
+  # Information for Evaluation
+
+  * **Question:**
+    {question}
+
+  * **Reference Answer:**
+    {reference_answer}
+
+  * **Key Memory Points:**
+    {key_memory_points}
+
+  * **Memory System Response:**
+    {response}
+
+  # Output Requirements
+
+  Please provide your evaluation result **strictly** in the JSON format below.
+  Do **not** add any extra explanation or comments outside the JSON block.
+
+  ```json
+  {{
+    "reasoning": "Provide a concise and traceable evaluation rationale: first verify that the system's response correctly includes all required elements from the Reference Answer, then check if any information contradicts the Key Memory Points or Reference Answer. Extra details not in Key Memory Points should be noted but not penalized unless they contradict or mislead. Finally state the classification basis.",
+    "evaluation_result": "Correct | Hallucination | Omission"
+  }}
+  ```
+  """

benchmark/longmemeval/eval_tools.py

@@ -0,0 +1,232 @@
+"""Evaluation tools for ReMe LongMemEval benchmark."""
+
+from pathlib import Path
+
+import yaml
+
+from reme.reme import ReMe
+
+
+# Load prompts from YAML file
+_YAML_PATH = Path(__file__).parent / "eval_reme.yaml"
+with open(_YAML_PATH, "r", encoding="utf-8") as f:
+    _PROMPTS = yaml.safe_load(f)
+
+
+async def evaluation_for_memory_integrity(
+    reme: ReMe,
+    extract_memories: str,
+    target_memory: str,
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Memory Integrity Evaluation
+
+    Args:
+        reme: ReMe instance
+        extract_memories: A formatted string concatenating all memory points extracted by the memory system.
+        target_memory: The target key memory point.
+        model_name: Model name for evaluation
+
+    Returns:
+        dict with 'reasoning' and 'score' fields
+    """
+    prompt = _PROMPTS["EVALUATION_PROMPT_FOR_MEMORY_INTEGRITY"].format(
+        memories=extract_memories,
+        expected_memory_point=target_memory,
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result
+
+
+async def evaluation_for_memory_accuracy(
+    reme: ReMe,
+    dialogue: str,
+    golden_memories: str,
+    candidate_memory: str,
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Memory Accuracy Evaluation
+
+    Args:
+        reme: ReMe instance
+        dialogue: The complete human-machine dialogue record.
+        golden_memories: The core memory points for this dialogue segment in the evaluation set .
+        candidate_memory: A specific memory point extracted by the memory system being evaluated.
+        model_name: Model name for evaluation
+
+    Returns:
+        dict with 'accuracy_score', 'is_included_in_golden_memories', and 'reason' fields
+    """
+    prompt = _PROMPTS["EVALUATION_PROMPT_FOR_MEMORY_ACCURACY"].format(
+        dialogue=dialogue,
+        golden_memories=golden_memories,
+        candidate_memory=candidate_memory,
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result
+
+
+async def evaluation_for_update_memory(
+    reme: ReMe,
+    extract_memories: str,
+    target_update_memory: str,
+    original_memory: str,
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Memory Update Evaluation
+
+    Args:
+        reme: ReMe instance
+        extract_memories: A formatted string concatenating all memory points extracted by the memory system .
+        target_update_memory: The target updated memory point.
+        original_memory: A formatted string concatenating all original memory points corresponding.
+        model_name: Model name for evaluation
+
+    Returns:
+        dict with 'reason' and 'evaluation_result' fields
+    """
+    prompt = _PROMPTS["EVALUATION_PROMPT_FOR_UPDATE_MEMORY"].format(
+        memories=extract_memories,
+        updated_memory=target_update_memory,
+        original_memory=original_memory,
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result
+
+
+async def evaluation_for_question(
+    reme: ReMe,
+    question: str,
+    reference_answer: str,
+    key_memory_points: str,
+    response: str,
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Question-Answering Evaluation
+
+    Args:
+        reme: ReMe instance
+        question: The question string to be evaluated.
+        reference_answer: The reference (gold-standard) answer.
+        key_memory_points: The memory points used to derive the reference answer.
+        response: The answer produced by the memory system.
+        model_name: Model name for evaluation
+
+    Returns:
+        dict with 'reasoning' and 'evaluation_result' fields
+    """
+    prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION"].format(
+        question=question,
+        reference_answer=reference_answer,
+        key_memory_points=key_memory_points,
+        response=response,
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result
+
+
+async def evaluation_for_question2(
+    reme: ReMe,
+    question: str,
+    reference_answer: str,
+    key_memory_points: str,
+    response: str,
+    dialogue: str = "",
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Question-Answering Evaluation with Dialogue Context (Version 2)
+
+    Args:
+        reme: ReMe instance
+        question: The question string to be evaluated.
+        reference_answer: The reference (gold-standard) answer.
+        key_memory_points: The memory points used to derive the reference answer.
+        response: The answer produced by the memory system.
+        dialogue: The formatted dialogue history (role, content, time_created).
+        model_name: Model name for evaluation
+
+    Returns:
+        dict with 'reasoning' and 'evaluation_result' fields
+    """
+    prompt = _PROMPTS["EVALUATION_PROMPT_FOR_QUESTION2"].format(
+        question=question,
+        reference_answer=reference_answer,
+        key_memory_points=key_memory_points,
+        response=response,
+        dialogue=dialogue if dialogue else "",
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result
+
+
+async def answer_question_with_memories(
+    reme: ReMe,
+    question: str,
+    memories: str,
+    user_id: str = None,
+    model_name: str = "qwen3-max",
+) -> dict:
+    """
+    Answer a question using retrieved memories with PROMPT_MEMZERO_JSON template.
+
+    Args:
+        reme: ReMe instance
+        question: The question to answer
+        memories: The retrieved memories (formatted as context)
+        user_id: Optional user ID for context formatting
+        model_name: Model name for LLM request
+
+    Returns:
+        dict with 'reasoning' and 'answer' fields
+    """
+    # Format context with memories
+    if user_id:
+        context = _PROMPTS["TEMPLATE_MEMOS"].format(
+            user_id=user_id,
+            memories=memories,
+        )
+    else:
+        context = f"Memories:\n{memories}"
+
+    # Use PROMPT_MEMZERO_JSON template for structured JSON response
+    prompt = _PROMPTS["PROMPT_MEMZERO_JSON"].format(
+        context=context,
+        question=question,
+    )
+
+    result = await reme.llm.simple_request_for_json(
+        prompt=prompt,
+        model_name=model_name,
+    )
+
+    return result

benchmark/longmemeval/kill.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-# 杀死指定进程及其所有子进程
-# Usage: bash kill.sh <PID>
-
-if [ -z "$1" ]; then
-    echo "Usage: bash kill.sh <PID>"
-    echo "  杀死指定进程及其所有子进程"
-    exit 1
-fi
-
-PID=$1
-
-# 检查进程是否存在
-if ! kill -0 "$PID" 2>/dev/null; then
-    echo "进程 $PID 不存在"
-    exit 1
-fi
-
-# 递归收集所有子进程（包括子进程的子进程）
-collect_children() {
-    local parent=$1
-    local children
-    children=$(ps -o pid= --ppid "$parent" 2>/dev/null | tr -d ' ')
-    for child in $children; do
-        collect_children "$child"
-    done
-    echo "$parent"
-}
-
-# 收集进程树（子进程在前，父进程在后，保证先杀子再杀父）
-PROCESS_TREE=$(collect_children "$PID")
-TOTAL=$(echo "$PROCESS_TREE" | wc -l | tr -d ' ')
-
-echo "进程树（共 $TOTAL 个进程）:"
-while read -r p; do
-    cmd=$(ps -o args= -p "$p" 2>/dev/null | head -c 80)
-    printf "  PID=%-8s %s\n" "$p" "$cmd"
-done <<< "$PROCESS_TREE"
-
-# 先 SIGTERM 优雅终止
-echo ""
-echo "发送 SIGTERM..."
-while read -r p; do
-    kill "$p" 2>/dev/null
-done <<< "$PROCESS_TREE"
-
-# 等待最多 5 秒
-for i in $(seq 1 5); do
-    alive=false
-    while read -r p; do
-        if kill -0 "$p" 2>/dev/null; then
-            alive=true
-        fi
-    done <<< "$PROCESS_TREE"
-    if [ "$alive" = false ]; then
-        break
-    fi
-    sleep 1
-done
-
-# 检查是否还有残留，强制 SIGKILL
-remaining=false
-while read -r p; do
-    if kill -0 "$p" 2>/dev/null; then
-        remaining=true
-    fi
-done <<< "$PROCESS_TREE"
-
-if [ "$remaining" = true ]; then
-    echo "部分进程未响应，发送 SIGKILL..."
-    while read -r p; do
-        kill -9 "$p" 2>/dev/null
-    done <<< "$PROCESS_TREE"
-fi
-
-echo "已终止进程树（根 PID=$PID，共 $TOTAL 个进程）"

benchmark/longmemeval/llms.py

@@ -0,0 +1,97 @@
+"""LLM utilities for LongMemEval benchmark evaluation."""
+
+import asyncio
+import json
+import logging
+import re
+
+from tenacity import retry, stop_after_attempt, wait_random_exponential, before_sleep_log
+
+from reme.core.schema import Message
+from reme.core.utils import load_env
+from reme.reme import ReMe
+
+logger = logging.getLogger(__name__)
+
+load_env()
+
+WAIT_TIME_LOWER = 1
+WAIT_TIME_UPPER = 60
+RETRY_TIMES = 5
+
+
+@retry(
+    wait=wait_random_exponential(min=WAIT_TIME_LOWER, max=WAIT_TIME_UPPER),
+    stop=stop_after_attempt(3),
+    reraise=True,
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+)
+async def llm_request(reme: ReMe, prompt: str, model_name: str = "qwen3-max", **kwargs) -> str:
+    """Make an LLM request using ReMe's LLM with optional model override.
+
+    Args:
+        reme: ReMe instance
+        prompt: The prompt to send to the LLM
+        model_name: Optional model name to override the default model (default: "qwen3-max")
+        **kwargs: Additional arguments to pass to the chat method
+
+    Returns:
+        The assistant's response content
+    """
+    assistant_message = await reme.llm.chat(
+        messages=[
+            Message(role="user", content=prompt),
+        ],
+        model_name=model_name,
+        **kwargs,
+    )
+    return assistant_message.content
+
+
+@retry(
+    wait=wait_random_exponential(min=WAIT_TIME_LOWER, max=WAIT_TIME_UPPER),
+    stop=stop_after_attempt(RETRY_TIMES),
+    reraise=True,
+    before_sleep=before_sleep_log(logger, logging.WARNING),
+)
+async def llm_request_for_json(reme: ReMe, prompt: str, model_name: str = "qwen-flash", **kwargs) -> dict:
+    """Make an LLM request expecting JSON response using ReMe's LLM.
+
+    Args:
+        reme: ReMe instance
+        prompt: The prompt to send to the LLM
+        model_name: Optional model name to override the default model (default: "qwen-flash")
+        **kwargs: Additional arguments to pass to the chat method
+
+    Returns:
+        Parsed JSON object from the LLM response
+
+    Raises:
+        ValueError: If no JSON block is found in the model output
+    """
+    content = await llm_request(reme, prompt, model_name=model_name, **kwargs)
+
+    match = re.search(r"```json\s*(\{.*?\})\s*```", content, re.DOTALL)
+    if not match:
+        raise ValueError(f"No JSON block found in model output: {content}")
+
+    json_str = match.group(1).strip()
+    return json.loads(json_str)
+
+
+if __name__ == "__main__":
+
+    async def test():
+        """Simple manual test for JSON LLM request."""
+        reme = ReMe()
+        await reme.start()
+        try:
+            r = await llm_request_for_json(
+                reme,
+                'hello? answer in ```json\n{"answer": "..."}```',
+            )
+            print(r)
+        finally:
+            await reme.close()
+
+    asyncio.run(test())

benchmark/longmemeval/run.py

@@ -1,816 +0,0 @@
-"""LongMemEval evaluation runner for ReMe.
-
-Evaluates ReMe's long-term memory capability using the LongMemEval dataset.
-Each item gets an isolated workspace; sessions are ingested in chronological order;
-dream is triggered when sessions cross midnight (23:00); finally questions are
-answered via an agentic (ReAct) approach and judged by an LLM.
-
-Usage:
-    python benchmark/longmemeval/run.py
-    python benchmark/longmemeval/run.py --config benchmark/longmemeval/config.yaml
-    python benchmark/longmemeval/run.py -q                          # quiet: only eval-level logs
-    python benchmark/longmemeval/run.py --log-level WARNING         # reduce eval runner logs
-    python benchmark/longmemeval/run.py --reme-log-level WARNING    # reduce reme internal logs
-    python benchmark/longmemeval/run.py --eval_only                 # query+judge only, reuse existing workspace
-"""
-
-import json
-import logging
-import os
-import re
-import shutil
-import time
-import threading
-from datetime import datetime
-from pathlib import Path
-
-import yaml
-from dotenv import load_dotenv
-
-# Load .env from project root
-_PROJECT_ROOT = Path(__file__).parent.parent.parent
-load_dotenv(_PROJECT_ROOT / ".env")
-
-# Workspace root for evaluation items — read from config.yaml (dataset.workspace_root)
-_WORKSPACE_ROOT_DEFAULT = "benchmark/longmemeval/workspaces/longmemeval-s"
-
-# ---------------------------------------------------------------------------
-# Logging
-# ---------------------------------------------------------------------------
-_DEFAULT_LOG_FORMAT = "%(asctime)s | %(levelname)s | %(message)s"
-
-logging.basicConfig(level=logging.INFO, format=_DEFAULT_LOG_FORMAT)
-logger = logging.getLogger("longmemeval")
-
-# Noisy library loggers silenced by default
-_NOISY_LOGGERS = [
-    "httpx",
-    "httpcore",
-    "openai",
-    "uvicorn",
-    "multipart",
-    "asyncio",
-    "watchfiles",
-    "filelock",
-]
-
-
-def setup_logging(
-    log_level: str,
-    reme_log_level: str,
-    log_dir: str | None = None,
-):
-    """Configure logging for the eval runner and reme internals.
-
-    Args:
-        log_level: Level for the eval runner logger (DEBUG/INFO/WARNING/ERROR).
-        reme_log_level: Level for reme's internal loguru logger.
-        log_dir: Per-run log directory (absolute path). None = no file logging.
-    """
-    numeric = getattr(logging, log_level.upper(), logging.INFO)
-    # Eval runner logger
-    logging.getLogger().setLevel(numeric)
-    logger.setLevel(numeric)
-
-    # Suppress noisy library loggers when above DEBUG
-    if numeric > logging.DEBUG:
-        for name in _NOISY_LOGGERS:
-            lib_logger = logging.getLogger(name)
-            lib_logger.setLevel(max(numeric, logging.WARNING))
-
-    # Add file handler for eval runner if log_dir is specified
-    if log_dir:
-        os.makedirs(log_dir, exist_ok=True)
-        log_filepath = os.path.join(log_dir, "runner.log")
-        file_handler = logging.FileHandler(log_filepath, encoding="utf-8")
-        file_handler.setLevel(numeric)
-        file_handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FORMAT))
-        logging.getLogger().addHandler(file_handler)
-        logger.info(f"Eval runner log file: {log_filepath}")
-
-    # Reme internal logger (loguru) — will be applied per-worker via _configure_worker
-    os.environ["REME_LOG_LEVEL"] = reme_log_level.upper()
-    if log_dir:
-        os.environ["REME_LOG_DIR"] = log_dir
-
-
-def _configure_worker(
-    log_level: str,
-    reme_log_level: str,
-    log_dir: str | None = None,
-):
-    """Set up logging inside a multiprocessing worker process.
-
-    Must be called at the top of each worker because child processes inherit
-    parent state but loguru sinks are NOT shared across fork/spawn.
-    """
-    numeric = getattr(logging, log_level.upper(), logging.INFO)
-    logging.basicConfig(level=numeric, format=_DEFAULT_LOG_FORMAT, force=True)
-    logging.getLogger("longmemeval").setLevel(numeric)
-    if numeric > logging.DEBUG:
-        for name in _NOISY_LOGGERS:
-            logging.getLogger(name).setLevel(max(numeric, logging.WARNING))
-
-    # Add file handler for eval runner in worker process
-    if log_dir:
-        os.makedirs(log_dir, exist_ok=True)
-        pid = os.getpid()
-        log_filepath = os.path.join(log_dir, f"worker-{pid}.log")
-        file_handler = logging.FileHandler(log_filepath, encoding="utf-8")
-        file_handler.setLevel(numeric)
-        file_handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FORMAT))
-        logging.getLogger().addHandler(file_handler)
-
-    # Re-initialize loguru for reme internals at the desired level
-    from reme.utils import get_logger
-
-    reme_log_dir = log_dir or "logs"
-    get_logger(log_dir=reme_log_dir, level=reme_log_level.upper(), force_init=True)
-
-
-# ---------------------------------------------------------------------------
-# Config loading
-# ---------------------------------------------------------------------------
-def load_eval_config(config_path: str | None = None) -> dict:
-    """Load evaluation config yaml with env-var expansion."""
-    if config_path is None:
-        config_path = str(Path(__file__).parent / "config.yaml")
-    with open(config_path, encoding="utf-8") as f:
-        raw = f.read()
-
-    # Expand ${VAR} and ${VAR:-default}
-    def _expand(m):
-        expr = m.group(1)
-        if ":-" in expr:
-            key, default = expr.split(":-", 1)
-            return os.environ.get(key, default)
-        return os.environ.get(expr, "")
-
-    raw = re.sub(r"\$\{([^}]+)\}", _expand, raw)
-    return yaml.safe_load(raw)
-
-
-# ---------------------------------------------------------------------------
-# Date utilities
-# ---------------------------------------------------------------------------
-def parse_haystack_date(date_str: str) -> datetime:
-    """Parse LongMemEval date format: '2023/05/20 (Sat) 02:21' -> datetime."""
-    m = re.match(r"(\d{4}/\d{2}/\d{2})\s+\(\w+\)\s+(\d{2}:\d{2})", date_str)
-    if not m:
-        raise ValueError(f"Cannot parse haystack date: {date_str!r}")
-    return datetime.strptime(f"{m.group(1)} {m.group(2)}", "%Y/%m/%d %H:%M")
-
-
-def to_iso(dt: datetime) -> str:
-    """Convert datetime to ISO-8601 string precise to seconds."""
-    return dt.strftime("%Y-%m-%dT%H:%M:%S")
-
-
-def should_trigger_dream(prev_dt: datetime, curr_dt: datetime, _trigger_hour: int = 23) -> bool:
-    """Check if the time gap between two sessions crosses trigger_hour (e.g. 23:00)."""
-    if prev_dt.date() == curr_dt.date():
-        return False
-    # There's at least one midnight crossing; check if trigger_hour is between them
-    # Simple heuristic: if dates differ, dream should run for the previous day
-    return True
-
-
-def sessions_sorted_by_time(item: dict) -> list[tuple[int, datetime, str, list[dict]]]:
-    """Return (original_index, parsed_datetime, session_id, messages) sorted by time."""
-    entries = []
-    for i, (date_str, sid, msgs) in enumerate(
-        zip(item["haystack_dates"], item["haystack_session_ids"], item["haystack_sessions"]),
-    ):
-        dt = parse_haystack_date(date_str)
-        entries.append((i, dt, sid, msgs))
-    # Sort by time (ascending)
-    entries.sort(key=lambda x: x[1])
-    return entries
-
-
-# ---------------------------------------------------------------------------
-# Message formatting
-# ---------------------------------------------------------------------------
-def format_messages_for_reme(messages: list[dict], session_dt: datetime) -> list[dict]:
-    """Convert LongMemEval messages to ReMe auto_memory format.
-
-    Adds: name, created_at (ISO seconds). All messages in a session share the
-    same created_at (the session timestamp).
-    """
-    formatted = []
-    for msg in messages:
-        role = msg["role"]
-        formatted.append(
-            {
-                "name": role,
-                "role": role,
-                "content": msg["content"],
-                "created_at": to_iso(session_dt),
-            },
-        )
-    return formatted
-
-
-# ---------------------------------------------------------------------------
-# LLM-as-Judge (delegated to answer_judge_step via app.run_job)
-# ---------------------------------------------------------------------------
-
-
-async def judge_response_via_job(
-    app,
-    question: str,
-    ground_truth: str,
-    response: str,
-    question_type: str,
-) -> dict:
-    """Use the answer_judge_step to evaluate a response against the golden answer."""
-    judge_resp = await app.run_job(
-        "answer_judge",
-        query=question,
-        agent_answer=response,
-        golden_answer=ground_truth,
-        question_type=question_type,
-    )
-
-    verdict = (judge_resp.answer or "").strip().lower()
-    raw_answer = (judge_resp.metadata or {}).get("raw_answer_judgement", "")
-
-    return {
-        "verdict": verdict,
-        "reason": raw_answer if verdict not in ("yes", "no") else "",
-        "metric": "binary",
-        "question_type": question_type,
-    }
-
-
-# ---------------------------------------------------------------------------
-# Main evaluation pipeline
-# ---------------------------------------------------------------------------
-async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_only: bool = False) -> dict:
-    """Evaluate a single LongMemEval item end-to-end.
-
-    Args:
-        item: The dataset item containing question, answer, sessions, etc.
-        eval_config: The evaluation configuration dict.
-        item_index: The index of this item in the dataset.
-        eval_only: If True, skip ingestion (phases 1-3) and only run query+judge
-            using the existing workspace. Useful for re-evaluating different query
-            configurations without re-ingesting sessions.
-    """
-    from reme import Application
-    from reme.config import resolve_app_config
-    from reme.utils.evaluation_interface import track_agent_token_usage, track_job_counts
-
-    reme_cfg = eval_config["reme"]
-    dream_trigger_hour = reme_cfg.get("dream_trigger_hour", 23)
-    dream_scan_days = reme_cfg.get("dream_scan_days", 2)
-    dream_max_units = reme_cfg.get("dream_max_units", 5)
-
-    # Sort sessions by time
-    sorted_sessions = sessions_sorted_by_time(item)
-
-    # Filter out sessions that occur after question_date (if enabled)
-    filter_future = eval_config["evaluation"].get("filter_future_sessions", True)
-    if filter_future and item.get("question_date"):
-        question_dt = parse_haystack_date(item["question_date"])
-        total_before_filter = len(sorted_sessions)
-        sorted_sessions = [(i, dt, sid, msgs) for i, dt, sid, msgs in sorted_sessions if dt <= question_dt]
-        if len(sorted_sessions) < total_before_filter:
-            logger.info(
-                f"[Item {item_index}] Filtered sessions: {total_before_filter} -> {len(sorted_sessions)} "
-                f"(removed {total_before_filter - len(sorted_sessions)} future sessions "
-                f"after question_date={item['question_date']})",
-            )
-
-    logger.info(
-        "[Item %s] question_id=%s type=%s sessions=%d%s",
-        item_index,
-        item["question_id"],
-        item["question_type"],
-        len(sorted_sessions),
-        " [eval_only]" if eval_only else "",
-    )
-
-    # Use fixed workspace directory (clean it for fresh evaluation)
-    workspace_root = _PROJECT_ROOT / eval_config["dataset"].get("workspace_root", _WORKSPACE_ROOT_DEFAULT)
-    item_dir = workspace_root / f"item_{item_index}"
-    workspace_dir = str(item_dir / ".reme")
-    if eval_only:
-        if not item_dir.exists() or not Path(workspace_dir).exists():
-            raise FileNotFoundError(
-                f"[Item {item_index}] eval_only: workspace not found at {item_dir}. "
-                f"Run without --eval_only first to build the workspace.",
-            )
-    else:
-        if item_dir.exists():
-            shutil.rmtree(item_dir)
-            logger.info(f"[Item {item_index}] Cleaned existing workspace: {item_dir}")
-        else:
-            logger.info(f"[Item {item_index}] Workspace not found, creating: {item_dir}")
-        item_dir.mkdir(parents=True, exist_ok=True)
-
-    # Pre-initialize ReMe's loguru logger with the correct log_dir
-    # (singleton — Application.__init__ will reuse this instance)
-    output_cfg = eval_config.get("output", {})
-    if output_cfg.get("log_to_file", False):
-        reme_log_dir = os.environ.get("REME_LOG_DIR")
-        if reme_log_dir:
-            from reme.utils import get_logger
-
-            get_logger(
-                log_dir=reme_log_dir,
-                level=os.environ.get("REME_LOG_LEVEL", "INFO"),
-                log_to_console=output_cfg.get("log_to_console", True),
-                log_to_file=True,
-                force_init=True,
-            )
-
-    cfg = resolve_app_config(
-        config=reme_cfg["config"],
-        workspace_dir=workspace_dir,
-        log_to_console=output_cfg.get("log_to_console", True),
-        log_to_file=output_cfg.get("log_to_file", False),
-        enable_logo=False,
-    )
-
-    app = Application(**cfg)
-    await app.start()
-
-    try:
-        dream_dates_triggered = set()
-        dream_available = True  # Set to False if auto_dream job is not found
-
-        if not eval_only:
-            # ── Phase 1: Ingest sessions ──────────────────────────────
-            prev_dt = None
-
-            for idx, (_, session_dt, session_id, messages) in enumerate(sorted_sessions):
-                # Check if dream should be triggered before this session
-                if (
-                    dream_available
-                    and prev_dt is not None
-                    and should_trigger_dream(prev_dt, session_dt, dream_trigger_hour)
-                ):
-                    dream_date = prev_dt.strftime("%Y-%m-%d")
-                    if dream_date not in dream_dates_triggered:
-                        logger.info(f"[Item {item_index}] Triggering dream for date={dream_date}")
-                        try:
-                            dream_resp = await app.run_job(
-                                "auto_dream",
-                                date=dream_date,
-                                scan_days=dream_scan_days,
-                                max_units=dream_max_units,
-                            )
-                            logger.info(
-                                f"[Item {item_index}] Dream done: success={dream_resp.success} "
-                                f"answer={dream_resp.answer[:100] if dream_resp.answer else ''}",
-                            )
-                        except Exception as e:
-                            if "not found" in str(e).lower():
-                                dream_available = False
-                                logger.warning(f"[Item {item_index}] auto_dream job not found, skipping all dreams")
-                            else:
-                                logger.warning(f"[Item {item_index}] Dream failed for {dream_date}: {e}")
-                        dream_dates_triggered.add(dream_date)
-                        # Index update after dream to pick up new digest nodes
-                        await app.run_job("index_update")
-
-                # Format and ingest the session
-                formatted_msgs = format_messages_for_reme(messages, session_dt)
-                date_str = session_dt.strftime("%Y-%m-%d")
-
-                logger.info(
-                    f"[Item {item_index}] Ingesting session {idx+1}/{len(sorted_sessions)} "
-                    f"id={session_id} date={date_str} msgs={len(formatted_msgs)}",
-                )
-                resp = await app.run_job(
-                    "auto_memory",
-                    messages=formatted_msgs,
-                    session_id=session_id,
-                    date=date_str,
-                )
-                if not resp.success:
-                    logger.warning(
-                        f"[Item {item_index}] auto_memory failed for session {session_id}: {resp.answer}",
-                    )
-
-                # Manual index update after each session
-                await app.run_job("index_update")
-
-                prev_dt = session_dt
-
-            # ── Phase 2: Final dream for the last day ─────────────────
-            if dream_available and prev_dt is not None:
-                last_dream_date = prev_dt.strftime("%Y-%m-%d")
-                if last_dream_date not in dream_dates_triggered:
-                    logger.info(f"[Item {item_index}] Final dream for date={last_dream_date}")
-                    try:
-                        await app.run_job(
-                            "auto_dream",
-                            date=last_dream_date,
-                            scan_days=dream_scan_days,
-                            max_units=dream_max_units,
-                        )
-                    except Exception as e:
-                        if "not found" in str(e).lower():
-                            dream_available = False
-                            logger.warning(f"[Item {item_index}] auto_dream job not found, skipping all dreams")
-                        else:
-                            logger.warning(f"[Item {item_index}] Final dream failed: {e}")
-                    dream_dates_triggered.add(last_dream_date)
-                    # Index update after final dream
-                    await app.run_job("index_update")
-
-            # ── Phase 3: Digest update ────────────────────────────────
-            await app.run_job("digest_update")
-
-        # ── Phase 4: Ask question via agentic_answer job (ReAct agent) ──
-        question = item["question"]
-        compress_session = bool(eval_config["evaluation"].get("compress_session", False))
-        question_date_raw = item.get("question_date", "")
-        question_dt = parse_haystack_date(question_date_raw) if question_date_raw else None
-        query_time = to_iso(question_dt) if question_dt else ""
-        logger.info(
-            f"[Item {item_index}] Asking (agentic): {question[:80]}... query_time={query_time}",
-        )
-
-        with (
-            track_job_counts(["search"], app.context) as tool_counts,
-            track_agent_token_usage(
-                ["bench"],
-                app.context,
-            ) as token_usages,
-        ):
-            query_resp = await app.run_job(
-                "agentic_answer",
-                query=question,
-                query_time=query_time,
-                compress_session=compress_session,
-            )
-        agentic_tool_counts = tool_counts
-        agentic_token_usage = token_usages["bench"]
-        agentic_response = (query_resp.answer or "").strip()
-        if not agentic_response:
-            agentic_response = "(no answer generated)"
-
-        logger.info(f"[Item {item_index}] Agentic response: {agentic_response[:200]}...")
-        logger.info(f"[Item {item_index}] Agentic tool calls: {agentic_tool_counts}")
-        logger.info(f"[Item {item_index}] Bench token usage: {agentic_token_usage}")
-
-        # ── Phase 5: Judge agentic response (via answer_judge_step) ──────────
-        logger.info(f"[Item {item_index}] Judging agentic (binary, type={item['question_type']})...")
-        agentic_judgment = await judge_response_via_job(
-            app=app,
-            question=question,
-            ground_truth=item["answer"],
-            response=agentic_response,
-            question_type=item["question_type"],
-        )
-        logger.info(f"[Item {item_index}] agentic binary result: {agentic_judgment}")
-
-    finally:
-        await app.close()
-
-    return {
-        "question_id": item["question_id"],
-        "question_type": item["question_type"],
-        "question": question,
-        "ground_truth": item["answer"],
-        "agentic_response": agentic_response,
-        "agentic_judgment": agentic_judgment,
-        "agentic_tool_counts": agentic_tool_counts,
-        "agentic_token_usage": agentic_token_usage,
-        "sessions_ingested": len(sorted_sessions),
-        "dreams_triggered": len(dream_dates_triggered),
-    }
-
-
-# ---------------------------------------------------------------------------
-# Worker: runs a single item in its own process with its own event loop
-# ---------------------------------------------------------------------------
-def _evaluate_item_worker(task_input: tuple) -> dict:
-    """Worker function for multiprocessing. Each process gets its own event loop."""
-    item, eval_config, item_index, log_level, reme_log_level, eval_only, log_dir = task_input
-    import asyncio  # pylint: disable=import-outside-toplevel
-
-    _configure_worker(log_level, reme_log_level, log_dir=log_dir)
-
-    # Permanently suppress "Task exception was never retrieved" /
-    # "Event loop is closed" noise from httpx AsyncClient GC cleanup.
-    # These fire AFTER asyncio.run() closes the loop, during Python's
-    # garbage collection of httpx connection-pool tasks — harmless.
-    logging.getLogger("asyncio").setLevel(logging.CRITICAL)
-
-    return asyncio.run(evaluate_item(item, eval_config, item_index, eval_only=eval_only))
-
-
-def _indexed_worker(indexed_input: tuple) -> tuple:
-    """Module-level wrapper for imap_unordered with index tracking."""
-    idx, task_input = indexed_input
-    return idx, _evaluate_item_worker(task_input)
-
-
-def _resolve_num_workers(configured: int) -> int:
-    """Resolve num_workers: 0=auto (cpu_count-2, min 1), 1=sequential, >1=parallel."""
-    if configured == 0:
-        return max(1, (os.cpu_count() or 4) - 2)
-    return max(1, configured)
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-def main(
-    config_path: str | None = None,
-    log_level: str = "INFO",
-    reme_log_level: str = "INFO",
-    eval_only: bool = False,
-):
-    """Run the LongMemEval evaluation pipeline.
-
-    Args:
-        config_path: Path to the YAML config file.
-        log_level: Log level for the eval runner.
-        reme_log_level: Log level for reme internal logs.
-        eval_only: If True, skip ingestion and only run query+judge using
-            existing workspaces.
-    """
-    from multiprocessing import Pool  # pylint: disable=import-outside-toplevel
-
-    # Load config BEFORE logging setup so log_dir is available
-    eval_config = load_eval_config(config_path)
-
-    # Resolve per-run log directory from config
-    output_cfg = eval_config.get("output", {})
-    log_dir_abs = None
-    if output_cfg.get("log_to_file", False):
-        log_dir_raw = output_cfg.get("log_dir", "logs")
-        log_prefix = output_cfg.get("log_prefix", "longmemeval")
-        run_ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        log_dir_abs = str(_PROJECT_ROOT / log_dir_raw / f"{log_prefix}_{run_ts}")
-
-    setup_logging(log_level, reme_log_level, log_dir=log_dir_abs)
-    dataset_cfg = eval_config["dataset"]
-
-    # Load dataset
-    dataset_path = _PROJECT_ROOT / dataset_cfg["path"]
-    logger.info(f"Loading dataset from {dataset_path}")
-    with open(dataset_path, encoding="utf-8") as f:
-        data = json.load(f)
-
-    start = dataset_cfg.get("start_index", 0)
-    num_items = dataset_cfg.get("num_items", 0)
-    if num_items > 0:
-        raw_items = data[start : start + num_items]
-    else:
-        raw_items = data[start:]
-
-    # Build item list
-    items_with_idx = [(start + i, item) for i, item in enumerate(raw_items)]
-
-    # Filter by question_type if specified
-    question_types = dataset_cfg.get("question_types") or []
-    if question_types:
-        before_filter = len(items_with_idx)
-        items_with_idx = [(idx, item) for idx, item in items_with_idx if item.get("question_type") in question_types]
-        logger.info(
-            f"Filtered by question_types={question_types}: {before_filter} -> {len(items_with_idx)} items",
-        )
-
-    # Filter by question_id if specified
-    question_ids = dataset_cfg.get("question_ids") or []
-    if question_ids:
-        qid_set = set(question_ids)
-        before_filter = len(items_with_idx)
-        items_with_idx = [(idx, item) for idx, item in items_with_idx if item.get("question_id") in qid_set]
-        logger.info(
-            f"Filtered by question_ids ({len(qid_set)} ids): {before_filter} -> {len(items_with_idx)} items",
-        )
-
-    logger.info(
-        "Evaluating %d item(s) starting from index %d%s",
-        len(items_with_idx),
-        start,
-        " [eval_only: query+judge only]" if eval_only else "",
-    )
-
-    # Resolve parallelism
-    num_workers = _resolve_num_workers(eval_config["evaluation"].get("num_workers", 1))
-    logger.info(f"Using {num_workers} worker(s)")
-
-    # Create output directory
-    output_dir = _PROJECT_ROOT / output_cfg.get("dir", "benchmark/longmemeval/results")
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Create workspace root directory
-    workspace_root = _PROJECT_ROOT / dataset_cfg.get("workspace_root", _WORKSPACE_ROOT_DEFAULT)
-    workspace_root.mkdir(parents=True, exist_ok=True)
-
-    # Pre-check: verify all workspaces exist in eval_only mode
-    if eval_only:
-        missing_items = []
-        for orig_idx, _ in items_with_idx:
-            item_dir = workspace_root / f"item_{orig_idx}"
-            if not item_dir.exists() or not (item_dir / ".reme").exists():
-                missing_items.append(orig_idx)
-        if missing_items:
-            preview = missing_items[:10]
-            suffix = "..." if len(missing_items) > 10 else ""
-            raise FileNotFoundError(
-                f"eval_only: {len(missing_items)} workspace(s) not found under {workspace_root}. "
-                f"Missing item indices: {preview}{suffix}. "
-                f"Run without --eval_only first to build the workspaces.",
-            )
-
-    # Build task args — include log levels, eval_only flag, and log paths (use original index for workspace lookup)
-    task_args = [
-        (item, eval_config, orig_idx, log_level, reme_log_level, eval_only, log_dir_abs)
-        for orig_idx, item in items_with_idx
-    ]
-
-    # Progress tracking (force print regardless of log level, every 10 minutes)
-    total_items = len(task_args)
-    completed_count = [0]  # use list for mutability in closure
-    start_time = time.time()
-    progress_lock = threading.Lock()
-
-    def _print_progress(prefix: str = "PROGRESS"):
-        elapsed = time.time() - start_time
-        elapsed_min = elapsed / 60
-        done = completed_count[0]
-        pct = 100.0 * done / total_items if total_items else 0
-        eta_str = "N/A"
-        if done > 0:
-            eta_sec = elapsed / done * (total_items - done)
-            eta_str = f"{eta_sec/60:.1f}min"
-        print(
-            f"[{prefix}] {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} | "
-            f"{done}/{total_items} ({pct:.1f}%) completed | "
-            f"elapsed={elapsed_min:.1f}min | ETA={eta_str}",
-            flush=True,
-        )
-
-    def _progress_timer():
-        """Background thread: print progress every 10 minutes."""
-        while not _timer_stop.is_set():
-            _timer_stop.wait(600)  # 10 minutes
-            if not _timer_stop.is_set():
-                with progress_lock:
-                    _print_progress()
-
-    _timer_stop = threading.Event()
-    timer_thread = threading.Thread(target=_progress_timer, daemon=True)
-    timer_thread.start()
-
-    # Run evaluation
-    if num_workers == 1:
-        # Sequential mode
-        results = []
-        for task_input in task_args:
-            result = _evaluate_item_worker(task_input)
-            results.append(result)
-            with progress_lock:
-                completed_count[0] += 1
-    else:
-        # Parallel mode — use imap_unordered for progress tracking
-        results = [None] * total_items
-        indexed_args = list(enumerate(task_args))
-
-        with Pool(processes=num_workers) as pool:
-            for idx, result in pool.imap_unordered(_indexed_worker, indexed_args):
-                results[idx] = result
-                with progress_lock:
-                    completed_count[0] += 1
-
-    # Stop progress timer
-    _timer_stop.set()
-    timer_thread.join(timeout=2)
-
-    # Save results
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    output_file = output_dir / f"results_{timestamp}.json"
-    with open(output_file, "w", encoding="utf-8") as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-    logger.info(f"Results saved to {output_file}")
-
-    # Final progress
-    _print_progress("FINAL")
-
-    _print_summary(results, start_time)
-
-
-# ---------------------------------------------------------------------------
-# Summary printing
-# ---------------------------------------------------------------------------
-def _print_summary(results: list[dict], start_time: float) -> None:
-    """Print per-item verdicts and per-type accuracy."""
-    print("\n" + "=" * 60)
-    print("EVALUATION RESULTS")
-    print("=" * 60)
-
-    def _accumulate(judgment_key):
-        correct = 0
-        stats: dict = {}  # {question_type: {correct: int, total: int}}
-        for r in results:
-            qtype = r["question_type"]
-            verdict = r.get(judgment_key, {}).get("verdict", "N/A")
-            if qtype not in stats:
-                stats[qtype] = {"correct": 0, "total": 0}
-            stats[qtype]["total"] += 1
-            if verdict == "yes":
-                correct += 1
-                stats[qtype]["correct"] += 1
-        return correct, stats
-
-    agentic_correct, agentic_type_stats = _accumulate("agentic_judgment")
-
-    total = len(results)
-
-    # Per-item verdict rows
-    for r in results:
-        a_verdict = r.get("agentic_judgment", {}).get("verdict", "N/A")
-        print(f"  [{r['question_id']}] type={r['question_type']}  agentic={a_verdict}")
-
-    print("\n" + "-" * 60)
-    print(f"  Items: {total}")
-
-    # Agentic stats
-    print("\n  ── Agentic (ReAct) ──")
-    print(f"  Overall accuracy: {agentic_correct}/{total} ({100*agentic_correct/total:.1f}%)")
-    tool_call_totals = [sum(r.get("agentic_tool_counts", {}).values()) for r in results]
-    tool_call_mean, tool_call_std = _mean_and_std(tool_call_totals)
-    print(f"  Tool calls/query: mean={tool_call_mean:.2f} std={tool_call_std:.2f}")
-    token_usages = [r.get("agentic_token_usage", {}) for r in results]
-    print("  Bench reported tokens/query:")
-    for metric in _TOKEN_USAGE_METRICS:
-        values = [usage[metric] for usage in token_usages if usage.get(metric) is not None]
-        if values:
-            mean, std = _mean_and_std(values)
-            print(f"    {metric}: mean={mean:.2f} std={std:.2f}")
-        else:
-            print(f"    {metric}: unavailable")
-    print("  Per-type accuracy:")
-    for qtype, stats in sorted(agentic_type_stats.items()):
-        acc = 100 * stats["correct"] / stats["total"] if stats["total"] else 0
-        print(f"    {qtype}: {stats['correct']}/{stats['total']} ({acc:.1f}%)")
-
-    print("=" * 60)
-    total_elapsed = time.time() - start_time
-    print(f"\n  Total time: {total_elapsed/60:.1f} min")
-    print("\n" + "=" * 60)
-    print("  [DONE] EVALUATION COMPLETED SUCCESSFULLY")
-    print("=" * 60 + "\n")
-
-
-_TOKEN_USAGE_METRICS = (
-    "input_tokens",
-    "output_tokens",
-    "total_tokens",
-)
-
-
-def _mean_and_std(values: list[int]) -> tuple[float, float]:
-    """Return population mean and standard deviation for one per-query metric."""
-    if not values:
-        return 0.0, 0.0
-    mean = sum(values) / len(values)
-    return mean, (sum((value - mean) ** 2 for value in values) / len(values)) ** 0.5
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser(description="LongMemEval evaluation runner")
-    parser.add_argument("--config", type=str, default=None, help="Path to config.yaml")
-    parser.add_argument(
-        "--log-level",
-        type=str,
-        default="INFO",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        help="Log level for the eval runner (default: INFO)",
-    )
-    parser.add_argument(
-        "--reme-log-level",
-        type=str,
-        default="INFO",
-        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-        help="Log level for reme internal logs — loguru (default: INFO)",
-    )
-    parser.add_argument(
-        "-q",
-        "--quiet",
-        action="store_true",
-        help="Shortcut for --log-level WARNING --reme-log-level WARNING",
-    )
-    parser.add_argument(
-        "--eval_only",
-        action="store_true",
-        help="Skip ingestion (phases 1-3). Reuse existing workspaces and only run query+judge.",
-    )
-    args = parser.parse_args()
-
-    if args.quiet:
-        args.log_level = "WARNING"
-        args.reme_log_level = "WARNING"
-
-    main(args.config, args.log_level, args.reme_log_level, eval_only=args.eval_only)

benchmark/pibench/.gitignore

@@ -1,14 +0,0 @@
-# 含真实 API key，绝不入库
-env.sh
-
-# 运行时产物（含对话内容，勿入库）
-logs/
-outputs/
-reme_workspace/
-nanobot_workspace/
-
-# 数据符号链接（指向外部 π-Bench 仓库）
-data
-
-__pycache__/
-*.pyc

benchmark/pibench/README.md

@@ -1,327 +0,0 @@
-[中文版 / Chinese version](./README_ZH.md)
-
-# π-Bench Evaluation Suite
-
-A glue layer that connects the **ReMe agent (with persistent memory)** to
-**π-Bench** (Proactive Personal Assistant Benchmark). This directory contains
-only the minimal code and configuration needed for the integration: the
-π-Bench framework (`src/`), evaluation data (`data/`), the AppWorld tool
-environment, and ReMe itself are all **external third-party dependencies**,
-referenced in place via symlink and environment variables and never bundled
-with this suite.
-
-- π-Bench: https://github.com/Simplified-Reasoning/Pi-Bench (arXiv: 2605.14678)
-- ReMe: the root of the ReMe repository this suite lives in (recommended
-  location: `ReMe/benchmark/pibench/`)
-
-## 1. Architecture
-
-```
-π-Bench runner (src.main --mode run)
-  │  user_agent (simulated-user LLM) walks data/{persona}/episode.yaml
-  │  task by task, chatting with the agent over multiple turns and judging
-  │  hidden intents (PROC) during the run phase
-  ▼
-test server (π-Bench scripts/test_server.py, HTTP long-polling)
-  ▲ /send            │ /poll
-  │                  ▼
-bridge_reme.py ──────────────► ReMe Application (embedded as a library)
-  │                             ├─ agent_wrapper: agent under test (AgentScope)
-  │                             ├─ jobs: search / auto_memory / daily_write
-  │                             └─ workspace: reme_workspace/{persona}/
-  │                                (isolated persistent memory per persona)
-  └──── MCP ────► AppWorld MCP ────► AppWorld APIs (tool/app environment)
-
-π-Bench runner (src.main --mode eval)
-  judger (judge LLM) reads the traces and scores each checklist item (COMP)
-```
-
-Key points:
-- The bridge runs on **ReMe's own venv python** and uses ReMe as a library
-  (`resolve_app_config` + `Application`); **no ReMe source modification** is
-  required.
-- Every incoming user message automatically triggers a ReMe memory `search`
-  and injects the matched memories (tuning knobs in §8); on task end (reset)
-  the session is distilled into daily notes by `auto_memory`.
-- Tool calls executed by the agent (AppWorld MCP + ReMe job tools) are
-  captured per turn into the trace as `tool_steps`, so π-Bench
-  `tools_evaluation_path` scripts can score tool behavior (§7).
-- π-Bench's `data/`, `src/` and AppWorld are not part of this suite; install
-  π-Bench first (§3.1).
-
-## 2. Directory layout
-
-```
-pibench/
-├── README.md / README_ZH.md  # this document (English / Chinese)
-├── env.sh.example            # environment template (copy to env.sh, fill TODOs)
-├── bridge_reme.py            # ReMe ↔ test server bridge (memory inject/save,
-│                             #   profile injection, tool-trace capture)
-├── run_persona.sh            # full pipeline for ONE persona (5 services + run + eval)
-├── run_all.sh                # batch over 5 personas (fresh/resume, default parallel=2)
-├── resume.py                 # checkpoint resume: completion detection + surgical
-│                             #   cleanup of interrupted tasks' residual memory
-├── fix_trace_logs.py         # run outputs → ~/.nanobot/trace_logs conversion,
-│                             #   merging tool sidecars into turn files (pre-eval)
-├── .gitignore                # excludes env.sh and all runtime artifacts
-└── config/
-    ├── models/reme.yaml      # runner model config (model_id=reme)
-    └── bench/evaluation/trace_history.yaml   # trace render policy (shipped with
-                                              #   the suite; passed via --history-config-path)
-```
-
-Generated at runtime (all git-ignored): `data` (symlink), `logs/`, `outputs/`,
-`reme_workspace/`, `nanobot_workspace/`.
-
-## 3. Prerequisites (third-party, install first)
-
-### 3.1 π-Bench repository (with AppWorld)
-
-```bash
-git clone https://github.com/Simplified-Reasoning/Pi-Bench.git <pi-bench-dir>
-cd <pi-bench-dir>
-python3.11 -m venv .venv            # scripts expect exactly this venv name
-source .venv/bin/activate
-pip install -e .                    # pibench runner (src.main)
-bash scripts/setup_appworld.sh      # install AppWorld and download its data (large)
-```
-
-Post-install sanity checks:
-```bash
-ls data/                    # should contain researcher marketer pharmacist law_trainee Financier
-.venv/bin/python -c "import src" && echo OK
-.venv/bin/appworld --help >/dev/null && echo OK
-```
-
-### 3.2 ReMe repository
-
-```bash
-cd <reme-dir>               # ReMe repository root (contains the reme/ package)
-python3.11 -m venv .venv    # scripts expect exactly this venv name
-source .venv/bin/activate
-pip install -e .            # or ReMe's own install flow; `import reme` must work
-```
-
-Sanity check: `.venv/bin/python -c "import reme; print('ok')"`
-
-## 4. Install this suite (step by step)
-
-1. **Place the suite** (recommended inside the ReMe repo so `REME_DIR` is
-   inferred automatically):
-   ```bash
-   cp -r pibench <reme-dir>/benchmark/pibench
-   cd <reme-dir>/benchmark/pibench
-   ```
-   If placed elsewhere, set `REME_DIR` explicitly in env.sh later.
-
-2. **Create the environment file and fill in the custom parameters**:
-   ```bash
-   cp env.sh.example env.sh
-   ```
-   Open `env.sh`; required items (marked TODO):
-   | Variable | Description |
-   |---|---|
-   | `PI_BENCH_ROOT` | π-Bench repo root (contains `src/` `data/` `.venv` `third_party/appworld`) |
-   | `USER_API_KEY` | API key of the simulated-user LLM (run phase, hidden-intent judging) |
-   | `JUDGER_API_KEY` | API key of the judger LLM (eval phase, checklist scoring) |
-   | `BRAVE_SEARCH_API_KEY` | optional; for the agent's web_search tool, `dummy` when unused |
-
-   Optional tuning: `REME_MODEL_NAME` (base model of the agent under test),
-   `REME_DIR`, `REME_LLM_BASE_URL` (default: DashScope OpenAI-compatible
-   endpoint).
-
-3. **Link the evaluation data** (referenced in place, never copied):
-   ```bash
-   ln -s "$PI_BENCH_ROOT/data" data
-   ```
-
-4. **(Optional) adjust model config** `config/models/reme.yaml`:
-   - `user_agent.model` / `judger.model`: model names for the simulated user
-     and the judger (literal values; π-Bench only expands `${ENV}` in
-     base_url/api_key).
-   - `run.turn_timeout`, `max_tool_iterations`, etc. as needed.
-
-5. **Smoke check** (does not start the evaluation):
-   ```bash
-   bash -n run_all.sh && bash -n run_persona.sh
-   source env.sh && "$REME_DIR/.venv/bin/python" -c "import reme; print('reme ok')"
-   ```
-
-## 5. Run the evaluation
-
-> ⚠️ For long runs use `screen`, **not nohup** (nohup loses the permission
-> context in sandboxed/restricted environments and breaks child processes).
-
-```bash
-# Full official run: wipe ALL personas' memory/outputs/traces first (default
-# fresh mode, parallel=2)
-mkdir -p logs   # on a fresh deployment logs/ does not exist yet
-screen -dmS pibench_suite bash -c "cd $(pwd) && bash run_all.sh > logs/run_all_master.log 2>&1"
-
-# Checkpoint continuation (after an interruption; no wipe, completed tasks skipped)
-bash run_all.sh --resume
-
-# Other usages
-bash run_all.sh --parallel 1          # sequential
-bash run_all.sh --resume --skip-eval  # run phase only
-bash run_persona.sh researcher        # single persona (default --resume semantics)
-bash run_persona.sh researcher --fresh
-```
-
-Time reference: 5 personas × 20 tasks, parallel=2, fresh full run ≈ 12–14 hours.
-
-`run_all.sh` exits non-zero when any persona fails, so upstream automation
-cannot mistake a partially failed suite run for a success.
-
-## 6. Port allocation (parallel personas never collide)
-
-| persona     | AppWorld API | AppWorld MCP | Test Server | ReMe internal service |
-|-------------|------|-------|------|-------|
-| marketer    | 9001 | 10001 | 9998 | 18766 |
-| law_trainee | 9002 | 10002 | 9997 | 18767 |
-| pharmacist  | 9003 | 10003 | 9996 | 18768 |
-| researcher  | 9004 | 10004 | 9995 | 18765 |
-| Financier   | 9005 | 10005 | 9994 | 18769 |
-
-## 7. Outputs and scores
-
-- **Results**: `outputs/reme/{persona}/{task}/eval/results/*_result.json`
-  - `overall_average_score`: checklist completeness (COMP; the judger scores
-    each criterion YES/NO, weighted across dependency groups)
-  - `overall_proactiveness_average_score`: proactiveness (PROC; the
-    user_agent judges hidden-intent coverage during the run phase; each task
-    file also carries the global average)
-- **Traces**: `~/.nanobot/trace_logs/reme/{persona}/{task}/...` (the scoring
-  input of the eval phase)
-- **Logs**: `logs/` (`suite_<persona>.log` per persona; `bridge_*`,
-  `runner_run/eval_*`, `appworld_*`, `test_server_*` per service)
-- **Memory store**: `reme_workspace/{persona}/` (daily/digest notes, raw
-  session dialogs, BM25 index, etc.; persistent across runs, wiped only in
-  fresh mode)
-
-Score summary:
-```bash
-grep -h "overall_average_score\|overall_proactiveness" \
-  outputs/reme/*/*/eval/results/*_result.json | head
-```
-
-### Tool-trace capture (tools_evaluation support)
-
-Some tasks define `objectives.tools_evaluation_path`: Python scripts that
-score tool behavior (e.g. "the temporary Todoist board was created and
-removed"). They need the executed tool calls in the trace. The pipeline:
-
-1. During `reply()`, the bridge reads the persisted AgentScope session state
-   after each turn and extracts the new `tool_call` / `tool_result` blocks
-   (tool name, arguments, result).
-2. Records are appended to
-   `outputs/reme/{persona}/{task}/history/{ts}-tools.jsonl`, tagged with the
-   turn number; AgentScope MCP names (`mcp__AppWorld__<tool>`) are normalized
-   to the π-Bench convention (`mcp_appworld_<tool>`).
-3. `fix_trace_logs.py` pairs each `{ts}-messages.jsonl` run with the
-   temporally closest tools sidecar and merges the records into the generated
-   `turn_N.json` files under the `tool_steps` key — one of the two
-   tool-history formats understood by π-Bench's `collect_tool_history()`.
-4. The eval phase then feeds `tool_steps` to both the tools_evaluation
-   scripts and the rendered `<tool_trace_extracts>` seen by the judger.
-
-## 8. Memory mechanism (core design of this suite)
-
-- **Persona isolation**: each persona has its own workspace
-  (`reme_workspace/{persona}/`); the bridge takes an exclusive
-  `.bridge.lock` on it at startup, so two bridges can never share one memory
-  store, and one persona's memory search can never reach another's memories.
-- **Writes**: on task end (runner sends reset), the session is distilled by
-  the `auto_memory` job into daily notes and indexed by the background
-  watcher (BM25). Saves are non-blocking background tasks; the first message
-  of a new session waits for in-flight writes before searching.
-- **Reads**: on every incoming user message the bridge runs one `search` and
-  injects matched memories (`[Relevant memories from previous sessions]`
-  prefix); without matches the message passes through unchanged. Retrieval
-  tuning (bridge CLI flags, adjustable in run_persona.sh):
-  - `--search-limit 3`: at most 3 memory chunks injected per message;
-  - `--search-min-score 2.0`: weak BM25 hits are filtered out;
-  - `tool_context_id` rotates per task: chunks already injected within the
-    same task are not re-injected (ReMe's seen-chunk dedup, 24h TTL); normal
-    recall resumes after task boundaries.
-- **No self-leakage**: the in-progress session is not in the store yet
-  (saves happen on reset), so a task can never retrieve its own unfinished
-  content.
-- The agent also holds `search`/`daily_write` tools and can retrieve/record
-  proactively.
-- **System prompt**: `bridge_reme.py:build_system_prompt()` embeds the
-  HIDDEN-NEEDS protocol (proactiveness-oriented) and injects the persona
-  profile from `data/{persona}/profile.yaml` into every turn's system prompt.
-
-## 9. Checkpoint resume and memory-cleanup semantics
-
-- **Completion detection** (resume.py): scans
-  `outputs/reme/{persona}/**/history/*-log.jsonl` and
-  `outputs/reme/{persona}/run/*-log.jsonl` for
-  `Task finished task_id=X status=Y`. The status with the **newest event
-  timestamp** wins per task (record `timestamp`, falling back to
-  `timestamp_iso`, then to the timestamp embedded in the log file name) —
-  file category and read order alone can never override a newer record, so an
-  old run-level SUCCESS cannot mask a newer per-task ERROR. `SUCCESS /
-  MAX_TURNS / TIMEOUT` count as completed; `ERROR` and never-started tasks
-  are re-run (passed to the runner as repeated `--task-id` flags in episode
-  order).
-- **Answer-leak prevention**: an interrupted task may already have been
-  distilled into daily notes during graceful shutdown; re-running it with
-  that memory injected would inflate scores. Before resuming,
-  `resume.py cleanup` therefore removes residual memory **only for tasks
-  about to be re-run** (daily/digest notes, session/dialog, mem_session;
-  matched via `session_id = pibench_{task}_*`). Completed tasks' memories are
-  never touched. Daily index files are refreshed **only for the dates that
-  lost notes**, by full workspace-relative wikilink path — and when the ReMe
-  package is importable, the refresh reuses ReMe's own daily-index rebuild
-  logic (`refresh_day_index`), so same-named notes on other dates are never
-  modified.
-- **fresh vs resume are mutually exclusive**: a full memory wipe belongs to
-  fresh mode only (`run_all.sh` default, executed before any service starts);
-  resume never wipes.
-
-## 10. Customization entry points
-
-| Goal | Location |
-|---|---|
-| Base model of the agent under test | `REME_MODEL_NAME` in `env.sh` |
-| user_agent / judger models | `config/models/reme.yaml` |
-| Agent system prompt | `bridge_reme.py` `build_system_prompt()` |
-| Memory retrieval limit/threshold | `--search-limit/--search-min-score` on the bridge command in `run_persona.sh` |
-| ReMe internal parameters | **Do not modify ReMe source**; write a dedicated config modeled on `reme/config/beam.yaml` and override via `resolve_app_config(config=...)` (see bridge `_init_reme_app`) |
-| Turn timeout / tool iteration cap | `config/models/reme.yaml` `run.turn_timeout`, `model.max_tool_iterations` |
-
-## 11. Troubleshooting
-
-- **Port already in use**: the scripts auto-kill residual processes on the
-  four port groups above; if another suite (e.g. a different π-Bench
-  experiment) holds them, stop it first or change the port table in
-  run_persona.sh.
-- **Bridge exits immediately with workspace locked**: another bridge already
-  holds the same workspace; make sure each persona uses its own
-  `--workspace-dir` (the scripts allocate one per persona).
-- **Runner reports `${USER_API_KEY} ... empty`**: env.sh is unfilled or not
-  sourced; run_persona.sh sources env.sh automatically — when running the
-  runner manually, `source env.sh` first.
-- **`Cannot import 'reme'`**: the bridge must run with
-  `${REME_DIR}/.venv/bin/python` (run_persona.sh already does); otherwise
-  check that `REME_DIR` points at the ReMe repository root.
-- **AppWorld fails to start**: run `bash scripts/setup_appworld.sh` in the
-  π-Bench repo first (downloads data); inspect
-  `logs/appworld_*_<persona>.log`.
-- **trace_history.yaml not found**: the runner needs
-  `config/bench/evaluation/trace_history.yaml`; this suite ships the file and
-  passes it explicitly via `--history-config-path`, and run_persona.sh fails
-  fast with a clear error if it is missing. Always launch run_persona.sh /
-  run_all.sh from the suite directory.
-
-## 12. Privacy and security
-
-- The suite code and config templates contain **no real API keys, user names
-  or absolute paths**; real keys live only in your local `env.sh`
-  (git-ignored).
-- `logs/`, `outputs/`, `reme_workspace/` and `nanobot_workspace/` contain
-  full conversations and model outputs; never commit or share them.
-- The `data` symlink points at the official π-Bench evaluation data; respect
-  its data license terms.

benchmark/pibench/README_ZH.md

@@ -1,284 +0,0 @@
-# π-Bench 评测说明
-
-[English version](./README.md)
-
-将 **ReMe agent（带持久记忆）** 接入 **π-Bench**（Proactive Personal Assistant
-Benchmark）的胶水层评测套件。只含对接所需的最小代码与配置；π-Bench 框架
-（`src/`）、评测数据（`data/`）、AppWorld 工具环境、ReMe 本体均为**外部第三方
-依赖**，通过符号链接与环境变量原位引用，不随本套件分发。
-
-- π-Bench: https://github.com/Simplified-Reasoning/Pi-Bench （arXiv: 2605.14678）
-- ReMe: 你所在 ReMe 仓库的根目录（本套件推荐放在 `ReMe/benchmark/pibench/`）
-
-## 1. 架构总览
-
-```
-π-Bench runner (src.main --mode run)
-  │  user_agent（模拟用户 LLM）按 data/{persona}/episode.yaml 顺序
-  │  逐任务、多轮地与 agent 对话，并在 run 阶段判定隐藏意图(PROC)
-  ▼
-test server (π-Bench scripts/test_server.py, HTTP 长轮询)
-  ▲ /send            │ /poll
-  │                  ▼
-bridge_reme.py ──────────────► ReMe Application（以库方式内嵌启动）
-  │                             ├─ agent_wrapper: 被测 agent（AgentScope）
-  │                             ├─ jobs: search / auto_memory / daily_write
-  │                             └─ workspace: reme_workspace/{persona}/
-  │                                （每 persona 独立持久记忆库，互不可见）
-  └──── MCP ────► AppWorld MCP ────► AppWorld API（工具/应用环境）
-
-π-Bench runner (src.main --mode eval)
-  judger（裁判 LLM）读取 trace，按 checklist 逐条 YES/NO 打分(COMP)
-```
-
-要点：
-- bridge 用 **ReMe 自己的 venv python** 运行，把 ReMe 当库用（`resolve_app_config`
-  + `Application`），**ReMe 源码零改动**。
-- 每条用户消息都会自动触发一次 ReMe memory `search` 并把命中记忆注入当前消息
-  （参数见 §8）；任务结束（reset）时会话被 `auto_memory` 提炼为 daily 笔记落盘。
-- agent 执行的每一轮工具调用（AppWorld MCP + ReMe job 工具）都会被采集并以
-  `tool_steps` 形式写入 trace，供 π-Bench 的 `tools_evaluation_path` 脚本
-  对工具行为评分（§7）。
-- π-Bench 的 `data/`、`src/`、AppWorld 均不属于本套件，需先装好 π-Bench（§3.1）。
-
-## 2. 目录结构
-
-```
-pibench/
-├── README.md / README_ZH.md  # 本文档（英文 / 中文）
-├── env.sh.example            # 环境配置模板（复制为 env.sh 后填写 TODO 项）
-├── bridge_reme.py            # ReMe ↔ test server 桥接（记忆注入/保存、
-│                             #   profile 注入、工具调用轨迹采集）
-├── run_persona.sh            # 单 persona 全流程（5 个服务 + run + eval）
-├── run_all.sh                # 5 个 persona 批跑（fresh/resume，默认 2 并行）
-├── resume.py                 # 断点续跑：完成判定 + 中断任务残留记忆的外科清理
-├── fix_trace_logs.py         # run 输出 → ~/.nanobot/trace_logs 转换，
-│                             #   并把工具轨迹合并进 turn 文件（eval 前置）
-├── .gitignore                # 排除 env.sh 与全部运行产物
-└── config/
-    ├── models/reme.yaml      # runner 模型配置（model_id=reme）
-    └── bench/evaluation/trace_history.yaml   # trace 渲染策略（随套件提供，
-                                              #   经 --history-config-path 显式传入）
-```
-
-运行时自动生成（均被 .gitignore 排除）：`data`（符号链接）、`logs/`、
-`outputs/`、`reme_workspace/`、`nanobot_workspace/`。
-
-## 3. 前置依赖（第三方，先装好）
-
-### 3.1 π-Bench 仓库（含 AppWorld）
-
-```bash
-git clone https://github.com/Simplified-Reasoning/Pi-Bench.git <pi-bench-dir>
-cd <pi-bench-dir>
-python3.11 -m venv .venv            # 脚本约定使用 .venv 这个目录名
-source .venv/bin/activate
-pip install -e .                    # pibench runner（src.main）
-bash scripts/setup_appworld.sh      # 安装 AppWorld 并下载其数据（体积较大，需网络）
-```
-
-装完自检：
-```bash
-ls data/                    # 应含 researcher marketer pharmacist law_trainee Financier
-.venv/bin/python -c "import src" && echo OK
-.venv/bin/appworld --help >/dev/null && echo OK
-```
-
-### 3.2 ReMe 仓库
-
-```bash
-cd <reme-dir>               # ReMe 仓库根目录（含 reme/ 包）
-python3.11 -m venv .venv    # 脚本约定使用 .venv 这个目录名
-source .venv/bin/activate
-pip install -e .            # 或按 ReMe 自身安装方式，保证 `import reme` 可用
-```
-
-自检：`.venv/bin/python -c "import reme; print('ok')"`
-
-## 4. 安装本套件（逐步）
-
-1. **放置套件**（推荐放进 ReMe 仓库，`REME_DIR` 可自动推断）：
-   ```bash
-   cp -r pibench <reme-dir>/benchmark/pibench
-   cd <reme-dir>/benchmark/pibench
-   ```
-   若放在其他位置，稍后在 env.sh 中显式设置 `REME_DIR`。
-
-2. **创建环境文件并填写自定义参数**：
-   ```bash
-   cp env.sh.example env.sh
-   ```
-   打开 `env.sh`，必填项（标 TODO 的）：
-   | 变量 | 说明 |
-   |---|---|
-   | `PI_BENCH_ROOT` | π-Bench 仓库根目录（含 `src/` `data/` `.venv` `third_party/appworld`） |
-   | `USER_API_KEY` | 模拟用户 LLM 的 API key（run 阶段判定隐藏意图） |
-   | `JUDGER_API_KEY` | 裁判 LLM 的 API key（eval 阶段 checklist 打分） |
-   | `BRAVE_SEARCH_API_KEY` | 可选；agent 的 web_search 工具用，不用填 `dummy` |
-
-   可选调整：`REME_MODEL_NAME`（被测 agent 基模）、`REME_DIR`、
-   `REME_LLM_BASE_URL`（默认 DashScope OpenAI 兼容端点）。
-
-3. **链接评测数据**（π-Bench 数据原位引用，不复制）：
-   ```bash
-   ln -s "$PI_BENCH_ROOT/data" data
-   ```
-
-4. **（可选）调整模型配置** `config/models/reme.yaml`：
-   - `user_agent.model` / `judger.model`：模拟用户与裁判的模型名（字面量，
-     π-Bench 仅对 base_url/api_key 做 `${ENV}` 展开）。
-   - `run.turn_timeout`、`max_tool_iterations` 等按需。
-
-5. **冒烟自检**（不启动评测）：
-   ```bash
-   bash -n run_all.sh && bash -n run_persona.sh
-   source env.sh && "$REME_DIR/.venv/bin/python" -c "import reme; print('reme ok')"
-   ```
-
-## 5. 运行评测
-
-> ⚠️ 长时间运行请放进 `screen`，**不要用 nohup**（nohup 在沙箱/受限环境下
-> 会丢失权限上下文导致子进程异常）。
-
-```bash
-# 完整正式评测：先清空全部 persona 的记忆/输出/trace，再从头跑（默认 fresh，2 并行）
-mkdir -p logs   # 全新部署时 logs/ 尚不存在，先建再重定向
-screen -dmS pibench_suite bash -c "cd $(pwd) && bash run_all.sh > logs/run_all_master.log 2>&1"
-
-# 断点续跑（中断后继续；不清记忆，跳过已完成任务）
-bash run_all.sh --resume
-
-# 其他用法
-bash run_all.sh --parallel 1        # 串行
-bash run_all.sh --resume --skip-eval  # 只跑 run 阶段
-bash run_persona.sh researcher      # 单 persona（默认 --resume 语义）
-bash run_persona.sh researcher --fresh
-```
-
-耗时参考：5 persona × 20 任务、2 并行，fresh 全量约 12–14 小时。
-
-任一 persona 失败时 `run_all.sh` 以非零状态退出，上层自动化不会把部分失败
-的评测误判为成功。
-
-## 6. 端口分配（多 persona 并行互不冲突）
-
-| persona     | AppWorld API | AppWorld MCP | Test Server | ReMe 内部服务 |
-|-------------|------|-------|------|-------|
-| marketer    | 9001 | 10001 | 9998 | 18766 |
-| law_trainee | 9002 | 10002 | 9997 | 18767 |
-| pharmacist  | 9003 | 10003 | 9996 | 18768 |
-| researcher  | 9004 | 10004 | 9995 | 18765 |
-| Financier   | 9005 | 10005 | 9994 | 18769 |
-
-## 7. 输出与分数
-
-- **结果**：`outputs/reme/{persona}/{task}/eval/results/*_result.json`
-  - `overall_average_score`：checklist 完整度（COMP，judger 逐条 YES/NO 按依赖组加权）
-  - `overall_proactiveness_average_score`：主动性（PROC，run 阶段 user_agent
-    判定隐藏意图覆盖率；每个任务文件同时携带全局均值）
-- **trace**：`~/.nanobot/trace_logs/reme/{persona}/{task}/...`（eval 的判分输入）
-- **日志**：`logs/`（`suite_<persona>.log` 为每 persona 总日志，`bridge_*`、
-  `runner_run/eval_*`、`appworld_*`、`test_server_*` 分服务）
-- **记忆库**：`reme_workspace/{persona}/`（daily/digest 笔记、session 原始对话、
-  BM25 索引等；跨运行持久，fresh 才清空）
-
-查看汇总：
-```bash
-grep -h "overall_average_score\|overall_proactiveness" \
-  outputs/reme/*/*/eval/results/*_result.json | head
-```
-
-### 工具轨迹采集（tools_evaluation 支持）
-
-部分任务定义了 `objectives.tools_evaluation_path`：用 Python 脚本对工具行为
-打分（例如"临时 Todoist 看板已创建并被删除"）。这些脚本需要 trace 里有真实
-的工具调用记录。采集链路：
-
-1. 每轮 `reply()` 之后，bridge 读取 AgentScope 落盘的会话状态，提取本轮新增
-   的 `tool_call` / `tool_result` 块（工具名、参数、结果）。
-2. 记录按 turn 编号追加写入
-   `outputs/reme/{persona}/{task}/history/{ts}-tools.jsonl`；AgentScope 的
-   MCP 工具名（`mcp__AppWorld__<tool>`）会规范化为 π-Bench 约定
-   （`mcp_appworld_<tool>`）。
-3. `fix_trace_logs.py` 将每个 `{ts}-messages.jsonl` 运行与时间上最接近的
-   tools 旁路文件配对，把记录合并进生成的 `turn_N.json` 的 `tool_steps`
-   字段——这是 π-Bench `collect_tool_history()` 支持的两种工具轨迹格式之一。
-4. eval 阶段 `tool_steps` 既提供给 tools_evaluation 脚本，也会被渲染为
-   judger 可见的 `<tool_trace_extracts>`。
-
-## 8. 记忆机制（本套件的核心设计）
-
-- **persona 隔离**：每个 persona 独立 workspace（`reme_workspace/{persona}/`），
-  bridge 启动时对 workspace 加 `.bridge.lock` 排他锁，两个 bridge 不可能共用
-  同一记忆库；一个 persona 的 memory search 永远接触不到其他 persona 的记忆。
-- **写入**：任务结束（runner 发送 reset）时，会话经 `auto_memory` job 提炼为
-  daily 笔记落盘，后台 watcher 建 BM25 索引。保存为非阻塞后台任务，
-  新会话首条消息会先等待在途写入完成再检索。
-- **读取**：bridge 每收到一条用户消息自动 `search` 一次并注入命中记忆
-  （`[Relevant memories from previous sessions]` 前缀），无命中则原样透传。
-  检索参数（bridge 命令行，可在 run_persona.sh 中调整）：
-  - `--search-limit 3`：每条消息最多注入 3 个记忆块；
-  - `--search-min-score 2.0`：过滤弱 BM25 命中；
-  - `tool_context_id` 按任务轮换：同一任务内已注入的记忆块不重复注入
-    （ReMe 自带 seen-chunk 去重，24h TTL），任务边界后恢复正常召回。
-- **无自泄漏**：进行中的会话尚未入库（save 发生在 reset），任务不会检索到
-  自己未完成的内容。
-- agent 同时持有 `search`/`daily_write` 工具，可主动检索/记录。
-- **system prompt**：`bridge_reme.py:build_system_prompt()` 内置
-  HIDDEN-NEEDS 协议（面向 proactiveness），并把 `data/{persona}/profile.yaml`
-  的 persona profile 注入每轮 system prompt。
-
-## 9. 断点续跑与记忆清理语义
-
-- **完成判定**（resume.py）：扫描 `outputs/reme/{persona}/**/history/*-log.jsonl`
-  与 `outputs/reme/{persona}/run/*-log.jsonl` 中的
-  `Task finished task_id=X status=Y`。每个任务以**事件时间最新**的记录为准
-  （优先取记录的 `timestamp`，回退 `timestamp_iso`，再回退日志文件名中的
-  时间戳）——文件类别与读取顺序本身不能覆盖更新的记录，因此旧的 run 级
-  SUCCESS 不会掩盖更新的 per-task ERROR。`SUCCESS/MAX_TURNS/TIMEOUT` 记为
-  完成，`ERROR`/未开始的任务重跑（按 episode 顺序以 `--task-id` 传给 runner）。
-- **防答案泄漏**：被中断的任务可能已在优雅退出时提炼成 daily 笔记，直接重跑会
-  把答案注入、抬高分数。因此 resume 启动前 `resume.py cleanup` **只删除待重跑
-  任务**的残留记忆（daily/digest 笔记、session/dialog、mem_session，按
-  `session_id = pibench_{task}_*` 匹配），已完成任务的记忆一律不动。daily
-  索引**只刷新实际发生删除的日期**，按完整的 workspace 相对 wikilink 路径
-  匹配；当 ReMe 包可导入时，刷新直接复用 ReMe 自带的 daily 索引重建逻辑
-  （`refresh_day_index`），不会误改其他日期下的同名笔记条目。
-- **fresh vs resume 互斥**：全量清记忆只属于 fresh 模式（`run_all.sh` 默认，
-  在任何服务启动前执行）；resume 永不清全量。
-
-## 10. 自定义与调优入口
-
-| 目标 | 位置 |
-|---|---|
-| 被测 agent 基模 | `env.sh` 的 `REME_MODEL_NAME` |
-| user_agent / judger 模型 | `config/models/reme.yaml` |
-| agent system prompt | `bridge_reme.py` `build_system_prompt()` |
-| 记忆检索条数/阈值 | `run_persona.sh` bridge 启动命令的 `--search-limit/--search-min-score` |
-| ReMe 内部参数 | **不要改 ReMe 源码**；仿照 `reme/config/beam.yaml` 写专有配置，经 `resolve_app_config(config=...)` 覆盖（见 bridge `_init_reme_app`） |
-| 轮超时/工具迭代上限 | `config/models/reme.yaml` `run.turn_timeout`、`model.max_tool_iterations` |
-
-## 11. 故障排查
-
-- **端口被占用**：脚本会自动 kill 上述 4 组端口上的残留进程；若与其他套件
-  （如别的 π-Bench 实验）冲突，请先停掉对方或改 run_persona.sh 的端口表。
-- **bridge 启动即退出，提示 workspace locked**：另一个 bridge 正占用同一
-  workspace；确认每个 persona 用各自的 `--workspace-dir`（脚本已按 persona 分配）。
-- **runner 报 `${USER_API_KEY} ... empty`**：env.sh 未填写或未生效；
-  run_persona.sh 会自动 source env.sh，手动运行 runner 时请先 `source env.sh`。
-- **`Cannot import 'reme'`**：bridge 必须用 `${REME_DIR}/.venv/bin/python` 运行
-  （run_persona.sh 已如此），或检查 `REME_DIR` 是否指向 ReMe 仓库根目录。
-- **AppWorld 启动失败**：先在 π-Bench 仓库执行 `bash scripts/setup_appworld.sh`
-  下载数据；查看 `logs/appworld_*_<persona>.log`。
-- **trace_history.yaml 找不到**：runner 需要
-  `config/bench/evaluation/trace_history.yaml`；本套件已随附该文件并通过
-  `--history-config-path` 显式传入，run_persona.sh 启动前会做存在性检查，
-  缺失时立即报出清晰错误。请始终从套件目录启动 run_persona.sh / run_all.sh。
-
-## 12. 隐私与安全
-
-- 套件代码与配置模板中**不含任何真实 API key、用户名或绝对路径**；
-  真实 key 只存在于你本地的 `env.sh`（已被 .gitignore 排除）。
-- `logs/`、`outputs/`、`reme_workspace/`、`nanobot_workspace/` 含完整对话内容
-  与模型输出，请勿提交仓库或外传。
-- `data` 符号链接指向 π-Bench 官方评测数据，请遵守其数据许可条款。

benchmark/pibench/config/bench/evaluation/trace_history.yaml

@@ -1,53 +0,0 @@
-version: 1
-
-format:
-  root_tag: trace
-  turn_tag: turn
-  message_tag: message
-  file_tag: file
-  tool_call_tag_prefix: tool_call
-  tool_result_tag_prefix: tool_result
-
-text_policy:
-  default:
-    truncate_chars: 1200
-    mask_newlines: false
-  field_overrides:
-    files_read:
-      truncate_chars: 40000
-    assistant_content:
-      truncate_chars: 40000
-    tool_result_content:
-      truncate_chars: 40000
-
-fields:
-  turn:
-    include_session_key: false
-
-  files:
-    enabled: true
-
-  messages:
-    enabled: true
-    include_message_role_attr: true
-    include_message_index_attr: false
-    include_system: false
-    include_user: true
-    include_assistant_thinking_content: false
-    include_assistant_thinking_reasoning: false
-    include_assistant_content: true
-    include_assistant_reasoning: false
-    include_assistant_tool_calls: false
-    require_matching_tool_call: true
-
-    tool_calls:
-      include_tool_call_id: false
-      tools:
-        web_fetch:
-          enabled: true
-          include_tool_call_keys: [url]
-          include_tool_result: false
-        web_search:
-          enabled: true
-          include_tool_call_keys: [query]
-          include_tool_result: false

benchmark/pibench/config/models/reme.yaml

@@ -1,40 +0,0 @@
-# ReMe model configuration for Pi-Bench
-# Uses ReMe's AgentScope agent with Dashscope as the LLM backend
-
-model:
-  model: reme
-  base_url: "http://localhost:8088"
-  api_key: "dummy"
-  provider: custom
-  max_tokens: 16384
-  max_tool_iterations: 120
-  memory_window: 100
-
-user_agent:
-  model: qwen3.8-max
-  base_url: "${USER_BASE_URL}"
-  api_key: "${USER_API_KEY}"
-  temperature: 0.0
-  request_timeout: 360.0
-
-judger:
-  model: qwen3.8-max
-  base_url: "${JUDGER_BASE_URL}"
-  api_key: "${JUDGER_API_KEY}"
-  temperature: 0.0
-  request_timeout: 360.0
-
-tools:
-  brave_search_api_key: "${BRAVE_SEARCH_API_KEY}"
-  web_search_max_results: 10
-
-nanobot:
-  trace_logs_dir: "~/.nanobot/trace_logs"
-  workspace_dir: "~/.nanobot/workspace"
-  copy_task_assets_to_workspace: true
-
-run:
-  output_dir: outputs
-  log_level: INFO
-  user_mode: llm
-  turn_timeout: 2400.0

benchmark/pibench/env.sh.example

@@ -1,57 +0,0 @@
-#!/bin/bash
-# ═══════════════════════════════════════════════════════════════════════
-# pibench evaluation suite - environment configuration template
-# Usage: cp env.sh.example env.sh, then fill in the TODO items below.
-# ⚠️  env.sh contains real API keys; never commit or share it
-#    (already excluded via .gitignore).
-# ═══════════════════════════════════════════════════════════════════════
-
-SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# ─── TODO: π-Bench repository root ────────────────────────────────────
-# Must contain src/, data/, scripts/test_server.py, third_party/appworld
-# and .venv (see README setup).
-export PI_BENCH_ROOT=""
-
-# ─── ReMe repository ──────────────────────────────────────────────────
-# Defaults to two levels above this directory (the layout this suite uses
-# when placed at ReMe/benchmark/pibench); point it at the actual ReMe
-# repository root if the suite lives elsewhere.
-export REME_DIR="${REME_DIR:-$(cd "${SUITE_DIR}/../.." && pwd)}"
-
-# ─── Base model of the agent under test (LLM used by the ReMe agent) ──
-export REME_MODEL_NAME="${REME_MODEL_NAME:-qwen3.6-plus}"
-
-# ─── LLM service endpoint (default: DashScope OpenAI-compatible; any
-#     OpenAI-compatible endpoint works) ────────────────────────────────
-DASHSCOPE_BASE_URL="https://dashscope.aliyuncs.com/compatible-mode/v1"
-export REME_LLM_BASE_URL="${REME_LLM_BASE_URL:-${DASHSCOPE_BASE_URL}}"
-
-# ─── TODO: API keys ───────────────────────────────────────────────────
-# USER_API_KEY : drives the simulated user LLM (run phase; judges whether
-#                hidden intents are satisfied and asks follow-ups)
-# JUDGER_API_KEY: drives the judger LLM (eval phase; scores the checklist)
-# The two may be identical; one strong model is recommended for both.
-export USER_BASE_URL="${DASHSCOPE_BASE_URL}"
-export USER_API_KEY="TODO-fill-in-user-agent-api-key"
-
-export JUDGER_BASE_URL="${DASHSCOPE_BASE_URL}"
-export JUDGER_API_KEY="TODO-fill-in-judger-api-key"
-
-# The ReMe agent's key reuses USER_API_KEY by default (no need to repeat
-# it when both use the same service and key).
-export REME_LLM_API_KEY="${REME_LLM_API_KEY:-${USER_API_KEY}}"
-
-# Brave Search (optional; used by the agent's web_search tool - use
-# "dummy" when not needed).
-export BRAVE_SEARCH_API_KEY="TODO-optional-brave-search-key-or-dummy"
-
-# ─── Persistent memory workspaces (one subdirectory per persona,
-#     created automatically) ───────────────────────────────────────────
-export REME_WORKSPACE_ROOT="${REME_WORKSPACE_ROOT:-${SUITE_DIR}/reme_workspace}"
-
-# ─── Variables consumed by ReMe's default.yaml model config expansion;
-#     do not remove ────────────────────────────────────────────────────
-export LLM_MODEL_NAME="${REME_MODEL_NAME}"
-export LLM_BASE_URL="${REME_LLM_BASE_URL}"
-export LLM_API_KEY="${REME_LLM_API_KEY}"

benchmark/pibench/fix_trace_logs.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-"""Convert reme_eval run outputs into eval-compatible trace logs.
-
-outputs/{model_id}/{user_id}/{task_id}/history/{ts}-messages.jsonl
-  ->  ~/.nanobot/trace_logs/{model_id}/{user_id}/{task_id}/{ts}/turn_N.json
-
-The bridge additionally writes {ts}-tools.jsonl sidecar files next to the
-message histories: one JSON object per executed tool call with fields
-{turn, name, arguments, result}. Each messages run is paired with the
-temporally closest sidecar, and the records are merged into the generated
-turn files under the "tool_steps" key, which is one of the tool-history
-formats π-Bench's collect_tool_history() understands. Without this step,
-tools_evaluation scripts would see no tool evidence at all.
-
-Usage: python fix_trace_logs.py [user_id ...]   (no args = all users)
-"""
-
-import json
-import re
-import sys
-from datetime import datetime
-from pathlib import Path
-
-SUITE_DIR = Path(__file__).resolve().parent
-OUTPUTS_DIR = SUITE_DIR / "outputs"
-TRACE_LOGS_DIR = Path.home() / ".nanobot" / "trace_logs"
-
-MESSAGES_FILE_RE = re.compile(r"^(\d{8}_\d{6})-messages\.jsonl$")
-TOOLS_FILE_RE = re.compile(r"^(\d{8}_\d{6})-tools\.jsonl$")
-TIME_FORMAT = "%Y%m%d_%H%M%S"
-# A tool sidecar belongs to the messages run that started at most this many
-# seconds earlier (the bridge stamps the sidecar when the task's first user
-# message arrives, shortly after the runner opened the messages file).
-MAX_PAIR_DELTA_SECONDS = 6 * 3600
-
-
-def _to_epoch(timestamp: str) -> float:
-    """Parse a YYYYMMDD_HHMMSS timestamp into epoch seconds."""
-    try:
-        return datetime.strptime(timestamp, TIME_FORMAT).timestamp()
-    except ValueError:
-        return 0.0
-
-
-def load_tool_records(tools_file: Path) -> dict:
-    """Group sidecar tool records by turn number."""
-    by_turn: dict = {}
-    try:
-        with open(tools_file, "r", encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    record = json.loads(line)
-                except json.JSONDecodeError:
-                    continue
-                if not isinstance(record, dict) or not record.get("name"):
-                    continue
-                turn = int(record.get("turn") or 0)
-                by_turn.setdefault(turn, []).append(
-                    {
-                        "name": record["name"],
-                        "arguments": record.get("arguments", {}),
-                        "result": record.get("result", ""),
-                    },
-                )
-    except OSError as exc:
-        print(f"  WARNING: cannot read tool sidecar {tools_file}: {exc}")
-    return by_turn
-
-
-def pair_tool_sidecars(message_runs: list, tool_runs: list) -> dict:
-    """Pair each messages run with the temporally closest unused tool sidecar.
-
-    Fresh runs produce exactly one messages file and one sidecar per task;
-    re-runs append matching pairs, so sorted greedy nearest-timestamp
-    matching is stable. Sidecars farther away than MAX_PAIR_DELTA_SECONDS
-    (e.g. leftovers of a crashed bridge) stay unpaired.
-    """
-    pairing: dict = {}
-    unused = list(tool_runs)
-    for msg_ts, _ in message_runs:
-        best_delta = None
-        best_item = None
-        for tool_ts, tool_path in unused:
-            delta = abs(_to_epoch(tool_ts) - _to_epoch(msg_ts))
-            if best_delta is None or delta < best_delta:
-                best_delta = delta
-                best_item = (tool_ts, tool_path)
-        if best_delta is not None and best_item is not None and best_delta <= MAX_PAIR_DELTA_SECONDS:
-            pairing[msg_ts] = best_item[1]
-            unused.remove(best_item)
-    return pairing
-
-
-def build_turns(messages: list) -> list:
-    """Split the flat message list into per-turn [user, assistant] groups."""
-    turns = []
-    i = 0
-    while i < len(messages):
-        turn_msgs = []
-        if messages[i]["role"] == "user":
-            turn_msgs.append({"role": "user", "content": messages[i]["message"]})
-            i += 1
-        if i < len(messages) and messages[i]["role"] == "assistant":
-            turn_msgs.append({"role": "assistant", "content": messages[i]["message"]})
-            i += 1
-        if not turn_msgs:
-            i += 1  # defensive: never spin on unexpected roles
-            continue
-        turns.append(turn_msgs)
-    return turns
-
-
-def convert_task(model_id: str, user_id: str, task_dir: Path) -> None:
-    """Convert one task's history dir into trace turn files with tool_steps."""
-    history_dir = task_dir / "history"
-    if not history_dir.is_dir():
-        return
-
-    message_runs = []
-    tool_runs = []
-    for msg_file in history_dir.glob("*-messages.jsonl"):
-        match = MESSAGES_FILE_RE.match(msg_file.name)
-        if match:
-            message_runs.append((match.group(1), msg_file))
-    for tools_file in history_dir.glob("*-tools.jsonl"):
-        match = TOOLS_FILE_RE.match(tools_file.name)
-        if match:
-            tool_runs.append((match.group(1), tools_file))
-    if not message_runs:
-        return
-
-    message_runs.sort(key=lambda item: item[0])
-    tool_runs.sort(key=lambda item: item[0])
-    pairing = pair_tool_sidecars(message_runs, tool_runs)
-
-    print(f"\n{model_id}/{user_id}/{task_dir.name}")
-    for timestamp, msg_file in message_runs:
-        trace_dir = TRACE_LOGS_DIR / model_id / user_id / task_dir.name / timestamp
-        trace_dir.mkdir(parents=True, exist_ok=True)
-
-        messages = []
-        with open(msg_file, "r", encoding="utf-8") as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                msg = json.loads(line)
-                if msg.get("role") == "user" and msg.get("message") == "/new":
-                    continue
-                messages.append(msg)
-
-        tools_file = pairing.get(timestamp)
-        tools_by_turn = load_tool_records(tools_file) if tools_file else {}
-        if tools_file is not None:
-            print(f"  {timestamp}: paired tool sidecar {tools_file.name}")
-
-        turns = build_turns(messages)
-        for turn_idx, turn_msgs in enumerate(turns, start=1):
-            turn_data = {"messages": turn_msgs}
-            tool_steps = tools_by_turn.get(turn_idx)
-            if tool_steps:
-                turn_data["tool_steps"] = tool_steps
-            turn_file = trace_dir / f"turn_{turn_idx}.json"
-            with open(turn_file, "w", encoding="utf-8") as f:
-                json.dump(turn_data, f, indent=2, ensure_ascii=False)
-        tool_total = sum(len(steps) for steps in tools_by_turn.values())
-        print(f"  {timestamp}: {len(turns)} turns, {tool_total} tool step(s) -> {trace_dir}")
-
-
-def convert_outputs(user_filter=None):
-    """Convert message history JSONL files into per-turn trace JSON files."""
-    if not OUTPUTS_DIR.exists():
-        print(f"outputs dir not found: {OUTPUTS_DIR}")
-        return
-
-    for model_dir in sorted(OUTPUTS_DIR.iterdir()):
-        if not model_dir.is_dir():
-            continue
-        model_id = model_dir.name
-
-        for user_dir in sorted(model_dir.iterdir()):
-            if not user_dir.is_dir():
-                continue
-            user_id = user_dir.name
-            if user_filter and user_id not in user_filter:
-                continue
-
-            for task_dir in sorted(user_dir.iterdir()):
-                if task_dir.is_dir():
-                    convert_task(model_id, user_id, task_dir)
-
-
-if __name__ == "__main__":
-    convert_outputs(set(sys.argv[1:]) or None)
-    print("\ndone")

benchmark/pibench/resume.py

@@ -1,332 +0,0 @@
-#!/usr/bin/env python3
-"""Checkpoint-resume support for the reme_eval suite.
-
-Completion source of truth:
-    - outputs/reme/<persona>/<task_id>/history/*-log.jsonl  (per-task logs,
-      flushed incrementally, survive mid-run kills)
-    - outputs/reme/<persona>/run/*-log.jsonl                (run-level logs,
-      may be truncated if the process was killed before flush)
-    lines: "Task finished task_id=<id> status=<STATUS>"
-    A task counts as COMPLETED when its latest terminal status is one of
-    SUCCESS / MAX_TURNS / TIMEOUT. ERROR or never-started tasks stay pending.
-
-    "Latest" is decided by EVENT TIME, not by file category or read order:
-    each record's "timestamp" (epoch seconds, or "timestamp_iso" as fallback)
-    is compared across per-task and run-level logs alike, with the timestamp
-    embedded in the log file name as a last-resort fallback. This keeps an
-    old run-level SUCCESS from overriding a newer per-task ERROR when the
-    re-run died before the new run-level log captured the task.
-
-Commands:
-    remaining <persona> [--json]
-        Print task_ids still to run, in data/<persona>/episode.yaml order
-        (one per line; --json prints {"completed": [...], "remaining": [...]}).
-
-    cleanup <persona> [--dry-run]
-        Surgically remove residual memory artifacts of tasks that are about
-        to be RE-RUN (i.e. pending tasks that left partial state because a
-        previous run was interrupted). This prevents answer leakage: an
-        interrupted task's conversation may already have been distilled into
-        daily notes during graceful shutdown, and re-running the task with
-        that memory injected would inflate scores.
-
-        Removed artifacts (only for pending tasks with residual state):
-          - daily/<date>/<note>.md whose frontmatter session_id matches
-            pibench_<task_id>_*, plus a refresh of ONLY the daily index of
-            the affected date(s) (daily/<date>.md), matched by the full
-            workspace-relative note path, never by bare file name
-          - digest notes with matching session_id
-          - session/dialog/pibench_<task_id>_*.jsonl
-          - mem_session/**.jsonl files containing pibench_<task_id>_
-        When the ReMe package is importable, the daily index refresh reuses
-        ReMe's own rebuild logic (reme.steps.file_io._daily_index.
-        refresh_day_index); otherwise index lines are dropped by exact
-        wikilink path match. Either way, indexes of other dates are never
-        touched. The ReMe watcher (init_changes_step) detects the deleted
-        daily notes on next bridge startup and removes them from the BM25
-        index itself.
-
-        Completed tasks' memories are NEVER touched by this command.
-
-Design note (resume vs memory-wipe conflict):
-    A full memory wipe is a suite-level action of fresh mode (run_all.sh
-    without --resume) and happens before any service starts. Resume mode
-    never wipes; it only performs the surgical cleanup above. The two modes
-    are mutually exclusive, so a resumed run can never lose the cross-session
-    memory accumulated by completed tasks.
-"""
-
-import asyncio
-import json
-import os
-import re
-import sys
-from datetime import datetime
-from pathlib import Path
-
-import yaml
-
-try:  # Reuse ReMe's daily-index rebuild when running inside the ReMe venv.
-    from reme.steps.file_io._daily_index import refresh_day_index
-except ImportError:  # pragma: no cover - depends on runtime venv
-    refresh_day_index = None
-
-SUITE_DIR = Path(__file__).resolve().parent
-DATA_DIR = Path(os.environ.get("REME_EVAL_DATA_DIR", SUITE_DIR / "data")).resolve()
-OUTPUTS_DIR = Path(os.environ.get("REME_EVAL_OUTPUTS_DIR", SUITE_DIR / "outputs")) / "reme"
-WORKSPACE_ROOT = Path(
-    os.environ.get("REME_WORKSPACE_ROOT", SUITE_DIR / "reme_workspace"),
-).resolve()
-
-COMPLETED_STATUSES = {"SUCCESS", "MAX_TURNS", "TIMEOUT"}
-TASK_FINISHED_RE = re.compile(r"Task finished task_id=(\S+) status=(\S+)")
-SESSION_ID_RE = re.compile(r"^session_id:\s*(\S+)", re.MULTILINE)
-NOTE_COUNT_RE = re.compile(r"(description:\s*)\d+(\s*note\(s\) today)")
-LOG_FILE_TS_RE = re.compile(r"^(\d{8}_\d{6})-log\.jsonl$")
-TIME_FORMAT = "%Y%m%d_%H%M%S"
-
-
-def log(msg: str) -> None:
-    """Print a status message to stderr."""
-    print(msg, file=sys.stderr)
-
-
-def episode_task_order(persona: str) -> list[str]:
-    """Return the ordered task ids from the persona's episode.yaml."""
-    episode_path = DATA_DIR / persona / "episode.yaml"
-    with open(episode_path, "r", encoding="utf-8") as f:
-        episode = yaml.safe_load(f)
-    return [task["task_id"] for task in episode.get("tasks", [])]
-
-
-def _event_time(record: dict, file_ts: str) -> float:
-    """Best-effort event time (epoch seconds) of one log record.
-
-    Prefers the record's own timestamp fields; falls back to the timestamp
-    embedded in the log file name so that even stripped records keep a
-    meaningful order. Returns 0.0 when nothing is parseable.
-    """
-    timestamp = record.get("timestamp")
-    if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool):
-        return float(timestamp)
-    iso = record.get("timestamp_iso")
-    if isinstance(iso, str):
-        try:
-            return datetime.fromisoformat(iso).timestamp()
-        except ValueError:
-            pass
-    if file_ts:
-        try:
-            return datetime.strptime(file_ts, TIME_FORMAT).timestamp()
-        except ValueError:
-            pass
-    return 0.0
-
-
-def latest_task_statuses(persona: str) -> dict[str, str]:
-    """Scan per-task and run-level logs; the newest EVENT TIME wins per task.
-
-    Every "Task finished" record across both log categories is keyed by
-    (event_time, file timestamp, file order, line number); the record with
-    the highest key decides the task's status. File category and read order
-    alone can never override a newer record from the other category.
-    """
-    persona_dir = OUTPUTS_DIR / persona
-    if not persona_dir.is_dir():
-        return {}
-
-    log_files = sorted(persona_dir.glob("*/history/*-log.jsonl"))
-    log_files += sorted(persona_dir.glob("run/*-log.jsonl"))
-
-    best: dict[str, tuple[tuple, str]] = {}
-    for file_order, log_file in enumerate(log_files):
-        ts_match = LOG_FILE_TS_RE.match(log_file.name)
-        file_ts = ts_match.group(1) if ts_match else ""
-        try:
-            with open(log_file, "r", encoding="utf-8") as f:
-                for line_no, line in enumerate(f):
-                    if "Task finished" not in line:
-                        continue
-                    try:
-                        record = json.loads(line)
-                    except json.JSONDecodeError:
-                        continue
-                    match = TASK_FINISHED_RE.search(str(record.get("message", "")))
-                    if not match:
-                        continue
-                    task_id, status = match.group(1), match.group(2)
-                    sort_key = (_event_time(record, file_ts), file_ts, file_order, line_no)
-                    current = best.get(task_id)
-                    if current is None or sort_key > current[0]:
-                        best[task_id] = (sort_key, status)
-        except OSError:
-            continue
-    return {task_id: status for task_id, (_, status) in best.items()}
-
-
-def split_tasks(persona: str) -> tuple[list[str], list[str]]:
-    """Split the episode task order into completed and remaining tasks."""
-    order = episode_task_order(persona)
-    statuses = latest_task_statuses(persona)
-    completed = [t for t in order if statuses.get(t) in COMPLETED_STATUSES]
-    remaining = [t for t in order if t not in set(completed)]
-    return completed, remaining
-
-
-def _daily_note_session_id(note_path: Path) -> str:
-    try:
-        text = note_path.read_text(encoding="utf-8")
-    except OSError:
-        return ""
-    match = SESSION_ID_RE.search(text)
-    return match.group(1) if match else ""
-
-
-class _WorkspaceFileStoreShim:
-    """Structural stand-in for ReMe's file store; only workspace_path is read."""
-
-    def __init__(self, workspace_path: Path):
-        self.workspace_path = workspace_path
-
-
-def _refresh_daily_indexes(
-    workspace: Path,
-    removed_by_date: dict[str, set[str]],
-    removed: list[str],
-) -> None:
-    """Rebuild the daily index of each affected date via ReMe's own logic."""
-    for date in sorted(removed_by_date):
-        result = asyncio.run(
-            refresh_day_index(_WorkspaceFileStoreShim(workspace), date, "daily"),
-        )
-        if result.get("error"):
-            log(f"[resume] WARNING: daily index refresh failed for {date}: {result['error']}")
-            continue
-        removed.append(f"daily/{date}.md (refreshed, {len(removed_by_date[date])} note(s) removed)")
-
-
-def _strip_index_lines(
-    workspace: Path,
-    removed_by_date: dict[str, set[str]],
-    removed: list[str],
-    dry_run: bool,
-) -> None:
-    """Fallback index edit: drop lines that reference removed notes by full
-    workspace-relative wikilink path, and fix the note count. Only the index
-    files of affected dates are touched."""
-    for date in sorted(removed_by_date):
-        index_path = workspace / "daily" / f"{date}.md"
-        if not index_path.is_file():
-            continue
-        wikilinks = [f"[[{rel_path}]]" for rel_path in sorted(removed_by_date[date])]
-        lines = index_path.read_text(encoding="utf-8").splitlines()
-        kept = [line for line in lines if not any(link in line for link in wikilinks)]
-        if len(kept) == len(lines):
-            continue
-        note_count = sum(1 for line in kept if line.startswith("- [[daily/"))
-        kept = [NOTE_COUNT_RE.sub(rf"\g<1>{note_count}\2", line) for line in kept]
-        removed.append(f"{index_path.relative_to(workspace)} (rewritten)")
-        if not dry_run:
-            index_path.write_text("\n".join(kept) + "\n", encoding="utf-8")
-
-
-def cleanup_partial_memory(persona: str, remaining: list[str], dry_run: bool = False) -> list[str]:
-    """Remove partial memory artifacts of remaining tasks so they can be re-run cleanly."""
-    workspace = WORKSPACE_ROOT / persona
-    removed: list[str] = []
-    if not workspace.is_dir() or not remaining:
-        return removed
-
-    prefixes = tuple(f"pibench_{task_id}_" for task_id in remaining)
-
-    def act(path: Path, label: str) -> None:
-        removed.append(label)
-        if not dry_run:
-            path.unlink()
-
-    # 1) daily / digest notes distilled from interrupted sessions. For daily
-    # notes, remember the full workspace-relative path grouped by date so only
-    # the affected daily indexes are refreshed below.
-    removed_by_date: dict[str, set[str]] = {}
-    for section in ("daily", "digest"):
-        section_root = workspace / section
-        if not section_root.is_dir():
-            continue
-        for note_path in section_root.rglob("*.md"):
-            if note_path.parent == section_root:
-                continue  # index files handled below
-            session_id = _daily_note_session_id(note_path)
-            if session_id.startswith(prefixes):
-                rel_path = note_path.relative_to(workspace).as_posix()
-                act(note_path, rel_path)
-                if section == "daily":
-                    removed_by_date.setdefault(note_path.parent.name, set()).add(rel_path)
-
-    # 2) daily index files: refresh only the dates that lost notes, matching
-    # notes by their full wikilink path instead of their bare file name.
-    if removed_by_date:
-        if dry_run:
-            for date in sorted(removed_by_date):
-                removed.append(f"daily/{date}.md (would refresh index)")
-        elif refresh_day_index is not None:
-            _refresh_daily_indexes(workspace, removed_by_date, removed)
-        else:
-            _strip_index_lines(workspace, removed_by_date, removed, dry_run)
-
-    # 3) raw dialog logs of interrupted sessions
-    dialog_dir = workspace / "session" / "dialog"
-    if dialog_dir.is_dir():
-        for task_id in remaining:
-            for dialog_path in dialog_dir.glob(f"pibench_{task_id}_*.jsonl"):
-                act(dialog_path, str(dialog_path.relative_to(workspace)))
-
-    # 4) agent-scope session states that contain interrupted-task sessions
-    mem_session_dir = workspace / "mem_session"
-    if mem_session_dir.is_dir():
-        for session_path in mem_session_dir.rglob("*.jsonl"):
-            try:
-                content = session_path.read_text(encoding="utf-8", errors="ignore")
-            except OSError:
-                continue
-            if any(prefix in content for prefix in prefixes):
-                act(session_path, str(session_path.relative_to(workspace)))
-
-    return removed
-
-
-def main() -> int:
-    """CLI entrypoint: run 'remaining' or 'cleanup' action for a persona."""
-    args = sys.argv[1:]
-    if len(args) < 2 or args[0] not in {"remaining", "cleanup"}:
-        print(__doc__, file=sys.stderr)
-        return 2
-
-    command, persona = args[0], args[1]
-    completed, remaining = split_tasks(persona)
-
-    if command == "remaining":
-        if "--json" in args:
-            print(json.dumps({"completed": completed, "remaining": remaining}))
-        else:
-            for task_id in remaining:
-                print(task_id)
-        log(
-            f"[resume] {persona}: completed={len(completed)} "
-            f"({', '.join(completed) if completed else '-'}) remaining={len(remaining)}",
-        )
-        return 0
-
-    dry_run = "--dry-run" in args
-    removed = cleanup_partial_memory(persona, remaining, dry_run=dry_run)
-    if removed:
-        verb = "would remove" if dry_run else "removed"
-        log(f"[resume] {persona}: {verb} {len(removed)} partial-memory artifact(s):")
-        for item in removed:
-            log(f"  - {item}")
-    else:
-        log(f"[resume] {persona}: no partial-memory artifacts to clean")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

benchmark/pibench/run_all.sh

@@ -1,119 +0,0 @@
-#!/bin/bash
-# Run all 5 personas with the ReMe agent, PARALLEL at a time (default 2).
-# Each persona's tasks follow data/{persona}/episode.yaml order.
-#
-# Usage:
-#   bash run_all.sh                  # FRESH official run: wipes ALL personas'
-#                                    # ReMe memory/outputs/trace logs first,
-#                                    # then runs everything from scratch.
-#   bash run_all.sh --resume         # Checkpoint continuation: no wipe; every
-#                                    # persona skips already-completed tasks.
-#   bash run_all.sh --parallel 1     # sequential (original behavior)
-#   bash run_all.sh --skip-eval      # run phase only
-#
-# Memory-wipe vs resume conflict resolution:
-#   The full ReMe memory wipe happens ONLY here, ONLY in fresh mode (the
-#   default), and ONLY before any service/bridge starts. --resume never
-#   wipes; run_persona.sh then additionally performs a surgical cleanup of
-#   residual memory belonging to interrupted (to-be-re-run) tasks, so a
-#   resumed run keeps all completed-task memory but never inherits a partial
-#   task's own answer. The two modes are mutually exclusive.
-set -uo pipefail
-
-SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PERSONAS=(researcher marketer law_trainee pharmacist Financier)
-TRACE_ROOT="${HOME}/.nanobot/trace_logs"
-
-PARALLEL=2
-MODE="fresh"
-PASS_ARGS=()
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --parallel)
-            PARALLEL="${2:-}"; shift 2 || true
-            case "$PARALLEL" in (""|*[!0-9]*) echo "--parallel needs a positive integer"; exit 2 ;; esac
-            [ "$PARALLEL" -lt 1 ] && PARALLEL=1
-            [ "$PARALLEL" -gt ${#PERSONAS[@]} ] && PARALLEL=${#PERSONAS[@]}
-            ;;
-        --resume)
-            if [ "$MODE" = "fresh_set" ]; then echo "--fresh and --resume are mutually exclusive"; exit 2; fi
-            MODE="resume"; shift ;;
-        --fresh)
-            if [ "$MODE" = "resume" ]; then echo "--fresh and --resume are mutually exclusive"; exit 2; fi
-            MODE="fresh_set"; shift ;;
-        --skip-eval) PASS_ARGS+=(--skip-eval); shift ;;
-        *) echo "Unknown option: $1"; exit 1 ;;
-    esac
-done
-[ "$MODE" = "fresh_set" ] && MODE="fresh"
-
-START_TS=$(date +%Y%m%d_%H%M%S)
-SUMMARY_LOG="${SUITE_DIR}/logs/run_all_${START_TS}.summary"
-mkdir -p "${SUITE_DIR}/logs"
-
-echo "############################################################"
-echo "# reme_eval suite | mode=${MODE} parallel=${PARALLEL} | ${START_TS}"
-echo "############################################################"
-
-# ─── Fresh mode: suite-level wipe BEFORE anything starts ──────────────
-if [ "$MODE" = "fresh" ]; then
-    echo "[fresh] wiping ALL personas' memory workspaces, outputs and trace logs..."
-    for persona in "${PERSONAS[@]}"; do
-        rm -rf "${SUITE_DIR}/reme_workspace/${persona}"
-        rm -rf "${SUITE_DIR}/outputs/reme/${persona}"
-        rm -rf "${TRACE_ROOT}/reme/${persona}"
-        rm -rf "${SUITE_DIR}/nanobot_workspace/${persona}"
-    done
-    echo "[fresh] wipe done."
-else
-    echo "[resume] no memory wipe; personas resume after their last completed task."
-fi
-
-# ─── Run personas in batches of PARALLEL ──────────────────────────────
-STATUS_LIST=()
-ANY_FAILED=0
-OVERALL_START=$(date +%s)
-TOTAL=${#PERSONAS[@]}
-
-for ((i = 0; i < TOTAL; i += PARALLEL)); do
-    BATCH=("${PERSONAS[@]:i:PARALLEL}")
-    BATCH_PIDS=()
-    BATCH_NAMES=()
-    echo ""
-    echo "============================================================"
-    echo "# BATCH $(( i / PARALLEL + 1 )): ${BATCH[*]}   started $(date '+%F %T')"
-    echo "============================================================"
-    for persona in "${BATCH[@]}"; do
-        bash "${SUITE_DIR}/run_persona.sh" "${persona}" --resume ${PASS_ARGS[@]+"${PASS_ARGS[@]}"} \
-            > "${SUITE_DIR}/logs/suite_${persona}.log" 2>&1 &
-        BATCH_PIDS+=($!)
-        BATCH_NAMES+=("$persona")
-    done
-    for j in $(seq 0 $(( ${#BATCH[@]} - 1 ))); do
-        pid=${BATCH_PIDS[$j]}
-        persona=${BATCH_NAMES[$j]}
-        if wait "$pid"; then
-            STATUS_LIST+=("${persona}: OK")
-        else
-            rc=$?
-            ANY_FAILED=1
-            STATUS_LIST+=("${persona}: FAILED rc=${rc}")
-            echo "[run_all] ${persona} FAILED (rc=${rc}); see logs/suite_${persona}.log"
-        fi
-    done
-done
-
-total=$(( $(date +%s) - OVERALL_START ))
-echo ""
-echo "================ FINAL SUMMARY (${total}s total) ================" | tee -a "${SUMMARY_LOG}"
-for line in "${STATUS_LIST[@]}"; do
-    echo "  ${line}" | tee -a "${SUMMARY_LOG}"
-done
-echo "Summary: ${SUMMARY_LOG}"
-
-if [ "${ANY_FAILED}" -ne 0 ]; then
-    FAILED_COUNT=$(printf '%s\n' "${STATUS_LIST[@]}" | grep -c "FAILED")
-    echo "[run_all] ${FAILED_COUNT} persona(s) FAILED; suite run is marked as failed." | tee -a "${SUMMARY_LOG}"
-    exit 1
-fi
-exit 0

benchmark/pibench/run_persona.sh

@@ -1,301 +0,0 @@
-#!/bin/bash
-# Run the full pi-bench evaluation for ONE persona with the ReMe agent.
-# Tasks follow data/{persona}/episode.yaml order (runner-native).
-#
-# Usage: bash run_persona.sh <persona> [--fresh|--resume] [--skip-eval]
-#
-# Modes (default: --resume):
-#   --resume  Checkpoint continuation. Never wipes memory. Tasks already
-#             finished (SUCCESS/MAX_TURNS/TIMEOUT in the task history logs)
-#             are skipped via repeated --task-id flags. Before starting, any
-#             residual memory of tasks that are about to be RE-RUN (partial
-#             sessions from an interrupted run) is surgically removed by
-#             resume.py cleanup, so re-runs don't inherit leaked answers.
-#   --fresh   Wipes THIS persona's ReMe memory, outputs and trace logs first,
-#             then runs all tasks from scratch.
-# The two flags are mutually exclusive. A full multi-persona memory wipe is a
-# suite-level action of `run_all.sh` (fresh mode), never done here implicitly.
-set -uo pipefail
-
-SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-TRACE_ROOT="${HOME}/.nanobot/trace_logs"
-
-# ─── External dependencies (pi-bench / ReMe are NOT bundled; see README) ──
-if [ ! -f "${SUITE_DIR}/env.sh" ]; then
-    echo "env.sh not found. Run: cp env.sh.example env.sh  (then fill in the TODO items)"
-    exit 1
-fi
-source "${SUITE_DIR}/env.sh"
-
-PIBENCH_DIR="${PI_BENCH_ROOT:-}"
-if [ -z "${PIBENCH_DIR}" ] || [ ! -f "${PIBENCH_DIR}/src/main.py" ]; then
-    echo "PI_BENCH_ROOT is unset or invalid (src/main.py not found). Set it in env.sh."
-    exit 1
-fi
-if [ ! -x "${PIBENCH_DIR}/.venv/bin/python" ] || [ ! -x "${PIBENCH_DIR}/.venv/bin/appworld" ]; then
-    echo "pi-bench venv incomplete: ${PIBENCH_DIR}/.venv must provide python + appworld (see README setup)."
-    exit 1
-fi
-if [ ! -x "${REME_DIR}/.venv/bin/python" ]; then
-    echo "ReMe venv not found: ${REME_DIR}/.venv/bin/python (check REME_DIR in env.sh)"
-    exit 1
-fi
-if [ ! -e "${SUITE_DIR}/data" ]; then
-    echo 'Benchmark data not linked. Run: ln -s "$PI_BENCH_ROOT/data" data'
-    exit 1
-fi
-
-# ─── Pre-flight: files the runner needs before any service starts ─────
-MODEL_CONFIG="${SUITE_DIR}/config/models/reme.yaml"
-HISTORY_CONFIG="${SUITE_DIR}/config/bench/evaluation/trace_history.yaml"
-if [ ! -f "${MODEL_CONFIG}" ]; then
-    echo "Model config not found: ${MODEL_CONFIG} (see README directory layout)."
-    exit 1
-fi
-if [ ! -f "${HISTORY_CONFIG}" ]; then
-    echo "Trace history config not found: ${HISTORY_CONFIG}"
-    echo "pi-bench requires config/bench/evaluation/trace_history.yaml; see README."
-    exit 1
-fi
-
-APPWORLD_DIR="${PIBENCH_DIR}/third_party/appworld"
-PI_PYTHON="${PIBENCH_DIR}/.venv/bin/python"
-APPWORLD_BIN="${PIBENCH_DIR}/.venv/bin/appworld"
-# resume.py runs on the ReMe venv so it can reuse ReMe's daily-index rebuild.
-REME_PYTHON="${REME_DIR}/.venv/bin/python"
-
-PERSONA="${1:-}"
-if [ -z "$PERSONA" ]; then
-    echo "Usage: $0 <persona> [--fresh|--resume] [--skip-eval]"
-    exit 1
-fi
-shift
-
-MODE="resume"
-SKIP_EVAL=false
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --fresh)
-            if [ "$MODE" = "resume_set" ]; then echo "--fresh and --resume are mutually exclusive"; exit 2; fi
-            MODE="fresh"; shift ;;
-        --resume)
-            if [ "$MODE" = "fresh" ]; then echo "--fresh and --resume are mutually exclusive"; exit 2; fi
-            MODE="resume_set"; shift ;;
-        --skip-eval) SKIP_EVAL=true; shift ;;
-        *) echo "Unknown option: $1"; exit 1 ;;
-    esac
-done
-[ "$MODE" = "resume_set" ] && MODE="resume"
-
-# ─── Per-persona ports (pi-bench AGENTS.md convention) ────────────────
-# REME_PORT: ReMe's internal HTTP service; must be unique per concurrent bridge.
-case "$PERSONA" in
-    marketer)     API_PORT=9001; MCP_PORT=10001; TEST_PORT=9998; REME_PORT=18766 ;;
-    law_trainee)  API_PORT=9002; MCP_PORT=10002; TEST_PORT=9997; REME_PORT=18767 ;;
-    pharmacist)   API_PORT=9003; MCP_PORT=10003; TEST_PORT=9996; REME_PORT=18768 ;;
-    researcher)   API_PORT=9004; MCP_PORT=10004; TEST_PORT=9995; REME_PORT=18765 ;;
-    Financier)    API_PORT=9005; MCP_PORT=10005; TEST_PORT=9994; REME_PORT=18769 ;;
-    *) echo "Unknown persona: $PERSONA"; exit 1 ;;
-esac
-
-API_URL="http://127.0.0.1:${API_PORT}"
-MCP_URL="http://127.0.0.1:${MCP_PORT}/mcp"
-TEST_URL="http://127.0.0.1:${TEST_PORT}"
-LOG_DIR="${SUITE_DIR}/logs"
-mkdir -p "${LOG_DIR}"
-
-# ─── Environment (env.sh already sourced at the top) ──────────────────
-WORKSPACE_DIR="${REME_WORKSPACE_ROOT}/${PERSONA}"
-NANOBOT_WORKSPACE_DIR="${SUITE_DIR}/nanobot_workspace/${PERSONA}"
-mkdir -p "${WORKSPACE_DIR}" "${NANOBOT_WORKSPACE_DIR}"
-
-echo "========================================="
-echo "ReMe x Pi-Bench | persona=${PERSONA} | mode=${MODE}"
-echo "  api=${API_PORT} mcp=${MCP_PORT} test=${TEST_PORT} reme=${REME_PORT}"
-echo "  model=${REME_MODEL_NAME}"
-echo "  memory workspace=${WORKSPACE_DIR}  (persistent)"
-echo "========================================="
-
-# ─── Fresh mode: wipe this persona's state ────────────────────────────
-if [ "$MODE" = "fresh" ]; then
-    echo "[fresh] wiping persona state: memory workspace, outputs, trace logs"
-    rm -rf "${WORKSPACE_DIR}"
-    rm -rf "${SUITE_DIR}/outputs/reme/${PERSONA}"
-    rm -rf "${TRACE_ROOT}/reme/${PERSONA}"
-    rm -rf "${NANOBOT_WORKSPACE_DIR}"
-    mkdir -p "${WORKSPACE_DIR}" "${NANOBOT_WORKSPACE_DIR}"
-fi
-
-# ─── Resume: determine remaining tasks + clean partial memories ───────
-TASK_ARGS=()
-RUN_PHASE_NEEDED=true
-if [ "$MODE" = "resume" ]; then
-    REMAINING_JSON="$("${REME_PYTHON}" "${SUITE_DIR}/resume.py" remaining "${PERSONA}" --json)"
-    if [ -z "$REMAINING_JSON" ]; then
-        echo "Failed to compute remaining tasks"; exit 1
-    fi
-    echo "[resume] ${REMAINING_JSON}"
-    REMAINING_TASKS=()
-    while IFS= read -r tid_line; do
-        [ -n "$tid_line" ] && REMAINING_TASKS+=("$tid_line")
-    done < <("${REME_PYTHON}" "${SUITE_DIR}/resume.py" remaining "${PERSONA}" 2>/dev/null)
-    if [ ${#REMAINING_TASKS[@]} -eq 0 ]; then
-        RUN_PHASE_NEEDED=false
-        echo "[resume] all tasks already completed; skipping run phase"
-    else
-        # Remove residual memory of interrupted (to-be-re-run) tasks so
-        # re-runs don't get their own partial answers injected.
-        "${REME_PYTHON}" "${SUITE_DIR}/resume.py" cleanup "${PERSONA}"
-        for tid in "${REMAINING_TASKS[@]}"; do
-            TASK_ARGS+=(--task-id "$tid")
-        done
-        echo "[resume] running ${#REMAINING_TASKS[@]} remaining task(s): ${REMAINING_TASKS[*]}"
-    fi
-fi
-
-# ─── Port cleanup from previous runs ──────────────────────────────────
-for port in ${API_PORT} ${MCP_PORT} ${TEST_PORT} ${REME_PORT}; do
-    pids=$(lsof -ti :${port} 2>/dev/null || true)
-    if [ -n "$pids" ]; then
-        echo "Killing stale processes on port ${port}: ${pids}"
-        kill -9 $pids 2>/dev/null || true
-    fi
-done
-sleep 2
-
-PIDS=()
-cleanup() {
-    echo "[${PERSONA}] cleaning up services..."
-    for pid in "${PIDS[@]:-}"; do
-        kill "$pid" 2>/dev/null || true
-    done
-    wait 2>/dev/null || true
-}
-trap cleanup EXIT INT TERM
-
-wait_for_service() {
-    local url="$1" name="$2" port="$3" timeout="${4:-180}"
-    echo -n "  waiting for ${name}..."
-    local start=$(date +%s)
-    while true; do
-        if curl -sf --max-time 5 "${url}" > /dev/null 2>&1; then
-            echo " ready"; return 0
-        fi
-        if [ -n "$port" ] && lsof -ti :${port} > /dev/null 2>&1; then
-            local elapsed=$(( $(date +%s) - start ))
-            if [ "$elapsed" -ge 10 ]; then echo " ready (port)"; return 0; fi
-        fi
-        if [ $(( $(date +%s) - start )) -ge "$timeout" ]; then
-            echo " TIMEOUT"; return 1
-        fi
-        sleep 2
-    done
-}
-
-# ─── [1/5] AppWorld API ────────────────────────────────────────────────
-echo "[1/5] AppWorld API (:${API_PORT})"
-(cd "${APPWORLD_DIR}" && exec "${APPWORLD_BIN}" serve apis --root . \
-    --port ${API_PORT}) > "${LOG_DIR}/appworld_api_${PERSONA}.log" 2>&1 &
-PIDS+=($!)
-if ! wait_for_service "${API_URL}/docs" "AppWorld API" "${API_PORT}" 180; then
-    tail -20 "${LOG_DIR}/appworld_api_${PERSONA}.log"; exit 1
-fi
-
-# ─── [2/5] AppWorld MCP ────────────────────────────────────────────────
-echo "[2/5] AppWorld MCP (:${MCP_PORT})"
-TOOLS_CONFIG="${SUITE_DIR}/data/${PERSONA}/tools.yaml"
-(cd "${APPWORLD_DIR}" && exec "${APPWORLD_BIN}" serve mcp http --root . \
-    --remote-apis-url "${API_URL}" --port ${MCP_PORT} \
-    --tools-config-file "${TOOLS_CONFIG}") > "${LOG_DIR}/appworld_mcp_${PERSONA}.log" 2>&1 &
-PIDS+=($!)
-if ! wait_for_service "${MCP_URL}" "AppWorld MCP" "${MCP_PORT}" 180; then
-    tail -20 "${LOG_DIR}/appworld_mcp_${PERSONA}.log"; exit 1
-fi
-
-# ─── [3/5] Test Server ─────────────────────────────────────────────────
-echo "[3/5] Test Server (:${TEST_PORT})"
-PORT=${TEST_PORT} "${PI_PYTHON}" "${PIBENCH_DIR}/scripts/test_server.py" \
-    > "${LOG_DIR}/test_server_${PERSONA}.log" 2>&1 &
-PIDS+=($!)
-if ! wait_for_service "${TEST_URL}/sent?after=-1" "Test Server" "${TEST_PORT}" 30; then
-    tail -20 "${LOG_DIR}/test_server_${PERSONA}.log"; exit 1
-fi
-
-# ─── [4/5] ReMe Bridge (ReMe venv) ─────────────────────────────────────
-echo "[4/5] ReMe Bridge (reme service port ${REME_PORT})"
-"${REME_DIR}/.venv/bin/python" "${SUITE_DIR}/bridge_reme.py" \
-    --test-server-url "${TEST_URL}" \
-    --appworld-mcp-url "${MCP_URL}" \
-    --reme-dir "${REME_DIR}" \
-    --data-root "${SUITE_DIR}/data" \
-    --user-id "${PERSONA}" \
-    --workspace-dir "${WORKSPACE_DIR}" \
-    --reme-port "${REME_PORT}" \
-    --model-name "${REME_MODEL_NAME}" \
-    --model-base-url "${REME_LLM_BASE_URL}" \
-    --model-api-key "${REME_LLM_API_KEY}" \
-    > "${LOG_DIR}/bridge_${PERSONA}.log" 2>&1 &
-BRIDGE_PID=$!
-PIDS+=(${BRIDGE_PID})
-sleep 5
-if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "Bridge failed to start:"; tail -30 "${LOG_DIR}/bridge_${PERSONA}.log"; exit 1
-fi
-for i in $(seq 1 12); do
-    if grep -q "Bridge started:" "${LOG_DIR}/bridge_${PERSONA}.log" 2>/dev/null; then
-        echo "  bridge initialized"; break
-    fi
-    sleep 5
-done
-grep -q "Bridge started:" "${LOG_DIR}/bridge_${PERSONA}.log" 2>/dev/null || {
-    echo "WARNING: bridge may not be ready:"; tail -20 "${LOG_DIR}/bridge_${PERSONA}.log"; }
-
-# ─── [5/5] Runner (run phase) ──────────────────────────────────────────
-if [ "$RUN_PHASE_NEEDED" = true ]; then
-    echo "[5/5] Runner: run phase (episode order from data/${PERSONA}/episode.yaml)"
-    cd "${SUITE_DIR}"
-    BENCH_TEST_SERVER_URL="${TEST_URL}" PYTHONPATH="${PIBENCH_DIR}" \
-        "${PI_PYTHON}" -m src.main \
-        --model-config "${MODEL_CONFIG}" \
-        --history-config-path "${HISTORY_CONFIG}" \
-        --mode run --user-id "${PERSONA}" \
-        --workspace-dir "${NANOBOT_WORKSPACE_DIR}" \
-        ${TASK_ARGS[@]+"${TASK_ARGS[@]}"} \
-        2>&1 | tee "${LOG_DIR}/runner_run_${PERSONA}.log"
-    RUN_EXIT=${PIPESTATUS[0]}
-    if [ ${RUN_EXIT} -ne 0 ]; then
-        echo "Run phase failed (exit ${RUN_EXIT}). Logs: ${LOG_DIR}/"
-        exit ${RUN_EXIT}
-    fi
-else
-    echo "[5/5] Runner: run phase skipped (all tasks completed)"
-fi
-
-if [ "$SKIP_EVAL" = true ]; then
-    echo "Skipping eval (--skip-eval)"
-    exit 0
-fi
-
-# ─── Trace conversion + eval phase (always over all available traces) ──
-echo "Converting trace logs..."
-"${PI_PYTHON}" "${SUITE_DIR}/fix_trace_logs.py" "${PERSONA}"
-
-echo "Runner: eval phase"
-cd "${SUITE_DIR}"
-BENCH_TEST_SERVER_URL="${TEST_URL}" PYTHONPATH="${PIBENCH_DIR}" \
-    "${PI_PYTHON}" -m src.main \
-    --model-config "${MODEL_CONFIG}" \
-    --history-config-path "${HISTORY_CONFIG}" \
-    --mode eval --user-id "${PERSONA}" \
-    --workspace-dir "${NANOBOT_WORKSPACE_DIR}" \
-    2>&1 | tee "${LOG_DIR}/runner_eval_${PERSONA}.log"
-EVAL_EXIT=${PIPESTATUS[0]}
-
-echo ""
-echo "========================================="
-echo "persona=${PERSONA} finished (eval exit=${EVAL_EXIT})"
-echo "  results : ${SUITE_DIR}/outputs/reme/${PERSONA}/"
-echo "  memory  : ${WORKSPACE_DIR}/"
-echo "  logs    : ${LOG_DIR}/"
-echo "========================================="
-exit ${EVAL_EXIT}

benchmark/toolmemory/README.md

@@ -1,98 +0,0 @@
-## Towards Robust Tool Use in Agents via Experience-Driven Adaptive Guidance
-
-**Language**: English (default) / [中文](./README_ZH.md)
-
-> Paper: [arXiv:2608.03403](https://arxiv.org/abs/2608.03403)
-> Code: [https://github.com/WangCan1178/ExpG](https://github.com/WangCan1178/ExpG)
-
-<p align="center">
-  <img src="gitcha.png" alt="ExpG challenges and overview" width="85%">
-</p>
-
-### Overview
-
-This folder archives **ExpG**, a tool-use enhancement built on [Agentscope ReMe](https://github.com/agentscope-ai/ReMe). ExpG mines, distills, and reuses experience from historical tool calls to provide **capability boundaries** and **best-practice guidance**, which helps agents:
-
-- Select and invoke tools more robustly under dynamic or noisy environments;
-- Let smaller models with guidance outperform larger, memoryless baselines;
-- Improve consistently across tool selection, tool calling, and response generation.
-
-**How ReMe is used:** Start the Tool Memory service; historical tool calls are written and evaluated via `add_tool_call_result`, distilled into tool-level guidance via `summary_tool_memory`, then retrieved and injected into later reasoning via `retrieve_tool_memory`. ReMe provides the vector store and service APIs; the acquisition / distillation / reuse strategy is implemented by ExpG. Full implementation and experiments are in [WangCan1178/ExpG](https://github.com/WangCan1178/ExpG).
-
----
-
-### ExpG Mechanism
-
-ExpG treats tool invocations as learnable experience and runs a three-stage pipeline:
-
-1. **Experience Acquisition**
-   - Analyze invocation quality from historical trajectories (success/failure, cost, latency, etc.);
-   - Build structured experience units per tool, recording context, parameter patterns, and outcomes.
-
-2. **Experience Distillation**
-   - Filter noisy or unhelpful experiences and keep representative patterns;
-   - Aggregate by equivalence classes to cover common and rare failure modes;
-   - Summarize with an LLM into generalizable textual guidance.
-
-3. **Experience Reuse**
-   - Retrieve relevant experience / guidance for future tasks;
-   - Inject guidance into tool selection, argument generation, and response synthesis;
-   - Improve stability under dynamic environments and imperfect feedback.
-
----
-
-### Main Results
-
-Performance comparison (%) across MetaTool, API-Bank, and BFCL-V3. **Bold** indicates the best results within each model.
-
-| Model | Method | MetaTool Pass@1 | MetaTool Avg@3 | MetaTool Pass@3 | API-Bank Pass@1 | API-Bank Avg@3 | API-Bank Pass@3 | BFCL-V3 Pass@1 | BFCL-V3 Avg@3 | BFCL-V3 Pass@3 | Total Pass@1 | Total Avg@3 | Total Pass@3 |
-| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
-| GPT-5 nano | No Method | 72.62 | 72.76 | 78.49 | 82.96 | 83.46 | 86.97 | 53.80 | 53.00 | 60.95 | 70.82 | 70.62 | 76.63 |
-| GPT-5 nano | Few-shot | 74.12 | 75.11 | 82.32 | 83.71 | 83.96 | **87.22** | 56.18 | 55.24 | 61.39 | 72.36 | 72.65 | 79.28 |
-| GPT-5 nano | DRAFT | 73.94 | 73.04 | 78.97 | 84.21 | 83.46 | **87.22** | 57.27 | 57.27 | 62.26 | 72.52 | 71.58 | 77.23 |
-| GPT-5 nano | Mem0 | 74.96 | 76.13 | 82.92 | 84.96 | 85.21 | **87.22** | 60.95 | 61.61 | 65.08 | 73.98 | 74.67 | 80.35 |
-| GPT-5 nano | **ExpG** | **81.67** | **82.07** | **84.60** | **86.72** | **86.55** | **87.22** | **64.43** | **63.99** | **66.38** | **79.32** | **79.22** | **81.69** |
-| DeepSeek-V3 | No Method | 83.10 | 82.94 | 84.66 | 84.71 | 84.38 | 85.46 | 58.79 | 59.65 | 65.94 | 78.92 | 78.66 | 81.37 |
-| DeepSeek-V3 | Few-shot | 82.74 | 83.90 | 86.28 | 85.21 | 84.63 | 86.22 | 60.52 | 60.30 | 67.90 | 79.08 | 79.45 | 82.92 |
-| DeepSeek-V3 | DRAFT | 80.23 | 80.79 | 82.44 | 84.96 | 85.63 | 86.47 | 62.26 | 61.61 | 68.55 | 77.70 | 77.80 | 80.54 |
-| DeepSeek-V3 | Mem0 | 83.88 | 84.56 | 86.40 | 85.46 | 85.55 | 86.47 | 65.08 | 65.15 | 68.33 | 80.70 | 80.91 | 83.12 |
-| DeepSeek-V3 | **ExpG** | **85.26** | **85.38** | **86.52** | **87.72** | **87.39** | **87.97** | **69.41** | **69.92** | **72.02** | **82.76** | **82.61** | **84.11** |
-| Qwen3-8B | No Method | 76.51 | 76.97 | 77.71 | 83.96 | 83.88 | 84.21 | 58.79 | 58.28 | 60.30 | 74.46 | 74.41 | 75.56 |
-| Qwen3-8B | Few-shot | 79.93 | 79.83 | 82.92 | 83.71 | 82.62 | 84.96 | 60.09 | 59.29 | 61.39 | 76.91 | 76.27 | 79.32 |
-| Qwen3-8B | DRAFT | 78.19 | 77.33 | 77.89 | 85.71 | 84.96 | 85.46 | 60.74 | 60.30 | 62.91 | 76.20 | 75.18 | 76.35 |
-| Qwen3-8B | Mem0 | 75.07 | 75.47 | 82.38 | 86.22 | 86.05 | 86.47 | 63.34 | 64.93 | 66.16 | 74.69 | 74.98 | 80.07 |
-| Qwen3-8B | **ExpG** | **83.52** | **84.88** | **85.08** | **86.47** | **87.89** | **87.97** | **67.46** | **66.96** | **68.33** | **81.06** | **81.82** | **82.48** |
-| Qwen3-32B | No Method | 80.05 | 79.43 | 80.17 | 84.71 | 84.88 | 85.21 | 65.15 | 65.08 | 66.16 | 78.05 | 77.55 | 78.41 |
-| Qwen3-32B | **ExpG** | **84.68** | **85.02** | **86.28** | **86.97** | **87.30** | **87.72** | **70.72** | **71.01** | **73.32** | **82.48** | **82.56** | **84.14** |
-| Qwen3-235B | No Method | 78.25 | 79.23 | 80.29 | 85.46 | 85.46 | 85.71 | 71.37 | 71.15 | 73.54 | 78.13 | 78.49 | 79.91 |
-| Qwen3-235B | **ExpG** | **86.34** | **86.70** | **86.94** | **87.47** | **86.97** | **88.22** | **79.61** | **78.52** | **80.04** | **85.29** | **84.98** | **85.69** |
-
----
-
-### Reference Code
-
-| Path | Role |
-| --- | --- |
-| [`tool_memory.py`](./tool_memory.py) | HTTP client for official ReMe Tool Memory APIs (`add_tool_call_result` / `summary_tool_memory` / `retrieve_tool_memory`) |
-| [`parse_tool_call_result_prompt.yaml`](./parse_tool_call_result_prompt.yaml) | Prompt for multi-aspect evaluation of each tool call |
-| [`summary_tool_memory_prompt.yaml`](./summary_tool_memory_prompt.yaml) | Prompt for summarizing tool call history into guidance |
-| [`tool_memory_flows.yaml`](./tool_memory_flows.yaml) | Tool Memory flow / op config excerpt |
-
-These are reference snippets. For the full runnable codebase, see [WangCan1178/ExpG](https://github.com/WangCan1178/ExpG).
-
----
-
-### Citation
-
-```bibtex
-@misc{wang2026expg,
-  title         = {Towards Robust Tool Use in Agents via Experience-Driven Adaptive Guidance},
-  author        = {Can Wang and Haoran Chen and Li Yu and Ding Hao and Bohai Zhao and Zhaoyang Liu and Zhiying Tu},
-  year          = {2026},
-  eprint        = {2608.03403},
-  archivePrefix = {arXiv},
-  primaryClass  = {cs.AI},
-  url           = {https://arxiv.org/abs/2608.03403},
-  howpublished  = {\url{https://github.com/WangCan1178/ExpG}}
-}
-```

benchmark/toolmemory/README_ZH.md

@@ -1,98 +0,0 @@
-## Towards Robust Tool Use in Agents via Experience-Driven Adaptive Guidance
-
-**语言**：中文 / [English](./README.md)
-
-> 论文：[arXiv:2608.03403](https://arxiv.org/abs/2608.03403)
-> 代码：[https://github.com/WangCan1178/ExpG](https://github.com/WangCan1178/ExpG)
-
-<p align="center">
-  <img src="gitcha.png" alt="ExpG 挑战与概览" width="85%">
-</p>
-
-### 简介
-
-本目录归档基于 [Agentscope ReMe](https://github.com/agentscope-ai/ReMe) 的工具使用增强工作 **ExpG**：在 ReMe 记忆框架之上，从历史工具调用中挖掘、提炼并复用经验，为智能体提供工具的 **能力边界** 与 **最佳实践指导**，从而：
-
-- 在动态或有噪环境下更鲁棒地选择和调用工具；
-- 让较小模型在带有经验指导时超越更大、但无记忆的基线；
-- 在工具选择、工具调用和响应生成等多个阶段带来一致收益。
-
-**如何使用 ReMe：** 启动 Tool Memory 服务后，历史工具调用经 `add_tool_call_result` 写入并评估，经 `summary_tool_memory` 蒸馏成工具级指导，再经 `retrieve_tool_memory` 取回并注入后续推理。向量存储与服务接口由 ReMe 提供，经验获取 / 蒸馏 / 复用策略由 ExpG 实现。完整实现与实验见 [WangCan1178/ExpG](https://github.com/WangCan1178/ExpG)。
-
----
-
-### ExpG 机制概览
-
-ExpG 将工具调用视为可学习经验，并通过三阶段流水线完成经验的获取、提炼与复用：
-
-1. **经验获取（Experience Acquisition）**
-   - 从历史工具调用轨迹中分析调用质量（成功/失败、代价、时间等）；
-   - 针对不同工具构建结构化的经验单元，记录调用上下文、参数模式和结果。
-
-2. **经验蒸馏（Experience Distillation）**
-   - 过滤无效 / 噪声经验，保留具有代表性的调用模式；
-   - 基于“等价类”视角对经验进行聚合，覆盖常见模式与稀有失败模式；
-   - 使用 LLM 对经验进行总结，形成可泛化的文本化指导（guidance）。
-
-3. **经验复用（Experience Reuse）**
-   - 在未来任务中，根据当前工具调用上下文检索相关经验 / 指导；
-   - 将经验引导融入到工具选择、参数生成和响应整理等环节；
-   - 使得代理在面对动态环境和不完美反馈时仍能保持稳定表现。
-
----
-
-### 主实验结果
-
-MetaTool、API-Bank、BFCL-V3 上的性能对比（%）。**加粗**为各模型组内最优。
-
-| Model | Method | MetaTool Pass@1 | MetaTool Avg@3 | MetaTool Pass@3 | API-Bank Pass@1 | API-Bank Avg@3 | API-Bank Pass@3 | BFCL-V3 Pass@1 | BFCL-V3 Avg@3 | BFCL-V3 Pass@3 | Total Pass@1 | Total Avg@3 | Total Pass@3 |
-| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
-| GPT-5 nano | No Method | 72.62 | 72.76 | 78.49 | 82.96 | 83.46 | 86.97 | 53.80 | 53.00 | 60.95 | 70.82 | 70.62 | 76.63 |
-| GPT-5 nano | Few-shot | 74.12 | 75.11 | 82.32 | 83.71 | 83.96 | **87.22** | 56.18 | 55.24 | 61.39 | 72.36 | 72.65 | 79.28 |
-| GPT-5 nano | DRAFT | 73.94 | 73.04 | 78.97 | 84.21 | 83.46 | **87.22** | 57.27 | 57.27 | 62.26 | 72.52 | 71.58 | 77.23 |
-| GPT-5 nano | Mem0 | 74.96 | 76.13 | 82.92 | 84.96 | 85.21 | **87.22** | 60.95 | 61.61 | 65.08 | 73.98 | 74.67 | 80.35 |
-| GPT-5 nano | **ExpG** | **81.67** | **82.07** | **84.60** | **86.72** | **86.55** | **87.22** | **64.43** | **63.99** | **66.38** | **79.32** | **79.22** | **81.69** |
-| DeepSeek-V3 | No Method | 83.10 | 82.94 | 84.66 | 84.71 | 84.38 | 85.46 | 58.79 | 59.65 | 65.94 | 78.92 | 78.66 | 81.37 |
-| DeepSeek-V3 | Few-shot | 82.74 | 83.90 | 86.28 | 85.21 | 84.63 | 86.22 | 60.52 | 60.30 | 67.90 | 79.08 | 79.45 | 82.92 |
-| DeepSeek-V3 | DRAFT | 80.23 | 80.79 | 82.44 | 84.96 | 85.63 | 86.47 | 62.26 | 61.61 | 68.55 | 77.70 | 77.80 | 80.54 |
-| DeepSeek-V3 | Mem0 | 83.88 | 84.56 | 86.40 | 85.46 | 85.55 | 86.47 | 65.08 | 65.15 | 68.33 | 80.70 | 80.91 | 83.12 |
-| DeepSeek-V3 | **ExpG** | **85.26** | **85.38** | **86.52** | **87.72** | **87.39** | **87.97** | **69.41** | **69.92** | **72.02** | **82.76** | **82.61** | **84.11** |
-| Qwen3-8B | No Method | 76.51 | 76.97 | 77.71 | 83.96 | 83.88 | 84.21 | 58.79 | 58.28 | 60.30 | 74.46 | 74.41 | 75.56 |
-| Qwen3-8B | Few-shot | 79.93 | 79.83 | 82.92 | 83.71 | 82.62 | 84.96 | 60.09 | 59.29 | 61.39 | 76.91 | 76.27 | 79.32 |
-| Qwen3-8B | DRAFT | 78.19 | 77.33 | 77.89 | 85.71 | 84.96 | 85.46 | 60.74 | 60.30 | 62.91 | 76.20 | 75.18 | 76.35 |
-| Qwen3-8B | Mem0 | 75.07 | 75.47 | 82.38 | 86.22 | 86.05 | 86.47 | 63.34 | 64.93 | 66.16 | 74.69 | 74.98 | 80.07 |
-| Qwen3-8B | **ExpG** | **83.52** | **84.88** | **85.08** | **86.47** | **87.89** | **87.97** | **67.46** | **66.96** | **68.33** | **81.06** | **81.82** | **82.48** |
-| Qwen3-32B | No Method | 80.05 | 79.43 | 80.17 | 84.71 | 84.88 | 85.21 | 65.15 | 65.08 | 66.16 | 78.05 | 77.55 | 78.41 |
-| Qwen3-32B | **ExpG** | **84.68** | **85.02** | **86.28** | **86.97** | **87.30** | **87.72** | **70.72** | **71.01** | **73.32** | **82.48** | **82.56** | **84.14** |
-| Qwen3-235B | No Method | 78.25 | 79.23 | 80.29 | 85.46 | 85.46 | 85.71 | 71.37 | 71.15 | 73.54 | 78.13 | 78.49 | 79.91 |
-| Qwen3-235B | **ExpG** | **86.34** | **86.70** | **86.94** | **87.47** | **86.97** | **88.22** | **79.61** | **78.52** | **80.04** | **85.29** | **84.98** | **85.69** |
-
----
-
-### 参考代码
-
-| 路径 | 作用 |
-| --- | --- |
-| [`tool_memory.py`](./tool_memory.py) | 官方风格 ReMe Tool Memory HTTP 客户端（`add_tool_call_result` / `summary_tool_memory` / `retrieve_tool_memory`） |
-| [`parse_tool_call_result_prompt.yaml`](./parse_tool_call_result_prompt.yaml) | 单次工具调用多维评估用的 prompt |
-| [`summary_tool_memory_prompt.yaml`](./summary_tool_memory_prompt.yaml) | 将工具调用历史总结为 guidance 的 prompt |
-| [`tool_memory_flows.yaml`](./tool_memory_flows.yaml) | Tool Memory 相关的 flow / op 配置摘录 |
-
-以上为参考片段。完整可运行代码见 [WangCan1178/ExpG](https://github.com/WangCan1178/ExpG)。
-
----
-
-### 引用
-
-```bibtex
-@misc{wang2026expg,
-  title         = {Towards Robust Tool Use in Agents via Experience-Driven Adaptive Guidance},
-  author        = {Can Wang and Haoran Chen and Li Yu and Ding Hao and Bohai Zhao and Zhaoyang Liu and Zhiying Tu},
-  year          = {2026},
-  eprint        = {2608.03403},
-  archivePrefix = {arXiv},
-  primaryClass  = {cs.AI},
-  url           = {https://arxiv.org/abs/2608.03403},
-  howpublished  = {\url{https://github.com/WangCan1178/ExpG}}
-}
-```

benchmark/toolmemory/parse_tool_call_result_prompt.yaml

@@ -1,49 +0,0 @@
-prompt: |
-  You are an expert in evaluating tool invocation process. The tool is invoked by an AI agent.
-
-  Tool invocation Information:
-  - Tool Name: {tool_name}
-  - Success Flag: {success_flag}
-  - Time Cost: {time_cost}s
-  - Token Cost: {token_cost} tokens
-  - Agent Context: {context}
-  - Input Parameters: {input_params}
-  - Tool Response: {response}
-  - Tool Schema: {schema}
-
-  Evaluation Method:
-  Start from a default score list of scores = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0].
-  For each item below that is satisfied, assign 1 point to the corresponding index.
-  The final scores should be a list of 10 integers, each being either 0 or 1.
-
-  1. Use Quality (total 2 points. If context is provided, use it as an aid when evaluating):
-  - Index 1: Should the tool be invoked now? Consider whether all necessary information for the tool's invocation is ready, and whether the tool execution environment is correct. If it is a multi-round conversation, also consider the dependency relationships of the tool chain.
-  - Index 2: If should, is the chosen tool appropriate?
-
-  2. Input Quality (total 4 points. When evaluating, consider both the context and the tool schema):
-  - Index 3: Are all required parameters provided?
-  - Index 4: Are the input parameters valid and supported by the tool?
-  - Index 5: Are the input parameters in the correct format for their respective fields?
-  - Index 6: Does the value (content) of input parameter correctly reflect and match the given context?
-
-  3. Response Quality (total 4 points):
-  - Index 7: Does the response provide meaningful and useful information? Or are there any error messages or information that can be used as guidance for agent invoking tool better?
-  - Index 8: Does the response match the tool's intended purpose/function?
-  - Index 9: Does the response value correct (content appropriate) given the input parameters?
-  - Index 10: Does the response help accomplish the task within the given context?
-
-  Important:
-  1. Sometimes there is not enough information in the context or schema to make a complete evaluation. In such cases, make your best judgment based on the available information.
-  2. Some tools (commonly system tools such as mkdir, touch, echo, etc.) modify the external environment. Since these results cannot be obtained, they return "None" as the response. At this point, all the scores in the quality of the response should be obtained and should not be seen as a problem for the tool.
-  3. Evaluation independently from the success flag. The success_flag indicates whether the tool executed without technical errors. The evaluation should evaluate the quality of the tool invocation. A tool can execute successfully (Success Flag=1) but still produce low-quality or irrelevant responses, leading to a low evaluation score.
-  4. Sometimes an agent will execute multiple steps and invoke multiple tools to complete a task, but you only need to evaluate the use of one tool for one of the steps, not whether the final task is completed or not.
-
-  Answer Format:
-  Please provide your answer in the following JSON format:
-
-  ```json
-  {
-    "scores": [0,0,0,0,0,0,0,0,0,0],
-    "explanation": "A brief evaluation (2-3 sentences) explaining the quality of the tool invocation, based on your evaluation. Low-quality aspects need to be reified, especially the causes of tool invocation errors."
-  }
-  ```

benchmark/toolmemory/summary_tool_memory_prompt.yaml

@@ -1,32 +0,0 @@
-prompt: |
-  You are an expert in analyzing tool usage patterns and generating practical usage guidance for agents.
-
-  Tool Information:
-  - Tool Name: {tool_name}
-  - Tool Schema: {tool_schema}
-
-  Recent Tool Invocation Experiences:
-  {experiences}
-
-  Important:
-  1. Assume the tool (tool schema) can't be changed, your task is to guide agent to use it better.
-  2. Your answer must be based on the information given, don't make it up. If not enough data, state "Not enough data to determine Core Function/Success Patterns/Common Issues/Best Practices."
-  3. Your answer will be used to guide the use of the tool in the future, so do not include content related to recent tool invocation experience such as "case #3" or "Call #2", but some values can be used as examples.
-  4. Pay attention to information not mentioned in the tool schema, such as the response upon successful tool invocation. It's also welcome to uncover insights, such as how tools can be used more effectively, and possible dependencies between tools. But if they aren't, don't make them up.
-  5. Finally, to avoid deriving incorrect guidance from individual invocation, check whether, if the agent follows the proposed guidance, it can perform better on all recent invocation histories. If not, revise the guidance until it can. Specifically:
-     - Don't write guidance in an absolute tone without a very deterministic message (meaning that all invocation histories are satisfied, otherwise it will result in failure).
-     - Sometimes there may be inconsistencies. Consider whether this is due to the context in which the tool is being used.
-
-  Your Task:
-  Based on the tool invocation history, generate a concise and logical tool usage guidance following this structure:
-  1. Core Function: What this tool does and when to use it.
-  2. Success Patterns: Parameter patterns and usage scenarios that work well.
-  3. Common Issues: Main pitfalls to avoid and why they fail.
-  4. Best Practices: 2-3 actionable recommendations.
-
-  Answer Format:
-  Provide a structured, concise guidance (max 200 words). Focus on actionable insights derived from actual usage data. Avoid generic advice and think step by step.
-
-  ```txt
-  Your concise, data-driven tool usage guidance
-  ```

benchmark/toolmemory/tool_memory.py

@@ -1,234 +0,0 @@
-"""Official-style ReMe Tool Memory HTTP helpers.
-
-Aligned with ReMe Tool Memory HTTP APIs (see ReMe cookbook
-``use_tool_memory_demo.py`` and docs under ``docs/tool_memory/``):
-
-- ``add_tool_call_result``
-- ``summary_tool_memory``
-- ``retrieve_tool_memory``
-
-Response memories are read from ``metadata.memory_list[].content``.
-This module does not use ExpG-only fields such as ``no_persist``,
-``source_task``, or ``add_to``.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any, Dict, List, Optional
-
-import httpx
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_BASE_URL = "http://localhost:8002"
-
-
-class ToolMemoryFetcher:
-    """HTTP client for ReMe Tool Memory endpoints."""
-
-    def __init__(
-        self,
-        workspace_id: str,
-        base_url: str = DEFAULT_BASE_URL,
-        timeout: float = 60.0,
-    ) -> None:
-        self.workspace_id = workspace_id
-        self.base_url = base_url.rstrip("/")
-        self.timeout = timeout
-
-    def _url(self, endpoint: str) -> str:
-        return f"{self.base_url}/{endpoint.lstrip('/')}"
-
-    @staticmethod
-    def _join_tool_names(tool_names: List[str] | str) -> str:
-        if isinstance(tool_names, str):
-            return tool_names
-        return ",".join(tool_names)
-
-    @staticmethod
-    def _memory_list(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
-        metadata = payload.get("metadata") or {}
-        if not isinstance(metadata, dict):
-            return []
-        memory_list = metadata.get("memory_list") or []
-        return memory_list if isinstance(memory_list, list) else []
-
-    @classmethod
-    def _content_by_tool(cls, payload: Dict[str, Any]) -> Dict[str, str]:
-        result: Dict[str, str] = {}
-        for memory in cls._memory_list(payload):
-            if not isinstance(memory, dict):
-                continue
-            tool_name = str(memory.get("when_to_use") or "").strip()
-            content = memory.get("content") or ""
-            if tool_name:
-                result[tool_name] = str(content)
-        return result
-
-    async def add_tool_call_result_async(
-        self,
-        tool_call_results: List[Dict[str, Any]],
-    ) -> Dict[str, Any]:
-        """Call ``add_tool_call_result``."""
-        async with httpx.AsyncClient() as client:
-            response = await client.post(
-                self._url("add_tool_call_result"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_call_results": tool_call_results,
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    async def summary_tool_memory_async(
-        self,
-        tool_names: List[str] | str,
-    ) -> Dict[str, Any]:
-        """Call ``summary_tool_memory``."""
-        async with httpx.AsyncClient() as client:
-            response = await client.post(
-                self._url("summary_tool_memory"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_names": self._join_tool_names(tool_names),
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    async def retrieve_tool_memory_async(
-        self,
-        tool_names: List[str] | str,
-    ) -> Dict[str, Any]:
-        """Call ``retrieve_tool_memory``."""
-        async with httpx.AsyncClient() as client:
-            response = await client.post(
-                self._url("retrieve_tool_memory"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_names": self._join_tool_names(tool_names),
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    async def collect_memory_async(
-        self,
-        tool_names: List[str],
-    ) -> Dict[str, str]:
-        """Summarize then retrieve guidance for tools.
-
-        Returns:
-            Mapping from tool name to memory ``content`` string.
-        """
-        if not tool_names:
-            return {}
-
-        names = self._join_tool_names(tool_names)
-        try:
-            summary = await self.summary_tool_memory_async(names)
-            if not summary.get("success"):
-                logger.warning("summary_tool_memory failed for %s", names)
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("summary_tool_memory error for %s: %s", names, exc)
-
-        try:
-            retrieved = await self.retrieve_tool_memory_async(names)
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("retrieve_tool_memory error for %s: %s", names, exc)
-            return {}
-
-        if not retrieved.get("success"):
-            logger.warning("retrieve_tool_memory failed for %s", names)
-            return {}
-
-        return self._content_by_tool(retrieved)
-
-    def add_tool_call_result(
-        self,
-        tool_call_results: List[Dict[str, Any]],
-    ) -> Dict[str, Any]:
-        """Sync wrapper for ``add_tool_call_result``."""
-        with httpx.Client() as client:
-            response = client.post(
-                self._url("add_tool_call_result"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_call_results": tool_call_results,
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    def summary_tool_memory(self, tool_names: List[str] | str) -> Dict[str, Any]:
-        """Sync wrapper for ``summary_tool_memory``."""
-        with httpx.Client() as client:
-            response = client.post(
-                self._url("summary_tool_memory"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_names": self._join_tool_names(tool_names),
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    def retrieve_tool_memory(self, tool_names: List[str] | str) -> Dict[str, Any]:
-        """Sync wrapper for ``retrieve_tool_memory``."""
-        with httpx.Client() as client:
-            response = client.post(
-                self._url("retrieve_tool_memory"),
-                json={
-                    "workspace_id": self.workspace_id,
-                    "tool_names": self._join_tool_names(tool_names),
-                },
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-            return response.json()
-
-    def collect_memory(self, tool_names: List[str]) -> Dict[str, str]:
-        """Sync wrapper for summarize + retrieve.
-
-        Prefer ``collect_memory_async`` inside an existing event loop.
-        """
-        if not tool_names:
-            return {}
-
-        names = self._join_tool_names(tool_names)
-        try:
-            summary = self.summary_tool_memory(names)
-            if not summary.get("success"):
-                logger.warning("summary_tool_memory failed for %s", names)
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("summary_tool_memory error for %s: %s", names, exc)
-
-        try:
-            retrieved = self.retrieve_tool_memory(names)
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("retrieve_tool_memory error for %s: %s", names, exc)
-            return {}
-
-        if not retrieved.get("success"):
-            logger.warning("retrieve_tool_memory failed for %s", names)
-            return {}
-
-        return self._content_by_tool(retrieved)
-
-    def get_memory_content(
-        self,
-        tool_names: List[str] | str,
-    ) -> Optional[str]:
-        """Retrieve and join memory contents for the given tools."""
-        payload = self.retrieve_tool_memory(tool_names)
-        if not payload.get("success"):
-            return None
-        contents = [content for content in self._content_by_tool(payload).values() if content]
-        return "\n\n".join(contents) if contents else None

benchmark/toolmemory/tool_memory_flows.yaml

@@ -1,45 +0,0 @@
-# Tool Memory flow / op config excerpt used by ExpG.
-# Full runnable code: https://github.com/WangCan1178/ExpG
-
-flow:
-  retrieve_tool_memory:
-    flow_content: retrieve_tool_memory_op
-    description: "Retrieves tool memories from the vector database based on tool names to provide tool usage patterns and best practices"
-    input_schema:
-      tool_names:
-        type: string
-        description: "Comma-separated tool names (e.g., 'tool_name1,tool_name2')"
-        required: true
-
-  add_tool_call_result:
-    flow_content: parse_tool_call_result_op >> update_vector_store_op
-    description: "Evaluates and adds tool call results to the tool memory database, creating new memory or updating existing memory for the specified tool"
-    input_schema:
-      tool_call_results:
-        type: array
-        description: "List of tool call result objects, each containing: tool_name, input, output, success, time_cost, token_cost, create_time"
-        required: true
-
-  summary_tool_memory:
-    flow_content: summary_tool_memory_op >> update_vector_store_op
-    description: "Analyzes tool call history and generates comprehensive usage patterns, best practices, and recommendations for the specified tools"
-    input_schema:
-      tool_names:
-        type: string
-        description: "Comma-separated tool names to summarize (e.g., 'tool_name1,tool_name2')"
-        required: true
-
-op:
-  parse_tool_call_result_op:
-    backend: parse_tool_call_result_op
-    llm: default
-    params:
-      max_history_tool_call_cnt: 100
-      evaluation_sleep_interval: 1.0
-
-  summary_tool_memory_op:
-    backend: summary_tool_memory_op
-    llm: default
-    params:
-      data_from: '2025-09-10 10:56:58'
-      summary_sleep_interval: 1.0

docs/README_0_2_x.md

@@ -0,0 +1,897 @@
+<p align="center">
+ <img src="docs/_static/figure/reme_logo.png" alt="ReMe Logo" width="50%">
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.10+-blue" alt="Python Version"></a>
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/pypi/v/reme-ai.svg?logo=pypi" alt="PyPI Version"></a>
+  <a href="https://pepy.tech/project/reme-ai/"><img src="https://img.shields.io/pypi/dm/reme-ai" alt="PyPI Downloads"></a>
+  <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/commit-activity/m/agentscope-ai/ReMe?style=flat-square" alt="GitHub commit activity"></a>
+</p>
+
+<p align="center">
+  <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-black" alt="License"></a>
+  <a href="./README.md"><img src="https://img.shields.io/badge/English-Click-yellow" alt="English"></a>
+  <a href="./README_ZH.md"><img src="https://img.shields.io/badge/简体中文-点击查看-orange" alt="简体中文"></a>
+  <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/stars/agentscope-ai/ReMe?style=social" alt="GitHub Stars"></a>
+</p>
+
+<p align="center">
+  <strong>Memory Management Kit for Agents, Remember Me, Refine Me.</strong><br>
+  <em><sub>If you find it useful, please give us a ⭐ Star.</sub></em>
+</p>
+
+---
+
+ReMe is a **modular memory management kit** that provides AI agents with unified memory capabilities—enabling the ability to extract, reuse, and share memories across users, tasks, and agents.
+Agent memory can be viewed as:
+
+```text
+Agent Memory = Long-Term Memory + Short-Term Memory
+             = (Personal + Task + Tool) Memory + (Working Memory)
+```
+
+- **Personal Memory**: Understand user preferences and adapt to context
+- **Task Memory**: Learn from experience and perform better on similar tasks
+- **Tool Memory**: Optimize tool selection and parameter usage based on historical performance
+- **Working Memory**: Manage short-term context for long-running agents without context overflow
+
+---
+
+## 📰 Latest Updates
+
+- **[2026-02]** 💻 ReMeCli: A terminal-based AI chat assistant with built-in memory management. Automatically compacts long conversations into summaries to free up context space, and persists important information as Markdown files for retrieval in future sessions. Memory design inspired by [OpenClaw](https://github.com/openclaw/openclaw).
+  - [Quick Start](docs/cli/quick_start_en.md)
+  - Type `/horse` to trigger the Year of the Horse Easter egg -- fireworks, a galloping horse animation, and a random blessing.
+<table border="0" cellspacing="0" cellpadding="0" style="border: none;">
+  <tr style="border: none;">
+    <td width="10%" style="border: none; vertical-align: middle; text-align: center;">
+      <strong>马<br>上<br>有<br>钱</strong>
+    </td>
+    <td width="80%" style="border: none;">
+      <video src="https://github.com/user-attachments/assets/d731ae5c-80eb-498b-a22c-8ab2b9169f87" autoplay muted loop controls></video>
+    </td>
+    <td width="10%" style="border: none; vertical-align: middle; text-align: center;">
+      <strong>马<br>到<br>成<br>功</strong>
+    </td>
+  </tr>
+</table>
+
+- **[2025-12]** 📄 Our procedural (task) memory paper has been released on [arXiv](https://arxiv.org/abs/2512.10696)
+- **[2025-11]** 🧠 React-agent with working-memory demo ([Intro](docs/work_memory/message_offload.md)) with ([Quick Start](docs/cookbook/working/quick_start.md)) and ([Code](cookbook/working_memory/work_memory_demo.py))
+- **[2025-10]** 🚀 Direct Python import support: use `from reme_ai import ReMeApp` without HTTP/MCP service
+- **[2025-10]** 🔧 Tool Memory: data-driven tool selection and parameter optimization ([Guide](docs/tool_memory/tool_memory.md))
+- **[2025-09]** 🎉 Async operations support, integrated into agentscope-runtime
+- **[2025-09]** 🎉 Task memory and personal memory integration
+- **[2025-09]** 🧪 Validated effectiveness in appworld, bfcl(v3), and frozenlake ([Experiments](docs/cookbook))
+- **[2025-08]** 🚀 MCP protocol support ([Quick Start](docs/mcp_quick_start.md))
+- **[2025-06]** 🚀 Multiple backend vector storage (Elasticsearch & ChromaDB) ([Guide](docs/vector_store_api_guide.md))
+- **[2024-09]** 🧠 Personalized and time-aware memory storage
+
+---
+
+## ✨ Architecture Design
+
+<p align="center">
+ <img src="docs/_static/figure/reme_structure.jpg" alt="ReMe Architecture" width="80%">
+</p>
+
+ReMe provides a **modular memory management kit** with pluggable components that can be integrated into any agent framework. The system consists of:
+
+#### 🧠 **Task Memory/Experience**
+
+Procedural knowledge reused across agents
+
+- **Success Pattern Recognition**: Identify effective strategies and understand their underlying principles
+- **Failure Analysis Learning**: Learn from mistakes and avoid repeating the same issues
+- **Comparative Patterns**: Different sampling trajectories provide more valuable memories through comparison
+- **Validation Patterns**: Confirm the effectiveness of extracted memories through validation modules
+
+Learn more about how to use task memory from [task memory](docs/task_memory/task_memory.md)
+
+#### 👤 **Personal Memory**
+
+Contextualized memory for specific users
+
+- **Individual Preferences**: User habits, preferences, and interaction styles
+- **Contextual Adaptation**: Intelligent memory management based on time and context
+- **Progressive Learning**: Gradually build deep understanding through long-term interaction
+- **Time Awareness**: Time sensitivity in both retrieval and integration
+
+Learn more about how to use personal memory from [personal memory](docs/personal_memory/personal_memory.md)
+
+#### 🔧 **Tool Memory**
+
+Data-driven tool selection and usage optimization
+
+- **Historical Performance Tracking**: Success rates, execution times, and token costs from real usage
+- **LLM-as-Judge Evaluation**: Qualitative insights on why tools succeed or fail
+- **Parameter Optimization**: Learn optimal parameter configurations from successful calls
+- **Dynamic Guidelines**: Transform static tool descriptions into living, learned manuals
+
+Learn more about how to use tool memory from [tool memory](docs/tool_memory/tool_memory.md)
+
+#### 🧠 Working Memory
+
+Short‑term contextual memory for long‑running agents via **message offload & reload**:
+- **Message Offload**: Compact large tool outputs to external files or LLM summaries
+- **Message Reload**: Search (`grep_working_memory`) and read (`read_working_memory`) offloaded content on demand
+📖 **Concept & API**:
+- Message offload overview: [Message Offload](docs/work_memory/message_offload.md)
+- Offload / reload operators: [Message Offload Ops](docs/work_memory/message_offload_ops.md), [Message Reload Ops](docs/work_memory/message_reload_ops.md)
+💻 **End‑to‑End Demo**:
+- Working memory quick start: [Working Memory Quick Start](docs/cookbook/working/quick_start.md)
+- ReAct agent with working memory: [react_agent_with_working_memory.py](cookbook/working_memory/react_agent_with_working_memory.py)
+- Runnable demo: [work_memory_demo.py](cookbook/working_memory/work_memory_demo.py)
+
+---
+
+## 🛠️ Installation
+
+### Install from PyPI (Recommended)
+
+```bash
+pip install reme-ai
+```
+
+### Install from Source
+
+```bash
+git clone https://github.com/agentscope-ai/ReMe.git
+cd ReMe
+pip install .
+```
+
+### Environment Configuration
+
+ReMe requires LLM and embedding model configurations. Copy `example.env` to `.env` and configure:
+
+```bash
+FLOW_LLM_API_KEY=sk-xxxx
+FLOW_LLM_BASE_URL=https://xxxx/v1
+FLOW_EMBEDDING_API_KEY=sk-xxxx
+FLOW_EMBEDDING_BASE_URL=https://xxxx/v1
+```
+
+---
+
+## 🚀 Quick Start
+
+### HTTP Service Startup
+
+```bash
+reme \
+  backend=http \
+  http.port=8002 \
+  llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
+  embedding_model.default.model_name=text-embedding-v4 \
+  vector_store.default.backend=local
+```
+
+### MCP Server Support
+
+```bash
+reme \
+  backend=mcp \
+  mcp.transport=stdio \
+  llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
+  embedding_model.default.model_name=text-embedding-v4 \
+  vector_store.default.backend=local
+```
+
+### Core API Usage
+
+#### Task Memory Management
+
+```python
+import requests
+
+# Experience Summarizer: Learn from execution trajectories
+response = requests.post("http://localhost:8002/summary_task_memory", json={
+    "workspace_id": "task_workspace",
+    "trajectories": [
+        {"messages": [{"role": "user", "content": "Help me create a project plan"}], "score": 1.0}
+    ]
+})
+
+# Retriever: Get relevant memories
+response = requests.post("http://localhost:8002/retrieve_task_memory", json={
+    "workspace_id": "task_workspace",
+    "query": "How to efficiently manage project progress?",
+    "top_k": 1
+})
+```
+
+<details>
+<summary>Python import version</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # Experience Summarizer: Learn from execution trajectories
+        result = await app.async_execute(
+            name="summary_task_memory",
+            workspace_id="task_workspace",
+            trajectories=[
+                {
+                    "messages": [
+                        {"role": "user", "content": "Help me create a project plan"}
+                    ],
+                    "score": 1.0
+                }
+            ]
+        )
+        print(result)
+
+        # Retriever: Get relevant memories
+        result = await app.async_execute(
+            name="retrieve_task_memory",
+            workspace_id="task_workspace",
+            query="How to efficiently manage project progress?",
+            top_k=1
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl version</summary>
+
+```bash
+# Experience Summarizer: Learn from execution trajectories
+curl -X POST http://localhost:8002/summary_task_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "trajectories": [
+      {"messages": [{"role": "user", "content": "Help me create a project plan"}], "score": 1.0}
+    ]
+  }'
+
+# Retriever: Get relevant memories
+curl -X POST http://localhost:8002/retrieve_task_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "query": "How to efficiently manage project progress?",
+    "top_k": 1
+  }'
+```
+
+</details>
+
+#### Personal Memory Management
+
+```python
+# Memory Integration: Learn from user interactions
+response = requests.post("http://localhost:8002/summary_personal_memory", json={
+    "workspace_id": "task_workspace",
+    "trajectories": [
+        {"messages":
+            [
+                {"role": "user", "content": "I like to drink coffee while working in the morning"},
+                {"role": "assistant",
+                 "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+            ]
+        }
+    ]
+})
+
+# Memory Retrieval: Get personal memory fragments
+response = requests.post("http://localhost:8002/retrieve_personal_memory", json={
+    "workspace_id": "task_workspace",
+    "query": "What are the user's work habits?",
+    "top_k": 5
+})
+```
+
+<details>
+<summary>Python import version</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # Memory Integration: Learn from user interactions
+        result = await app.async_execute(
+            name="summary_personal_memory",
+            workspace_id="task_workspace",
+            trajectories=[
+                {
+                    "messages": [
+                        {"role": "user", "content": "I like to drink coffee while working in the morning"},
+                        {"role": "assistant",
+                         "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+                    ]
+                }
+            ]
+        )
+        print(result)
+
+        # Memory Retrieval: Get personal memory fragments
+        result = await app.async_execute(
+            name="retrieve_personal_memory",
+            workspace_id="task_workspace",
+            query="What are the user's work habits?",
+            top_k=5
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl version</summary>
+
+```bash
+# Memory Integration: Learn from user interactions
+curl -X POST http://localhost:8002/summary_personal_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "trajectories": [
+      {"messages": [
+        {"role": "user", "content": "I like to drink coffee while working in the morning"},
+        {"role": "assistant", "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+      ]}
+    ]
+  }'
+
+# Memory Retrieval: Get personal memory fragments
+curl -X POST http://localhost:8002/retrieve_personal_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "query": "What are the user'\''s work habits?",
+    "top_k": 5
+  }'
+```
+
+</details>
+
+#### Tool Memory Management
+
+```python
+import requests
+
+# Record tool execution results
+response = requests.post("http://localhost:8002/add_tool_call_result", json={
+    "workspace_id": "tool_workspace",
+    "tool_call_results": [
+        {
+            "create_time": "2025-10-21 10:30:00",
+            "tool_name": "web_search",
+            "input": {"query": "Python asyncio tutorial", "max_results": 10},
+            "output": "Found 10 relevant results...",
+            "token_cost": 150,
+            "success": True,
+            "time_cost": 2.3
+        }
+    ]
+})
+
+# Generate usage guidelines from history
+response = requests.post("http://localhost:8002/summary_tool_memory", json={
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+})
+
+# Retrieve tool guidelines before use
+response = requests.post("http://localhost:8002/retrieve_tool_memory", json={
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+})
+```
+
+<details>
+<summary>Python import version</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # Record tool execution results
+        result = await app.async_execute(
+            name="add_tool_call_result",
+            workspace_id="tool_workspace",
+            tool_call_results=[
+                {
+                    "create_time": "2025-10-21 10:30:00",
+                    "tool_name": "web_search",
+                    "input": {"query": "Python asyncio tutorial", "max_results": 10},
+                    "output": "Found 10 relevant results...",
+                    "token_cost": 150,
+                    "success": True,
+                    "time_cost": 2.3
+                }
+            ]
+        )
+        print(result)
+
+        # Generate usage guidelines from history
+        result = await app.async_execute(
+            name="summary_tool_memory",
+            workspace_id="tool_workspace",
+            tool_names="web_search"
+        )
+        print(result)
+
+        # Retrieve tool guidelines before use
+        result = await app.async_execute(
+            name="retrieve_tool_memory",
+            workspace_id="tool_workspace",
+            tool_names="web_search"
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl version</summary>
+
+```bash
+# Record tool execution results
+curl -X POST http://localhost:8002/add_tool_call_result \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_call_results": [
+      {
+        "create_time": "2025-10-21 10:30:00",
+        "tool_name": "web_search",
+        "input": {"query": "Python asyncio tutorial", "max_results": 10},
+        "output": "Found 10 relevant results...",
+        "token_cost": 150,
+        "success": true,
+        "time_cost": 2.3
+      }
+    ]
+  }'
+
+# Generate usage guidelines from history
+curl -X POST http://localhost:8002/summary_tool_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+  }'
+
+# Retrieve tool guidelines before use
+curl -X POST http://localhost:8002/retrieve_tool_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+  }'
+```
+
+</details>
+
+#### Working Memory Management
+
+```python
+import requests
+
+# Summarize and compact working memory for a long-running conversation
+response = requests.post("http://localhost:8002/summary_working_memory", json={
+    "messages": [
+        {
+            "role": "system",
+            "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+        },
+        {
+            "role": "user",
+            "content": "搜索下reme项目的的README内容"
+        },
+        {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {
+                    "index": 0,
+                    "id": "call_6596dafa2a6a46f7a217da",
+                    "function": {
+                        "arguments": "{\"query\": \"readme\"}",
+                        "name": "web_search"
+                    },
+                    "type": "function"
+                }
+            ]
+        },
+        {
+            "role": "tool",
+            "content": "ultra large context , over 50000 tokens......"
+        },
+        {
+            "role": "user",
+            "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+        }
+    ],
+    "working_summary_mode": "auto",
+    "compact_ratio_threshold": 0.75,
+    "max_total_tokens": 20000,
+    "max_tool_message_tokens": 2000,
+    "group_token_threshold": 4000,
+    "keep_recent_count": 2,
+    "store_dir": "test_working_memory",
+    "chat_id": "demo_chat_id"
+})
+```
+
+<details>
+<summary>Python import version</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # Summarize and compact working memory for a long-running conversation
+        result = await app.async_execute(
+            name="summary_working_memory",
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+                },
+                {
+                    "role": "user",
+                    "content": "搜索下reme项目的的README内容"
+                },
+                {
+                    "role": "assistant",
+                    "content": "",
+                    "tool_calls": [
+                        {
+                            "index": 0,
+                            "id": "call_6596dafa2a6a46f7a217da",
+                            "function": {
+                                "arguments": "{\"query\": \"readme\"}",
+                                "name": "web_search"
+                            },
+                            "type": "function"
+                        }
+                    ]
+                },
+                {
+                    "role": "tool",
+                    "content": "ultra large context , over 50000 tokens......"
+                },
+                {
+                    "role": "user",
+                    "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+                }
+            ],
+            working_summary_mode="auto",
+            compact_ratio_threshold=0.75,
+            max_total_tokens=20000,
+            max_tool_message_tokens=2000,
+            group_token_threshold=4000,
+            keep_recent_count=2,
+            store_dir="test_working_memory",
+            chat_id="demo_chat_id",
+        )
+        print(result)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl version</summary>
+
+```bash
+curl -X POST http://localhost:8002/summary_working_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      {
+        "role": "system",
+        "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+      },
+      {
+        "role": "user",
+        "content": "搜索下reme项目的的README内容"
+      },
+      {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+          {
+            "index": 0,
+            "id": "call_6596dafa2a6a46f7a217da",
+            "function": {
+              "arguments": "{\"query\": \"readme\"}",
+              "name": "web_search"
+            },
+            "type": "function"
+          }
+        ]
+      },
+      {
+        "role": "tool",
+        "content": "ultra large context , over 50000 tokens......"
+      },
+      {
+        "role": "user",
+        "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+      }
+    ],
+    "working_summary_mode": "auto",
+    "compact_ratio_threshold": 0.75,
+    "max_total_tokens": 20000,
+    "max_tool_message_tokens": 2000,
+    "group_token_threshold": 4000,
+    "keep_recent_count": 2,
+    "store_dir": "test_working_memory",
+    "chat_id": "demo_chat_id"
+  }'
+```
+
+</details>
+
+---
+
+## 📦 Pre-built Memory Library
+
+ReMe provides a **memory library** with pre-extracted, production-ready memories that agents can load and use immediately:
+
+### Available Memory Packs
+
+| Memory Pack          | Domain         | Size          | Description                                                                         |
+|----------------------|----------------|---------------|-------------------------------------------------------------------------------------|
+| **`appworld.jsonl`** | Task Execution | ~100 memories | Complex task planning patterns, multi-step workflows, and error recovery strategies |
+| **`bfcl_v3.jsonl`**  | Tool Usage     | ~150 memories | Function calling patterns, parameter optimization, and tool selection strategies    |
+
+### Loading Pre-built Memories
+
+```python
+# Load pre-built memories
+response = requests.post("http://localhost:8002/vector_store", json={
+    "workspace_id": "appworld",
+    "action": "load",
+    "path": "./docs/library/"
+})
+
+# Query relevant memories
+response = requests.post("http://localhost:8002/retrieve_task_memory", json={
+    "workspace_id": "appworld",
+    "query": "How to navigate to settings and update user profile?",
+    "top_k": 1
+})
+```
+
+<details>
+<summary>Python import version</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # Load pre-built memories
+        result = await app.async_execute(
+            name="vector_store",
+            workspace_id="appworld",
+            action="load",
+            path="./docs/library/"
+        )
+        print(result)
+
+        # Query relevant memories
+        result = await app.async_execute(
+            name="retrieve_task_memory",
+            workspace_id="appworld",
+            query="How to navigate to settings and update user profile?",
+            top_k=1
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+## 🧪 Experiments
+
+### 🌍 [Appworld Experiment](docs/cookbook/appworld/quickstart.md)
+
+We tested ReMe on Appworld using Qwen3-8B (non-thinking mode):
+
+| Method       | Avg@4               | Pass@4              |
+|--------------|---------------------|---------------------|
+| without ReMe | 0.1497              | 0.3285              |
+| with ReMe    | 0.1706 **(+2.09%)** | 0.3631 **(+3.46%)** |
+
+Pass@K measures the probability that at least one of the K generated samples successfully completes the task (
+score=1).
+The current experiment uses an internal AppWorld environment, which may have slight differences.
+
+You can find more details on reproducing the experiment in [quickstart.md](docs/cookbook/appworld/quickstart.md).
+
+### 🔧 [BFCL-V3 Experiment](docs/cookbook/bfcl/quickstart.md)
+
+We tested ReMe on BFCL-V3 multi-turn-base (randomly split 50train/150val) using Qwen3-8B (thinking mode):
+
+| Method       | Avg@4               | Pass@4              |
+|--------------|---------------------|---------------------|
+| without ReMe | 0.4033              | 0.5955              |
+| with ReMe    | 0.4450 **(+4.17%)** | 0.6577 **(+6.22%)** |
+
+### 🧊 [Frozenlake Experiment](docs/cookbook/frozenlake/quickstart.md)
+
+|                                             without ReMe                                             |                                              with ReMe                                               |
+|:----------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------:|
+| <p align="center"><img src="docs/_static/figure/frozenlake_failure.gif" alt="GIF 1" width="30%"></p> | <p align="center"><img src="docs/_static/figure/frozenlake_success.gif" alt="GIF 2" width="30%"></p> |
+
+We tested on 100 random frozenlake maps using qwen3-8b:
+
+| Method       | pass rate        |
+|--------------|------------------|
+| without ReMe | 0.66             |
+| with ReMe    | 0.72 **(+6.0%)** |
+
+You can find more details on reproducing the experiment in [quickstart.md](docs/cookbook/frozenlake/quickstart.md).
+
+### 🛠️ [Tool Memory Benchmark](docs/tool_memory/tool_bench.md)
+
+We evaluated Tool Memory effectiveness using a controlled benchmark with three mock search tools using Qwen3-30B-Instruct:
+
+| Scenario               | Avg Score | Improvement |
+|------------------------|-----------|-------------|
+| Train (No Memory)      | 0.650     | -           |
+| Test (No Memory)       | 0.672     | Baseline    |
+| **Test (With Memory)** | **0.772** | **+14.88%** |
+
+**Key Findings:**
+- Tool Memory enables data-driven tool selection based on historical performance
+- Success rates improved by ~15% with learned parameter configurations
+
+You can find more details in [tool_bench.md](docs/tool_memory/tool_bench.md) and the implementation at [run_reme_tool_bench.py](cookbook/tool_memory/run_reme_tool_bench.py).
+
+## 📚 Resources
+
+### Getting Started
+- **[Quick Start](./cookbook/simple_demo)**: Practical examples for immediate use
+  - [Tool Memory Demo](cookbook/simple_demo/use_tool_memory_demo.py): Complete lifecycle demonstration of tool memory
+  - [Tool Memory Benchmark](cookbook/tool_memory/run_reme_tool_bench.py): Evaluate tool memory effectiveness
+
+### Integration Guides
+- **[Direct Python Import](docs/cookbook/working/quick_start.md)**: Embed ReMe directly into your agent code
+- **[HTTP Service API](docs/vector_store_api_guide.md)**: RESTful API for multi-agent systems
+- **[MCP Protocol](docs/mcp_quick_start.md)**: Integration with Claude Desktop and MCP-compatible clients
+
+### Memory System Configuration
+- **[Personal Memory](docs/personal_memory)**: User preference learning and contextual adaptation
+- **[Task Memory](docs/task_memory)**: Procedural knowledge extraction and reuse
+- **[Tool Memory](docs/tool_memory)**: Data-driven tool selection and optimization
+- **[Working Memory](docs/work_memory/message_offload.md)**: Short-term context management for long-running agents
+
+### Advanced Topics
+- **[Operator Pipelines](reme_ai/config/default.yaml)**: Customize memory processing workflows by modifying operator chains
+- **[Vector Store Backends](docs/vector_store_api_guide.md)**: Configure local, Elasticsearch, Qdrant, or ChromaDB storage
+- **[Example Collection](./cookbook)**: Real-world use cases and best practices
+
+---
+
+## ⭐ Support & Community
+
+- **Star & Watch**: Stars surface ReMe to more agent builders; watching keeps you updated on new releases.
+- **Share your wins**: Open an issue or discussion with what ReMe unlocked for your agents—we love showcasing community builds.
+- **Need a feature?** File a request and we’ll help shape it together.
+
+---
+
+## 🤝 Contribution
+
+We believe the best memory systems come from collective wisdom. Contributions welcome 👉[Guide](docs/contribution.md):
+
+### Code Contributions
+
+- **New Operators**: Develop custom memory processing operators (retrieval, summarization, etc.)
+- **Backend Implementations**: Add support for new vector stores or LLM providers
+- **Memory Services**: Extend with new memory types or capabilities
+- **API Enhancements**: Improve existing endpoints or add new ones
+
+### Documentation Improvements
+
+- **Integration Examples**: Show how to integrate ReMe with different agent frameworks
+- **Operator Tutorials**: Document custom operator development
+- **Best Practice Guides**: Share effective memory management patterns
+- **Use Case Studies**: Demonstrate ReMe in real-world applications
+
+
+---
+
+## 📄 Citation
+
+```bibtex
+@software{AgentscopeReMe2025,
+  title = {AgentscopeReMe: Memory Management Kit for Agents},
+  author = {Li Yu and
+            Jiaji Deng and
+            Zouying Cao and
+            Weikang Zhou and
+            Tiancheng Qin and
+            Qingxu Fu and
+            Sen Huang and
+            Xianzhe Xu and
+            Zhaoyang Liu and
+            Boyin Liu},
+  url = {https://reme.agentscope.io},
+  year = {2025}
+}
+
+@misc{AgentscopeReMe2025Paper,
+  title={Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution},
+  author={Zouying Cao and
+          Jiaji Deng and
+          Li Yu and
+          Weikang Zhou and
+          Zhaoyang Liu and
+          Bolin Ding and
+          Hai Zhao},
+  year={2025},
+  eprint={2512.10696},
+  archivePrefix={arXiv},
+  primaryClass={cs.AI},
+  url={https://arxiv.org/abs/2512.10696},
+}
+```
+
+---
+
+## ⚖️ License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE](./LICENSE) file for details.
+
+---
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=agentscope-ai/ReMe&type=Date)](https://www.star-history.com/#agentscope-ai/ReMe&Date)

docs/README_0_2_x_ZH.md

@@ -0,0 +1,902 @@
+<p align="center">
+ <img src="docs/_static/figure/reme_logo.png" alt="ReMe 标志" width="50%">
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/badge/python-3.10+-blue" alt="Python Version"></a>
+  <a href="https://pypi.org/project/reme-ai/"><img src="https://img.shields.io/pypi/v/reme-ai.svg?logo=pypi" alt="PyPI Version"></a>
+  <a href="https://pepy.tech/project/reme-ai/"><img src="https://img.shields.io/pypi/dm/reme-ai" alt="PyPI Downloads"></a>
+  <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/commit-activity/m/agentscope-ai/ReMe?style=flat-square" alt="GitHub commit activity"></a>
+</p>
+
+<p align="center">
+  <a href="./LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-black" alt="License"></a>
+  <a href="./README.md"><img src="https://img.shields.io/badge/English-Click-yellow" alt="English"></a>
+  <a href="./README_ZH.md"><img src="https://img.shields.io/badge/简体中文-点击查看-orange" alt="简体中文"></a>
+  <a href="https://github.com/agentscope-ai/ReMe"><img src="https://img.shields.io/github/stars/agentscope-ai/ReMe?style=social" alt="GitHub Stars"></a>
+</p>
+
+<p align="center">
+  <strong>面向智能体的记忆管理工具包, Remember Me, Refine Me.</strong><br>
+  <em><sub>如果 ReMe 对你有帮助，欢迎点一个 ⭐ Star，你的支持是我们持续改进的动力。</sub></em>
+</p>
+
+---
+
+ReMe 是一个**模块化的记忆管理工具包**，为 AI 智能体提供统一的记忆能力——支持在用户、任务与智能体之间提取、复用与共享记忆。
+
+智能体的记忆可以被视为：
+
+```text
+Agent Memory = Long-Term Memory + Short-Term Memory
+             = (Personal + Task + Tool) Memory + (Working Memory)
+```
+
+- **个人记忆（Personal Memory）**：理解用户偏好并适应上下文
+- **任务记忆（Task Memory）**：从经验中学习并在类似任务中表现更好
+- **工具记忆（Tool Memory）**：基于历史表现优化工具选择和参数使用
+- **工作记忆（Working Memory）**：管理长运行智能体的短期上下文，避免上下文溢出
+
+---
+
+## 📰 最新进展
+
+- **[2026-02]** 💻 ReMeCli：终端 AI 聊天助手，内置记忆管理能力。当对话过长时自动将旧内容压缩为摘要以释放上下文空间，同时将重要信息以 Markdown 文件持久化存储，供未来会话自动检索使用。记忆设计灵感来源于 [OpenClaw](https://github.com/openclaw/openclaw)。
+  - [快速开始](docs/cli/quick_start_en.md)
+  - 输入 `/horse` 触发马年彩蛋——烟花、奔马动画和随机马年祝福。
+<table border="0" cellspacing="0" cellpadding="0" style="border: none;">
+  <tr style="border: none;">
+    <td width="10%" style="border: none; vertical-align: middle; text-align: center;">
+      <strong>马<br>上<br>有<br>钱</strong>
+    </td>
+    <td width="80%" style="border: none;">
+      <video src="https://github.com/user-attachments/assets/befa7e40-63ba-4db2-8251-516024616e00" autoplay muted loop controls></video>
+    </td>
+    <td width="10%" style="border: none; vertical-align: middle; text-align: center;">
+      <strong>马<br>到<br>成<br>功</strong>
+    </td>
+  </tr>
+</table>
+
+- **[2025-12]** 📄 我们的程序性（任务）记忆论文已在 [arXiv](https://arxiv.org/abs/2512.10696) 发布
+- **[2025-11]** 🧠 基于工作记忆的 react-agent demo（[介绍](docs/work_memory/message_offload.md)、[Quick Start](docs/cookbook/working/quick_start.md)、[代码](cookbook/working_memory/work_memory_demo.py)）
+- **[2025-10]** 🚀 直接 Python 导入：支持 `from reme_ai import ReMeApp`，无需 HTTP/MCP 服务
+- **[2025-10]** 🔧 工具记忆：支持基于数据驱动的工具选择与参数优化（[指南](docs/tool_memory/tool_memory.md)）
+- **[2025-09]** 🎉 支持异步操作，并已集成至 agentscope-runtime
+- **[2025-09]** 🎉 集成任务记忆与个人记忆
+- **[2025-09]** 🧪 在 appworld、bfcl(v3)、frozenlake 等环境中验证有效性（[实验文档](docs/cookbook)）
+- **[2025-08]** 🚀 支持 MCP 协议（[快速开始](docs/mcp_quick_start.md)）
+- **[2025-06]** 🚀 支持多种向量存储后端（Elasticsearch & ChromaDB）（[向量库指南](docs/vector_store_api_guide.md)）
+- **[2024-09]** 🧠 支持个性化与时间敏感的记忆存储
+
+---
+
+## ✨ 架构设计
+
+<p align="center">
+ <img src="docs/_static/figure/reme_structure.jpg" alt="ReMe 架构" width="80%">
+</p>
+
+ReMe 提供了一个**模块化的记忆管理工具包**，具有可插拔的组件，可以集成到任何智能体框架中。系统包括：
+
+#### 🧠 **任务记忆 / 经验记忆（Task Memory/Experience）**
+
+可在不同智能体之间复用的程序性知识：
+
+- **成功模式识别**：识别有效策略并理解其背后的原理
+- **失败分析学习**：从错误中学习，避免重复踩坑
+- **对比式模式**：通过多条采样轨迹的对比获取更有价值的记忆
+- **验证模式**：通过验证模块确认提炼出的经验是否有效
+
+了解如何使用任务记忆可参考：[任务记忆文档](docs/task_memory/task_memory.md)
+
+#### 👤 **个人记忆（Personal Memory）**
+
+面向特定用户的情境化长期记忆：
+
+- **个体偏好**：记录用户的习惯、偏好与交互风格
+- **情境自适应**：基于时间与上下文动态管理记忆
+- **渐进式学习**：在长期多轮交互中不断加深对用户的理解
+- **时间敏感**：在记忆检索与整合中考虑时间因素
+
+了解如何使用个人记忆可参考：[个人记忆文档](docs/personal_memory/personal_memory.md)
+
+#### 🔧 **工具记忆（Tool Memory）**
+
+基于真实调用数据的工具选择与使用优化：
+
+- **历史表现追踪**：记录成功率、调用耗时与 Token 成本
+- **LLM-as-Judge 评估**：提供工具成功 / 失败原因的定性洞察
+- **参数优化**：从历史成功调用中学习最优参数配置
+- **动态指南**：将静态工具描述演化为可持续更新的「活文档」
+
+了解如何使用工具记忆可参考：[工具记忆文档](docs/tool_memory/tool_memory.md)
+
+#### 🧠 **工作记忆（Working Memory）**
+
+面向长流程智能体的短期上下文记忆，通过**消息卸载与重载（message offload & reload）**实现：
+- **消息卸载（Message Offload）**：将体积巨大的工具输出压缩为外部文件或 LLM 摘要
+- **消息重载（Message Reload）**：按需搜索（`grep_working_memory`）并读取（`read_working_memory`）已卸载的内容
+
+📖 **概念与 API：**
+- 消息卸载概览：[Message Offload](docs/work_memory/message_offload.md)
+- 卸载 / 重载算子：[Message Offload Ops](docs/work_memory/message_offload_ops.md)、[Message Reload Ops](docs/work_memory/message_reload_ops.md)
+
+💻 **端到端 Demo：**
+- 工作记忆快速上手：[Working Memory Quick Start](docs/cookbook/working/quick_start.md)
+- 带工作记忆的 ReAct 智能体：[react_agent_with_working_memory.py](cookbook/working_memory/react_agent_with_working_memory.py)
+- 可运行 Demo：[work_memory_demo.py](cookbook/working_memory/work_memory_demo.py)
+
+---
+
+## 🛠️ 安装
+
+### 通过 PyPI 安装（推荐）
+
+```bash
+pip install reme-ai
+```
+
+### 从源码安装
+
+```bash
+git clone https://github.com/agentscope-ai/ReMe.git
+cd ReMe
+pip install .
+```
+
+### 环境变量配置
+
+复制 `example.env` 为 `.env` 并按需修改：
+
+```bash
+FLOW_LLM_API_KEY=sk-xxxx
+FLOW_LLM_BASE_URL=https://xxxx/v1
+FLOW_EMBEDDING_API_KEY=sk-xxxx
+FLOW_EMBEDDING_BASE_URL=https://xxxx/v1
+```
+
+---
+
+## 🚀 快速开始
+
+### 启动 HTTP 服务
+
+```bash
+reme \
+  backend=http \
+  http.port=8002 \
+  llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
+  embedding_model.default.model_name=text-embedding-v4 \
+  vector_store.default.backend=local
+```
+
+### 启动 MCP Server
+
+```bash
+reme \
+  backend=mcp \
+  mcp.transport=stdio \
+  llm.default.model_name=qwen3-30b-a3b-thinking-2507 \
+  embedding_model.default.model_name=text-embedding-v4 \
+  vector_store.default.backend=local
+```
+
+### 核心 API 用法
+
+#### 任务记忆管理
+
+```python
+import requests
+
+# 经验总结：从执行轨迹中学习
+response = requests.post("http://localhost:8002/summary_task_memory", json={
+    "workspace_id": "task_workspace",
+    "trajectories": [
+        {"messages": [{"role": "user", "content": "Help me create a project plan"}], "score": 1.0}
+    ]
+})
+
+# 记忆检索：获取相关经验
+response = requests.post("http://localhost:8002/retrieve_task_memory", json={
+    "workspace_id": "task_workspace",
+    "query": "How to efficiently manage project progress?",
+    "top_k": 1
+})
+```
+
+<details>
+<summary>Python 导入版本</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # 经验总结：从执行轨迹中学习
+        result = await app.async_execute(
+            name="summary_task_memory",
+            workspace_id="task_workspace",
+            trajectories=[
+                {
+                    "messages": [
+                        {"role": "user", "content": "Help me create a project plan"}
+                    ],
+                    "score": 1.0
+                }
+            ]
+        )
+        print(result)
+
+        # 记忆检索：获取相关经验
+        result = await app.async_execute(
+            name="retrieve_task_memory",
+            workspace_id="task_workspace",
+            query="How to efficiently manage project progress?",
+            top_k=1
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl 版本</summary>
+
+```bash
+# 经验总结：从执行轨迹中学习
+curl -X POST http://localhost:8002/summary_task_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "trajectories": [
+      {"messages": [{"role": "user", "content": "Help me create a project plan"}], "score": 1.0}
+    ]
+  }'
+
+# 记忆检索：获取相关经验
+curl -X POST http://localhost:8002/retrieve_task_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "query": "How to efficiently manage project progress?",
+    "top_k": 1
+  }'
+```
+
+</details>
+
+#### 个人记忆管理
+
+```python
+# 记忆整合：从用户交互中学习
+response = requests.post("http://localhost:8002/summary_personal_memory", json={
+    "workspace_id": "task_workspace",
+    "trajectories": [
+        {"messages":
+            [
+                {"role": "user", "content": "I like to drink coffee while working in the morning"},
+                {"role": "assistant",
+                 "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+            ]
+        }
+    ]
+})
+
+# 记忆检索：获取个人记忆片段
+response = requests.post("http://localhost:8002/retrieve_personal_memory", json={
+    "workspace_id": "task_workspace",
+    "query": "What are the user's work habits?",
+    "top_k": 5
+})
+```
+
+<details>
+<summary>Python 导入版本</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # 记忆整合：从用户交互中学习
+        result = await app.async_execute(
+            name="summary_personal_memory",
+            workspace_id="task_workspace",
+            trajectories=[
+                {
+                    "messages": [
+                        {"role": "user", "content": "I like to drink coffee while working in the morning"},
+                        {"role": "assistant",
+                         "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+                    ]
+                }
+            ]
+        )
+        print(result)
+
+        # 记忆检索：获取个人记忆片段
+        result = await app.async_execute(
+            name="retrieve_personal_memory",
+            workspace_id="task_workspace",
+            query="What are the user's work habits?",
+            top_k=5
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl 版本</summary>
+
+```bash
+# 记忆整合：从用户交互中学习
+curl -X POST http://localhost:8002/summary_personal_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "trajectories": [
+      {"messages": [
+        {"role": "user", "content": "I like to drink coffee while working in the morning"},
+        {"role": "assistant", "content": "I understand, you prefer to start your workday with coffee to stay energized"}
+      ]}
+    ]
+  }'
+
+# 记忆检索：获取个人记忆片段
+curl -X POST http://localhost:8002/retrieve_personal_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "task_workspace",
+    "query": "What are the user'\''s work habits?",
+    "top_k": 5
+  }'
+```
+
+</details>
+
+#### 工具记忆管理
+
+```python
+import requests
+
+# 记录工具调用结果
+response = requests.post("http://localhost:8002/add_tool_call_result", json={
+    "workspace_id": "tool_workspace",
+    "tool_call_results": [
+        {
+            "create_time": "2025-10-21 10:30:00",
+            "tool_name": "web_search",
+            "input": {"query": "Python asyncio tutorial", "max_results": 10},
+            "output": "Found 10 relevant results...",
+            "token_cost": 150,
+            "success": True,
+            "time_cost": 2.3
+        }
+    ]
+})
+
+# 从历史生成使用指南
+response = requests.post("http://localhost:8002/summary_tool_memory", json={
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+})
+
+# 在使用前检索工具指南
+response = requests.post("http://localhost:8002/retrieve_tool_memory", json={
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+})
+```
+
+<details>
+<summary>Python 导入版本</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # 记录工具调用结果
+        result = await app.async_execute(
+            name="add_tool_call_result",
+            workspace_id="tool_workspace",
+            tool_call_results=[
+                {
+                    "create_time": "2025-10-21 10:30:00",
+                    "tool_name": "web_search",
+                    "input": {"query": "Python asyncio tutorial", "max_results": 10},
+                    "output": "Found 10 relevant results...",
+                    "token_cost": 150,
+                    "success": True,
+                    "time_cost": 2.3
+                }
+            ]
+        )
+        print(result)
+
+        # 从历史生成使用指南
+        result = await app.async_execute(
+            name="summary_tool_memory",
+            workspace_id="tool_workspace",
+            tool_names="web_search"
+        )
+        print(result)
+
+        # 在使用前检索工具指南
+        result = await app.async_execute(
+            name="retrieve_tool_memory",
+            workspace_id="tool_workspace",
+            tool_names="web_search"
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl 版本</summary>
+
+```bash
+# 记录工具调用结果
+curl -X POST http://localhost:8002/add_tool_call_result \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_call_results": [
+      {
+        "create_time": "2025-10-21 10:30:00",
+        "tool_name": "web_search",
+        "input": {"query": "Python asyncio tutorial", "max_results": 10},
+        "output": "Found 10 relevant results...",
+        "token_cost": 150,
+        "success": true,
+        "time_cost": 2.3
+      }
+    ]
+  }'
+
+# 从历史生成使用指南
+curl -X POST http://localhost:8002/summary_tool_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+  }'
+
+# 在使用前检索工具指南
+curl -X POST http://localhost:8002/retrieve_tool_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "workspace_id": "tool_workspace",
+    "tool_names": "web_search"
+  }'
+```
+
+</details>
+
+#### 工作记忆管理
+
+```python
+import requests
+
+# 对长对话 / 长流程的工作记忆进行压缩与总结
+response = requests.post("http://localhost:8002/summary_working_memory", json={
+    "messages": [
+        {
+            "role": "system",
+            "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+        },
+        {
+            "role": "user",
+            "content": "搜索下reme项目的的README内容"
+        },
+        {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {
+                    "index": 0,
+                    "id": "call_6596dafa2a6a46f7a217da",
+                    "function": {
+                        "arguments": "{\"query\": \"readme\"}",
+                        "name": "web_search"
+                    },
+                    "type": "function"
+                }
+            ]
+        },
+        {
+            "role": "tool",
+            "content": "ultra large context , over 50000 tokens......"
+        },
+        {
+            "role": "user",
+            "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+        }
+    ],
+    "working_summary_mode": "auto",
+    "compact_ratio_threshold": 0.75,
+    "max_total_tokens": 20000,
+    "max_tool_message_tokens": 2000,
+    "group_token_threshold": 4000,
+    "keep_recent_count": 2,
+    "store_dir": "test_working_memory",
+    "chat_id": "demo_chat_id"
+})
+```
+
+<details>
+<summary>Python 导入版本</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # 对长对话 / 长流程的工作记忆进行压缩与总结
+        result = await app.async_execute(
+            name="summary_working_memory",
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+                },
+                {
+                    "role": "user",
+                    "content": "搜索下reme项目的的README内容"
+                },
+                {
+                    "role": "assistant",
+                    "content": "",
+                    "tool_calls": [
+                        {
+                            "index": 0,
+                            "id": "call_6596dafa2a6a46f7a217da",
+                            "function": {
+                                "arguments": "{\"query\": \"readme\"}",
+                                "name": "web_search"
+                            },
+                            "type": "function"
+                        }
+                    ]
+                },
+                {
+                    "role": "tool",
+                    "content": "ultra large context , over 50000 tokens......"
+                },
+                {
+                    "role": "user",
+                    "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+                }
+            ],
+            working_summary_mode="auto",
+            compact_ratio_threshold=0.75,
+            max_total_tokens=20000,
+            max_tool_message_tokens=2000,
+            group_token_threshold=4000,
+            keep_recent_count=2,
+            store_dir="test_working_memory",
+            chat_id="demo_chat_id",
+        )
+        print(result)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>curl 版本</summary>
+
+```bash
+curl -X POST http://localhost:8002/summary_working_memory \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      {
+        "role": "system",
+        "content": "You are a helpful assistant. First use `Grep` to find the line numbers that match the keywords or regular expressions, and then use `ReadFile` to read the code around those locations. If no matches are found, never give up; try different parameters, such as searching with only part of the keywords. After `Grep`, use the `ReadFile` command to view content starting from a specified `offset` and `limit`, and do not exceed 100 lines. If the current content is insufficient, you can continue trying different `offset` and `limit` values with the `ReadFile` command."
+      },
+      {
+        "role": "user",
+        "content": "搜索下reme项目的的README内容"
+      },
+      {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+          {
+            "index": 0,
+            "id": "call_6596dafa2a6a46f7a217da",
+            "function": {
+              "arguments": "{\"query\": \"readme\"}",
+              "name": "web_search"
+            },
+            "type": "function"
+          }
+        ]
+      },
+      {
+        "role": "tool",
+        "content": "ultra large context , over 50000 tokens......"
+      },
+      {
+        "role": "user",
+        "content": "根据readme回答task memory在appworld的效果是多少，需要具体的数值"
+      }
+    ],
+    "working_summary_mode": "auto",
+    "compact_ratio_threshold": 0.75,
+    "max_total_tokens": 20000,
+    "max_tool_message_tokens": 2000,
+    "group_token_threshold": 4000,
+    "keep_recent_count": 2,
+    "store_dir": "test_working_memory",
+    "chat_id": "demo_chat_id"
+  }'
+```
+
+</details>
+
+---
+
+## 📦 开箱即用的记忆库
+
+ReMe 提供一个**记忆库**，包含预先提取的、生产就绪的记忆，智能体可以立即加载和使用：
+
+### 可用记忆包
+
+| 记忆包                | 领域       | 规模           | 描述                                                   |
+|----------------------|------------|----------------|--------------------------------------------------------|
+| **`appworld.jsonl`** | 任务执行   | ~100 条记忆    | 复杂任务规划模式、多步骤工作流和错误恢复策略              |
+| **`bfcl_v3.jsonl`**  | 工具使用   | ~150 条记忆    | 函数调用模式、参数优化和工具选择策略                      |
+
+### 加载预构建记忆
+
+```python
+# 加载内置记忆
+response = requests.post("http://localhost:8002/vector_store", json={
+    "workspace_id": "appworld",
+    "action": "load",
+    "path": "./docs/library/"
+})
+
+# 查询相关记忆
+response = requests.post("http://localhost:8002/retrieve_task_memory", json={
+    "workspace_id": "appworld",
+    "query": "How to navigate to settings and update user profile?",
+    "top_k": 1
+})
+```
+
+<details>
+<summary>Python 导入版本</summary>
+
+```python
+import asyncio
+from reme_ai import ReMeApp
+
+async def main():
+    async with ReMeApp(
+        "llm.default.model_name=qwen3-30b-a3b-thinking-2507",
+        "embedding_model.default.model_name=text-embedding-v4",
+        "vector_store.default.backend=memory"
+    ) as app:
+        # 加载内置记忆
+        result = await app.async_execute(
+            name="vector_store",
+            workspace_id="appworld",
+            action="load",
+            path="./docs/library/"
+        )
+        print(result)
+
+        # 查询相关记忆
+        result = await app.async_execute(
+            name="retrieve_task_memory",
+            workspace_id="appworld",
+            query="How to navigate to settings and update user profile?",
+            top_k=1
+        )
+        print(result)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+---
+
+## 🧪 实验结果
+
+### 🌍 [Appworld 实验](docs/cookbook/appworld/quickstart.md)
+
+我们在 Appworld 环境上使用 Qwen3-8B（非思考模式）进行评测：
+
+| 方法       | Avg@4               | Pass@4              |
+|-----------|-------------------|-------------------|
+| 无 ReMe   |  0.1497              | 0.3285              |
+| 使用 ReMe  | 0.1706 **(+2.09%)** | 0.3631 **(+3.46%)** |
+
+Pass@K 衡量在生成 K 个候选中，至少一个成功完成任务（score=1）的概率。
+当前实验使用的是内部 AppWorld 环境，可能与对外版本存在轻微差异。
+
+关于如何复现实验的更多细节，见 [quickstart.md](docs/cookbook/appworld/quickstart.md)。
+
+### 🔧 [BFCL-V3 实验](docs/cookbook/bfcl/quickstart.md)
+
+我们在 BFCL-V3 multi-turn-base 任务（随机划分 50 train / 150 val）上，使用 Qwen3-8B（思考模式）进行评测：
+
+| 方法       | Avg@4            | Pass@4              |
+|------------|-----------------|---------------------|
+| 无 ReMe    | 0.4033              | 0.5955              |
+| 使用 ReMe  | 0.4450 **(+4.17%)** | 0.6577 **(+6.22%)** |
+
+### 🧊 [Frozenlake 实验](docs/cookbook/frozenlake/quickstart.md)
+
+|                                             无 ReMe                                              |                                              使用 ReMe                                               |
+|:------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------:|
+| <p align="center"><img src="docs/_static/figure/frozenlake_failure.gif" alt="失败示例" width="30%"></p> | <p align="center"><img src="docs/_static/figure/frozenlake_success.gif" alt="成功示例" width="30%"></p> |
+
+我们在 100 张随机 frozenlake 地图上，使用 qwen3-8b 进行测试：
+
+| 方法       | 通过率          |
+|------------|-----------------|
+| 无 ReMe    | 0.66            |
+| 使用 ReMe  | 0.72 **(+6.0%)** |
+
+更多复现实验细节见 [quickstart.md](docs/cookbook/frozenlake/quickstart.md)。
+
+### 🛠️ [工具记忆基准](docs/tool_memory/tool_bench.md)
+
+我们在一个受控基准上，使用三个模拟搜索工具与 Qwen3-30B-Instruct 评估工具记忆的效果：
+
+| 场景                  | 平均分 | 提升       |
+|-----------------------|--------|------------|
+| 训练集（无记忆）      | 0.650  | -          |
+| 测试集（无记忆）      | 0.672  | 基线       |
+| **测试集（使用记忆）** | **0.772** | **+14.88%** |
+
+**关键结论：**
+- 工具记忆可以基于历史表现进行数据驱动的工具选择
+- 通过学习参数配置，成功率约提升 15%
+
+更多细节见 [tool_bench.md](docs/tool_memory/tool_bench.md) 与实现代码 [run_reme_tool_bench.py](cookbook/tool_memory/run_reme_tool_bench.py)。
+
+---
+
+## 📚 资源
+
+### 快速入门
+- **[Quick Start](./cookbook/simple_demo)**：实用示例，可立即使用
+  - [工具记忆 Demo](cookbook/simple_demo/use_tool_memory_demo.py)：工具记忆的完整生命周期演示
+  - [工具记忆基准](cookbook/tool_memory/run_reme_tool_bench.py)：评估工具记忆效果
+
+### 集成指南
+- **[直接 Python 导入](docs/cookbook/working/quick_start.md)**：将 ReMe 直接嵌入到你的智能体代码中
+- **[HTTP 服务 API](docs/vector_store_api_guide.md)**：用于多智能体系统的 RESTful API
+- **[MCP 协议](docs/mcp_quick_start.md)**：与 Claude Desktop 和 MCP 兼容客户端集成
+
+### 记忆系统配置
+- **[个人记忆](docs/personal_memory)**：用户偏好学习和上下文自适应
+- **[任务记忆](docs/task_memory)**：程序性知识提取和复用
+- **[工具记忆](docs/tool_memory)**：数据驱动的工具选择和优化
+- **[工作记忆](docs/work_memory/message_offload.md)**：长流程智能体的短期上下文管理
+
+### 高级主题
+- **[算子管道](reme_ai/config/default.yaml)**：通过修改算子链来自定义记忆处理工作流
+- **[向量存储后端](docs/vector_store_api_guide.md)**：配置本地、Elasticsearch、Qdrant 或 ChromaDB 存储
+- **[案例集](./cookbook)**：真实场景的用例和最佳实践
+
+---
+
+## ⭐ 社区与支持
+
+- **Star & Watch**：Star 可以让更多智能体开发者发现 ReMe；Watch 能帮助你第一时间获知新版本与特性。
+- **分享你的成果**：在 Issue 或 Discussion 中分享 ReMe 为你的智能体解锁了什么——我们非常乐意展示社区的优秀案例。
+- **需要新功能？** 提交 Feature Request，我们将一起完善它。
+
+---
+
+## 🤝 参与贡献
+
+我们相信，最好的记忆系统来自社区的集体智慧。欢迎贡献 👉[贡献指南](docs/contribution.md)：
+
+### 代码贡献
+
+- **新算子**：开发自定义记忆处理算子（检索、总结等）
+- **后端实现**：添加对新向量存储或 LLM 提供商的支持
+- **记忆服务**：扩展新的记忆类型或能力
+- **API 增强**：改进现有端点或添加新端点
+
+### 文档改进
+
+- **集成示例**：展示如何将 ReMe 与不同智能体框架集成
+- **算子教程**：记录自定义算子开发
+- **最佳实践指南**：分享有效的记忆管理模式
+- **用例研究**：展示 ReMe 在实际应用中的使用
+
+---
+
+## 📄 引用
+
+```bibtex
+@software{AgentscopeReMe2025,
+  title = {AgentscopeReMe: Memory Management Kit for Agents},
+  author = {Li Yu and
+            Jiaji Deng and
+            Zouying Cao and
+            Weikang Zhou and
+            Tiancheng Qin and
+            Qingxu Fu and
+            Sen Huang and
+            Xianzhe Xu and
+            Zhaoyang Liu and
+            Boyin Liu},
+  url = {https://reme.agentscope.io},
+  year = {2025}
+}
+
+@misc{AgentscopeReMe2025Paper,
+  title={Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution},
+  author={Zouying Cao and
+          Jiaji Deng and
+          Li Yu and
+          Weikang Zhou and
+          Zhaoyang Liu and
+          Bolin Ding and
+          Hai Zhao},
+  year={2025},
+  eprint={2512.10696},
+  archivePrefix={arXiv},
+  primaryClass={cs.AI},
+  url={https://arxiv.org/abs/2512.10696},
+}
+```
+
+---
+
+## ⚖️ 许可证
+
+本项目基于 Apache License 2.0 开源，详情参见 [LICENSE](./LICENSE) 文件。
+
+---
+
+## Star 历史
+
+[![Star History Chart](https://api.star-history.com/svg?repos=agentscope-ai/ReMe&type=Date)](https://www.star-history.com/#agentscope-ai/ReMe&Date)

docs/_config.yml

@@ -0,0 +1,132 @@
+# Book settings
+# Learn more at https://jupyterbook.org/customize/config.html
+
+project: "ReMe"
+title: "<div style='text-align:center'>
+  <span style='font-weight:700;color:#2196f3;'>AgentScope</span><br>
+  <span style='font-weight:900;color:#ff5722;'>ReMe</span>
+</div>"
+author: Alibaba Tongyi Lab
+logo: _static/figure/logo.svg
+copyright: "2025, Tongyi Lab, Alibaba Inc."
+only_build_toc_files: true
+
+# Force re-execution of notebooks on each build.
+# See https://jupyterbook.org/content/execute.html
+execute:
+  execute_notebooks: off
+
+parse:
+  myst_enable_extensions:
+    - colon_fence
+    - deflist
+    - attrs_inline
+    - dollarmath
+
+# Define the name of the latex output file for PDF builds
+latex:
+  latex_documents:
+    targetname: book.tex
+
+# Add a bibtex file so that we can create citations
+bibtex_bibfiles:
+  - references.bib
+
+html:
+  extra_js:
+    - _static/memory-lib/memory-lib.js
+  extra_css:
+    - _static/memory-lib/memory-lib.css
+    - _static/custom.css
+
+# Sphinx settings
+sphinx:
+  extra_extensions:
+    - sphinx.ext.autodoc
+    - sphinx.ext.viewcode
+    - sphinx.ext.napoleon
+    - sphinx.ext.intersphinx
+    - sphinx.ext.autosummary
+    - sphinxcontrib.mermaid
+    - sphinx_design
+  config:
+    # API Documentation Configuration
+    autosummary_generate: True
+    autosummary_imported_members: True
+
+    # Autodoc Configuration
+    autodoc_typehints: 'description'
+    autodoc_member_order: 'bysource'
+    autodoc_default_options:
+      members: True
+      member-order: 'bysource'
+      special-members: '__init__'
+      undoc-members: True
+      exclude-members: '__weakref__'
+
+    # Napoleon Configuration
+    napoleon_google_docstring: True
+    napoleon_numpy_docstring: True
+    napoleon_include_init_with_doc: False
+    napoleon_include_private_with_doc: False
+    napoleon_include_special_with_doc: True
+    napoleon_use_admonition_for_examples: False
+    napoleon_use_admonition_for_notes: False
+    napoleon_use_admonition_for_references: False
+    napoleon_use_ivar: False
+    napoleon_use_param: True
+    napoleon_use_rtype: True
+
+    # Intersphinx Configuration
+    intersphinx_mapping:
+      python: ['https://docs.python.org/3', null]
+      numpy: ['https://numpy.org/doc/stable/', null]
+
+    # Theme Configuration
+    html_theme: furo
+    pygments_style: "friendly"
+    html_show_sphinx: false
+    html_last_updated_fmt: "%Y-%m-%d"
+    html_copy_source: false
+    html_show_sourcelink: false
+    templates_path: ["./_templates"]
+    html_static_path:
+      - "_static"
+    use_multitoc_numbering: false
+    html_js_files:
+      - language.js
+    html_css_files:
+      - custom.css
+    html_sidebars:
+      "**":
+        - "sidebar/scroll-start.html"
+        - "sidebar/brand.html"
+        - "sidebar/search.html"
+        - "sidebar/navigation.html"
+        - "sidebar/ethical-ads.html"
+        - "sidebar/scroll-end.html"
+    html_theme_options:
+      top_of_page_buttons: ["view"]
+      sidebar_hide_name: false
+      source_repository: "https://reme.agentscope.io"
+      source_branch: "main"
+      source_directory: "docs/"
+      footer_icons:
+        - name: GitHub
+          url: "https://reme.agentscope.io"
+          html: |
+            <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
+                <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>
+            </svg>
+          class: ""
+      light_css_variables:
+        color-brand-primary: "#2196f3"
+        color-brand-content: "#2196f3"
+        color-admonition-background: "#f8f9fa"
+      dark_css_variables:
+        color-brand-primary: "#64b5f6"
+        color-brand-content: "#64b5f6"
+
+# jupyter-book build --all .
+# echo "reme.agentscope.io" > _build/html/CNAME
+# ghp-import -n -p -f _build/html

docs/_static/custom.css

@@ -0,0 +1,33 @@
+h1, .bd-article h1 {
+  font-size: 1.8rem !important;
+}
+
+h2, .bd-article h2 {
+  font-size: 1.5rem !important;
+}
+
+h3, .bd-article h3 {
+  font-size: 1.25rem !important;
+}
+
+h4, .bd-article h4 {
+  font-size: 1.1rem !important;
+}
+
+h5, .bd-article h5 {
+  font-size: 1rem !important;
+}
+
+h6, .bd-article h6 {
+  font-size: 0.9rem !important;
+}
+
+div.bd-sidebar .navbar-brand {
+  text-align: center;
+  width: 100%;
+}
+
+div.bd-sidebar .navbar-brand span {
+  display: block;
+  text-align: center;
+}

docs/_static/figure/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" fill="none" version="1.1" width="550" height="550" viewBox="0 0 550 550"><defs><linearGradient x1="0.01500389538705349" y1="0.4831196665763855" x2="0.9407116114637801" y2="0.3076102348892569" id="master_svg0_8_2390"><stop offset="0%" stop-color="#01C5FF" stop-opacity="1"/><stop offset="100%" stop-color="#019DFB" stop-opacity="1"/></linearGradient><linearGradient x1="0.21085502207279205" y1="0.38703426718711853" x2="0.9523109409029081" y2="0.390888421140005" id="master_svg1_8_1638"><stop offset="0%" stop-color="#4701EF" stop-opacity="1"/><stop offset="100%" stop-color="#395EEF" stop-opacity="1"/></linearGradient></defs><g><g></g><g><g><path d="M275.4998779296875,211.25Q275.4998779296875,279.5,373.4999779296875,310.5Q338.4998779296875,287.5,343.9998779296875,232Q344.4969779296875,227.19400000000002,345.9004779296875,222.498Q352.96577792968753,198.857,382.9998779296875,178L415.9998779296875,193.5L469.7738779296875,193.5C477.0958779296875,193.5,481.9218779296875,185.91559999999998,478.3148779296875,179.543Q441.5068779296875,114.5,372.9998779296875,114.5C343.9998779296875,114.5,275.4998779296875,143,275.4998779296875,211.25Z" fill="url(#master_svg0_8_2390)" fill-opacity="1"/></g><g><path d="M343.9999162890625,231.99999791015625Q337.9999162890625,287.5000079101562,373.5000462890625,310.5000079101562L433.4998462890625,333.00010791015626Q449.4994462890625,314.2500079101562,449.4994462890625,295.5000079101562Q449.4994462890625,276.7500079101562,433.4998462890625,258.0000079101562L345.9004862890625,222.49810791015625Q344.4970462890625,227.19412791015625,343.9999162890625,231.99999791015625Z" fill="#0064FC" fill-opacity="1"/></g><g><path d="M122.9998779296875,351.5C124.3900479296875,350.6567,125.7727179296875,349.8325,127.1479779296875,349.0269Q125.0392779296875,350.2098,122.9998779296875,351.5ZM127.1479779296875,349.0269Q150.3716779296875,336,181.9998779296875,336C186.2692779296875,335.7983,190.4211779296875,335.9808,194.4638779296875,336.502C250.5488779296875,343.7327,285.6218779296875,416.14300000000003,321.9998779296875,432Q350.1248779296875,444,377.9998779296875,444Q405.8748779296875,444,433.4998779296875,432Q489.9998779296875,400,489.9998779296875,344.5Q489.9998779296875,283,433.4998779296875,258Q449.4998779296875,276.75,449.4998779296875,295.5Q449.4998779296875,314.25,433.4998779296875,333C394.3168779296875,374.257,360.65987792968747,359.947,319.4498779296875,342.4251C286.9518779296875,328.6077,249.7568779296875,312.7935,201.4527779296875,320.6594C179.2413779296875,324.2763,154.6808779296875,332.9,127.1479779296875,349.0269Z" fill="url(#master_svg1_8_1638)" fill-opacity="1"/></g><g><path d="M61,437.99973876953123L133.8305,437.99973876953123C141.5661,437.99973876953123,148.608,433.53913876953123,151.9131,426.54513876953126L194.464,336.50202976953125C190.421,335.98083496953126,186.269,335.79829376953126,182,335.99999996953125Q147.4999,335.99999996953125,122.9999,351.5000387695313C93.5,373.5000387695313,87,387.0000387695313,61,437.99973876953123Z" fill="#0064FC" fill-opacity="1"/></g><g><path d="M61,438.00038301849366C87,387.00038301849366,93.5,373.50038301849366,122.9999,351.50038301849366C152.2215,333.77438301849367,178.132,324.45738301849366,201.453,320.65938301849366L256.597,207.04148301849364C259.081,201.92438301849364,259.26800000000003,195.99188301849364,257.11199999999997,190.72838301849367L227.948,119.52337301849366C225.653,113.91851301849366,217.805,113.67667301849366,215.169,119.12954301849365L61,438.00038301849366Z" fill="#01C8FF" fill-opacity="1"/></g></g></g></svg>