mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-09-14 23:21:04 +00:00
9975bb37b9
Some checks failed
CI / Python tests / Unit Tests - py3.12 (push) Waiting to run
CI / Python tests / Unit Tests - py3.13 (push) Waiting to run
CI / Windows / CLI smoke - py3.11 (push) Waiting to run
Deploy / Documentation / Build documentation (push) Waiting to run
Deploy / Documentation / deploy (push) Blocked by required conditions
CI / Python packages / Build and verify distributions (push) Waiting to run
CI / Python quality / Pre-commit (push) Waiting to run
CI / Python tests / Unit Tests - py3.11 (push) Waiting to run
Security / CodeQL / Analyze javascript-typescript (push) Waiting to run
Security / CodeQL / Analyze python (push) Waiting to run
CI / Documentation / Test and build documentation (push) Has been cancelled
34 lines
1.6 KiB
YAML
34 lines
1.6 KiB
YAML
# LongMemEval evaluation configuration
|
|
# This file controls what/how to evaluate.
|
|
|
|
dataset:
|
|
path: "benchmark/longmemeval/dataset/longmemeval_s_reme_cleaned.json"
|
|
start_index: 0 # first item index
|
|
num_items: 500 # how many items to evaluate (starting from start_index)
|
|
max_sessions: 0 # 0 = all sessions; >0 = limit sessions per item for testing
|
|
question_types: [] # filter by question_type; empty list = no filtering (all types)
|
|
workspace_root: "benchmark/longmemeval/workspaces/longmemeval-s" # workspace root for item workspaces
|
|
|
|
evaluation:
|
|
# LLM-as-judge uses the 'judge' as_llm component defined in benchmark.yaml
|
|
# Model and credentials are configured there (reading from .env)
|
|
# Judgment is always binary (yes/no) — defined in lme/llm_judge.yaml
|
|
num_workers: 32 # 0 = auto (cpu_count - 2, min 1); 1 = sequential; >1 = parallel
|
|
filter_future_sessions: true # true = only ingest sessions with timestamp <= question_date
|
|
compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression
|
|
|
|
reme:
|
|
config: "benchmark" # runner enables both plugins below
|
|
plugins: [lme, lme-judge]
|
|
# Dream trigger: when gap between consecutive sessions crosses this hour (23:00)
|
|
dream_trigger_hour: 23
|
|
# Dream scan_days for each trigger
|
|
dream_scan_days: 2
|
|
dream_max_units: 5
|
|
|
|
output:
|
|
dir: "benchmark/longmemeval/results"
|
|
log_dir: "logs" # log directory (relative to project root)
|
|
log_prefix: "longmemeval" # benchmark name used in log filenames
|
|
log_to_console: true
|
|
log_to_file: true
|