ReMe/reme/config/benchmark.yaml
xyf2020 9975bb37b9
Some checks failed
CI / Python tests / Unit Tests - py3.12 (push) Waiting to run
CI / Python tests / Unit Tests - py3.13 (push) Waiting to run
CI / Windows / CLI smoke - py3.11 (push) Waiting to run
Deploy / Documentation / Build documentation (push) Waiting to run
Deploy / Documentation / deploy (push) Blocked by required conditions
CI / Python packages / Build and verify distributions (push) Waiting to run
CI / Python quality / Pre-commit (push) Waiting to run
CI / Python tests / Unit Tests - py3.11 (push) Waiting to run
Security / CodeQL / Analyze javascript-typescript (push) Waiting to run
Security / CodeQL / Analyze python (push) Waiting to run
CI / Documentation / Test and build documentation (push) Has been cancelled
Separate benchmark judge plugins (#535)
2026-09-10 20:16:46 +08:00

491 lines
13 KiB
YAML

# Shared benchmark application preset; independent of the default service config.
# Background and cron jobs are omitted in favor of manually callable base jobs.
# Benchmark plugins contribute their own jobs through plugin.yaml.
service:
backend: http
jobs:
# ── Manual index update (replaces index_update_loop background) ──
index_update:
backend: base
description: "Manually trigger incremental index update for watched dirs."
watch_dirs: [daily_dir, digest_dir, session_dir/dialog]
watch_suffixes: [md, jsonl]
parameters:
type: object
properties: {}
steps:
- backend: init_changes_step
monitor_type: file_store
monitor_name: default
dispatch_steps: [update_index_step]
# ── Manual digest catalog update (replaces digest_watch_loop background) ──
digest_update:
backend: base
description: "Manually trigger digest catalog update."
watch_dirs: [daily_dir, digest_dir]
watch_suffixes: [md]
parameters:
type: object
properties: {}
steps:
- backend: init_changes_step
monitor_type: file_catalog
monitor_name: digest
dispatch_steps:
- backend: update_catalog_step
file_catalog: digest
- backend: log_changes_step
# ── Auto dream (same as default.yaml auto_dream, base mode) ──
# auto_dream:
# backend: base
# description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units, integrate digest units, and persist the dream catalog."
# parameters:
# type: object
# properties:
# date:
# type: string
# description: "YYYY-MM-DD to scan; defaults to today in the dreamer's timezone"
# default: ""
# hint:
# type: string
# description: "caller guidance passed through to dream extract/integrate"
# default: ""
# scan_days:
# type: integer
# description: "number of recent daily directories to scan, ending at date"
# default: 2
# max_units:
# type: integer
# description: "maximum number of extracted memory units"
# default: 5
# steps:
# - backend: dream_extract_step
# file_catalog: dream
# scan_days: 2
# max_units: 5
# - backend: dream_integrate_step
# - backend: dream_finish_step
# file_catalog: dream
# ── Text compression (direct LLM call, no agent) ──
compressor:
backend: base
description: "Compress text via a direct LLM call, optionally guided by queries as relevance filter"
parameters:
type: object
properties:
text:
type: string
description: "the text to compress"
queries:
type: array
description: "optional list of queries; content potentially relevant to any query is kept, content certainly irrelevant to all queries may be dropped"
items:
type: string
default: []
required:
- text
steps:
- backend: compressor_step
as_llm: compressor
# ── Reindex (derived search indexes only) ──
reindex:
backend: base
description: "rebuild BM25, embedding, and/or tag indexes without rescanning workspace files"
parameters:
type: object
properties:
scope:
type: string
enum: [all, bm25, embedding, tag]
default: all
steps:
- backend: reindex_step
add_draft:
backend: base
description: "Append text to the current draft list."
parameters:
type: object
properties:
text:
type: string
description: "draft text to append"
required:
- text
steps:
- backend: add_draft_step
read_all_draft:
backend: base
description: "Read all draft text previously appended in the current tool context."
parameters:
type: object
properties: { }
steps:
- backend: read_all_draft_step
python_execute:
backend: base
description: "Execute Python code and return printed stdout."
parameters:
type: object
properties:
code:
type: string
description: "Python code to execute. Print the final result to stdout."
timeout:
type: number
description: "Execution timeout in seconds; defaults to 60."
required:
- code
steps:
- backend: python_execute_step
# ── File I/O jobs (needed by auto_memory agent tools) ──
daily_list:
backend: base
description: "List notes under a single day."
parameters:
type: object
properties:
date:
type: string
description: "YYYY-MM-DD; empty = today"
default: ""
steps:
- backend: daily_list_step
daily_reindex:
backend: base
description: "Rebuild the day-index page daily/<date>.md."
parameters:
type: object
properties:
date:
type: string
description: "YYYY-MM-DD; empty = today"
default: ""
steps:
- backend: daily_reindex_step
frontmatter_update:
backend: base
description: "Merge key-values into a file's frontmatter."
parameters:
type: object
properties:
path:
type: string
description: "workspace-relative path"
metadata:
type: object
description: "key-values to merge"
required:
- path
- metadata
steps:
- backend: frontmatter_update_step
move:
backend: base
description: "Move / rename a workspace file; rewrites inbound wikilinks by default."
parameters:
type: object
properties:
src_path:
type: string
description: "workspace-relative source"
dst_path:
type: string
description: "workspace-relative destination"
overwrite:
type: boolean
description: "overwrite if dst exists"
default: false
retarget:
type: boolean
description: "rewrite [[src]] → [[dst]] across the workspace"
default: true
required:
- src_path
- dst_path
steps:
- backend: move_step
read:
backend: base
description: "Read a markdown file under the workspace."
parameters:
type: object
properties:
path:
type: string
description: "workspace-relative path; markdown only"
start_line:
type: integer
description: "first line (1-based, inclusive)"
end_line:
type: integer
description: "last line (1-based, inclusive)"
required:
- path
steps:
- backend: read_step
with_neighbors: false
max_neighbors_per_direction: 10
write:
backend: base
description: "Write a markdown file (create or overwrite) with name/description frontmatter."
parameters:
type: object
properties:
path:
type: string
description: "workspace-relative path; markdown only"
name:
type: string
description: "frontmatter name"
description:
type: string
description: "frontmatter description"
content:
type: string
description: "body"
metadata:
type: object
description: "Optional extra frontmatter fields (md only)."
required:
- path
- name
- description
- content
steps:
- backend: write_step
daily_write:
backend: base
description: "Write a daily markdown note with conversation source frontmatter."
parameters:
type: object
properties:
name:
type: string
description: "daily note filename stem and frontmatter name"
description:
type: string
description: "frontmatter description"
session_id:
type: string
description: "source conversation session identifier"
content:
type: string
description: "body"
date:
type: string
description: "YYYY-MM-DD daily note date; empty = today"
default: ""
metadata:
type: object
description: "Optional extra frontmatter fields."
required:
- name
- description
- session_id
- content
steps:
- backend: daily_write_step
edit:
backend: base
description: "Find-and-replace in a markdown file (all occurrences)."
parameters:
type: object
properties:
path:
type: string
description: "workspace-relative path"
old:
type: string
description: "text to find"
new:
type: string
description: "replacement"
default: ""
required:
- path
- old
- new
steps:
- backend: edit_step
frontmatter_read:
backend: base
description: "Read a file's frontmatter as a dict."
parameters:
type: object
properties:
path:
type: string
description: "workspace-relative path"
required:
- path
steps:
- backend: frontmatter_read_step
node_search:
backend: base
description: "Digest node recall — given a candidate abstraction's name+description, surface existing digest nodes similar enough to either dedup against or link to as related."
parameters:
type: object
properties:
query:
type: string
description: "search query"
limit:
type: integer
description: "max digest nodes to return"
default: 20
required:
- query
steps:
- backend: node_search_step
vector_weight: 0.7
candidate_multiplier: 5.0
components:
tokenizer:
default:
backend: regex
as_embedding:
default:
backend: ${EMBEDDING_BACKEND:-openai}
model: ${EMBEDDING_MODEL_NAME:-text-embedding-v4}
credential:
api_key: ${EMBEDDING_API_KEY:-}
base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1}
dimensions: 1024
embedding_store:
default:
backend: local
as_embedding: default
as_llm:
default:
backend: ${LLM_BACKEND:-openai}
model: ${LLM_MODEL_NAME:-qwen3.6-flash}
stream: true
context_size: 200000
retry_delay: 5.0
credential:
api_key: ${LLM_API_KEY:-}
base_url: ${LLM_BASE_URL:-}
parameters:
max_tokens: 65536
thinking_enable: false
bench:
backend: ${LLM_BACKEND:-openai}
model: ${BENCH_MODEL_NAME:-qwen3.7-max}
stream: true
context_size: 400000
max_retries: 5
retry_delay: 5.0
credential:
api_key: ${LLM_API_KEY:-}
base_url: ${LLM_BASE_URL:-}
parameters:
max_tokens: 65536
thinking_enable: true
compressor:
backend: ${LLM_BACKEND:-openai}
model: ${LLM_MODEL_NAME:-qwen3.6-flash}
stream: false
context_size: 200000
max_retries: 5
retry_delay: 5.0
credential:
api_key: ${LLM_API_KEY:-}
base_url: ${LLM_BASE_URL:-}
parameters:
max_tokens: 65536
thinking_enable: false
agent_wrapper:
default:
backend: agentscope
as_llm: default
permission_mode: bypass
react_config:
max_iters: 30
context_config:
trigger_ratio: 0.8
reserve_ratio: 0.1
tool_result_limit: 50000
model_config:
max_retries: 1
bench:
backend: agentscope
as_llm: bench
permission_mode: bypass
react_config:
max_iters: 30
context_config:
trigger_ratio: 0.8
reserve_ratio: 0.1
tool_result_limit: 50000
model_config:
max_retries: 1
file_graph:
default:
backend: local
file_catalog:
default:
backend: local
resource:
backend: local
digest:
backend: local
dream:
backend: local
file_chunker:
markdown:
backend: markdown
supported_extensions: [ "md" ]
embed_toc: true
max_ast_sections: 100
include_frontmatter_in_metadata: false
include_frontmatter_keys_in_metadata: [] # empty = all non-empty frontmatter keys
json:
backend: json
supported_extensions: [ "json" ]
jsonl:
backend: jsonl
supported_extensions: [ "jsonl" ] # noqa: keep #314 chunker scope intact after #325
max_chars: 4000
default:
backend: default
supported_extensions: ["txt","log"]
keyword_index:
default:
backend: bm25
tokenizer: default
file_store:
default:
backend: local
store_name: local
embedding_store: default
keyword_index: default
file_graph: default