mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-10-13 04:01:35 +00:00
e4c381c7b1
benchmark.yaml's python_execute job advertised a `timeout` parameter, but `timeout` is one of the CLI's client-selection kwargs (reme/reme.py pops it before the payload is sent), so `reme python_execute code=... timeout=300` never delivered the timeout to the job. The value silently became the HTTP client's request timeout while the subprocess kept the 60s default, so benchmark code that needs longer was killed at 60s with no hint why. HTTP and MCP callers were unaffected. Rename the parameter to `python_timeout`, mirroring how the shell job already avoids the same collision with `shell_timeout`. The step prefers the new name and still accepts the legacy `timeout` context key so existing benchmark configs keep working. A new guard test fails if any job in reme/config/*.yaml or a plugin plugin.yaml declares a client-reserved parameter name again.
491 lines
13 KiB
YAML
491 lines
13 KiB
YAML
# Shared benchmark application preset; independent of the default service config.
|
|
# Background and cron jobs are omitted in favor of manually callable base jobs.
|
|
# Benchmark plugins contribute their own jobs through plugin.yaml.
|
|
|
|
service:
|
|
backend: http
|
|
|
|
jobs:
|
|
# ── Manual index update (replaces index_update_loop background) ──
|
|
index_update:
|
|
backend: base
|
|
description: "Manually trigger incremental index update for watched dirs."
|
|
watch_dirs: [daily_dir, digest_dir, session_dir/dialog]
|
|
watch_suffixes: [md, jsonl]
|
|
parameters:
|
|
type: object
|
|
properties: {}
|
|
steps:
|
|
- backend: init_changes_step
|
|
monitor_type: file_store
|
|
monitor_name: default
|
|
dispatch_steps: [update_index_step]
|
|
|
|
# ── Manual digest catalog update (replaces digest_watch_loop background) ──
|
|
digest_update:
|
|
backend: base
|
|
description: "Manually trigger digest catalog update."
|
|
watch_dirs: [daily_dir, digest_dir]
|
|
watch_suffixes: [md]
|
|
parameters:
|
|
type: object
|
|
properties: {}
|
|
steps:
|
|
- backend: init_changes_step
|
|
monitor_type: file_catalog
|
|
monitor_name: digest
|
|
dispatch_steps:
|
|
- backend: update_catalog_step
|
|
file_catalog: digest
|
|
- backend: log_changes_step
|
|
|
|
# ── Auto dream (same as default.yaml auto_dream, base mode) ──
|
|
# auto_dream:
|
|
# backend: base
|
|
# description: "Auto-dream: scan today's day-index and daily notes, globally extract merged units, integrate digest units, and persist the dream catalog."
|
|
# parameters:
|
|
# type: object
|
|
# properties:
|
|
# date:
|
|
# type: string
|
|
# description: "YYYY-MM-DD to scan; defaults to today in the dreamer's timezone"
|
|
# default: ""
|
|
# hint:
|
|
# type: string
|
|
# description: "caller guidance passed through to dream extract/integrate"
|
|
# default: ""
|
|
# scan_days:
|
|
# type: integer
|
|
# description: "number of recent daily directories to scan, ending at date"
|
|
# default: 2
|
|
# max_units:
|
|
# type: integer
|
|
# description: "maximum number of extracted memory units"
|
|
# default: 5
|
|
# steps:
|
|
# - backend: dream_extract_step
|
|
# file_catalog: dream
|
|
# scan_days: 2
|
|
# max_units: 5
|
|
# - backend: dream_integrate_step
|
|
# - backend: dream_finish_step
|
|
# file_catalog: dream
|
|
|
|
# ── Text compression (direct LLM call, no agent) ──
|
|
compressor:
|
|
backend: base
|
|
description: "Compress text via a direct LLM call, optionally guided by queries as relevance filter"
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
text:
|
|
type: string
|
|
description: "the text to compress"
|
|
queries:
|
|
type: array
|
|
description: "optional list of queries; content potentially relevant to any query is kept, content certainly irrelevant to all queries may be dropped"
|
|
items:
|
|
type: string
|
|
default: []
|
|
required:
|
|
- text
|
|
steps:
|
|
- backend: compressor_step
|
|
as_llm: compressor
|
|
|
|
# ── Reindex (derived search indexes only) ──
|
|
reindex:
|
|
backend: base
|
|
description: "rebuild BM25, embedding, and/or tag indexes without rescanning workspace files"
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
scope:
|
|
type: string
|
|
enum: [all, bm25, embedding, tag]
|
|
default: all
|
|
steps:
|
|
- backend: reindex_step
|
|
|
|
add_draft:
|
|
backend: base
|
|
description: "Append text to the current draft list."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
text:
|
|
type: string
|
|
description: "draft text to append"
|
|
required:
|
|
- text
|
|
steps:
|
|
- backend: add_draft_step
|
|
|
|
read_all_draft:
|
|
backend: base
|
|
description: "Read all draft text previously appended in the current tool context."
|
|
parameters:
|
|
type: object
|
|
properties: { }
|
|
steps:
|
|
- backend: read_all_draft_step
|
|
|
|
python_execute:
|
|
backend: base
|
|
description: "Execute Python code and return printed stdout."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
code:
|
|
type: string
|
|
description: "Python code to execute. Print the final result to stdout."
|
|
python_timeout:
|
|
type: number
|
|
description: "Execution timeout in seconds; defaults to 60."
|
|
required:
|
|
- code
|
|
steps:
|
|
- backend: python_execute_step
|
|
|
|
# ── File I/O jobs (needed by auto_memory agent tools) ──
|
|
daily_list:
|
|
backend: base
|
|
description: "List notes under a single day."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
date:
|
|
type: string
|
|
description: "YYYY-MM-DD; empty = today"
|
|
default: ""
|
|
steps:
|
|
- backend: daily_list_step
|
|
|
|
daily_reindex:
|
|
backend: base
|
|
description: "Rebuild the day-index page daily/<date>.md."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
date:
|
|
type: string
|
|
description: "YYYY-MM-DD; empty = today"
|
|
default: ""
|
|
steps:
|
|
- backend: daily_reindex_step
|
|
|
|
frontmatter_update:
|
|
backend: base
|
|
description: "Merge key-values into a file's frontmatter."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
path:
|
|
type: string
|
|
description: "workspace-relative path"
|
|
metadata:
|
|
type: object
|
|
description: "key-values to merge"
|
|
required:
|
|
- path
|
|
- metadata
|
|
steps:
|
|
- backend: frontmatter_update_step
|
|
|
|
move:
|
|
backend: base
|
|
description: "Move / rename a workspace file; rewrites inbound wikilinks by default."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
src_path:
|
|
type: string
|
|
description: "workspace-relative source"
|
|
dst_path:
|
|
type: string
|
|
description: "workspace-relative destination"
|
|
overwrite:
|
|
type: boolean
|
|
description: "overwrite if dst exists"
|
|
default: false
|
|
retarget:
|
|
type: boolean
|
|
description: "rewrite [[src]] → [[dst]] across the workspace"
|
|
default: true
|
|
required:
|
|
- src_path
|
|
- dst_path
|
|
steps:
|
|
- backend: move_step
|
|
|
|
read:
|
|
backend: base
|
|
description: "Read a markdown file under the workspace."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
path:
|
|
type: string
|
|
description: "workspace-relative path; markdown only"
|
|
start_line:
|
|
type: integer
|
|
description: "first line (1-based, inclusive)"
|
|
end_line:
|
|
type: integer
|
|
description: "last line (1-based, inclusive)"
|
|
required:
|
|
- path
|
|
steps:
|
|
- backend: read_step
|
|
with_neighbors: false
|
|
max_neighbors_per_direction: 10
|
|
|
|
write:
|
|
backend: base
|
|
description: "Write a markdown file (create or overwrite) with name/description frontmatter."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
path:
|
|
type: string
|
|
description: "workspace-relative path; markdown only"
|
|
name:
|
|
type: string
|
|
description: "frontmatter name"
|
|
description:
|
|
type: string
|
|
description: "frontmatter description"
|
|
content:
|
|
type: string
|
|
description: "body"
|
|
metadata:
|
|
type: object
|
|
description: "Optional extra frontmatter fields (md only)."
|
|
required:
|
|
- path
|
|
- name
|
|
- description
|
|
- content
|
|
steps:
|
|
- backend: write_step
|
|
|
|
daily_write:
|
|
backend: base
|
|
description: "Write a daily markdown note with conversation source frontmatter."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
name:
|
|
type: string
|
|
description: "daily note filename stem and frontmatter name"
|
|
description:
|
|
type: string
|
|
description: "frontmatter description"
|
|
session_id:
|
|
type: string
|
|
description: "source conversation session identifier"
|
|
content:
|
|
type: string
|
|
description: "body"
|
|
date:
|
|
type: string
|
|
description: "YYYY-MM-DD daily note date; empty = today"
|
|
default: ""
|
|
metadata:
|
|
type: object
|
|
description: "Optional extra frontmatter fields."
|
|
required:
|
|
- name
|
|
- description
|
|
- session_id
|
|
- content
|
|
steps:
|
|
- backend: daily_write_step
|
|
|
|
edit:
|
|
backend: base
|
|
description: "Find-and-replace in a markdown file (all occurrences)."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
path:
|
|
type: string
|
|
description: "workspace-relative path"
|
|
old:
|
|
type: string
|
|
description: "text to find"
|
|
new:
|
|
type: string
|
|
description: "replacement"
|
|
default: ""
|
|
required:
|
|
- path
|
|
- old
|
|
- new
|
|
steps:
|
|
- backend: edit_step
|
|
|
|
frontmatter_read:
|
|
backend: base
|
|
description: "Read a file's frontmatter as a dict."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
path:
|
|
type: string
|
|
description: "workspace-relative path"
|
|
required:
|
|
- path
|
|
steps:
|
|
- backend: frontmatter_read_step
|
|
|
|
node_search:
|
|
backend: base
|
|
description: "Digest node recall — given a candidate abstraction's name+description, surface existing digest nodes similar enough to either dedup against or link to as related."
|
|
parameters:
|
|
type: object
|
|
properties:
|
|
query:
|
|
type: string
|
|
description: "search query"
|
|
limit:
|
|
type: integer
|
|
description: "max digest nodes to return"
|
|
default: 20
|
|
required:
|
|
- query
|
|
steps:
|
|
- backend: node_search_step
|
|
vector_weight: 0.7
|
|
candidate_multiplier: 5.0
|
|
|
|
components:
|
|
tokenizer:
|
|
default:
|
|
backend: regex
|
|
|
|
as_embedding:
|
|
default:
|
|
backend: ${EMBEDDING_BACKEND:-openai}
|
|
model: ${EMBEDDING_MODEL_NAME:-text-embedding-v4}
|
|
credential:
|
|
api_key: ${EMBEDDING_API_KEY:-}
|
|
base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1}
|
|
dimensions: 1024
|
|
|
|
embedding_store:
|
|
default:
|
|
backend: local
|
|
as_embedding: default
|
|
|
|
as_llm:
|
|
default:
|
|
backend: ${LLM_BACKEND:-openai}
|
|
model: ${LLM_MODEL_NAME:-qwen3.6-flash}
|
|
stream: true
|
|
context_size: 200000
|
|
retry_delay: 5.0
|
|
credential:
|
|
api_key: ${LLM_API_KEY:-}
|
|
base_url: ${LLM_BASE_URL:-}
|
|
parameters:
|
|
max_tokens: 65536
|
|
thinking_enable: false
|
|
bench:
|
|
backend: ${LLM_BACKEND:-openai}
|
|
model: ${BENCH_MODEL_NAME:-qwen3.7-max}
|
|
stream: true
|
|
context_size: 400000
|
|
max_retries: 5
|
|
retry_delay: 5.0
|
|
credential:
|
|
api_key: ${LLM_API_KEY:-}
|
|
base_url: ${LLM_BASE_URL:-}
|
|
parameters:
|
|
max_tokens: 65536
|
|
thinking_enable: true
|
|
compressor:
|
|
backend: ${LLM_BACKEND:-openai}
|
|
model: ${LLM_MODEL_NAME:-qwen3.6-flash}
|
|
stream: false
|
|
context_size: 200000
|
|
max_retries: 5
|
|
retry_delay: 5.0
|
|
credential:
|
|
api_key: ${LLM_API_KEY:-}
|
|
base_url: ${LLM_BASE_URL:-}
|
|
parameters:
|
|
max_tokens: 65536
|
|
thinking_enable: false
|
|
|
|
agent_wrapper:
|
|
default:
|
|
backend: agentscope
|
|
as_llm: default
|
|
permission_mode: bypass
|
|
react_config:
|
|
max_iters: 30
|
|
context_config:
|
|
trigger_ratio: 0.8
|
|
reserve_ratio: 0.1
|
|
tool_result_limit: 50000
|
|
model_config:
|
|
max_retries: 1
|
|
bench:
|
|
backend: agentscope
|
|
as_llm: bench
|
|
permission_mode: bypass
|
|
react_config:
|
|
max_iters: 30
|
|
context_config:
|
|
trigger_ratio: 0.8
|
|
reserve_ratio: 0.1
|
|
tool_result_limit: 50000
|
|
model_config:
|
|
max_retries: 1
|
|
|
|
file_graph:
|
|
default:
|
|
backend: local
|
|
|
|
file_catalog:
|
|
default:
|
|
backend: local
|
|
resource:
|
|
backend: local
|
|
digest:
|
|
backend: local
|
|
dream:
|
|
backend: local
|
|
|
|
file_chunker:
|
|
markdown:
|
|
backend: markdown
|
|
supported_extensions: [ "md" ]
|
|
embed_toc: true
|
|
max_ast_sections: 100
|
|
include_frontmatter_in_metadata: false
|
|
include_frontmatter_keys_in_metadata: [] # empty = all non-empty frontmatter keys
|
|
json:
|
|
backend: json
|
|
supported_extensions: [ "json" ]
|
|
jsonl:
|
|
backend: jsonl
|
|
supported_extensions: [ "jsonl" ] # noqa: keep #314 chunker scope intact after #325
|
|
max_chars: 4000
|
|
default:
|
|
backend: default
|
|
supported_extensions: ["txt","log"]
|
|
|
|
keyword_index:
|
|
default:
|
|
backend: bm25
|
|
tokenizer: default
|
|
|
|
file_store:
|
|
default:
|
|
backend: local
|
|
store_name: local
|
|
embedding_store: default
|
|
keyword_index: default
|
|
file_graph: default
|