From 9975bb37b9450d6cf2762ababf57ee98db96d939 Mon Sep 17 00:00:00 2001 From: xyf2020 <75460675+xyf2020@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:16:46 +0800 Subject: [PATCH] Separate benchmark judge plugins (#535) --- benchmark/beam/README.md | 1 + benchmark/beam/README_ZH.md | 1 + benchmark/beam/config.yaml | 3 +- benchmark/beam/run.py | 8 ++-- benchmark/longmemeval/README.md | 1 + benchmark/longmemeval/README_ZH.md | 1 + benchmark/longmemeval/config.yaml | 3 +- benchmark/longmemeval/run.py | 8 ++-- plugins/beam-judge/README.md | 3 ++ plugins/beam-judge/pyproject.toml | 22 +++++++++ .../src/judge_beam/__init__.py | 2 +- .../src/judge_beam/llm_judge.py | 1 + .../src/judge_beam/llm_judge.yaml | 0 plugins/beam-judge/src/judge_beam/plugin.yaml | 47 +++++++++++++++++++ plugins/beam/README.md | 16 ++++--- plugins/beam/README_ZH.md | 13 +++-- plugins/beam/pyproject.toml | 5 +- plugins/beam/src/reme_beam/plugin.yaml | 34 -------------- plugins/lme-judge/README.md | 3 ++ plugins/lme-judge/pyproject.toml | 22 +++++++++ .../src/judge_lme/__init__.py | 2 +- .../src/judge_lme/llm_judge.py | 1 + .../src/judge_lme/llm_judge.yaml | 0 plugins/lme-judge/src/judge_lme/plugin.yaml | 44 +++++++++++++++++ plugins/lme/README.md | 16 ++++--- plugins/lme/README_ZH.md | 13 +++-- plugins/lme/pyproject.toml | 3 +- plugins/lme/src/reme_lme/plugin.yaml | 30 ------------ reme/config/benchmark.yaml | 24 ---------- tests/unit/test_plugin.py | 37 +++++++++++++++ 30 files changed, 236 insertions(+), 128 deletions(-) create mode 100644 plugins/beam-judge/README.md create mode 100644 plugins/beam-judge/pyproject.toml rename plugins/{beam => beam-judge}/src/judge_beam/__init__.py (68%) rename plugins/{beam => beam-judge}/src/judge_beam/llm_judge.py (99%) rename plugins/{beam => beam-judge}/src/judge_beam/llm_judge.yaml (100%) create mode 100644 plugins/beam-judge/src/judge_beam/plugin.yaml create mode 100644 plugins/lme-judge/README.md create mode 100644 plugins/lme-judge/pyproject.toml rename plugins/{lme => lme-judge}/src/judge_lme/__init__.py (64%) rename plugins/{lme => lme-judge}/src/judge_lme/llm_judge.py (97%) rename plugins/{lme => lme-judge}/src/judge_lme/llm_judge.yaml (100%) create mode 100644 plugins/lme-judge/src/judge_lme/plugin.yaml diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 014427c2..287616b0 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -20,6 +20,7 @@ Install ReMe and the BEAM plugin in editable mode from the repository root: ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/beam --editable +reme plugins install ./plugins/beam-judge --editable reme plugins validate beam ``` diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index 1c063497..781d30ba 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -18,6 +18,7 @@ knowledge update（知识更新）、multi-session reasoning（多会话推理 ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/beam --editable +reme plugins install ./plugins/beam-judge --editable reme plugins validate beam ``` diff --git a/benchmark/beam/config.yaml b/benchmark/beam/config.yaml index cca633b8..3f289747 100644 --- a/benchmark/beam/config.yaml +++ b/benchmark/beam/config.yaml @@ -14,7 +14,8 @@ evaluation: compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression reme: - config: "benchmark" # shared ReMe benchmark preset; runner enables the installed beam plugin + config: "benchmark" # shared ReMe benchmark preset + plugins: [beam, beam-judge] output: dir: "benchmark/beam/results" diff --git a/benchmark/beam/run.py b/benchmark/beam/run.py index b49284fd..2bb8e6c6 100644 --- a/benchmark/beam/run.py +++ b/benchmark/beam/run.py @@ -152,7 +152,7 @@ def load_eval_config(config_path: str | None = None) -> dict: def create_reme_app(config: str = "benchmark", **overrides): - """Create an app with the installed BEAM plugin explicitly enabled. + """Create an app with the BEAM candidate and judge plugins enabled. Plugin discovery remains environment-based; editable installation keeps local plugin source changes visible to every multiprocessing worker. @@ -161,8 +161,9 @@ def create_reme_app(config: str = "benchmark", **overrides): from reme.config import resolve_app_config enabled_plugins = list(overrides.pop("plugins", ()) or ()) - if "beam" not in enabled_plugins: - enabled_plugins.append("beam") + for plugin in ("beam", "beam-judge"): + if plugin not in enabled_plugins: + enabled_plugins.append(plugin) app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides) return Application(**app_config) @@ -391,6 +392,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False app = create_reme_app( config=eval_config["reme"]["config"], + plugins=eval_config["reme"].get("plugins", ()), workspace_dir=workspace_dir, log_to_console=output_cfg.get("log_to_console", True), log_to_file=output_cfg.get("log_to_file", False), diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 7f07d170..09185db7 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -17,6 +17,7 @@ Install ReMe and the LongMemEval plugin in editable mode from the repository roo ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/lme --editable +reme plugins install ./plugins/lme-judge --editable reme plugins validate lme ``` diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index 4c66df67..cd17ee95 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -13,6 +13,7 @@ LongMemEval 是一个面向**多轮多会话历史的长期记忆能力**的评 ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/lme --editable +reme plugins install ./plugins/lme-judge --editable reme plugins validate lme ``` diff --git a/benchmark/longmemeval/config.yaml b/benchmark/longmemeval/config.yaml index 36260479..84b1ceb5 100644 --- a/benchmark/longmemeval/config.yaml +++ b/benchmark/longmemeval/config.yaml @@ -18,7 +18,8 @@ evaluation: compress_session: false # true = compress session chunks in search_v2 (query-aware); false = no compression reme: - config: "benchmark" # shared ReMe benchmark preset; runner enables the installed lme plugin + config: "benchmark" # runner enables both plugins below + plugins: [lme, lme-judge] # Dream trigger: when gap between consecutive sessions crosses this hour (23:00) dream_trigger_hour: 23 # Dream scan_days for each trigger diff --git a/benchmark/longmemeval/run.py b/benchmark/longmemeval/run.py index 97b10ae4..581c1673 100644 --- a/benchmark/longmemeval/run.py +++ b/benchmark/longmemeval/run.py @@ -151,7 +151,7 @@ def load_eval_config(config_path: str | None = None) -> dict: def create_reme_app(config: str = "benchmark", **overrides): - """Create an app with the installed LongMemEval plugin explicitly enabled. + """Create an app with the LongMemEval candidate and judge plugins enabled. Plugin discovery remains environment-based; editable installation keeps local plugin source changes visible to every multiprocessing worker. @@ -160,8 +160,9 @@ def create_reme_app(config: str = "benchmark", **overrides): from reme.config import resolve_app_config enabled_plugins = list(overrides.pop("plugins", ()) or ()) - if "lme" not in enabled_plugins: - enabled_plugins.append("lme") + for plugin in ("lme", "lme-judge"): + if plugin not in enabled_plugins: + enabled_plugins.append(plugin) app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides) return Application(**app_config) @@ -341,6 +342,7 @@ async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_onl app = create_reme_app( config=reme_cfg["config"], + plugins=reme_cfg.get("plugins", ()), workspace_dir=workspace_dir, log_to_console=output_cfg.get("log_to_console", True), log_to_file=output_cfg.get("log_to_file", False), diff --git a/plugins/beam-judge/README.md b/plugins/beam-judge/README.md new file mode 100644 index 00000000..b953f210 --- /dev/null +++ b/plugins/beam-judge/README.md @@ -0,0 +1,3 @@ +# ReMe BEAM Judge + +Trusted BEAM scoring plugin. Install it alongside `plugins/beam` for a full benchmark run. Meta-ReMe candidate bundles must not include this plugin. diff --git a/plugins/beam-judge/pyproject.toml b/plugins/beam-judge/pyproject.toml new file mode 100644 index 00000000..0dff94b2 --- /dev/null +++ b/plugins/beam-judge/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "reme-beam-judge" +version = "0.1.0" +description = "Trusted BEAM judge plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11", "json-repair", "numpy>=2.2.6"] + +[project.entry-points."reme.plugins"] +beam-judge = "judge_beam" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["judge_beam*"] + +[tool.setuptools.package-data] +judge_beam = ["plugin.yaml", "*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/beam/src/judge_beam/__init__.py b/plugins/beam-judge/src/judge_beam/__init__.py similarity index 68% rename from plugins/beam/src/judge_beam/__init__.py rename to plugins/beam-judge/src/judge_beam/__init__.py index 7090e1ce..cf89af27 100644 --- a/plugins/beam/src/judge_beam/__init__.py +++ b/plugins/beam-judge/src/judge_beam/__init__.py @@ -1,4 +1,4 @@ -"""BEAM benchmark judge backend.""" +"""BEAM benchmark judge plugin.""" from .llm_judge import BeamRubricJudgeStep diff --git a/plugins/beam/src/judge_beam/llm_judge.py b/plugins/beam-judge/src/judge_beam/llm_judge.py similarity index 99% rename from plugins/beam/src/judge_beam/llm_judge.py rename to plugins/beam-judge/src/judge_beam/llm_judge.py index fdadabbb..40e420eb 100644 --- a/plugins/beam/src/judge_beam/llm_judge.py +++ b/plugins/beam-judge/src/judge_beam/llm_judge.py @@ -253,6 +253,7 @@ class BeamRubricJudgeStep(BaseStep): ) async def execute(self): + """Evaluate a BEAM response against its rubric.""" assert self.context is not None llm_response: str = self.context.get("llm_response", "") rubric: list[str] = self.context.get("rubric", []) diff --git a/plugins/beam/src/judge_beam/llm_judge.yaml b/plugins/beam-judge/src/judge_beam/llm_judge.yaml similarity index 100% rename from plugins/beam/src/judge_beam/llm_judge.yaml rename to plugins/beam-judge/src/judge_beam/llm_judge.yaml diff --git a/plugins/beam-judge/src/judge_beam/plugin.yaml b/plugins/beam-judge/src/judge_beam/plugin.yaml new file mode 100644 index 00000000..43f1d9e0 --- /dev/null +++ b/plugins/beam-judge/src/judge_beam/plugin.yaml @@ -0,0 +1,47 @@ +backends: + beam_rubric_judge_step: judge_beam.llm_judge:BeamRubricJudgeStep + +application_defaults: + jobs: + answer_judge: + backend: base + description: "BEAM rubric-based LLM-as-Judge" + parameters: + type: object + properties: + llm_response: {type: string} + rubric: + type: array + items: {type: string} + probing_question: {type: string, default: ""} + question_type: {type: string, default: ""} + required: [llm_response, rubric] + steps: + - backend: beam_rubric_judge_step + agent_wrapper: judge + components: + as_llm: + judge: + backend: ${LLM_BACKEND:-openai} + model: ${JUDGE_MODEL_NAME:-qwen3.7-max} + stream: false + context_size: 200000 + max_retries: 5 + retry_delay: 5.0 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false + agent_wrapper: + judge: + backend: agentscope + as_llm: judge + permission_mode: bypass + react_config: {max_iters: 1} + context_config: + trigger_ratio: 0.8 + reserve_ratio: 0.1 + tool_result_limit: 50000 + model_config: {max_retries: 1} diff --git a/plugins/beam/README.md b/plugins/beam/README.md index fb2f9559..341cd79a 100644 --- a/plugins/beam/README.md +++ b/plugins/beam/README.md @@ -2,8 +2,8 @@ [中文说明](./README_ZH.md) -This plugin owns the BEAM memory, agentic-answer and judge Steps, their prompts, -and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the +This plugin owns the BEAM memory and agentic-answer Steps and their Job defaults. +The trusted judge Step, prompts and `answer_judge` Job live in `plugins/beam-judge`. ReMe's built-in `benchmark.yaml` owns the shared evaluation Jobs and components. Dataset handling, the runner and results remain in [`benchmark/beam`](../../benchmark/beam/README.md). @@ -12,17 +12,19 @@ From the repository root, install ReMe and this plugin in editable mode before r ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/beam --editable +reme plugins install ./plugins/beam-judge --editable reme plugins validate beam +reme plugins validate beam-judge python benchmark/beam/run.py ``` Editable installation registers the `beam` entry point while keeping source changes immediately -visible. The runner selects the built-in `benchmark` preset and explicitly enables `beam` for +visible. The runner selects the built-in `benchmark` preset and explicitly enables `beam` and `beam-judge` for each Application. Installing the plugin makes it discoverable but does not enable it globally. `plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`, -`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with -`reme start config=benchmark plugins='["beam"]'`. The shared preset does not inherit +`agentic_answer` Job defaults. Start a full benchmark application with +`reme start config=benchmark plugins='["beam", "beam-judge"]'`. The shared preset does not inherit `default`: only declared Jobs run, indexing is manual, and neither scheduled dream nor the optional `auto_dream` Job is enabled. The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge` @@ -31,8 +33,8 @@ still take precedence. Installing this plugin does not start an evaluation. The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`. The old core-owned `reme.steps.benchmark.beam` Python import path is removed. -Custom Python callers should import memory, search and answer Steps from `reme_beam`, and the -judge Step from `judge_beam`. After uninstalling, +Custom Python callers should import memory, search and answer Steps from `reme_beam`, and install +`beam-judge` before importing the judge Step from `judge_beam`. After uninstalling, Applications and CLI services must omit the plugin until it is installed again. Uninstallation never removes datasets, workspaces or results. Restart an existing service after changing plugins. diff --git a/plugins/beam/README_ZH.md b/plugins/beam/README_ZH.md index 4d695715..86558936 100644 --- a/plugins/beam/README_ZH.md +++ b/plugins/beam/README_ZH.md @@ -2,7 +2,8 @@ [English](./README.md) -插件包含 BEAM 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。 +插件包含 BEAM 的记忆、回答 Step 及对应 Job 默认配置。可信评分 Step、提示词和 +`answer_judge` Job 位于独立的 `plugins/beam-judge` 插件中。 ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在 [`benchmark/beam`](../../benchmark/beam/README_ZH.md)。 @@ -11,22 +12,24 @@ ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集 ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/beam --editable +reme plugins install ./plugins/beam-judge --editable reme plugins validate beam +reme plugins validate beam-judge python benchmark/beam/run.py ``` editable 安装会注册 `beam` entry point，并让源码修改立即生效。runner 选择内置 `benchmark` -配置，并为每个 Application 显式启用 `beam`。安装只让插件可被发现，不会在所有应用中全局启用。 +配置，并为每个 Application 显式启用 `beam` 和 `beam-judge`。安装只让插件可被发现，不会在所有应用中全局启用。 `plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、 -`agentic_answer` 和 `answer_judge` Job。安装后使用 -`reme start config=benchmark plugins='["beam"]'`。公共评测配置不继承 `default`，只运行声明的 Job： +`agentic_answer` Job。完整评测使用 +`reme start config=benchmark plugins='["beam", "beam-judge"]'`。公共评测配置不继承 `default`，只运行声明的 Job： 索引手动更新，dream 定时任务和可选的 `auto_dream` 均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量 保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。 共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。 原 `reme.steps.benchmark.beam` Python 导入路径已移除。自定义 Python 调用应从 `reme_beam` -导入记忆、搜索和回答 Step，并从 `judge_beam` 导入评判 Step。 +导入记忆、搜索和回答 Step；安装 `beam-judge` 后再从 `judge_beam` 导入评判 Step。 卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。 卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。 diff --git a/plugins/beam/pyproject.toml b/plugins/beam/pyproject.toml index 2b35f5aa..82fec7f9 100644 --- a/plugins/beam/pyproject.toml +++ b/plugins/beam/pyproject.toml @@ -8,8 +8,6 @@ license-files = ["LICENSE"] requires-python = ">=3.11" dependencies = [ "reme-ai[as]>=0.4.1.11", - "json-repair", - "numpy>=2.2.6", ] [project.entry-points."reme.plugins"] @@ -17,11 +15,10 @@ beam = "reme_beam" [tool.setuptools.packages.find] where = ["src"] -include = ["reme_beam*", "judge_beam*"] +include = ["reme_beam*"] [tool.setuptools.package-data] reme_beam = ["plugin.yaml", "*.yaml"] -judge_beam = ["*.yaml"] [build-system] requires = ["setuptools>=77", "wheel"] diff --git a/plugins/beam/src/reme_beam/plugin.yaml b/plugins/beam/src/reme_beam/plugin.yaml index 389653fc..cd2a1402 100644 --- a/plugins/beam/src/reme_beam/plugin.yaml +++ b/plugins/beam/src/reme_beam/plugin.yaml @@ -1,7 +1,6 @@ backends: beam_auto_memory_step: reme_beam.auto_memory:BeamAutoMemoryStep beam_agentic_answer_step: reme_beam.agentic_answer:BeamAgenticAnswerStep - beam_rubric_judge_step: judge_beam.llm_judge:BeamRubricJudgeStep beam_search_v2_step: reme_beam.search_v2:SearchV2Step application_defaults: @@ -51,37 +50,6 @@ application_defaults: - backend: beam_agentic_answer_step agent_wrapper: bench - answer_judge: - backend: base - description: "BEAM rubric-based LLM-as-Judge: evaluate response against rubric criteria" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - llm_response: - type: string - description: "The model's response to evaluate" - rubric: - type: array - description: "List of rubric criteria to check" - items: - type: string - probing_question: - type: string - description: "The original probing question" - default: "" - question_type: - type: string - description: "BEAM question type (e.g. event_ordering)" - default: "" - required: - - llm_response - - rubric - steps: - - backend: beam_rubric_judge_step - agent_wrapper: judge - auto_memory: backend: base description: "Auto-memory: record conversation facts into a daily note" @@ -116,5 +84,3 @@ application_defaults: as_llm: default: max_retries: 5 - judge: - retry_delay: 5.0 diff --git a/plugins/lme-judge/README.md b/plugins/lme-judge/README.md new file mode 100644 index 00000000..51503a7e --- /dev/null +++ b/plugins/lme-judge/README.md @@ -0,0 +1,3 @@ +# ReMe LongMemEval Judge + +Trusted LongMemEval scoring plugin. Install it alongside `plugins/lme` for a full benchmark run. Meta-ReMe candidate bundles must not include this plugin. diff --git a/plugins/lme-judge/pyproject.toml b/plugins/lme-judge/pyproject.toml new file mode 100644 index 00000000..33d73408 --- /dev/null +++ b/plugins/lme-judge/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "reme-lme-judge" +version = "0.1.0" +description = "Trusted LongMemEval judge plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11"] + +[project.entry-points."reme.plugins"] +lme-judge = "judge_lme" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["judge_lme*"] + +[tool.setuptools.package-data] +judge_lme = ["plugin.yaml", "*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/lme/src/judge_lme/__init__.py b/plugins/lme-judge/src/judge_lme/__init__.py similarity index 64% rename from plugins/lme/src/judge_lme/__init__.py rename to plugins/lme-judge/src/judge_lme/__init__.py index 127e5621..56eb459c 100644 --- a/plugins/lme/src/judge_lme/__init__.py +++ b/plugins/lme-judge/src/judge_lme/__init__.py @@ -1,4 +1,4 @@ -"""LongMemEval benchmark judge backend.""" +"""LongMemEval benchmark judge plugin.""" from .llm_judge import LmeAnswerJudgeStep diff --git a/plugins/lme/src/judge_lme/llm_judge.py b/plugins/lme-judge/src/judge_lme/llm_judge.py similarity index 97% rename from plugins/lme/src/judge_lme/llm_judge.py rename to plugins/lme-judge/src/judge_lme/llm_judge.py index 10dbb06e..00fcd9c3 100644 --- a/plugins/lme/src/judge_lme/llm_judge.py +++ b/plugins/lme-judge/src/judge_lme/llm_judge.py @@ -27,6 +27,7 @@ class LmeAnswerJudgeStep(BaseStep): return raw_answer.strip().lower() async def execute(self): + """Evaluate an agent answer against the LongMemEval reference answer.""" assert self.context is not None query: str = self.context.get("query", "") agent_answer: str = self.context.get("agent_answer", "") diff --git a/plugins/lme/src/judge_lme/llm_judge.yaml b/plugins/lme-judge/src/judge_lme/llm_judge.yaml similarity index 100% rename from plugins/lme/src/judge_lme/llm_judge.yaml rename to plugins/lme-judge/src/judge_lme/llm_judge.yaml diff --git a/plugins/lme-judge/src/judge_lme/plugin.yaml b/plugins/lme-judge/src/judge_lme/plugin.yaml new file mode 100644 index 00000000..58fcb464 --- /dev/null +++ b/plugins/lme-judge/src/judge_lme/plugin.yaml @@ -0,0 +1,44 @@ +backends: + lme_answer_judge_step: judge_lme.llm_judge:LmeAnswerJudgeStep + +application_defaults: + jobs: + answer_judge: + backend: base + description: "LLM-as-Judge: evaluate agent answer against golden answer" + parameters: + type: object + properties: + query: {type: string} + agent_answer: {type: string} + golden_answer: {type: string} + question_type: {type: string, default: ""} + required: [query, agent_answer, golden_answer] + steps: + - backend: lme_answer_judge_step + agent_wrapper: judge + components: + as_llm: + judge: + backend: ${LLM_BACKEND:-openai} + model: ${JUDGE_MODEL_NAME:-qwen3.7-max} + stream: false + context_size: 200000 + max_retries: 5 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false + agent_wrapper: + judge: + backend: agentscope + as_llm: judge + permission_mode: bypass + react_config: {max_iters: 1} + context_config: + trigger_ratio: 0.8 + reserve_ratio: 0.1 + tool_result_limit: 50000 + model_config: {max_retries: 1} diff --git a/plugins/lme/README.md b/plugins/lme/README.md index 284ac7a4..a2310caf 100644 --- a/plugins/lme/README.md +++ b/plugins/lme/README.md @@ -2,8 +2,8 @@ [中文说明](./README_ZH.md) -This plugin owns the LongMemEval memory, agentic-answer and judge Steps, their prompts, -and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the +This plugin owns the LongMemEval memory and agentic-answer Steps and their Job defaults. +The trusted judge Step, prompts and `answer_judge` Job live in `plugins/lme-judge`. ReMe's built-in `benchmark.yaml` owns the shared evaluation Jobs and components. Dataset handling, the runner and results remain in [`benchmark/longmemeval`](../../benchmark/longmemeval/README.md). @@ -12,17 +12,19 @@ From the repository root, install ReMe and this plugin in editable mode before r ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/lme --editable +reme plugins install ./plugins/lme-judge --editable reme plugins validate lme +reme plugins validate lme-judge python benchmark/longmemeval/run.py ``` Editable installation registers the `lme` entry point while keeping source changes immediately -visible. The runner selects the built-in `benchmark` preset and explicitly enables `lme` for +visible. The runner selects the built-in `benchmark` preset and explicitly enables `lme` and `lme-judge` for each Application. Installing the plugin makes it discoverable but does not enable it globally. `plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`, -`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with -`reme start config=benchmark plugins='["lme"]'`. The shared preset does not inherit +`agentic_answer` Job defaults. Start a full benchmark application with +`reme start config=benchmark plugins='["lme", "lme-judge"]'`. The shared preset does not inherit `default`: only declared Jobs run, indexing is manual, and neither scheduled dream nor the optional `auto_dream` Job is enabled. The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge` @@ -31,8 +33,8 @@ still take precedence. Installing this plugin does not start an evaluation. The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`. The old core-owned `reme.steps.benchmark.lme` Python import path is removed. -Custom Python callers should import memory, search and answer Steps from `reme_lme`, and the -judge Step from `judge_lme`. After uninstalling, +Custom Python callers should import memory, search and answer Steps from `reme_lme`, and install +`lme-judge` before importing the judge Step from `judge_lme`. After uninstalling, Applications and CLI services must omit the plugin until it is installed again. Uninstallation never removes datasets, workspaces or results. Restart an existing service after changing plugins. diff --git a/plugins/lme/README_ZH.md b/plugins/lme/README_ZH.md index 176b2e80..9d0bbbb0 100644 --- a/plugins/lme/README_ZH.md +++ b/plugins/lme/README_ZH.md @@ -2,7 +2,8 @@ [English](./README.md) -插件包含 LongMemEval 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。 +插件包含 LongMemEval 的记忆、回答 Step 及对应 Job 默认配置。可信评分 Step、提示词和 +`answer_judge` Job 位于独立的 `plugins/lme-judge` 插件中。 ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在 [`benchmark/longmemeval`](../../benchmark/longmemeval/README_ZH.md)。 @@ -11,22 +12,24 @@ ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集 ```bash python -m pip install -e ".[as]" reme plugins install ./plugins/lme --editable +reme plugins install ./plugins/lme-judge --editable reme plugins validate lme +reme plugins validate lme-judge python benchmark/longmemeval/run.py ``` editable 安装会注册 `lme` entry point，并让源码修改立即生效。runner 选择内置 `benchmark` -配置，并为每个 Application 显式启用 `lme`。安装只让插件可被发现，不会在所有应用中全局启用。 +配置，并为每个 Application 显式启用 `lme` 和 `lme-judge`。安装只让插件可被发现，不会在所有应用中全局启用。 `plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、 -`agentic_answer` 和 `answer_judge` Job。安装后使用 -`reme start config=benchmark plugins='["lme"]'`。公共评测配置不继承 `default`，只运行声明的 Job： +`agentic_answer` Job。完整评测使用 +`reme start config=benchmark plugins='["lme", "lme-judge"]'`。公共评测配置不继承 `default`，只运行声明的 Job： 索引手动更新，dream 定时任务和可选的 `auto_dream` 均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量 保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。 共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。 原 `reme.steps.benchmark.lme` Python 导入路径已移除。自定义 Python 调用应从 `reme_lme` -导入记忆、搜索和回答 Step，并从 `judge_lme` 导入评判 Step。 +导入记忆、搜索和回答 Step；安装 `lme-judge` 后再从 `judge_lme` 导入评判 Step。 卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。 卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。 diff --git a/plugins/lme/pyproject.toml b/plugins/lme/pyproject.toml index fa91dd74..1171f4a8 100644 --- a/plugins/lme/pyproject.toml +++ b/plugins/lme/pyproject.toml @@ -15,11 +15,10 @@ lme = "reme_lme" [tool.setuptools.packages.find] where = ["src"] -include = ["reme_lme*", "judge_lme*"] +include = ["reme_lme*"] [tool.setuptools.package-data] reme_lme = ["plugin.yaml", "*.yaml"] -judge_lme = ["*.yaml"] [build-system] requires = ["setuptools>=77", "wheel"] diff --git a/plugins/lme/src/reme_lme/plugin.yaml b/plugins/lme/src/reme_lme/plugin.yaml index e8f9513c..a0621101 100644 --- a/plugins/lme/src/reme_lme/plugin.yaml +++ b/plugins/lme/src/reme_lme/plugin.yaml @@ -1,7 +1,6 @@ backends: lme_auto_memory_step: reme_lme.auto_memory:LmeAutoMemoryStep lme_agentic_answer_step: reme_lme.agentic_answer:LmeAgenticAnswerStep - lme_answer_judge_step: judge_lme.llm_judge:LmeAnswerJudgeStep lme_search_v2_step: reme_lme.search_v2:SearchV2Step application_defaults: @@ -51,35 +50,6 @@ application_defaults: - backend: lme_agentic_answer_step agent_wrapper: bench - answer_judge: - backend: base - description: "LLM-as-Judge: evaluate agent answer against golden answer" - watch_dirs: [] - watch_suffixes: [] - parameters: - type: object - properties: - query: - type: string - description: "The question being asked" - agent_answer: - type: string - description: "The model's answer to evaluate" - golden_answer: - type: string - description: "The correct/golden answer" - question_type: - type: string - description: "The question type for prompt selection" - default: "" - required: - - query - - agent_answer - - golden_answer - steps: - - backend: lme_answer_judge_step - agent_wrapper: judge - auto_memory: backend: base description: "Auto-memory: record conversation facts into a daily note" diff --git a/reme/config/benchmark.yaml b/reme/config/benchmark.yaml index d9d51c3e..99cb05ef 100644 --- a/reme/config/benchmark.yaml +++ b/reme/config/benchmark.yaml @@ -391,18 +391,6 @@ components: parameters: max_tokens: 65536 thinking_enable: false - judge: - backend: ${LLM_BACKEND:-openai} - model: ${JUDGE_MODEL_NAME:-qwen3.7-max} - stream: false - context_size: 200000 - max_retries: 5 - credential: - api_key: ${LLM_API_KEY:-} - base_url: ${LLM_BASE_URL:-} - parameters: - max_tokens: 65536 - thinking_enable: false bench: backend: ${LLM_BACKEND:-openai} model: ${BENCH_MODEL_NAME:-qwen3.7-max} @@ -443,18 +431,6 @@ components: tool_result_limit: 50000 model_config: max_retries: 1 - judge: - backend: agentscope - as_llm: judge - permission_mode: bypass - react_config: - max_iters: 1 - context_config: - trigger_ratio: 0.8 - reserve_ratio: 0.1 - tool_result_limit: 50000 - model_config: - max_retries: 1 bench: backend: agentscope as_llm: bench diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 595b32e0..2df7c1dc 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -13,6 +13,7 @@ from reme.config.config_parser import _load_config from reme.enumeration import ComponentEnum from reme.plugin import Backend, Plugin, PluginManager, _load_backend from reme.plugin_manifest import parse_plugin_manifest +from reme.schema import ApplicationConfig class _PluginStep(ComponentMixin): @@ -191,6 +192,42 @@ def test_plugin_manager_loads_package_manifest(monkeypatch, tmp_path): assert manager.merge_config({})["jobs"]["example"]["backend"] == "base" +@pytest.mark.parametrize( + ("candidate", "judge", "candidate_package", "judge_package", "judge_backend"), + [ + ("lme", "lme-judge", "reme_lme", "judge_lme", "lme_answer_judge_step"), + ("beam", "beam-judge", "reme_beam", "judge_beam", "beam_rubric_judge_step"), + ], +) +def test_benchmark_candidate_and_judge_plugins_compose( + monkeypatch, + candidate, + judge, + candidate_package, + judge_package, + judge_backend, +): + root = Path(__file__).resolve().parents[2] + monkeypatch.syspath_prepend(str(root / "plugins" / candidate / "src")) + monkeypatch.syspath_prepend(str(root / "plugins" / judge / "src")) + _set_entry_points( + monkeypatch, + _FakeEntryPoint(candidate, candidate_package, lambda: None, "reme.plugins"), + _FakeEntryPoint(judge, judge_package, lambda: None, "reme.plugins"), + ) + + manager = PluginManager.discover([candidate, judge]) + registry = ComponentRegistry() + manager.register(registry) + merged = manager.merge_config(_load_config("benchmark")) + + assert {"auto_memory", "search", "agentic_answer", "answer_judge"} <= merged["jobs"].keys() + assert registry.get(ComponentEnum.STEP, judge_backend) is not None + assert "judge" in merged["components"]["as_llm"] + assert "judge" in merged["components"]["agent_wrapper"] + ApplicationConfig.model_validate(merged) + + def test_plugin_manifest_rejects_legacy_defaults_field(): with pytest.raises(ValueError, match="unknown keys: defaults"): parse_plugin_manifest("defaults: {}\n", plugin_name="example")