Separate benchmark judge plugins (#535)

benchmark/beam/README.md

@@ -20,6 +20,7 @@ Install ReMe and the BEAM plugin in editable mode from the repository root:
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/beam --editable
+reme plugins install ./plugins/beam-judge --editable
 reme plugins validate beam
 ```
 

benchmark/beam/README_ZH.md

@@ -18,6 +18,7 @@ knowledge update（知识更新）、multi-session reasoning（多会话推理
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/beam --editable
+reme plugins install ./plugins/beam-judge --editable
 reme plugins validate beam
 ```
 

benchmark/beam/config.yaml

@@ -14,7 +14,8 @@ evaluation:
   compress_session: false               # true = compress session chunks in search_v2 (query-aware); false = no compression
 
 reme:
-  config: "benchmark"                   # shared ReMe benchmark preset; runner enables the installed beam plugin
+  config: "benchmark"                   # shared ReMe benchmark preset
+  plugins: [beam, beam-judge]
 
 output:
   dir: "benchmark/beam/results"

benchmark/beam/run.py

@@ -152,7 +152,7 @@ def load_eval_config(config_path: str | None = None) -> dict:
 
 
 def create_reme_app(config: str = "benchmark", **overrides):
-    """Create an app with the installed BEAM plugin explicitly enabled.
+    """Create an app with the BEAM candidate and judge plugins enabled.
 
     Plugin discovery remains environment-based; editable installation keeps local
     plugin source changes visible to every multiprocessing worker.
@@ -161,8 +161,9 @@ def create_reme_app(config: str = "benchmark", **overrides):
     from reme.config import resolve_app_config
 
     enabled_plugins = list(overrides.pop("plugins", ()) or ())
-    if "beam" not in enabled_plugins:
-        enabled_plugins.append("beam")
+    for plugin in ("beam", "beam-judge"):
+        if plugin not in enabled_plugins:
+            enabled_plugins.append(plugin)
     app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides)
     return Application(**app_config)
 
@@ -391,6 +392,7 @@ async def evaluate_case(eval_config: dict, case_id: str, eval_only: bool = False
 
     app = create_reme_app(
         config=eval_config["reme"]["config"],
+        plugins=eval_config["reme"].get("plugins", ()),
         workspace_dir=workspace_dir,
         log_to_console=output_cfg.get("log_to_console", True),
         log_to_file=output_cfg.get("log_to_file", False),

benchmark/longmemeval/README.md

@@ -17,6 +17,7 @@ Install ReMe and the LongMemEval plugin in editable mode from the repository roo
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/lme --editable
+reme plugins install ./plugins/lme-judge --editable
 reme plugins validate lme
 ```
 

benchmark/longmemeval/README_ZH.md

@@ -13,6 +13,7 @@ LongMemEval 是一个面向**多轮多会话历史的长期记忆能力**的评
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/lme --editable
+reme plugins install ./plugins/lme-judge --editable
 reme plugins validate lme
 ```
 

benchmark/longmemeval/config.yaml

@@ -18,7 +18,8 @@ evaluation:
   compress_session: false   # true = compress session chunks in search_v2 (query-aware); false = no compression
 
 reme:
-  config: "benchmark" # shared ReMe benchmark preset; runner enables the installed lme plugin
+  config: "benchmark" # runner enables both plugins below
+  plugins: [lme, lme-judge]
   # Dream trigger: when gap between consecutive sessions crosses this hour (23:00)
   dream_trigger_hour: 23
   # Dream scan_days for each trigger

benchmark/longmemeval/run.py

@@ -151,7 +151,7 @@ def load_eval_config(config_path: str | None = None) -> dict:
 
 
 def create_reme_app(config: str = "benchmark", **overrides):
-    """Create an app with the installed LongMemEval plugin explicitly enabled.
+    """Create an app with the LongMemEval candidate and judge plugins enabled.
 
     Plugin discovery remains environment-based; editable installation keeps local
     plugin source changes visible to every multiprocessing worker.
@@ -160,8 +160,9 @@ def create_reme_app(config: str = "benchmark", **overrides):
     from reme.config import resolve_app_config
 
     enabled_plugins = list(overrides.pop("plugins", ()) or ())
-    if "lme" not in enabled_plugins:
-        enabled_plugins.append("lme")
+    for plugin in ("lme", "lme-judge"):
+        if plugin not in enabled_plugins:
+            enabled_plugins.append(plugin)
     app_config = resolve_app_config(config=config, plugins=enabled_plugins, **overrides)
     return Application(**app_config)
 
@@ -341,6 +342,7 @@ async def evaluate_item(item: dict, eval_config: dict, item_index: int, eval_onl
 
     app = create_reme_app(
         config=reme_cfg["config"],
+        plugins=reme_cfg.get("plugins", ()),
         workspace_dir=workspace_dir,
         log_to_console=output_cfg.get("log_to_console", True),
         log_to_file=output_cfg.get("log_to_file", False),

plugins/beam-judge/README.md

@@ -0,0 +1,3 @@
+# ReMe BEAM Judge
+
+Trusted BEAM scoring plugin. Install it alongside `plugins/beam` for a full benchmark run. Meta-ReMe candidate bundles must not include this plugin.

plugins/beam-judge/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "reme-beam-judge"
+version = "0.1.0"
+description = "Trusted BEAM judge plugin for ReMe."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11"
+dependencies = ["reme-ai[as]>=0.4.1.11", "json-repair", "numpy>=2.2.6"]
+
+[project.entry-points."reme.plugins"]
+beam-judge = "judge_beam"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["judge_beam*"]
+
+[tool.setuptools.package-data]
+judge_beam = ["plugin.yaml", "*.yaml"]
+
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"

plugins/beam-judge/src/judge_beam/__init__.py

@@ -1,4 +1,4 @@
-"""BEAM benchmark judge backend."""
+"""BEAM benchmark judge plugin."""
 
 from .llm_judge import BeamRubricJudgeStep
 

plugins/beam-judge/src/judge_beam/llm_judge.py

@@ -253,6 +253,7 @@ class BeamRubricJudgeStep(BaseStep):
     )
 
     async def execute(self):
+        """Evaluate a BEAM response against its rubric."""
         assert self.context is not None
         llm_response: str = self.context.get("llm_response", "")
         rubric: list[str] = self.context.get("rubric", [])

plugins/beam-judge/src/judge_beam/plugin.yaml

@@ -0,0 +1,47 @@
+backends:
+  beam_rubric_judge_step: judge_beam.llm_judge:BeamRubricJudgeStep
+
+application_defaults:
+  jobs:
+    answer_judge:
+      backend: base
+      description: "BEAM rubric-based LLM-as-Judge"
+      parameters:
+        type: object
+        properties:
+          llm_response: {type: string}
+          rubric:
+            type: array
+            items: {type: string}
+          probing_question: {type: string, default: ""}
+          question_type: {type: string, default: ""}
+        required: [llm_response, rubric]
+      steps:
+        - backend: beam_rubric_judge_step
+          agent_wrapper: judge
+  components:
+    as_llm:
+      judge:
+        backend: ${LLM_BACKEND:-openai}
+        model: ${JUDGE_MODEL_NAME:-qwen3.7-max}
+        stream: false
+        context_size: 200000
+        max_retries: 5
+        retry_delay: 5.0
+        credential:
+          api_key: ${LLM_API_KEY:-}
+          base_url: ${LLM_BASE_URL:-}
+        parameters:
+          max_tokens: 65536
+          thinking_enable: false
+    agent_wrapper:
+      judge:
+        backend: agentscope
+        as_llm: judge
+        permission_mode: bypass
+        react_config: {max_iters: 1}
+        context_config:
+          trigger_ratio: 0.8
+          reserve_ratio: 0.1
+          tool_result_limit: 50000
+        model_config: {max_retries: 1}

plugins/beam/README.md

@@ -2,8 +2,8 @@
 
 [中文说明](./README_ZH.md)
 
-This plugin owns the BEAM memory, agentic-answer and judge Steps, their prompts,
-and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the
+This plugin owns the BEAM memory and agentic-answer Steps and their Job defaults.
+The trusted judge Step, prompts and `answer_judge` Job live in `plugins/beam-judge`. ReMe's built-in `benchmark.yaml` owns the
 shared evaluation Jobs and components. Dataset handling, the runner and results
 remain in [`benchmark/beam`](../../benchmark/beam/README.md).
 
@@ -12,17 +12,19 @@ From the repository root, install ReMe and this plugin in editable mode before r
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/beam --editable
+reme plugins install ./plugins/beam-judge --editable
 reme plugins validate beam
+reme plugins validate beam-judge
 python benchmark/beam/run.py
 ```
 
 Editable installation registers the `beam` entry point while keeping source changes immediately
-visible. The runner selects the built-in `benchmark` preset and explicitly enables `beam` for
+visible. The runner selects the built-in `benchmark` preset and explicitly enables `beam` and `beam-judge` for
 each Application. Installing the plugin makes it discoverable but does not enable it globally.
 
 `plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`,
-`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with
-`reme start config=benchmark plugins='["beam"]'`. The shared preset does not inherit
+`agentic_answer` Job defaults. Start a full benchmark application with
+`reme start config=benchmark plugins='["beam", "beam-judge"]'`. The shared preset does not inherit
 `default`: only declared Jobs run, indexing is manual, and neither scheduled dream
 nor the optional `auto_dream` Job is enabled.
 The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge`
@@ -31,8 +33,8 @@ still take precedence. Installing this plugin does not start an evaluation.
 
 The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`.
 The old core-owned `reme.steps.benchmark.beam` Python import path is removed.
-Custom Python callers should import memory, search and answer Steps from `reme_beam`, and the
-judge Step from `judge_beam`. After uninstalling,
+Custom Python callers should import memory, search and answer Steps from `reme_beam`, and install
+`beam-judge` before importing the judge Step from `judge_beam`. After uninstalling,
 Applications and CLI services must omit the plugin until it is installed again.
 Uninstallation never removes datasets, workspaces or results.
 Restart an existing service after changing plugins.

plugins/beam/README_ZH.md

@@ -2,7 +2,8 @@
 
 [English](./README.md)
 
-插件包含 BEAM 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。
+插件包含 BEAM 的记忆、回答 Step 及对应 Job 默认配置。可信评分 Step、提示词和
+`answer_judge` Job 位于独立的 `plugins/beam-judge` 插件中。
 ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在
 [`benchmark/beam`](../../benchmark/beam/README_ZH.md)。
 
@@ -11,22 +12,24 @@ ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/beam --editable
+reme plugins install ./plugins/beam-judge --editable
 reme plugins validate beam
+reme plugins validate beam-judge
 python benchmark/beam/run.py
 ```
 
 editable 安装会注册 `beam` entry point，并让源码修改立即生效。runner 选择内置 `benchmark`
-配置，并为每个 Application 显式启用 `beam`。安装只让插件可被发现，不会在所有应用中全局启用。
+配置，并为每个 Application 显式启用 `beam` 和 `beam-judge`。安装只让插件可被发现，不会在所有应用中全局启用。
 
 `plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、
-`agentic_answer` 和 `answer_judge` Job。安装后使用
-`reme start config=benchmark plugins='["beam"]'`。公共评测配置不继承 `default`，只运行声明的 Job：
+`agentic_answer` Job。完整评测使用
+`reme start config=benchmark plugins='["beam", "beam-judge"]'`。公共评测配置不继承 `default`，只运行声明的 Job：
 索引手动更新，dream 定时任务和可选的 `auto_dream`
 均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量
 保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。
 
 共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。
 原 `reme.steps.benchmark.beam` Python 导入路径已移除。自定义 Python 调用应从 `reme_beam`
-导入记忆、搜索和回答 Step，并从 `judge_beam` 导入评判 Step。
+导入记忆、搜索和回答 Step；安装 `beam-judge` 后再从 `judge_beam` 导入评判 Step。
 卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。
 卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。

plugins/beam/pyproject.toml

@@ -8,8 +8,6 @@ license-files = ["LICENSE"]
 requires-python = ">=3.11"
 dependencies = [
     "reme-ai[as]>=0.4.1.11",
-    "json-repair",
-    "numpy>=2.2.6",
 ]
 
 [project.entry-points."reme.plugins"]
@@ -17,11 +15,10 @@ beam = "reme_beam"
 
 [tool.setuptools.packages.find]
 where = ["src"]
-include = ["reme_beam*", "judge_beam*"]
+include = ["reme_beam*"]
 
 [tool.setuptools.package-data]
 reme_beam = ["plugin.yaml", "*.yaml"]
-judge_beam = ["*.yaml"]
 
 [build-system]
 requires = ["setuptools>=77", "wheel"]

plugins/beam/src/reme_beam/plugin.yaml

@@ -1,7 +1,6 @@
 backends:
   beam_auto_memory_step: reme_beam.auto_memory:BeamAutoMemoryStep
   beam_agentic_answer_step: reme_beam.agentic_answer:BeamAgenticAnswerStep
-  beam_rubric_judge_step: judge_beam.llm_judge:BeamRubricJudgeStep
   beam_search_v2_step: reme_beam.search_v2:SearchV2Step
 
 application_defaults:
@@ -51,37 +50,6 @@ application_defaults:
         - backend: beam_agentic_answer_step
           agent_wrapper: bench
 
-    answer_judge:
-      backend: base
-      description: "BEAM rubric-based LLM-as-Judge: evaluate response against rubric criteria"
-      watch_dirs: []
-      watch_suffixes: []
-      parameters:
-        type: object
-        properties:
-          llm_response:
-            type: string
-            description: "The model's response to evaluate"
-          rubric:
-            type: array
-            description: "List of rubric criteria to check"
-            items:
-              type: string
-          probing_question:
-            type: string
-            description: "The original probing question"
-            default: ""
-          question_type:
-            type: string
-            description: "BEAM question type (e.g. event_ordering)"
-            default: ""
-        required:
-          - llm_response
-          - rubric
-      steps:
-        - backend: beam_rubric_judge_step
-          agent_wrapper: judge
-
     auto_memory:
       backend: base
       description: "Auto-memory: record conversation facts into a daily note"
@@ -116,5 +84,3 @@ application_defaults:
     as_llm:
       default:
         max_retries: 5
-      judge:
-        retry_delay: 5.0

plugins/lme-judge/README.md

@@ -0,0 +1,3 @@
+# ReMe LongMemEval Judge
+
+Trusted LongMemEval scoring plugin. Install it alongside `plugins/lme` for a full benchmark run. Meta-ReMe candidate bundles must not include this plugin.

plugins/lme-judge/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "reme-lme-judge"
+version = "0.1.0"
+description = "Trusted LongMemEval judge plugin for ReMe."
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11"
+dependencies = ["reme-ai[as]>=0.4.1.11"]
+
+[project.entry-points."reme.plugins"]
+lme-judge = "judge_lme"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["judge_lme*"]
+
+[tool.setuptools.package-data]
+judge_lme = ["plugin.yaml", "*.yaml"]
+
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"

plugins/lme-judge/src/judge_lme/__init__.py

@@ -1,4 +1,4 @@
-"""LongMemEval benchmark judge backend."""
+"""LongMemEval benchmark judge plugin."""
 
 from .llm_judge import LmeAnswerJudgeStep
 

plugins/lme-judge/src/judge_lme/llm_judge.py

@@ -27,6 +27,7 @@ class LmeAnswerJudgeStep(BaseStep):
         return raw_answer.strip().lower()
 
     async def execute(self):
+        """Evaluate an agent answer against the LongMemEval reference answer."""
         assert self.context is not None
         query: str = self.context.get("query", "")
         agent_answer: str = self.context.get("agent_answer", "")

plugins/lme-judge/src/judge_lme/plugin.yaml

@@ -0,0 +1,44 @@
+backends:
+  lme_answer_judge_step: judge_lme.llm_judge:LmeAnswerJudgeStep
+
+application_defaults:
+  jobs:
+    answer_judge:
+      backend: base
+      description: "LLM-as-Judge: evaluate agent answer against golden answer"
+      parameters:
+        type: object
+        properties:
+          query: {type: string}
+          agent_answer: {type: string}
+          golden_answer: {type: string}
+          question_type: {type: string, default: ""}
+        required: [query, agent_answer, golden_answer]
+      steps:
+        - backend: lme_answer_judge_step
+          agent_wrapper: judge
+  components:
+    as_llm:
+      judge:
+        backend: ${LLM_BACKEND:-openai}
+        model: ${JUDGE_MODEL_NAME:-qwen3.7-max}
+        stream: false
+        context_size: 200000
+        max_retries: 5
+        credential:
+          api_key: ${LLM_API_KEY:-}
+          base_url: ${LLM_BASE_URL:-}
+        parameters:
+          max_tokens: 65536
+          thinking_enable: false
+    agent_wrapper:
+      judge:
+        backend: agentscope
+        as_llm: judge
+        permission_mode: bypass
+        react_config: {max_iters: 1}
+        context_config:
+          trigger_ratio: 0.8
+          reserve_ratio: 0.1
+          tool_result_limit: 50000
+        model_config: {max_retries: 1}

plugins/lme/README.md

@@ -2,8 +2,8 @@
 
 [中文说明](./README_ZH.md)
 
-This plugin owns the LongMemEval memory, agentic-answer and judge Steps, their prompts,
-and their Job defaults in `plugin.yaml`. ReMe's built-in `benchmark.yaml` owns the
+This plugin owns the LongMemEval memory and agentic-answer Steps and their Job defaults.
+The trusted judge Step, prompts and `answer_judge` Job live in `plugins/lme-judge`. ReMe's built-in `benchmark.yaml` owns the
 shared evaluation Jobs and components. Dataset handling, the runner and results remain
 in [`benchmark/longmemeval`](../../benchmark/longmemeval/README.md).
 
@@ -12,17 +12,19 @@ From the repository root, install ReMe and this plugin in editable mode before r
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/lme --editable
+reme plugins install ./plugins/lme-judge --editable
 reme plugins validate lme
+reme plugins validate lme-judge
 python benchmark/longmemeval/run.py
 ```
 
 Editable installation registers the `lme` entry point while keeping source changes immediately
-visible. The runner selects the built-in `benchmark` preset and explicitly enables `lme` for
+visible. The runner selects the built-in `benchmark` preset and explicitly enables `lme` and `lme-judge` for
 each Application. Installing the plugin makes it discoverable but does not enable it globally.
 
 `plugin.yaml` registers backends and contributes the plugin-owned `auto_memory`,
-`agentic_answer` and `answer_judge` Job defaults. Start the installed plugin with
-`reme start config=benchmark plugins='["lme"]'`. The shared preset does not inherit
+`agentic_answer` Job defaults. Start a full benchmark application with
+`reme start config=benchmark plugins='["lme", "lme-judge"]'`. The shared preset does not inherit
 `default`: only declared Jobs run, indexing is manual, and neither scheduled dream
 nor the optional `auto_dream` Job is enabled.
 The existing `auto_memory`, `agentic_answer`, `answer_judge`, `bench` and `judge`
@@ -31,8 +33,8 @@ still take precedence. Installing this plugin does not start an evaluation.
 
 The shared answer base class lives in `reme.steps.benchmark.base_agentic_answer`.
 The old core-owned `reme.steps.benchmark.lme` Python import path is removed.
-Custom Python callers should import memory, search and answer Steps from `reme_lme`, and the
-judge Step from `judge_lme`. After uninstalling,
+Custom Python callers should import memory, search and answer Steps from `reme_lme`, and install
+`lme-judge` before importing the judge Step from `judge_lme`. After uninstalling,
 Applications and CLI services must omit the plugin until it is installed again.
 Uninstallation never removes datasets, workspaces or results.
 Restart an existing service after changing plugins.

plugins/lme/README_ZH.md

@@ -2,7 +2,8 @@
 
 [English](./README.md)
 
-插件包含 LongMemEval 的记忆、回答、评分 Step、提示词，以及 `plugin.yaml` 中对应的 Job 默认配置。
+插件包含 LongMemEval 的记忆、回答 Step 及对应 Job 默认配置。可信评分 Step、提示词和
+`answer_judge` Job 位于独立的 `plugins/lme-judge` 插件中。
 ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集处理、runner 和结果仍留在
 [`benchmark/longmemeval`](../../benchmark/longmemeval/README_ZH.md)。
 
@@ -11,22 +12,24 @@ ReMe 内置的 `benchmark.yaml` 负责公共评测 Job 和 Component；数据集
 ```bash
 python -m pip install -e ".[as]"
 reme plugins install ./plugins/lme --editable
+reme plugins install ./plugins/lme-judge --editable
 reme plugins validate lme
+reme plugins validate lme-judge
 python benchmark/longmemeval/run.py
 ```
 
 editable 安装会注册 `lme` entry point，并让源码修改立即生效。runner 选择内置 `benchmark`
-配置，并为每个 Application 显式启用 `lme`。安装只让插件可被发现，不会在所有应用中全局启用。
+配置，并为每个 Application 显式启用 `lme` 和 `lme-judge`。安装只让插件可被发现，不会在所有应用中全局启用。
 
 `plugin.yaml` 注册 backend，并通过 `application_defaults` 提供插件拥有的 `auto_memory`、
-`agentic_answer` 和 `answer_judge` Job。安装后使用
-`reme start config=benchmark plugins='["lme"]'`。公共评测配置不继承 `default`，只运行声明的 Job：
+`agentic_answer` Job。完整评测使用
+`reme start config=benchmark plugins='["lme", "lme-judge"]'`。公共评测配置不继承 `default`，只运行声明的 Job：
 索引手动更新，dream 定时任务和可选的 `auto_dream`
 均保持关闭。原有 `auto_memory`、`agentic_answer`、`answer_judge`、`bench`、`judge` 名称及模型环境变量
 保持不变，显式应用参数和 CLI 覆盖仍优先。安装或启用插件不会自动开始评测。
 
 共享回答基类位于 `reme.steps.benchmark.base_agentic_answer`。
 原 `reme.steps.benchmark.lme` Python 导入路径已移除。自定义 Python 调用应从 `reme_lme`
-导入记忆、搜索和回答 Step，并从 `judge_lme` 导入评判 Step。
+导入记忆、搜索和回答 Step；安装 `lme-judge` 后再从 `judge_lme` 导入评判 Step。
 卸载插件后，Application 和 CLI 服务必须移除插件选择，直到再次安装。
 卸载不会删除数据集、工作区或结果。修改插件后需重启已有服务。

plugins/lme/pyproject.toml

@@ -15,11 +15,10 @@ lme = "reme_lme"
 
 [tool.setuptools.packages.find]
 where = ["src"]
-include = ["reme_lme*", "judge_lme*"]
+include = ["reme_lme*"]
 
 [tool.setuptools.package-data]
 reme_lme = ["plugin.yaml", "*.yaml"]
-judge_lme = ["*.yaml"]
 
 [build-system]
 requires = ["setuptools>=77", "wheel"]

plugins/lme/src/reme_lme/plugin.yaml

@@ -1,7 +1,6 @@
 backends:
   lme_auto_memory_step: reme_lme.auto_memory:LmeAutoMemoryStep
   lme_agentic_answer_step: reme_lme.agentic_answer:LmeAgenticAnswerStep
-  lme_answer_judge_step: judge_lme.llm_judge:LmeAnswerJudgeStep
   lme_search_v2_step: reme_lme.search_v2:SearchV2Step
 
 application_defaults:
@@ -51,35 +50,6 @@ application_defaults:
         - backend: lme_agentic_answer_step
           agent_wrapper: bench
 
-    answer_judge:
-      backend: base
-      description: "LLM-as-Judge: evaluate agent answer against golden answer"
-      watch_dirs: []
-      watch_suffixes: []
-      parameters:
-        type: object
-        properties:
-          query:
-            type: string
-            description: "The question being asked"
-          agent_answer:
-            type: string
-            description: "The model's answer to evaluate"
-          golden_answer:
-            type: string
-            description: "The correct/golden answer"
-          question_type:
-            type: string
-            description: "The question type for prompt selection"
-            default: ""
-        required:
-          - query
-          - agent_answer
-          - golden_answer
-      steps:
-        - backend: lme_answer_judge_step
-          agent_wrapper: judge
-
     auto_memory:
       backend: base
       description: "Auto-memory: record conversation facts into a daily note"

reme/config/benchmark.yaml

@@ -391,18 +391,6 @@ components:
       parameters:
         max_tokens: 65536
         thinking_enable: false
-    judge:
-      backend: ${LLM_BACKEND:-openai}
-      model: ${JUDGE_MODEL_NAME:-qwen3.7-max}
-      stream: false
-      context_size: 200000
-      max_retries: 5
-      credential:
-        api_key: ${LLM_API_KEY:-}
-        base_url: ${LLM_BASE_URL:-}
-      parameters:
-        max_tokens: 65536
-        thinking_enable: false
     bench:
       backend: ${LLM_BACKEND:-openai}
       model: ${BENCH_MODEL_NAME:-qwen3.7-max}
@@ -443,18 +431,6 @@ components:
         tool_result_limit: 50000
       model_config:
         max_retries: 1
-    judge:
-      backend: agentscope
-      as_llm: judge
-      permission_mode: bypass
-      react_config:
-        max_iters: 1
-      context_config:
-        trigger_ratio: 0.8
-        reserve_ratio: 0.1
-        tool_result_limit: 50000
-      model_config:
-        max_retries: 1
     bench:
       backend: agentscope
       as_llm: bench

tests/unit/test_plugin.py

@@ -13,6 +13,7 @@ from reme.config.config_parser import _load_config
 from reme.enumeration import ComponentEnum
 from reme.plugin import Backend, Plugin, PluginManager, _load_backend
 from reme.plugin_manifest import parse_plugin_manifest
+from reme.schema import ApplicationConfig
 
 
 class _PluginStep(ComponentMixin):
@@ -191,6 +192,42 @@ def test_plugin_manager_loads_package_manifest(monkeypatch, tmp_path):
     assert manager.merge_config({})["jobs"]["example"]["backend"] == "base"
 
 
+@pytest.mark.parametrize(
+    ("candidate", "judge", "candidate_package", "judge_package", "judge_backend"),
+    [
+        ("lme", "lme-judge", "reme_lme", "judge_lme", "lme_answer_judge_step"),
+        ("beam", "beam-judge", "reme_beam", "judge_beam", "beam_rubric_judge_step"),
+    ],
+)
+def test_benchmark_candidate_and_judge_plugins_compose(
+    monkeypatch,
+    candidate,
+    judge,
+    candidate_package,
+    judge_package,
+    judge_backend,
+):
+    root = Path(__file__).resolve().parents[2]
+    monkeypatch.syspath_prepend(str(root / "plugins" / candidate / "src"))
+    monkeypatch.syspath_prepend(str(root / "plugins" / judge / "src"))
+    _set_entry_points(
+        monkeypatch,
+        _FakeEntryPoint(candidate, candidate_package, lambda: None, "reme.plugins"),
+        _FakeEntryPoint(judge, judge_package, lambda: None, "reme.plugins"),
+    )
+
+    manager = PluginManager.discover([candidate, judge])
+    registry = ComponentRegistry()
+    manager.register(registry)
+    merged = manager.merge_config(_load_config("benchmark"))
+
+    assert {"auto_memory", "search", "agentic_answer", "answer_judge"} <= merged["jobs"].keys()
+    assert registry.get(ComponentEnum.STEP, judge_backend) is not None
+    assert "judge" in merged["components"]["as_llm"]
+    assert "judge" in merged["components"]["agent_wrapper"]
+    ApplicationConfig.model_validate(merged)
+
+
 def test_plugin_manifest_rejects_legacy_defaults_field():
     with pytest.raises(ValueError, match="unknown keys: defaults"):
         parse_plugin_manifest("defaults: {}\n", plugin_name="example")