diff --git a/.github/workflows/auto-fin-publish.yml b/.github/workflows/auto-fin-publish.yml new file mode 100644 index 00000000..042ca965 --- /dev/null +++ b/.github/workflows/auto-fin-publish.yml @@ -0,0 +1,138 @@ +# 发布操作手册： +# 1. 先将 plugin/auto-fin/pyproject.toml 中的 project.version 更新为待发布版本并合入目标分支。 +# 2. 确认插件依赖的 reme-ai 版本已经发布到 PyPI；本工作流会在构建阶段验证该依赖可下载。 +# 3. 确认仓库 Actions Secret 已配置 PYPI_API_TOKEN，且 PyPI 上不存在相同版本。 +# 4. 在 GitHub 仓库的 Actions 页面选择“Publish reme-auto-fin to PyPI”，点击“Run workflow”。 +# 5. 输入与 project.version 完全一致的版本号（例如 0.1.0）后运行；版本也可以带 v 前缀。 +# +# 推荐发布顺序：reme-ai -> reme-auto-fin -> QwenPaw 更新依赖并通过 plugins: [auto-fin] 启用。 +# 当前仅支持 workflow_dispatch 手动触发，不会因 push、tag 或 release 自动发布。 + +name: Publish reme-auto-fin to PyPI + +run-name: Publish reme-auto-fin ${{ inputs.version }} + +on: + workflow_dispatch: + inputs: + version: + description: Version from plugin/auto-fin/pyproject.toml (for example, 0.1.0) + required: true + type: string + +permissions: + contents: read + +concurrency: + group: publish-reme-auto-fin + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ inputs.version }} + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Install test and build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build packaging pytest pytest-asyncio twine + python -m pip install -e ".[core]" + python -m pip install --no-deps -e plugin/auto-fin + + - name: Validate package name and release version + id: package + run: | + python - "${RELEASE_VERSION}" <<'PY' + import os + import sys + import tomllib + from pathlib import Path + + from packaging.requirements import Requirement + from packaging.version import Version + + project = tomllib.loads(Path("plugin/auto-fin/pyproject.toml").read_text(encoding="utf-8"))["project"] + expected = Version(sys.argv[1].removeprefix("v")) + actual = Version(project["version"]) + if project["name"] != "reme-auto-fin": + raise SystemExit(f"Expected project name 'reme-auto-fin', found {project['name']!r}") + if actual != expected: + raise SystemExit(f"Package version is {actual}, but workflow input is {expected}") + requirements = [requirement for requirement in project["dependencies"] if requirement.startswith("reme-ai")] + if len(requirements) != 1: + raise SystemExit(f"Expected one reme-ai dependency, found {requirements!r}") + reme_requirement = Requirement(requirements[0]) + if reme_requirement.name != "reme-ai" or set(reme_requirement.extras) != {"core"}: + raise SystemExit(f"Expected a reme-ai[core] dependency, found {requirements[0]!r}") + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: + print(f"reme_requirement={reme_requirement}", file=output) + print(f"Publishing {project['name']} {actual}") + PY + + - name: Run Auto Fin tests + run: python -m pytest plugin/auto-fin -q + + - name: Require the plugin-enabled ReMe release on PyPI + run: | + python -m pip download --no-deps \ + --dest "${RUNNER_TEMP}/reme-auto-fin-core" \ + "${{ steps.package.outputs.reme_requirement }}" + + - name: Build and check distributions + run: | + mkdir -p dist/auto-fin + python -m build plugin/auto-fin --outdir dist/auto-fin + python -m twine check dist/auto-fin/* + + - name: Verify distributions and isolated installation + run: | + AUTO_FIN_WHEEL="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.whl)" + AUTO_FIN_SDIST="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.tar.gz)" + python -m zipfile -l "${AUTO_FIN_WHEEL}" | grep 'dist-info/licenses/LICENSE' + python -m tarfile -l "${AUTO_FIN_SDIST}" | grep '/LICENSE' + python -m venv "${RUNNER_TEMP}/reme-auto-fin-smoke" + "${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" -m pip install "${AUTO_FIN_WHEEL}" + cd "${RUNNER_TEMP}" + "${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" - <<'PY' + from importlib.metadata import distribution + + package = distribution("reme-auto-fin") + plugins = {entry.name: entry for entry in package.entry_points if entry.group == "reme.plugins"} + configs = {entry.name: entry for entry in package.entry_points if entry.group == "reme.configs"} + assert plugins["auto-fin"].load().name == "auto-fin" + assert configs["auto-fin"].load().is_file() + PY + + - name: Upload distributions + uses: actions/upload-artifact@v4 + with: + name: reme-auto-fin-${{ inputs.version }} + path: dist/auto-fin/ + if-no-files-found: error + + publish: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Download distributions + uses: actions/download-artifact@v4 + with: + name: reme-auto-fin-${{ inputs.version }} + path: dist/auto-fin + + - name: Publish reme-auto-fin + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: dist/auto-fin diff --git a/.github/workflows/github-pages-check.yml b/.github/workflows/github-pages-check.yml index 1b7ca454..7b52a411 100644 --- a/.github/workflows/github-pages-check.yml +++ b/.github/workflows/github-pages-check.yml @@ -12,7 +12,7 @@ on: - 'github-pages/**' - 'website/README*.md' - 'website/public/og.jpg' - - 'cookbook/*/README*.md' + - 'plugin/*/README*.md' - 'benchmark/*/README*.md' - 'skills/reme_memory/SKILL.md' pull_request: @@ -26,7 +26,7 @@ on: - 'github-pages/**' - 'website/README*.md' - 'website/public/og.jpg' - - 'cookbook/*/README*.md' + - 'plugin/*/README*.md' - 'benchmark/*/README*.md' - 'skills/reme_memory/SKILL.md' workflow_dispatch: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 5337b01f..30b1b4e7 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -10,7 +10,7 @@ on: - "README_ZH.md" - "website/README*.md" - "website/public/og.jpg" - - "cookbook/*/README*.md" + - "plugin/*/README*.md" - "benchmark/*/README*.md" - "skills/reme_memory/SKILL.md" - "AGENTS.md" diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 4fd5886e..6ce9bffd 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -33,11 +33,12 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel pip install -e packages/reme_ai_studio -e ".[dev,core]" + pip install --no-deps -e plugin/auto-fin pip install coverage - name: Run unit tests run: | - coverage run -m pytest tests/unit \ + coverage run -m pytest tests/unit plugin/auto-fin \ -v \ --tb=long \ -s \ diff --git a/AGENTS.md b/AGENTS.md index 931204fc..54523d0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ and concise documentation together. - `reme/components/application_context.py`: application-wide wiring and in-memory shared state. - `reme/components/runtime_context.py`: request-scoped data, response, streaming queue, and stop event. - `reme/components/base_component.py`: component lifecycle, dependency binding, and workspace helpers. -- `reme/components/component_registry.py`: the process-wide `(component type, backend)` registry. +- `reme/components/component_registry.py`: the frozen built-in registry template and application-local registry factory. - `reme/components/job/`: base, stream, background, and cron job implementations. - `reme/components/service/`: local CLI, HTTP, and MCP service backends. - `reme/components/`: agent wrappers, model adapters, stores, catalogs, graphs, indexes, clients, tokenizers, and @@ -56,7 +56,7 @@ and concise documentation together. - `plugins/claude_code/` and `plugins/hermes_agent/`: agent integrations. - `skills/`: standalone skills; `reme_memory` calls ReMe, while other skills may use separate tools or direct-file conventions. -- `benchmark/` and `cookbook/`: runnable evaluation and example workflows. +- `benchmark/` and `plugin/`: runnable evaluations and external plugin examples. - `docs/`: README-linked supporting pages and figures. ## Development Setup diff --git a/README.md b/README.md index 94861322..ab22913d 100644 --- a/README.md +++ b/README.md @@ -200,13 +200,13 @@ These Markdown guides cover the main user workflows and the runtime contracts im | [Framework](docs/en/framework.md) | Understand Application, Job, Step, Component, service, configuration, and lifecycle boundaries. | | [ReMe Blog](https://agentscope-ai.github.io/ReMe/?doc=en-reme-blog) | Read the product story, design rationale, examples, and benchmark summary. | -## 🧑‍🍳 Cookbooks +## 🔌 Plugins -Cookbooks are optional, end-to-end workflows assembled from ReMe jobs and steps. They are not enabled by the default -configuration; select the cookbook's standalone configuration when starting ReMe. Each new cookbook will be added as -another row in this table. +Plugins are optional Python distributions that contribute Component, Step, or Job backends and configuration. They are +installed separately and enabled explicitly by configuration. Auto Fin is the complete external-plugin example; Daily +Paper remains an optional research workflow while it is migrated to the same packaging model. -| Cookbook | Capability | +| Plugin / workflow | Capability | |-----------------------------------------------|---------------------------------------------------------------------------------------------------------------| | [Daily Paper](https://reme.agentscope.io/?doc=daily-paper-en) | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. | | [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-en) | Fetch topic-related CLS news, search ReMe history, and generate wikilink-backed Markdown reports. | diff --git a/README_ZH.md b/README_ZH.md index e23837b5..6620df8c 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -193,12 +193,12 @@ ReMe 会把 Agent 记忆保存为可读的 Markdown。 | [框架说明](docs/zh/framework.md) | 理解 Application、Job、Step、Component、service、配置和生命周期边界。 | | [ReMe 博客](https://agentscope-ai.github.io/ReMe/?doc=zh-reme-blog) | 了解完整产品故事、设计动机、使用示例和评测摘要。 | -## 🧑‍🍳 Cookbooks +## 🔌 插件 -Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们；启动 ReMe 时选择对应的独立配置即可启用。后续新增的 -cookbook 会继续在表格中按行追加。 +插件是可选的独立 Python distribution，可以贡献 Component、Step、Job backend 和配置，并通过配置显式启用。Auto Fin +是完整的外部插件示例；每日论文在迁移到同一打包模型前仍作为可选研究工作流提供。 -| Cookbook | 能力 | +| 插件 / 工作流 | 能力 | |-----------------------------------------------|--------------------------------------------------------------------------------| | [每日论文](https://reme.agentscope.io/?doc=daily-paper-zh) | 发现并排序论文，使用 Agent 解读 PDF，生成文件化论文笔记和五分钟简报。 | | [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-zh) | 拉取主题相关财联社新闻，搜索 ReMe 历史材料并生成带 wikilink 的 Markdown 报告。 | diff --git a/docs/en/framework.md b/docs/en/framework.md index d95cab1d..a5d77b85 100644 --- a/docs/en/framework.md +++ b/docs/en/framework.md @@ -55,11 +55,12 @@ Core layers: reme/ reme.py # CLI entry point application.py # Application assembly and lifecycle + plugin.py # installed plugin contract and entry-point loader config/ default.yaml # default service / jobs / components config_parser.py # config=, dot notation, and env placeholder parsing components/ - component_registry.py # global registry R + component_registry.py # backend registry and application-local copies base_component.py # ComponentMixin / BaseComponent / bind dependency declarations runtime_context.py # context for one Job execution job/ # BaseJob / StreamJob / BackgroundJob / CronJob @@ -81,6 +82,8 @@ reme/ index/ # watch/init/update/search/traverse evolve/ # auto_memory, auto_resource, auto_dream, proactive transfer/ # upload/download +plugin/ + auto-fin/ # independent example plugin distribution ``` The default workspace directories are defined by `ApplicationConfig`: @@ -216,14 +219,23 @@ The registry key is: The same backend name can therefore exist under different component types. For example, `http` can be both a service backend and a client backend. -### 4.2 Registration Through Module Imports +`ComponentEnum` provides the built-in identifiers, but installed plugins may declare a new type with a namespaced +string such as `example.reranker`. Custom identifiers use lowercase letters and numbers separated by `.`, `_`, or `-`. +They are configured under `components` and participate in the same dependency ordering and lifecycle as built-ins. -Registration happens when a module is imported. `reme/components/__init__.py` imports component packages, while -`reme/steps/__init__.py` imports `benchmark/common/cookbook/evolve/file_io/index/transfer`. Each package's `__init__.py` -then imports its concrete modules, causing `@R.register(...)` to execute. +### 4.2 Built-in and Plugin Registration -After adding a Step file, make sure the package's `__init__.py` imports it. Otherwise, the backend will not appear in -the registry. +Built-in implementations populate the built-in registry through package imports. ReMe freezes that template after +bootstrap, and each `Application` receives a mutable copy. Runtime code resolves backends through the application's +registry rather than changing the process-wide template. ReMe then loads only the installed plugins explicitly named by +`plugins` in the resolved configuration. A plugin +is exposed through the `reme.plugins` Python entry-point group and returns a declarative `reme.plugin.Plugin` containing +its named backend classes and default configuration. Plugin registration therefore stays local to one application; +duplicate `(component_type, backend)` providers fail during assembly instead of overwriting each other. + +Installed plugins may also expose named configuration files through `reme.configs`. Configuration can use `extends` to +inherit another built-in, plugin, or file-based configuration. The [Auto Fin plugin](../../plugin/auto-fin/README.md) +is the complete packaging example. ### 4.3 Component.bind diff --git a/docs/zh/framework.md b/docs/zh/framework.md index 6c17612d..d2cce659 100644 --- a/docs/zh/framework.md +++ b/docs/zh/framework.md @@ -50,11 +50,12 @@ flowchart LR reme/ reme.py # CLI 入口 application.py # Application 装配与生命周期 + plugin.py # 已安装插件契约与 entry-point loader config/ default.yaml # 默认 service / jobs / components config_parser.py # config=、dot notation、env 占位符解析 components/ - component_registry.py # 全局注册表 R + component_registry.py # backend 注册表及 Application 局部副本 base_component.py # ComponentMixin / BaseComponent / bind 依赖声明 runtime_context.py # 单次 Job 执行上下文 job/ # BaseJob / StreamJob / BackgroundJob / CronJob @@ -76,6 +77,8 @@ reme/ index/ # watch/init/update/search/traverse evolve/ # auto_memory、auto_resource、auto_dream、proactive transfer/ # upload/download +plugin/ + auto-fin/ # 独立发布的示例插件 ``` 默认 workspace 目录由 `ApplicationConfig` 定义： @@ -208,13 +211,19 @@ class VersionStep(BaseStep): 所以同名 backend 在不同 component type 下可以共存。例如 `http` 同时可以是 service backend 和 client backend。 -### 4.2 模块导入触发注册 +`ComponentEnum` 提供内置类型标识；已安装插件也可以用 `example.reranker` 这样的命名空间字符串声明新类型。自定义标识仅使用 +小写字母和数字，并以 `.`、`_` 或 `-` 分隔。它们配置在 `components` 下，与内置组件参与相同的依赖排序和生命周期。 -注册发生在模块 import 时。`reme/components/__init__.py` 会 import 各组件包，`reme/steps/__init__.py` 会 import -`benchmark/common/cookbook/evolve/file_io/index/transfer`。这些包的 `__init__.py` 再 import 具体模块，从而执行 -`@R.register(...)`。 +### 4.2 内置注册与插件注册 -新增 Step 文件后，必须保证它所在包的 `__init__.py` 会 import 该模块，否则注册表里找不到这个 backend。 +内置实现通过 package import 填充内置注册表；bootstrap 完成后 ReMe 会冻结这个模板，并为每个 `Application` 创建可写副本。 +运行期代码通过当前 Application 的注册表解析 backend，不能修改进程级模板。随后只加载最终配置中 `plugins` 明确启用的已安装插件。 +插件通过 Python `reme.plugins` entry-point group 暴露，返回声明式 `reme.plugin.Plugin`，其中包含命名 +backend class 和默认配置。插件注册因此只影响当前 Application；两个插件提供相同 `(component_type, backend)` 时会在装配阶段失败， +不会互相覆盖。 + +插件还可以通过 `reme.configs` 暴露命名配置；配置的 `extends` 可以继承内置配置、插件配置或文件配置。完整打包示例见 +[Auto Fin 插件](../../plugin/auto-fin/README_ZH.md)。 ### 4.3 Component.bind diff --git a/github-pages/README.md b/github-pages/README.md index fca537ee..bbbbb6a0 100644 --- a/github-pages/README.md +++ b/github-pages/README.md @@ -51,7 +51,7 @@ The build script reads the canonical repository files directly. Do not edit gene - `docs/en/` and `docs/zh/`: English and Chinese guides - `docs/figure/`: documentation images - `website/README.md` and `website/README_ZH.md`: ReMe Studio guide -- `cookbook/*/README*.md`: research workflow guides +- `plugin/*/README*.md`: plugin and research workflow guides - `benchmark/{beam,longmemeval,pibench,toolmemory}/README*.md`: benchmark guides and results - `skills/reme_memory/SKILL.md`: ReMe Memory skill guide - `AGENTS.md`: repository development guide diff --git a/github-pages/scripts/generate-content.mjs b/github-pages/scripts/generate-content.mjs index 4a3130d6..42faa556 100644 --- a/github-pages/scripts/generate-content.mjs +++ b/github-pages/scripts/generate-content.mjs @@ -64,7 +64,7 @@ const productDocuments = [ }, { slug: "daily-paper", - source: "cookbook/daily_paper", + source: "plugin/daily_paper", titles: { zh: "每日论文", en: "Daily Paper" }, descriptions: { zh: "发现论文、解析 PDF，并生成阅读笔记与每日简报。", @@ -74,7 +74,7 @@ const productDocuments = [ }, { slug: "auto-fin", - source: "cookbook/auto-fin", + source: "plugin/auto-fin", titles: { zh: "财经研究", en: "Auto Fin" }, descriptions: { zh: "结合最新财联社新闻与本地历史记忆生成研究报告。", diff --git a/plugin/auto-fin/LICENSE b/plugin/auto-fin/LICENSE new file mode 100644 index 00000000..65c2c5cf --- /dev/null +++ b/plugin/auto-fin/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Alibaba Group + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cookbook/auto-fin/README.md b/plugin/auto-fin/README.md similarity index 83% rename from cookbook/auto-fin/README.md rename to plugin/auto-fin/README.md index f8ec45c8..74c1cd93 100644 --- a/cookbook/auto-fin/README.md +++ b/plugin/auto-fin/README.md @@ -1,12 +1,12 @@ -# Auto Fin Cookbook +# Auto Fin Plugin [中文](README_ZH.md) Auto Fin fetches a rolling window of CLS telegraph news (24 hours by default), selects items related to configured topics, searches ReMe for useful historical context, and writes one Chinese Markdown report with validated wikilinks. -Current news and topic selection stay in runtime memory; only the final report becomes durable memory. The -implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/) and is assembled by -[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml). +Current news and topic selection stay in runtime memory; only the final report becomes durable memory. This directory +is an independent Python distribution. Its `reme.plugins` entry point contributes the three Step backends and their Job +configuration; its `reme.configs` entry point exposes the runnable `auto-fin` configuration. > Auto Fin has no reliable market-price feed. It does not calculate returns, targets, or entry points and is not > investment advice. @@ -14,11 +14,11 @@ implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookb ## Quick start ```bash -python -m pip install -e ".[core]" +python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin export LLM_API_KEY="your-api-key" export LLM_MODEL_NAME="qwen3.7-plus" export LLM_BASE_URL="https://your-provider.example/v1" -reme start config=daily_cookbook job=auto_fin +reme start config=auto-fin job=auto_fin ``` `LLM_MODEL_NAME` defaults to `qwen3.7-plus`. There is no built-in `LLM_BASE_URL`, so set the OpenAI-compatible endpoint @@ -27,7 +27,7 @@ required by the selected provider. The default topics are `黄金,机器人,半导体`. Override them per run: ```bash -reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片" +reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片" ``` An empty value also uses the defaults. @@ -76,23 +76,22 @@ refreshes the daily index. No JSONL, intermediate Markdown, or structured Agent | `request_interval` | `10` | Minimum delay in seconds after every CLS request attempt; may be zero | | `max_retries` | `3` | Maximum attempts for each CLS page request; must be at least one | -The built-in schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`. +The plugin-provided schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`. ## Output ```text -reme_workspace/daily/YYYY-MM-DD/auto_fin.md +.reme/daily/YYYY-MM-DD/auto_fin.md ``` The report includes a title, description, current CLS evidence, historical analysis, contextual wikilinks, and a fixed non-investment disclaimer. Network errors and invalid Agent output fail explicitly; no relevant current news is a -successful skip. If `DINGTALK_CONVERSATION_IDS` is empty, delivery is a no-op. If it is set, the DingTalk credentials -described in the [Daily Paper cookbook](../daily_paper/README.md#6-dingtalk) are required. +successful skip. ## Validation ```bash -pytest tests/unit/test_auto_fin.py -v +python -m pytest plugin/auto-fin -v ``` Unit tests mock the CLS and Agent boundaries and do not contact external services. diff --git a/cookbook/auto-fin/README_ZH.md b/plugin/auto-fin/README_ZH.md similarity index 81% rename from cookbook/auto-fin/README_ZH.md rename to plugin/auto-fin/README_ZH.md index 0809ee50..d61abeb0 100644 --- a/cookbook/auto-fin/README_ZH.md +++ b/plugin/auto-fin/README_ZH.md @@ -1,22 +1,22 @@ -# Auto Fin Cookbook +# Auto Fin 插件 [English](README.md) Auto Fin 自动拉取一个滚动时间窗口内的财联社电报（默认 24 小时），按配置 topics 筛选相关新闻，搜索 ReMe 中有回顾价值的历史材料，最后写入一份带校验 -wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中，只有最终报告成为持久记忆。实现位于 -[`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/)，并由 -[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配。 +wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中，只有最终报告成为持久记忆。本目录是一个独立 Python +distribution：`reme.plugins` entry point 贡献三个 Step backend 及其 Job 配置，`reme.configs` entry point 暴露可直接运行的 +`auto-fin` 配置。 > Auto Fin 没有可靠行情数据，不计算收益、目标价或买卖点，也不提供投资建议。 ## 快速开始 ```bash -python -m pip install -e ".[core]" +python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin export LLM_API_KEY="your-api-key" export LLM_MODEL_NAME="qwen3.7-plus" export LLM_BASE_URL="https://your-provider.example/v1" -reme start config=daily_cookbook job=auto_fin +reme start config=auto-fin job=auto_fin ``` `LLM_MODEL_NAME` 默认是 `qwen3.7-plus`。代码没有内置 `LLM_BASE_URL`，请设置所选服务商提供的 OpenAI 兼容 endpoint。 @@ -24,7 +24,7 @@ reme start config=daily_cookbook job=auto_fin 默认 topics 是 `黄金,机器人,半导体`。可在运行时覆盖： ```bash -reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片" +reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片" ``` 传入空值也会使用默认 topics。 @@ -69,22 +69,21 @@ workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠 | `request_interval` | `10` | 每次财联社请求尝试后的最小等待秒数，可设为 0 | | `max_retries` | `3` | 每页财联社请求的最大尝试次数，至少为 1 | -内置定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。 +插件提供的定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。 ## 产物 ```text -reme_workspace/daily/YYYY-MM-DD/auto_fin.md +.reme/daily/YYYY-MM-DD/auto_fin.md ``` 报告包含标题、说明、当前 CLS 证据、历史分析、上下文 wikilink 和固定非投资建议声明。网络错误与无效 Agent 输出 -会明确失败；没有相关当前新闻则成功跳过。`DINGTALK_CONVERSATION_IDS` 为空时发送步骤无副作用；设置该变量后， -还必须提供[每日论文 Cookbook](../daily_paper/README_ZH.md#6-dingtalk)中列出的钉钉凭据。 +会明确失败；没有相关当前新闻则成功跳过。 ## 验证 ```bash -pytest tests/unit/test_auto_fin.py -v +python -m pytest plugin/auto-fin -v ``` 单元测试 mock CLS 与 Agent 边界，不访问外部服务。 diff --git a/plugin/auto-fin/pyproject.toml b/plugin/auto-fin/pyproject.toml new file mode 100644 index 00000000..8c222bfa --- /dev/null +++ b/plugin/auto-fin/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "reme-auto-fin" +version = "0.1.0" +description = "Auto Fin example plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.11" +dependencies = [ + "reme-ai[core]>=0.4.1.8", +] + +[project.entry-points."reme.plugins"] +auto-fin = "reme_auto_fin:plugin" + +[project.entry-points."reme.configs"] +auto-fin = "reme_auto_fin:CONFIG_PATH" + +[tool.setuptools] +package-dir = { "" = "src" } +packages = ["reme_auto_fin"] +include-package-data = true + +[tool.setuptools.package-data] +reme_auto_fin = ["*.yaml"] + +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" +pythonpath = ["src", "../.."] +testpaths = ["tests"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugin/auto-fin/src/reme_auto_fin/__init__.py b/plugin/auto-fin/src/reme_auto_fin/__init__.py new file mode 100644 index 00000000..3e226af3 --- /dev/null +++ b/plugin/auto-fin/src/reme_auto_fin/__init__.py @@ -0,0 +1,21 @@ +"""Auto Fin news research workflow.""" + +from pathlib import Path + +from .data import AutoFinDataStep +from .merge import AutoFinMergeStep +from .plugin import plugin +from .schema import AutoFinReportOutput, AutoFinTopicOutput +from .topic import AutoFinTopicStep + +CONFIG_PATH = Path(__file__).with_name("config.yaml") + +__all__ = [ + "AutoFinReportOutput", + "AutoFinDataStep", + "AutoFinMergeStep", + "AutoFinTopicOutput", + "AutoFinTopicStep", + "CONFIG_PATH", + "plugin", +] diff --git a/reme/steps/cookbook/auto_fin/_base.py b/plugin/auto-fin/src/reme_auto_fin/base.py similarity index 99% rename from reme/steps/cookbook/auto_fin/_base.py rename to plugin/auto-fin/src/reme_auto_fin/base.py index 6c38b561..31d3ea95 100644 --- a/reme/steps/cookbook/auto_fin/_base.py +++ b/plugin/auto-fin/src/reme_auto_fin/base.py @@ -12,7 +12,7 @@ from uuid import uuid4 from pydantic import BaseModel -from ...base_step import BaseStep +from reme.steps import BaseStep AGENT_INPUT_LOG_LIMIT = 2000 AGENT_OUTPUT_LOG_LIMIT = 4000 diff --git a/plugin/auto-fin/src/reme_auto_fin/config.yaml b/plugin/auto-fin/src/reme_auto_fin/config.yaml new file mode 100644 index 00000000..35148071 --- /dev/null +++ b/plugin/auto-fin/src/reme_auto_fin/config.yaml @@ -0,0 +1,2 @@ +extends: default +plugins: [auto-fin] diff --git a/reme/steps/cookbook/auto_fin/data.py b/plugin/auto-fin/src/reme_auto_fin/data.py similarity index 98% rename from reme/steps/cookbook/auto_fin/data.py rename to plugin/auto-fin/src/reme_auto_fin/data.py index 4c1701ea..79551b92 100644 --- a/reme/steps/cookbook/auto_fin/data.py +++ b/plugin/auto-fin/src/reme_auto_fin/data.py @@ -9,8 +9,7 @@ from typing import Any import httpx -from ....components import R -from ._base import AutoFinStep, _plain_text +from .base import AutoFinStep, _plain_text API_URL = "https://www.cls.cn/v1/roll/get_roll_list" HEADERS = { @@ -23,7 +22,6 @@ DEFAULT_TOPICS = ("黄金", "机器人", "半导体") WINDOW = timedelta(hours=24) -@R.register("auto_fin_data_step") class AutoFinDataStep(AutoFinStep): """Fetch and normalize one rolling day of CLS news without writing files.""" diff --git a/plugin/auto-fin/src/reme_auto_fin/defaults.yaml b/plugin/auto-fin/src/reme_auto_fin/defaults.yaml new file mode 100644 index 00000000..14773f4e --- /dev/null +++ b/plugin/auto-fin/src/reme_auto_fin/defaults.yaml @@ -0,0 +1,54 @@ +jobs: + auto_fin: + backend: base + description: "Fetch and research recent topic-related CLS news." + parameters: + type: object + properties: + date: + type: string + description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai." + default: "" + now: + type: string + description: "Optional simulated current time in ISO 8601 format; empty means the real current time." + default: "" + topics: + type: string + description: "Comma-separated topics used to filter current CLS news." + default: "黄金,机器人,半导体" + window_hours: + type: number + exclusiveMinimum: 0 + description: "Rolling number of hours of CLS news to fetch." + default: 24 + request_interval: + type: number + minimum: 0 + description: "Minimum delay in seconds after each CLS request attempt." + default: 10 + max_retries: + type: integer + minimum: 1 + description: "Maximum attempts for each CLS page request." + default: 3 + steps: &auto_fin_steps + - backend: auto_fin_data_step + - backend: auto_fin_topic_step + - backend: auto_fin_merge_step + job_tools: [memory_search, read] + + auto_fin_0930_cron: + backend: cron + cron: "30 9 * * *" + steps: *auto_fin_steps + + auto_fin_1130_cron: + backend: cron + cron: "30 11 * * *" + steps: *auto_fin_steps + + auto_fin_1800_cron: + backend: cron + cron: "0 18 * * *" + steps: *auto_fin_steps diff --git a/reme/steps/cookbook/auto_fin/merge.py b/plugin/auto-fin/src/reme_auto_fin/merge.py similarity index 97% rename from reme/steps/cookbook/auto_fin/merge.py rename to plugin/auto-fin/src/reme_auto_fin/merge.py index 9aebf7a8..05297190 100644 --- a/reme/steps/cookbook/auto_fin/merge.py +++ b/plugin/auto-fin/src/reme_auto_fin/merge.py @@ -8,10 +8,10 @@ from datetime import date, timedelta from pathlib import Path from types import SimpleNamespace -from ....components import R -from ....schema import AutoFinReportOutput -from ...file_io import refresh_day_index -from ._base import AutoFinStep, _write +from reme.steps.file_io import refresh_day_index + +from .base import AutoFinStep, _write +from .schema import AutoFinReportOutput _WIKILINK_RE = re.compile(r"\[\[([^\[\]\n]+)\]\]") _HYBRID_WIKILINK_RE = re.compile( @@ -19,7 +19,6 @@ _HYBRID_WIKILINK_RE = re.compile( ) -@R.register("auto_fin_merge_step") class AutoFinMergeStep(AutoFinStep): """Give one Agent read-only ReMe tools, then validate links in its Markdown.""" diff --git a/reme/steps/cookbook/auto_fin/merge.yaml b/plugin/auto-fin/src/reme_auto_fin/merge.yaml similarity index 100% rename from reme/steps/cookbook/auto_fin/merge.yaml rename to plugin/auto-fin/src/reme_auto_fin/merge.yaml diff --git a/plugin/auto-fin/src/reme_auto_fin/plugin.py b/plugin/auto-fin/src/reme_auto_fin/plugin.py new file mode 100644 index 00000000..04d929d4 --- /dev/null +++ b/plugin/auto-fin/src/reme_auto_fin/plugin.py @@ -0,0 +1,31 @@ +"""Auto Fin plugin declaration.""" + +from pathlib import Path + +import yaml + +from reme.plugin import Backend, Plugin + +from .data import AutoFinDataStep +from .merge import AutoFinMergeStep +from .topic import AutoFinTopicStep + + +def _default_config() -> dict: + path = Path(__file__).with_name("defaults.yaml") + with path.open(encoding="utf-8") as stream: + config = yaml.safe_load(stream) + if not isinstance(config, dict): + raise ValueError(f"Auto Fin defaults must be a mapping: {path}") + return config + + +plugin = Plugin( + name="auto-fin", + backends=( + Backend("auto_fin_data_step", AutoFinDataStep), + Backend("auto_fin_topic_step", AutoFinTopicStep), + Backend("auto_fin_merge_step", AutoFinMergeStep), + ), + config=_default_config(), +) diff --git a/reme/schema/auto_fin.py b/plugin/auto-fin/src/reme_auto_fin/schema.py similarity index 100% rename from reme/schema/auto_fin.py rename to plugin/auto-fin/src/reme_auto_fin/schema.py diff --git a/reme/steps/cookbook/auto_fin/topic.py b/plugin/auto-fin/src/reme_auto_fin/topic.py similarity index 92% rename from reme/steps/cookbook/auto_fin/topic.py rename to plugin/auto-fin/src/reme_auto_fin/topic.py index 10163edf..d5c5e680 100644 --- a/reme/steps/cookbook/auto_fin/topic.py +++ b/plugin/auto-fin/src/reme_auto_fin/topic.py @@ -4,12 +4,10 @@ from __future__ import annotations import json -from ....components import R -from ....schema import AutoFinTopicOutput -from ._base import AutoFinStep +from .base import AutoFinStep +from .schema import AutoFinTopicOutput -@R.register("auto_fin_topic_step") class AutoFinTopicStep(AutoFinStep): """Filter current news in bounded Agent batches without writing files.""" diff --git a/reme/steps/cookbook/auto_fin/topic.yaml b/plugin/auto-fin/src/reme_auto_fin/topic.yaml similarity index 100% rename from reme/steps/cookbook/auto_fin/topic.yaml rename to plugin/auto-fin/src/reme_auto_fin/topic.yaml diff --git a/tests/unit/test_auto_fin.py b/plugin/auto-fin/tests/test_auto_fin.py similarity index 92% rename from tests/unit/test_auto_fin.py rename to plugin/auto-fin/tests/test_auto_fin.py index e84ecff6..6efe5b54 100644 --- a/tests/unit/test_auto_fin.py +++ b/plugin/auto-fin/tests/test_auto_fin.py @@ -8,14 +8,15 @@ from zoneinfo import ZoneInfo import pytest +from reme_auto_fin.base import _plain_text, _write +from reme_auto_fin.data import AutoFinDataStep +from reme_auto_fin.merge import AutoFinMergeStep +from reme_auto_fin.plugin import plugin +from reme_auto_fin.schema import AutoFinReportOutput, AutoFinTopicOutput +from reme_auto_fin.topic import AutoFinTopicStep from reme.components import ApplicationContext from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper from reme.components.runtime_context import RuntimeContext -from reme.schema import AutoFinReportOutput, AutoFinTopicOutput -from reme.steps.cookbook.auto_fin._base import _plain_text, _write -from reme.steps.cookbook.auto_fin.data import AutoFinDataStep -from reme.steps.cookbook.auto_fin.merge import AutoFinMergeStep -from reme.steps.cookbook.auto_fin.topic import AutoFinTopicStep SHANGHAI = ZoneInfo("Asia/Shanghai") @@ -28,7 +29,7 @@ def test_atomic_write_preserves_existing_file_on_failure(tmp_path: Path, monkeyp path = tmp_path / "result.md" path.write_text("existing", encoding="utf-8") monkeypatch.setattr( - "reme.steps.cookbook.auto_fin._base.os.replace", + "reme_auto_fin.base.os.replace", lambda *_args: (_ for _ in ()).throw(OSError()), ) @@ -210,7 +211,7 @@ async def test_merge_writes_only_final_report_and_validates_historical_links(tmp def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path: Path, monkeypatch): - import reme.steps.cookbook.auto_fin.merge as merge_module + import reme_auto_fin.merge as merge_module step = AutoFinMergeStep( app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"), @@ -237,11 +238,8 @@ def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path assert step._normalize_hybrid_wikilinks(body) == body -def test_config_has_default_topics_and_no_intermediate_index_step(): - from reme.config.config_parser import _load_config - - config = _load_config("daily_cookbook") - job = config["jobs"]["auto_fin"] +def test_plugin_config_has_default_topics_and_no_intermediate_index_step(): + job = plugin.config["jobs"]["auto_fin"] assert job["parameters"]["properties"]["topics"]["default"] == "黄金,机器人,半导体" assert job["parameters"]["properties"]["window_hours"]["default"] == 24 assert job["parameters"]["properties"]["request_interval"]["default"] == 10 @@ -251,7 +249,6 @@ def test_config_has_default_topics_and_no_intermediate_index_step(): "auto_fin_data_step", "auto_fin_topic_step", "auto_fin_merge_step", - "dingtalk_markdown_send_step", ] assert job["steps"][2]["job_tools"] == ["memory_search", "read"] for name, schedule in { @@ -259,8 +256,8 @@ def test_config_has_default_topics_and_no_intermediate_index_step(): "auto_fin_1130_cron": "30 11 * * *", "auto_fin_1800_cron": "0 18 * * *", }.items(): - assert config["jobs"][name]["cron"] == schedule - assert config["jobs"][name]["steps"] == job["steps"] + assert plugin.config["jobs"][name]["cron"] == schedule + assert plugin.config["jobs"][name]["steps"] == job["steps"] def test_agent_schemas_are_small_and_required(): diff --git a/cookbook/daily_paper/README.md b/plugin/daily_paper/README.md similarity index 100% rename from cookbook/daily_paper/README.md rename to plugin/daily_paper/README.md diff --git a/cookbook/daily_paper/README_ZH.md b/plugin/daily_paper/README_ZH.md similarity index 100% rename from cookbook/daily_paper/README_ZH.md rename to plugin/daily_paper/README_ZH.md diff --git a/reme/__init__.py b/reme/__init__.py index a80ee594..1aa33dad 100644 --- a/reme/__init__.py +++ b/reme/__init__.py @@ -8,13 +8,20 @@ from . import enumeration from . import schema from . import steps from . import utils +from .components import BaseComponent, R from .application import Application -from .components import BaseComponent +from .plugin import Backend, Plugin from .reme import ReMe +# Component and Step packages above have completed their decorator-driven +# bootstrap. Runtime code receives mutable copies of this immutable template. +R.freeze() + __all__ = [ "Application", "BaseComponent", + "Backend", + "Plugin", "ReMe", # submodules "config", diff --git a/reme/application.py b/reme/application.py index a0e763f5..57fae78b 100644 --- a/reme/application.py +++ b/reme/application.py @@ -7,22 +7,27 @@ from pathlib import Path from typing import AsyncGenerator, TypeVar from . import __version__ -from .components import BaseComponent, ApplicationContext +from .components import ApplicationContext, BaseComponent, create_application_registry from .components.job import BackgroundJob, BaseJob, CronJob, StreamJob from .components.service import BaseService -from .enumeration import ComponentEnum +from .enumeration import ComponentEnum, ComponentType, component_type_name +from .plugin import PluginManager from .schema import ComponentConfig, Response, StreamChunk from .utils import execute_stream_task, print_logo, get_logger T = TypeVar("T", bound=BaseComponent) -_NodeKey = tuple[ComponentEnum, str] +_NodeKey = tuple[str, str] class Application(BaseComponent): """Wires components from config and runs jobs against them.""" def __init__(self, **kwargs) -> None: - self.context = ApplicationContext(**kwargs) + plugin_manager = PluginManager.discover(kwargs.get("plugins") or ()) + resolved = plugin_manager.merge_config(kwargs) + registry = create_application_registry() + plugin_manager.register(registry) + self.context = ApplicationContext(registry=registry, **resolved) self._started_components: list[BaseComponent] = [] self._setup_workspace_directories() @@ -98,7 +103,7 @@ class Application(BaseComponent): def _instantiate( self, - ctype: ComponentEnum, + ctype: ComponentType, cfg: ComponentConfig, *, label: str, @@ -109,16 +114,13 @@ class Application(BaseComponent): `label` is the human-readable identifier used only in error messages. `expected_type` narrows the return type and guards against a backend - registered under the wrong ComponentEnum. + registered under the wrong component type. `name` is forwarded to the constructor for named components/jobs; leave it None for the service, which is keyed solely by type. """ - # Lazy import: the registry self-populates as component modules load. - from .components import R - if not cfg.backend: raise ValueError(f"{label} is missing the required 'backend' field") - backend_cls = R.get(ctype, cfg.backend) + backend_cls = self.context.registry.get(ctype, cfg.backend) if backend_cls is None: raise ValueError(f"Unregistered backend '{cfg.backend}' for {label}") @@ -153,7 +155,7 @@ class Application(BaseComponent): heapq.heappush(ready, downstream) if len(ordered) != len(nodes): - unresolved = [f"{k[0].value}:{k[1]}" for k, d in in_degree.items() if d > 0] + unresolved = [f"{k[0]}:{k[1]}" for k, d in in_degree.items() if d > 0] raise ValueError(f"Circular dependency detected among: {unresolved}") return ordered @@ -172,7 +174,7 @@ class Application(BaseComponent): in_degree[key] += 1 elif not dep.optional: raise ValueError( - f"Component {key[0].value}:{key[1]} depends on unregistered {dep.ctype.value}:{dep.name}", + f"Component {key[0]}:{key[1]} depends on unregistered {dep.ctype}:{dep.name}", ) return in_degree, dependents @@ -205,7 +207,7 @@ class Application(BaseComponent): await c.start() self._started_components.append(c) except Exception as e: - self.logger.exception(f"Failed to start {c.component_type.value}:{c.name}: {e}") + self.logger.exception(f"Failed to start {component_type_name(c.component_type)}:{c.name}: {e}") raise async def _close(self) -> None: @@ -214,23 +216,23 @@ class Application(BaseComponent): try: await c.close() except Exception as e: - self.logger.exception(f"Failed to close {c.component_type.value}:{c.name}: {e}") + self.logger.exception(f"Failed to close {component_type_name(c.component_type)}:{c.name}: {e}") self._started_components.clear() if self.context.thread_pool is not None: self.context.thread_pool.shutdown(wait=True) self.context.thread_pool = None - async def update_component(self, component_enum: ComponentEnum | str, name: str, /, **kwargs) -> BaseComponent: + async def update_component(self, component_enum: ComponentType, name: str, /, **kwargs) -> BaseComponent: """Update an existing component by type/name; never creates missing components.""" - component_enum = ComponentEnum(component_enum) - group = self.context.components.get(component_enum) + component_type = component_type_name(component_enum) + group = self.context.components.get(component_type) if not group or name not in group: - raise KeyError(f"Component '{name}' not found in {component_enum.value}") + raise KeyError(f"Component '{name}' not found in {component_type}") component = group[name] for key, value in kwargs.items(): if not hasattr(component, key): - raise AttributeError(f"Component {component_enum.value}:{name} has no attribute '{key}'") + raise AttributeError(f"Component {component_type}:{name} has no attribute '{key}'") setattr(component, key, value) return component diff --git a/reme/components/__init__.py b/reme/components/__init__.py index bd2c7c98..eada5ce5 100644 --- a/reme/components/__init__.py +++ b/reme/components/__init__.py @@ -16,7 +16,7 @@ from . import service from . import tokenizer from .application_context import ApplicationContext from .base_component import BaseComponent, ComponentMixin -from .component_registry import ComponentRegistry, R +from .component_registry import ComponentRegistry, R, create_application_registry from .prompt_handler import PromptHandler from .runtime_context import RuntimeContext @@ -26,6 +26,7 @@ __all__ = [ "ComponentMixin", "ComponentRegistry", "R", + "create_application_registry", "PromptHandler", "RuntimeContext", # base components diff --git a/reme/components/application_context.py b/reme/components/application_context.py index 317e81e7..0e0c393f 100644 --- a/reme/components/application_context.py +++ b/reme/components/application_context.py @@ -3,8 +3,8 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any -from ..enumeration import ComponentEnum from ..schema import ApplicationConfig +from .component_registry import ComponentRegistry, create_application_registry if TYPE_CHECKING: from .base_component import BaseComponent @@ -20,13 +20,14 @@ class ApplicationContext: components, jobs, and the service can find each other at runtime. """ - def __init__(self, **kwargs): + def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs): # Parse raw kwargs into a typed, validated config object. self.app_config: ApplicationConfig = ApplicationConfig(**kwargs) + self.registry = registry or create_application_registry() # Populated by Application during initialization. self.service: "BaseService | None" = None - self.components: dict[ComponentEnum, dict[str, "BaseComponent"]] = {} + self.components: dict[str, dict[str, "BaseComponent"]] = {} self.jobs: dict[str, "BaseJob"] = {} self.thread_pool: ThreadPoolExecutor | None = None diff --git a/reme/components/base_component.py b/reme/components/base_component.py index dcb67b19..efd45f6a 100644 --- a/reme/components/base_component.py +++ b/reme/components/base_component.py @@ -5,7 +5,7 @@ from abc import ABC from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast -from ..enumeration import ComponentEnum +from ..enumeration import ComponentEnum, ComponentType, component_type_name from ..utils import get_logger if TYPE_CHECKING: @@ -61,24 +61,24 @@ class Dependency: def __init__( self, - ctype: ComponentEnum, + ctype: ComponentType, name: str, default_factory: Callable[[], Any] | None = None, optional: bool = True, ) -> None: - self.ctype = ctype + self.ctype = component_type_name(ctype) self.name = name self.default_factory = default_factory self.optional = optional def __repr__(self) -> str: suffix = "?" if self.optional else "" - return f"" + return f"" def __getattr__(self, item: str) -> Any: # Catches accidental use of the placeholder before start() resolves it. raise RuntimeError( - f"Dependency {self.ctype.value}:{self.name} accessed before start() " f"(attribute '{item}')", + f"Dependency {self.ctype}:{self.name} accessed before start() " f"(attribute '{item}')", ) @@ -126,10 +126,14 @@ class BaseComponent(ComponentMixin, ABC): if not name: return None ctype = getattr(base_cls, "component_type", None) - if not isinstance(ctype, ComponentEnum) or ctype is ComponentEnum.BASE: + try: + ctype = component_type_name(ctype) + except (TypeError, ValueError) as exc: raise TypeError( - f"{base_cls.__name__} must declare a non-BASE ComponentEnum 'component_type'", - ) + f"{base_cls.__name__} must declare a non-BASE string 'component_type'", + ) from exc + if ctype == ComponentEnum.BASE.value: + raise TypeError(f"{base_cls.__name__} must declare a non-BASE 'component_type'") return cast(T, Dependency(ctype, name, default_factory, optional)) @property @@ -172,7 +176,7 @@ class BaseComponent(ComponentMixin, ABC): elif dep.optional: setattr(self, attr, None) else: - raise ValueError(f"{dep.ctype.value} '{dep.name}' not found.") + raise ValueError(f"{dep.ctype} '{dep.name}' not found.") # ----- Workspace path helpers -------------------------------------------- @@ -186,7 +190,7 @@ class BaseComponent(ComponentMixin, ABC): @property def component_metadata_path(self) -> Path: """Per-component metadata directory under the workspace.""" - return self.workspace_metadata_path / self.component_type.value + return self.workspace_metadata_path / component_type_name(self.component_type) # ----- Lifecycle hooks (override in subclasses) ---------------------- diff --git a/reme/components/component_registry.py b/reme/components/component_registry.py index 2a7d044f..76630672 100644 --- a/reme/components/component_registry.py +++ b/reme/components/component_registry.py @@ -1,12 +1,14 @@ -"""Global registry mapping ``(ComponentEnum, name) -> component class``.""" +"""Registry mapping ``(component type, backend)`` to implementation classes.""" +from collections.abc import Iterator +from contextlib import contextmanager +from threading import RLock from typing import Callable, TypeVar, cast -from .base_component import BaseComponent -from ..enumeration import ComponentEnum -from ..utils import get_logger +from .base_component import ComponentMixin +from ..enumeration import ComponentType, component_type_name -T = TypeVar("T", bound=BaseComponent) +T = TypeVar("T", bound=ComponentMixin) class ComponentRegistry: @@ -17,27 +19,46 @@ class ComponentRegistry: """ def __init__(self) -> None: - self._registry: dict[ComponentEnum, dict[str, type[BaseComponent]]] = {} - self.logger = get_logger(log_to_file=False) + self._registry: dict[str, dict[str, type[ComponentMixin]]] = {} + self._owners: dict[tuple[str, str], str] = {} + self._lock = RLock() + self._frozen = False - def _do_register(self, cls: type[T], name: str) -> type[T]: - """Insert `cls` under its ``component_type`` group; warn on overwrite.""" - component_type = getattr(cls, "component_type", None) - if not isinstance(component_type, ComponentEnum): - raise TypeError( - f"{cls.__name__} must have a ComponentEnum 'component_type' attribute", - ) + def _ensure_mutable(self) -> None: + """Reject changes after this registry becomes an immutable template.""" + if self._frozen: + raise RuntimeError("Component registry is frozen") + + def _do_register(self, cls: type[T], name: str, *, owner: str | None = None) -> type[T]: + """Insert ``cls`` under its component type and reject ambiguous providers.""" + try: + component_type = component_type_name(getattr(cls, "component_type", None)) + except (TypeError, ValueError) as exc: + raise TypeError(f"{cls.__name__} must have a non-empty string 'component_type' attribute") from exc if not name: raise ValueError("Component name cannot be empty") - group = self._registry.setdefault(component_type, {}) - if name in group: - self.logger.warning( - f"Component '{name}' already registered for {component_type}, overwriting", - ) - group[name] = cls + with self._lock: + self._ensure_mutable() + group = self._registry.setdefault(component_type, {}) + key = (component_type, name) + if name in group: + existing = group[name] + existing_owner = self._owners[key] + new_owner = owner or cls.__module__ + if existing is cls and existing_owner == new_owner: + return cls + raise ValueError( + f"Backend '{component_type}:{name}' is provided by both " f"'{existing_owner}' and '{new_owner}'", + ) + group[name] = cls + self._owners[key] = owner or cls.__module__ return cls + def add(self, name: str, cls: type[T], *, owner: str) -> type[T]: + """Register one explicitly owned plugin contribution.""" + return self._do_register(cls, name, owner=owner) + def register( self, cls_or_name: type[T] | str, @@ -60,25 +81,76 @@ class ComponentRegistry: return decorator - def get(self, component_type: ComponentEnum, name: str) -> type[BaseComponent] | None: + def get(self, component_type: ComponentType, name: str) -> type[ComponentMixin] | None: """Look up a registered class; return None if not found.""" - return self._registry.get(component_type, {}).get(name) + with self._lock: + return self._registry.get(component_type_name(component_type), {}).get(name) - def get_all(self, component_type: ComponentEnum) -> dict[str, type[BaseComponent]]: + def get_all(self, component_type: ComponentType) -> dict[str, type[ComponentMixin]]: """Return a shallow copy of all classes registered under `component_type`.""" - return dict(self._registry.get(component_type, {})) + with self._lock: + return dict(self._registry.get(component_type_name(component_type), {})) - def unregister(self, component_type: ComponentEnum, name: str) -> bool: + def unregister(self, component_type: ComponentType, name: str) -> bool: """Remove an entry; return True if it existed, False otherwise.""" - if (group := self._registry.get(component_type)) and name in group: - del group[name] - return True - return False + component_type = component_type_name(component_type) + with self._lock: + self._ensure_mutable() + if (group := self._registry.get(component_type)) and name in group: + del group[name] + self._owners.pop((component_type, name), None) + return True + return False def clear(self) -> None: """Drop every registered entry.""" - self._registry.clear() + with self._lock: + self._ensure_mutable() + self._registry.clear() + self._owners.clear() + + def freeze(self) -> None: + """Make this registry an immutable template for future copies.""" + with self._lock: + self._frozen = True + + @property + def frozen(self) -> bool: + """Whether mutating operations are disabled.""" + with self._lock: + return self._frozen + + def copy(self) -> "ComponentRegistry": + """Return an independent registry containing the same providers.""" + copied = ComponentRegistry() + with self._lock: + for component_type, group in self._registry.items(): + for name, implementation in group.items(): + copied.add(name, implementation, owner=self._owners[(component_type, name)]) + return copied + + @contextmanager + def preserve(self, *, allow_mutation: bool = False) -> Iterator[None]: + """Restore the registry after code that may register through import side effects.""" + with self._lock: + registry = {component_type: dict(group) for component_type, group in self._registry.items()} + owners = dict(self._owners) + frozen = self._frozen + if allow_mutation: + self._frozen = False + try: + yield + finally: + self._registry = registry + self._owners = owners + self._frozen = frozen -# Process-wide singleton used throughout the codebase. +# Import-time registry for built-in implementations. Runtime code should use +# ``create_application_registry`` rather than mutate this template. R = ComponentRegistry() + + +def create_application_registry() -> ComponentRegistry: + """Return a mutable registry initialized from the frozen built-in template.""" + return R.copy() diff --git a/reme/components/job/base_job.py b/reme/components/job/base_job.py index 5133111d..bb026a6c 100644 --- a/reme/components/job/base_job.py +++ b/reme/components/job/base_job.py @@ -47,7 +47,7 @@ class BaseJob(BaseComponent): config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw) if not config.backend: raise ValueError("Step is missing the required 'backend' field") - step_cls = R.get(ComponentEnum.STEP, config.backend) + step_cls = self.app_context.registry.get(ComponentEnum.STEP, config.backend) if not step_cls: raise ValueError(f"Unregistered backend '{config.backend}' of type '{ComponentEnum.STEP}'") params = config.model_dump() diff --git a/reme/config/__init__.py b/reme/config/__init__.py index c2903189..ce106a77 100644 --- a/reme/config/__init__.py +++ b/reme/config/__init__.py @@ -1,8 +1,10 @@ """Config""" -from .config_parser import parse_args, resolve_app_config +from .config_parser import deep_merge_config, expand_env_vars, parse_args, resolve_app_config __all__ = [ + "deep_merge_config", + "expand_env_vars", "parse_args", "resolve_app_config", ] diff --git a/reme/config/config_parser.py b/reme/config/config_parser.py index 3f6ec1b4..e70ad805 100644 --- a/reme/config/config_parser.py +++ b/reme/config/config_parser.py @@ -3,15 +3,20 @@ import json import os import re +from collections.abc import Mapping +from importlib import metadata from pathlib import Path from typing import Any import yaml +from ..entry_point import load_entry_point + # Config files are looked up relative to this module's directory _CONFIG_DIR = Path(__file__).parent # Extensions in priority order: yaml > yml > json when stems collide _SUPPORTED_EXTS = (".yaml", ".yml", ".json") +_CONFIG_ENTRY_POINT_GROUP = "reme.configs" _ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?}") # Strings like "007" / "00501" must stay as strings, not be coerced to numbers _LEADING_ZERO_RE = re.compile(r"^-?0\d") @@ -41,6 +46,11 @@ def _expand_env_vars(value: Any) -> Any: return value +def expand_env_vars(value: Any) -> Any: + """Expand environment placeholders in an arbitrary plugin config value.""" + return _expand_env_vars(value) + + def _discover_configs() -> dict[str, Path]: """Pre-scan config directory: maps file stem (name without ext) -> Path.""" discovered: dict[str, Path] = {} @@ -117,17 +127,43 @@ def _convert_value(value_str: str) -> Any: return s -def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict: - """Load a YAML or JSON config file. +def _external_config_entries(name: str) -> list[metadata.EntryPoint]: + """Find installed config providers without importing their packages.""" + return list(metadata.entry_points().select(group=_CONFIG_ENTRY_POINT_GROUP, name=name)) - First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY). - If not, treat as a file path and load directly. - """ - # 1. Try pre-discovered configs first - if name_or_path in _CONFIG_REGISTRY: - return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding) - # 2. Treat as file path +def _external_config_path(name: str, entries: list[metadata.EntryPoint] | None = None) -> Path | None: + """Resolve an installed plugin config exposed through ``reme.configs``.""" + entries = _external_config_entries(name) if entries is None else entries + if not entries: + return None + if len(entries) > 1: + providers = ", ".join(sorted(entry.value for entry in entries)) + raise ValueError(f"Config '{name}' has multiple installed providers: {providers}") + value = load_entry_point(entries[0], invoke=True) + path = Path(value) + if path.suffix not in _SUPPORTED_EXTS or not path.is_file(): + raise ValueError(f"Config entry point '{name}' did not resolve to a YAML or JSON file") + return path + + +def _load_config(name_or_path: str, encoding: str = "utf-8", _stack: tuple[str, ...] = ()) -> dict: + """Load a built-in, installed-plugin, or direct YAML/JSON config.""" + if name_or_path in _stack: + chain = " -> ".join((*_stack, name_or_path)) + raise ValueError(f"Circular config inheritance: {chain}") + + built_in = _CONFIG_REGISTRY.get(name_or_path) + external_entries = _external_config_entries(name_or_path) + if built_in is not None and external_entries: + raise ValueError(f"Config '{name_or_path}' is provided by both ReMe and an installed distribution") + if built_in is not None: + return _load_config_path(built_in, name_or_path, encoding, _stack) + + external = _external_config_path(name_or_path, external_entries) + if external is not None: + return _load_config_path(external, name_or_path, encoding, _stack) + p = Path(name_or_path) if p.suffix in _SUPPORTED_EXTS: candidates = [p] @@ -135,13 +171,31 @@ def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict: candidates.append(_CONFIG_DIR / p) for candidate in candidates: if candidate.exists(): - return _read_config_file(candidate, encoding) + identity = str(candidate.resolve()) + return _load_config_path(candidate, identity, encoding, _stack) raise FileNotFoundError(f"Config file not found: {p}") known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none" raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}") +def _load_config_path(path: Path, identity: str, encoding: str, stack: tuple[str, ...]) -> dict: + """Load one config and merge its optional parents before its own values.""" + config = _read_config_file(path, encoding) + raw_parents = config.pop("extends", ()) + parents = [raw_parents] if isinstance(raw_parents, str) else list(raw_parents or ()) + merged: dict = {} + for parent in parents: + if not isinstance(parent, str) or not parent: + raise ValueError(f"Config 'extends' entries must be non-empty strings: {path}") + parent_name = parent + relative = path.parent / parent + if Path(parent).suffix in _SUPPORTED_EXTS and relative.is_file(): + parent_name = str(relative.resolve()) + merged = deep_merge_config(merged, _load_config(parent_name, encoding, (*stack, identity))) + return deep_merge_config(merged, config) + + def _read_config_file(path: Path, encoding: str = "utf-8") -> dict: """Read YAML or JSON file based on extension. Expands ${ENV_VAR}.""" with path.open(encoding=encoding) as f: @@ -156,12 +210,12 @@ def _read_config_file(path: Path, encoding: str = "utf-8") -> dict: return _expand_env_vars(result) -def _deep_merge(base: dict, update: dict) -> dict: - """Recursively merge dicts.""" - result = base.copy() +def deep_merge_config(base: Mapping[str, Any], update: Mapping[str, Any]) -> dict[str, Any]: + """Recursively merge configuration mappings without mutating either input.""" + result = dict(base) for k, v in update.items(): - if k in result and isinstance(result[k], dict) and isinstance(v, dict): - result[k] = _deep_merge(result[k], v) + if k in result and isinstance(result[k], Mapping) and isinstance(v, Mapping): + result[k] = deep_merge_config(result[k], v) else: result[k] = v return result @@ -230,6 +284,6 @@ def resolve_app_config(*, log_config: bool = True, **kwargs) -> dict: merged: dict = {} for cfg in configs: - merged = _deep_merge(merged, cfg) + merged = deep_merge_config(merged, cfg) return merged diff --git a/reme/config/daily_cookbook.yaml b/reme/config/daily_cookbook.yaml index 953024ca..aa15e890 100644 --- a/reme/config/daily_cookbook.yaml +++ b/reme/config/daily_cookbook.yaml @@ -316,71 +316,6 @@ jobs: steps: - backend: edit_step - auto_fin: - backend: base - description: "Fetch and research the latest 24 hours of topic-related CLS news." - parameters: - type: object - properties: - date: - type: string - description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai." - default: "" - now: - type: string - description: "Optional simulated current time in ISO 8601 format; empty means the real current time." - default: "" - topics: - type: string - description: "Comma-separated topics used to filter current CLS news." - default: "黄金,机器人,半导体" - window_hours: - type: number - exclusiveMinimum: 0 - description: "Rolling number of hours of CLS news to fetch." - default: 24 - request_interval: - type: number - minimum: 0 - description: "Minimum delay in seconds after each CLS request attempt." - default: 10 - max_retries: - type: integer - minimum: 1 - description: "Maximum attempts for each CLS page request." - default: 3 - steps: &auto_fin_steps - - backend: auto_fin_data_step - - backend: auto_fin_topic_step - - backend: auto_fin_merge_step - job_tools: [memory_search, read] - - backend: dingtalk_markdown_send_step - input_mapping: - auto_fin_digest_path: markdown_path - app_key: ${DINGTALK_APP_KEY:-} - app_secret: ${DINGTALK_APP_SECRET:-} - robot_code: ${DINGTALK_ROBOT_CODE:-} - conversation_ids: ${DINGTALK_CONVERSATION_IDS:-} - title: ReMe Auto Fin - timeout: 15 - - # Three intraday runs (Asia/Shanghai). News remains useful outside market - # sessions. Each rerun refines the same day's report. - auto_fin_0930_cron: - backend: cron - cron: "30 9 * * *" - steps: *auto_fin_steps - - auto_fin_1130_cron: - backend: cron - cron: "30 11 * * *" - steps: *auto_fin_steps - - auto_fin_1800_cron: - backend: cron - cron: "0 18 * * *" - steps: *auto_fin_steps - daily_paper: backend: base description: "Build detailed readings and a five-minute brief from Hugging Face weekly/monthly papers." diff --git a/reme/entry_point.py b/reme/entry_point.py new file mode 100644 index 00000000..298b2793 --- /dev/null +++ b/reme/entry_point.py @@ -0,0 +1,14 @@ +"""Safe loading boundary for third-party Python entry points.""" + +from importlib import metadata +from typing import Any + + +def load_entry_point(entry: metadata.EntryPoint, *, invoke: bool = False) -> Any: + """Load and optionally invoke an entry point without retaining registrations.""" + # Import lazily so config parser imports do not pull in the component graph. + from .components.component_registry import R + + with R.preserve(allow_mutation=True): + loaded = entry.load() + return loaded() if invoke and callable(loaded) else loaded diff --git a/reme/enumeration/__init__.py b/reme/enumeration/__init__.py index 695210c3..2845d5fb 100644 --- a/reme/enumeration/__init__.py +++ b/reme/enumeration/__init__.py @@ -2,12 +2,15 @@ from .chunk_enum import ChunkEnum from .component_enum import ComponentEnum +from .component_type import ComponentType, component_type_name from .dream_bucket_enum import DreamBucketEnum from .link_scope_enum import LinkScopeEnum __all__ = [ "ChunkEnum", "ComponentEnum", + "ComponentType", "DreamBucketEnum", "LinkScopeEnum", + "component_type_name", ] diff --git a/reme/enumeration/component_type.py b/reme/enumeration/component_type.py new file mode 100644 index 00000000..59436037 --- /dev/null +++ b/reme/enumeration/component_type.py @@ -0,0 +1,25 @@ +"""Extensible component type identifiers.""" + +import re +from typing import TypeAlias + +from .component_enum import ComponentEnum + +ComponentType: TypeAlias = ComponentEnum | str +_COMPONENT_TYPE_PATTERN = re.compile(r"[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*") + + +def component_type_name(value: ComponentType) -> str: + """Return the canonical string name for a built-in or plugin component type.""" + if isinstance(value, ComponentEnum): + return value.value + if not isinstance(value, str): + raise TypeError(f"Component type must be a string, got {type(value).__name__}") + name = value.strip() + if not name: + raise ValueError("Component type cannot be empty") + if _COMPONENT_TYPE_PATTERN.fullmatch(name) is None: + raise ValueError( + f"Invalid component type {name!r}; use lowercase letters and numbers separated by '.', '_' or '-'", + ) + return name diff --git a/reme/plugin.py b/reme/plugin.py new file mode 100644 index 00000000..78a88186 --- /dev/null +++ b/reme/plugin.py @@ -0,0 +1,84 @@ +"""Installed plugin discovery and application-local registration.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from importlib import metadata +from typing import Any + +from .components.base_component import ComponentMixin +from .components.component_registry import ComponentRegistry +from .config import deep_merge_config, expand_env_vars +from .entry_point import load_entry_point + +PLUGIN_ENTRY_POINT_GROUP = "reme.plugins" + + +@dataclass(frozen=True) +class Backend: + """One named component, step, or job backend contributed by a plugin.""" + + name: str + implementation: type[ComponentMixin] + + +@dataclass(frozen=True) +class Plugin: + """Declarative plugin loaded from the ``reme.plugins`` entry-point group.""" + + name: str + backends: tuple[Backend, ...] = () + config: Mapping[str, Any] = field(default_factory=dict) + + +def _entry_points(group: str, name: str) -> list[metadata.EntryPoint]: + """Return matching entry points for one group and name.""" + return list(metadata.entry_points().select(group=group, name=name)) + + +class PluginManager: + """Resolve enabled plugins and apply their contributions to one application.""" + + def __init__(self, plugins: Iterable[Plugin] = ()) -> None: + self.plugins = tuple(plugins) + + @classmethod + def discover(cls, specs: Iterable[str]) -> "PluginManager": + """Load explicitly enabled plugins by entry-point name.""" + plugins: list[Plugin] = [] + seen: set[str] = set() + for name in specs: + if not isinstance(name, str): + raise TypeError(f"Invalid plugin name: {name!r}") + if not name: + raise ValueError("Plugin name cannot be empty") + if name in seen: + raise ValueError(f"Plugin '{name}' is enabled more than once") + entries = _entry_points(PLUGIN_ENTRY_POINT_GROUP, name) + if not entries: + raise ValueError(f"Plugin '{name}' is not installed") + if len(entries) > 1: + providers = ", ".join(sorted(entry.value for entry in entries)) + raise ValueError(f"Plugin '{name}' has multiple installed providers: {providers}") + plugin = load_entry_point(entries[0], invoke=True) + if not isinstance(plugin, Plugin): + raise TypeError(f"Plugin entry point '{name}' did not return reme.plugin.Plugin") + if plugin.name != name: + raise ValueError(f"Plugin entry point '{name}' returned plugin '{plugin.name}'") + plugins.append(plugin) + seen.add(name) + return cls(plugins) + + def merge_config(self, application_config: Mapping[str, Any]) -> dict[str, Any]: + """Place plugin defaults below the user's resolved application config.""" + merged: dict[str, Any] = {} + for plugin in self.plugins: + merged = deep_merge_config(merged, expand_env_vars(plugin.config)) + return deep_merge_config(merged, application_config) + + def register(self, registry: ComponentRegistry) -> None: + """Register every backend into an application-local registry.""" + for plugin in self.plugins: + for backend in plugin.backends: + registry.add(backend.name, backend.implementation, owner=plugin.name) diff --git a/reme/reme.py b/reme/reme.py index a1f98efe..64166be7 100644 --- a/reme/reme.py +++ b/reme/reme.py @@ -4,11 +4,12 @@ import asyncio import sys from .application import Application -from .components import R +from .components import create_application_registry from .components.service.cli_service import prepare_start_config, should_precheck_start from .config import parse_args, resolve_app_config from .enumeration import ComponentEnum -from .utils import cli_find_reme, load_env, precheck_start, running_service_config +from .plugin import PluginManager +from .utils import cli_find_reme, load_env, precheck_start, running_app_config _CLIENT_KWARGS = {"host", "port", "timeout", "transport", "command", "args", "show_metadata"} @@ -36,10 +37,14 @@ async def call_server(action: str, **kwargs): if isinstance(kwargs.get("service"), dict): resolve_kwargs["service"] = kwargs.pop("service") - # Prefer the running server's real config; fall back to the local config file. - service = running_service_config() - if service is None: - service = resolve_app_config(log_config=False, **resolve_kwargs).get("service") + # Prefer the running server's complete config so its enabled client plugins + # are available even when the caller does not repeat config=. + app_config = running_app_config() + if app_config is None: + app_config = resolve_app_config(log_config=False, **resolve_kwargs) + plugin_manager = PluginManager.discover(app_config.get("plugins") or ()) + app_config = plugin_manager.merge_config(app_config) + service = app_config.get("service") service = service if isinstance(service, dict) else {} backend: str = kwargs.pop("backend", None) or service.get("backend", "http") @@ -50,7 +55,9 @@ async def call_server(action: str, **kwargs): client_kwargs = {k: seed[k] for k in _CLIENT_KWARGS if k in seed} client_kwargs.update({key: kwargs.pop(key) for key in list(kwargs) if key in _CLIENT_KWARGS}) - client_cls = R.get(ComponentEnum.CLIENT, backend) + registry = create_application_registry() + plugin_manager.register(registry) + client_cls = registry.get(ComponentEnum.CLIENT, backend) if client_cls is None: raise ValueError(f"Unknown client backend: {backend!r}") async with client_cls(**client_kwargs) as client: diff --git a/reme/schema/__init__.py b/reme/schema/__init__.py index 9285c802..c6e0e1f2 100644 --- a/reme/schema/__init__.py +++ b/reme/schema/__init__.py @@ -1,7 +1,6 @@ """Schema""" from .application_config import ApplicationConfig, ComponentConfig, JobConfig -from .auto_fin import AutoFinReportOutput, AutoFinTopicOutput from .daily_paper import ( AnalyzedPaper, DailyPaperMarkdownOutput, @@ -32,8 +31,6 @@ from .traverse_graph import TraverseGraph, TraverseGraphEdge, TraverseGraphNode __all__ = [ "ApplicationConfig", - "AutoFinReportOutput", - "AutoFinTopicOutput", "ComponentConfig", "AnalyzedPaper", "DailyPaperMarkdownOutput", diff --git a/reme/schema/application_config.py b/reme/schema/application_config.py index 5f008ce6..0800ef30 100644 --- a/reme/schema/application_config.py +++ b/reme/schema/application_config.py @@ -6,7 +6,7 @@ from pathlib import PurePosixPath, PureWindowsPath from pydantic import BaseModel, ConfigDict, Field, field_validator -from ..enumeration import ComponentEnum +from ..enumeration import component_type_name class ComponentConfig(BaseModel): @@ -52,14 +52,25 @@ class ApplicationConfig(BaseModel): log_to_console: bool = Field(default=True, description="Log to console") log_to_file: bool = Field(default=True, description="Log to file") mcp_servers: dict[str, dict] = Field(default_factory=dict, description="MCP server configs by name") + plugins: list[str] = Field(default_factory=list, description="Installed plugins enabled for this app") service: ComponentConfig = Field(default_factory=ComponentConfig, description="Service endpoint config") jobs: dict[str, JobConfig] = Field(default_factory=dict, description="Job definitions keyed by job name") thread_pool_max_workers: int = Field(default=0, description="Max worker threads; 0 to disable") - components: dict[ComponentEnum, dict[str, ComponentConfig]] = Field( + components: dict[str, dict[str, ComponentConfig]] = Field( default_factory=dict, description="Component registry keyed by type then name", ) + @field_validator("components", mode="before") + @classmethod + def normalize_component_types(cls, value): + """Canonicalize built-in enums and allow plugin-defined component type names.""" + if value is None: + return {} + if not isinstance(value, dict): + return value + return {component_type_name(component_type): group for component_type, group in value.items()} + @field_validator("workspace_dir", mode="before") @classmethod def normalize_workspace_dir(cls, value) -> str: diff --git a/reme/steps/base_step.py b/reme/steps/base_step.py index 2baa1fc9..da9bce91 100644 --- a/reme/steps/base_step.py +++ b/reme/steps/base_step.py @@ -201,7 +201,8 @@ class BaseStep(ComponentMixin, ABC): backend = params.get("backend", "") if not backend: raise ValueError("Dispatch step is missing the required 'backend' field") - step_cls = R.get(ComponentEnum.STEP, backend) + registry = self.app_context.registry if self.app_context is not None else R + step_cls = registry.get(ComponentEnum.STEP, backend) if step_cls is None: raise RuntimeError(f"Unregistered step '{backend}'") params["app_context"] = self.app_context diff --git a/reme/steps/cookbook/__init__.py b/reme/steps/cookbook/__init__.py index 782776c2..77dc8bd5 100644 --- a/reme/steps/cookbook/__init__.py +++ b/reme/steps/cookbook/__init__.py @@ -1,5 +1,5 @@ """Optional, end-to-end cookbook workflows.""" -from . import auto_fin, daily_paper, dingtalk +from . import daily_paper, dingtalk -__all__ = ["auto_fin", "daily_paper", "dingtalk"] +__all__ = ["daily_paper", "dingtalk"] diff --git a/reme/steps/cookbook/auto_fin/__init__.py b/reme/steps/cookbook/auto_fin/__init__.py deleted file mode 100644 index 908dd635..00000000 --- a/reme/steps/cookbook/auto_fin/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Auto Fin news research workflow.""" - -from .data import AutoFinDataStep -from .merge import AutoFinMergeStep -from .topic import AutoFinTopicStep - -__all__ = [ - "AutoFinDataStep", - "AutoFinMergeStep", - "AutoFinTopicStep", -] diff --git a/reme/utils/__init__.py b/reme/utils/__init__.py index 2f7570fd..60b68968 100644 --- a/reme/utils/__init__.py +++ b/reme/utils/__init__.py @@ -12,7 +12,14 @@ from .link_expansion import expand_links, render_expansion_lines from .line_anchor import format_line_anchor, parse_line_anchor from .logger_utils import get_logger from .logo_utils import print_logo -from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme, running_service_config +from .service_utils import ( + cli_find_reme, + find_reme, + locate_reme, + precheck_start, + running_app_config, + running_service_config, +) from .similarity_utils import cosine_similarity, batch_cosine_similarity from .token_utils import estimate_token_count from .web_static import REME_WEB_STATIC_DIR, resolve_web_static_dir @@ -43,6 +50,7 @@ __all__ = [ "locate_reme", "precheck_start", "cli_find_reme", + "running_app_config", "running_service_config", "cosine_similarity", "batch_cosine_similarity", diff --git a/reme/utils/service_utils.py b/reme/utils/service_utils.py index d7edb8fe..6b44b838 100644 --- a/reme/utils/service_utils.py +++ b/reme/utils/service_utils.py @@ -88,13 +88,12 @@ def _reme_start_argv() -> list[list[str]]: return argvs -def running_service_config() -> dict | None: - """Resolve the ``service`` config of a running reme by replaying its start args. +def running_app_config() -> dict | None: + """Resolve the full config of a running ReMe by replaying its start args. Reads the live ``reme start ...`` process cmdline and re-runs the same - ``resolve_app_config`` the server used, so the result matches the running - server's real backend/transport/host/port even when those were passed on the - command line and are absent from (or differ from) the on-disk config file. + ``resolve_app_config`` the server used. Returning the full config preserves + enabled plugins as well as service connection settings for CLI clients. Returns ``None`` when no running reme is found or its args can't be parsed. """ from ..config import parse_args, resolve_app_config @@ -102,14 +101,23 @@ def running_service_config() -> dict | None: for argv in _reme_start_argv(): try: _, kwargs = parse_args("start", *argv) + config = resolve_app_config(log_config=False, **kwargs) except ValueError: continue - service = resolve_app_config(log_config=False, **kwargs).get("service") - if isinstance(service, dict): - return service + if isinstance(config, dict): + return config return None +def running_service_config() -> dict | None: + """Return only the service section of the running ReMe configuration.""" + config = running_app_config() + if config is None: + return None + service = config.get("service") + return service if isinstance(service, dict) else None + + async def locate_reme() -> tuple[str, int, int | None] | None: """Find a running reme: try default port, then scanned processes.""" if await find_reme(REME_DEFAULT_HOST, REME_DEFAULT_PORT) == "reme": diff --git a/tests/unit/test_application_config.py b/tests/unit/test_application_config.py index 69272d9a..3feed8a8 100644 --- a/tests/unit/test_application_config.py +++ b/tests/unit/test_application_config.py @@ -26,6 +26,34 @@ def test_workspace_dir_expands_user_home(monkeypatch, tmp_path): assert config.workspace_dir == str(tmp_path / ".copaw/workspaces/default") +def test_application_config_accepts_plugin_names_only(): + """Plugin configuration remains a simple list of installed entry-point names.""" + config = ApplicationConfig(plugins=["example"]) + + assert config.plugins == ["example"] + with pytest.raises(ValidationError): + ApplicationConfig(plugins=[{"name": "example"}]) + + +def test_application_config_accepts_plugin_defined_component_type(): + """Plugin component type names remain typed configuration buckets.""" + config = ApplicationConfig( + components={ + "example.reranker": { + "default": {"backend": "cross_encoder"}, + }, + }, + ) + + assert config.components["example.reranker"]["default"].backend == "cross_encoder" + + +def test_application_config_rejects_unsafe_component_type(): + """Component type names cannot escape their workspace metadata directory.""" + with pytest.raises(ValidationError, match="Invalid component type"): + ApplicationConfig(components={"../outside": {"default": {"backend": "unsafe"}}}) + + def test_dialog_dir_is_not_an_application_config_field(): """The removed option is absent from schemas and ignored when supplied.""" custom = ApplicationConfig(session_dir="sessions/", dialog_dir="somewhere/else") diff --git a/tests/unit/test_background_steps.py b/tests/unit/test_background_steps.py index 2d515ae7..4a326378 100644 --- a/tests/unit/test_background_steps.py +++ b/tests/unit/test_background_steps.py @@ -23,6 +23,7 @@ import numpy as np import pytest from watchfiles import Change +from reme.components import R from reme.components.agent_wrapper import BaseAgentWrapper from reme.components.file_chunker import DefaultFileChunker from reme.components.file_catalog import LocalFileCatalog @@ -115,6 +116,7 @@ def _make_app_context(workspace_path: Path, daily_dir="daily", digest_dir="diges ctx.app_config.resource_dir = resource_dir ctx.app_config.session_dir = "session" ctx.app_config.timezone = None + ctx.registry = R.copy() return ctx diff --git a/tests/unit/test_base_component.py b/tests/unit/test_base_component.py index 27ac4dd8..e510f032 100644 --- a/tests/unit/test_base_component.py +++ b/tests/unit/test_base_component.py @@ -86,6 +86,16 @@ def test_bind_rejects_no_component_type(): BaseComponent.bind("x", NoType) +def test_bind_accepts_plugin_defined_component_type(): + class PluginTarget(BaseComponent): + component_type = "example.reranker" + + result = BaseComponent.bind("default", PluginTarget, optional=False) + + assert isinstance(result, Dependency) + assert result.ctype == "example.reranker" + + def test_bind_with_default_factory(): def factory(): return DepTarget(name="default") diff --git a/tests/unit/test_component_registry.py b/tests/unit/test_component_registry.py index d7cda5d1..cf46ff5d 100644 --- a/tests/unit/test_component_registry.py +++ b/tests/unit/test_component_registry.py @@ -5,7 +5,7 @@ import pytest from reme.components.base_component import BaseComponent -from reme.components.component_registry import ComponentRegistry +from reme.components.component_registry import ComponentRegistry, R, create_application_registry from reme.enumeration import ComponentEnum @@ -52,10 +52,32 @@ def test_register_decorator(): def test_register_rejects_missing_component_type(): reg = ComponentRegistry() - with pytest.raises(TypeError, match="ComponentEnum"): + with pytest.raises(TypeError, match="component_type"): reg.register(_NoComponentType, "bad") +def test_register_plugin_defined_component_type(): + reg = ComponentRegistry() + + class PluginComponent(BaseComponent): + component_type = "example.reranker" + + reg.register(PluginComponent, "cross_encoder") + + assert reg.get("example.reranker", "cross_encoder") is PluginComponent + assert reg.get_all("example.reranker") == {"cross_encoder": PluginComponent} + + +def test_register_rejects_unsafe_plugin_component_type(): + reg = ComponentRegistry() + + class UnsafePluginComponent(BaseComponent): + component_type = "../outside" + + with pytest.raises(TypeError, match="component_type"): + reg.register(UnsafePluginComponent, "unsafe") + + def test_register_rejects_empty_name(): reg = ComponentRegistry() with pytest.raises(ValueError, match="empty"): @@ -127,6 +149,21 @@ def test_clear(): assert not reg.get_all(ComponentEnum.KEYWORD_INDEX) +def test_builtin_registry_is_frozen_after_package_bootstrap(): + assert R.frozen is True + with pytest.raises(RuntimeError, match="frozen"): + R.register(_DummyComponent, "runtime-mutation") + + +def test_application_registry_copy_remains_mutable(): + reg = create_application_registry() + + assert reg.frozen is False + reg.register(_DummyComponent, "runtime-component") + assert reg.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is _DummyComponent + assert R.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is None + + if __name__ == "__main__": print("\n=== ComponentRegistry Tests ===") test_register_direct_with_explicit_name() diff --git a/tests/unit/test_config_parser.py b/tests/unit/test_config_parser.py index 8ea287b5..ecf2b68a 100644 --- a/tests/unit/test_config_parser.py +++ b/tests/unit/test_config_parser.py @@ -21,6 +21,37 @@ def test_load_builtin_config_by_filename_with_suffix(): assert cfg["service"]["backend"] == "http" +def test_builtin_and_external_config_name_collision_fails(monkeypatch): + """An installed config cannot be silently shadowed by a built-in name.""" + + class FakeEntryPoint: + """Installed config entry point with a built-in name.""" + + name = "default" + value = "example:CONFIG_PATH" + + @staticmethod + def load(): + """The provider need not be imported to detect the collision.""" + raise AssertionError("colliding provider should not be loaded") + + class FakeEntryPoints(list): + """Minimal selectable entry-point collection.""" + + def select(self, *, group, name): + """Return entries matching the requested group and name.""" + assert group == "reme.configs" + return [entry for entry in self if entry.name == name] + + monkeypatch.setattr( + "reme.config.config_parser.metadata.entry_points", + lambda: FakeEntryPoints([FakeEntryPoint()]), + ) + + with pytest.raises(ValueError, match="provided by both ReMe and an installed distribution"): + _load_config("default") + + def test_resolve_app_config_can_suppress_config_log(monkeypatch): """Client-side config resolution can avoid polluting command output.""" messages = [] diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py index b1749975..8686e503 100644 --- a/tests/unit/test_job.py +++ b/tests/unit/test_job.py @@ -47,7 +47,7 @@ def test_resolve_step_missing_backend(): def test_resolve_step_unregistered_backend(): job = BaseJob(name="j") - job.app_context = MagicMock() + job.app_context = SimpleNamespace(registry=ComponentRegistry()) with pytest.raises(ValueError, match="Unregistered backend"): job._resolve_step(ComponentConfig(backend="nonexistent_step")) diff --git a/tests/unit/test_package_versions.py b/tests/unit/test_package_versions.py index 5b69038d..25efb0ed 100644 --- a/tests/unit/test_package_versions.py +++ b/tests/unit/test_package_versions.py @@ -5,6 +5,7 @@ from pathlib import Path import tomllib from types import ModuleType +from packaging.requirements import Requirement import pytest REPOSITORY = Path(__file__).resolve().parents[2] @@ -181,6 +182,23 @@ def test_studio_package_preparation_copies_license(monkeypatch, tmp_path: Path) ) +def test_auto_fin_license_matches_repository() -> None: + """Keep the independently distributed Auto Fin license complete and current.""" + assert (REPOSITORY / "plugin" / "auto-fin" / "LICENSE").read_text(encoding="utf-8") == ( + REPOSITORY / "LICENSE" + ).read_text(encoding="utf-8") + + +def test_auto_fin_requires_reme_core() -> None: + """Install the optional runtime packages needed while loading Auto Fin's entry points.""" + config = tomllib.loads((REPOSITORY / "plugin" / "auto-fin" / "pyproject.toml").read_text(encoding="utf-8")) + requirements = [Requirement(value) for value in config["project"]["dependencies"]] + reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"] + + assert len(reme_requirements) == 1 + assert set(reme_requirements[0].extras) == {"core"} + + def test_studio_package_preparation_preserves_static_gitignore(monkeypatch, tmp_path: Path) -> None: """Keep generated static assets ignored after staging the Studio build.""" package_dir = tmp_path / "reme_ai_studio" diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py new file mode 100644 index 00000000..9a3845bd --- /dev/null +++ b/tests/unit/test_plugin.py @@ -0,0 +1,238 @@ +"""Tests for installed plugin discovery and application-local registration.""" + +# pylint: disable=missing-class-docstring,missing-function-docstring + +from importlib.metadata import EntryPoint +from pathlib import Path + +import pytest + +from reme.application import Application +from reme.components.base_component import BaseComponent, ComponentMixin +from reme.components.component_registry import ComponentRegistry, R +from reme.config.config_parser import _load_config +from reme.enumeration import ComponentEnum +from reme.plugin import Backend, Plugin, PluginManager + + +class _PluginStep(ComponentMixin): + component_type = ComponentEnum.STEP + + +class _PluginComponent(BaseComponent): + component_type = "example.reranker" + + +def test_plugin_defaults_are_below_application_config(): + manager = PluginManager([Plugin(name="example", config={"jobs": {"task": {"backend": "base", "value": 1}}})]) + + merged = manager.merge_config({"jobs": {"task": {"value": 2}}}) + + assert merged["jobs"]["task"] == {"backend": "base", "value": 2} + + +def test_plugin_defaults_expand_environment(monkeypatch): + monkeypatch.setenv("PLUGIN_LIMIT", "12") + manager = PluginManager([Plugin(name="example", config={"limit": "${PLUGIN_LIMIT}"})]) + + assert manager.merge_config({})["limit"] == 12 + + +def test_plugin_registers_into_only_the_supplied_registry(): + manager = PluginManager([Plugin(name="example", backends=(Backend("example_step", _PluginStep),))]) + first = ComponentRegistry() + second = ComponentRegistry() + + manager.register(first) + + assert first.get(ComponentEnum.STEP, "example_step") is _PluginStep + assert second.get(ComponentEnum.STEP, "example_step") is None + + +@pytest.mark.asyncio +async def test_plugin_registers_and_runs_custom_component_type(monkeypatch, tmp_path): + manager = PluginManager( + [ + Plugin( + name="example", + backends=(Backend("cross_encoder", _PluginComponent),), + config={ + "components": { + "example.reranker": { + "default": {"backend": "cross_encoder"}, + }, + }, + }, + ), + ], + ) + monkeypatch.setattr(PluginManager, "discover", classmethod(lambda cls, specs: manager)) + + app = Application( + plugins=["example"], + workspace_dir=str(tmp_path), + enable_logo=False, + log_to_console=False, + log_to_file=False, + service={"backend": "cli"}, + ) + + component = app.context.components["example.reranker"]["default"] + assert isinstance(component, _PluginComponent) + assert app.context.registry.get("example.reranker", "cross_encoder") is _PluginComponent + + await app.start() + assert component.is_started is True + await app.update_component("example.reranker", "default", backend="updated") + assert component.backend == "updated" + await app.close() + assert component.is_started is False + + +def test_plugin_backend_collision_fails_with_both_owners(): + registry = ComponentRegistry() + registry.add("same", _PluginStep, owner="first") + + class OtherStep(ComponentMixin): + component_type = ComponentEnum.STEP + + with pytest.raises(ValueError, match="both 'first' and 'second'"): + registry.add("same", OtherStep, owner="second") + + +def test_plugin_manager_loads_explicit_entry_point(monkeypatch): + descriptor = Plugin(name="example", backends=(Backend("example_step", _PluginStep),)) + + class FakeEntryPoint: + name = "example" + value = "example:plugin" + + @staticmethod + def load(): + return descriptor + + class FakeEntryPoints(list): + def select(self, *, group, name): + assert group == "reme.plugins" + return [entry for entry in self if entry.name == name] + + monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()])) + + manager = PluginManager.discover(["example"]) + + assert manager.plugins == (descriptor,) + + +def test_plugin_entry_point_import_side_effect_does_not_leak(monkeypatch, tmp_path): + class UndeclaredClient(ComponentMixin): + component_type = ComponentEnum.CLIENT + + descriptor = Plugin(name="example") + + class FakeEntryPoint: + name = "example" + value = "example:plugin" + + @staticmethod + def load(): + R.register(UndeclaredClient, "undeclared-client") + return descriptor + + class FakeEntryPoints(list): + def select(self, *, group, name): + assert group == "reme.plugins" + return [entry for entry in self if entry.name == name] + + monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()])) + + app = Application( + plugins=["example"], + workspace_dir=str(tmp_path), + enable_logo=False, + log_to_console=False, + log_to_file=False, + service={"backend": "cli"}, + ) + + assert R.get(ComponentEnum.CLIENT, "undeclared-client") is None + assert app.context.registry.get(ComponentEnum.CLIENT, "undeclared-client") is None + + +def test_plugin_manager_rejects_non_string_name(): + with pytest.raises(TypeError, match="Invalid plugin name"): + PluginManager.discover([{"name": "example"}]) + + +def test_config_can_extend_another_config(tmp_path: Path): + parent = tmp_path / "parent.yaml" + child = tmp_path / "child.yaml" + parent.write_text("service:\n backend: http\n port: 8000\n", encoding="utf-8") + child.write_text("extends: parent.yaml\nservice:\n port: 9000\n", encoding="utf-8") + + assert _load_config(str(child))["service"] == {"backend": "http", "port": 9000} + + +def test_config_can_come_from_installed_entry_point(tmp_path: Path, monkeypatch): + config = tmp_path / "example.yaml" + config.write_text("plugins: [example]\n", encoding="utf-8") + entry = EntryPoint(name="example", value="pathlib:Path", group="reme.configs") + + class LoadedEntryPoint: + name = entry.name + value = entry.value + + @staticmethod + def load(): + return config + + class FakeEntryPoints(list): + def select(self, *, group, name): + assert group == "reme.configs" + return [item for item in self if item.name == name] + + monkeypatch.setattr( + "reme.config.config_parser.metadata.entry_points", + lambda: FakeEntryPoints([LoadedEntryPoint()]), + ) + + assert _load_config("example") == {"plugins": ["example"]} + + +def test_config_entry_point_import_side_effect_does_not_leak(tmp_path: Path, monkeypatch): + config = tmp_path / "side-effect.yaml" + config.write_text("service:\n backend: cli\n", encoding="utf-8") + + class UndeclaredClient(ComponentMixin): + component_type = ComponentEnum.CLIENT + + class LoadedEntryPoint: + name = "side-effect" + value = "example:CONFIG_PATH" + + @staticmethod + def load(): + R.register(UndeclaredClient, "config-side-effect-client") + return config + + class FakeEntryPoints(list): + def select(self, *, group, name): + assert group == "reme.configs" + return [item for item in self if item.name == name] + + monkeypatch.setattr( + "reme.config.config_parser.metadata.entry_points", + lambda: FakeEntryPoints([LoadedEntryPoint()]), + ) + + loaded = _load_config("side-effect") + app = Application( + **loaded, + workspace_dir=str(tmp_path / "workspace"), + enable_logo=False, + log_to_console=False, + log_to_file=False, + ) + + assert loaded == {"service": {"backend": "cli"}} + assert R.get(ComponentEnum.CLIENT, "config-side-effect-client") is None + assert app.context.registry.get(ComponentEnum.CLIENT, "config-side-effect-client") is None diff --git a/tests/unit/test_reme_cli.py b/tests/unit/test_reme_cli.py index ecc1e0ce..f4dfbda0 100644 --- a/tests/unit/test_reme_cli.py +++ b/tests/unit/test_reme_cli.py @@ -11,6 +11,9 @@ import pytest from reme.components.service import cli_service from reme.components.service.cli_service import CliService from reme import reme as reme_module +from reme.components.base_component import ComponentMixin +from reme.enumeration import ComponentEnum +from reme.plugin import Backend, Plugin, PluginManager def test_package_import_does_not_load_optional_core_dependencies(): @@ -257,8 +260,12 @@ def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys): seen["payload"] = kwargs yield "ok" - monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient) - monkeypatch.setattr(reme_module, "running_service_config", lambda: None) + monkeypatch.setattr( + reme_module, + "create_application_registry", + lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient), + ) + monkeypatch.setattr(reme_module, "running_app_config", lambda: None) async def run(): await reme_module.call_server( @@ -299,8 +306,12 @@ def test_call_server_treats_show_metadata_as_client_kwarg(monkeypatch, capsys): seen["payload"] = kwargs yield "ok" - monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient) - monkeypatch.setattr(reme_module, "running_service_config", lambda: None) + monkeypatch.setattr( + reme_module, + "create_application_registry", + lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient), + ) + monkeypatch.setattr(reme_module, "running_app_config", lambda: None) async def run(): await reme_module.call_server("version", backend="http", show_metadata=True) @@ -334,8 +345,12 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys): seen["payload"] = kwargs yield "ok" - monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient) - monkeypatch.setattr(reme_module, "running_service_config", lambda: None) + monkeypatch.setattr( + reme_module, + "create_application_registry", + lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient), + ) + monkeypatch.setattr(reme_module, "running_app_config", lambda: None) async def run(): await reme_module.call_server("shell", backend="http", cmd="ls", shell_timeout=5) @@ -345,3 +360,86 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys): assert seen["action"] == "shell" assert seen["payload"] == {"cmd": "ls", "shell_timeout": 5} assert capsys.readouterr().out == "ok\n" + + +def test_call_server_uses_running_plugins_and_their_service_defaults(monkeypatch, capsys): + """A bare client call can load the Client backend enabled by the running app.""" + seen = {} + + class PluginClient(ComponentMixin): + """Client backend supplied by an enabled plugin.""" + + component_type = ComponentEnum.CLIENT + + def __init__(self, **kwargs): + super().__init__(**kwargs) + seen["client_kwargs"] = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + return None + + async def __call__(self, action: str, **kwargs): + seen["action"] = action + seen["payload"] = kwargs + yield "plugin-ok" + + manager = PluginManager( + [ + Plugin( + name="example", + backends=(Backend("plugin-client", PluginClient),), + config={"service": {"backend": "plugin-client", "host": "127.0.0.9", "port": 9911}}, + ), + ], + ) + monkeypatch.setattr(reme_module, "resolve_app_config", lambda **_kwargs: {"service": {"backend": "http"}}) + monkeypatch.setattr(reme_module, "running_app_config", lambda: {"plugins": ["example"]}) + monkeypatch.setattr(reme_module.PluginManager, "discover", lambda _specs: manager) + + asyncio.run(reme_module.call_server("search", query="hello")) + + assert seen["client_kwargs"]["host"] == "127.0.0.9" + assert seen["client_kwargs"]["port"] == 9911 + assert seen["action"] == "search" + assert seen["payload"] == {"query": "hello"} + assert capsys.readouterr().out == "plugin-ok\n" + + +def test_call_server_skips_local_fallback_when_server_is_running(monkeypatch, capsys): + """A usable running config prevents eager parsing of the local fallback.""" + + class FakeClient: + """Minimal running-service client.""" + + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + return None + + async def __call__(self, action: str, **kwargs): + assert action == "version" + assert not kwargs + yield "ok" + + monkeypatch.setattr( + reme_module, + "create_application_registry", + lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient), + ) + monkeypatch.setattr(reme_module, "running_app_config", lambda: {"service": {"backend": "http"}}) + + def fail_local_resolution(**_kwargs): + raise AssertionError("local fallback should not be resolved") + + monkeypatch.setattr(reme_module, "resolve_app_config", fail_local_resolution) + + asyncio.run(reme_module.call_server("version")) + + assert capsys.readouterr().out == "ok\n" diff --git a/tests/unit/test_service_utils.py b/tests/unit/test_service_utils.py index 15bdb1f0..b61a1fca 100644 --- a/tests/unit/test_service_utils.py +++ b/tests/unit/test_service_utils.py @@ -156,3 +156,22 @@ def test_scan_reme_procs_skips_access_denied(monkeypatch): ] _patch_iter(monkeypatch, procs) assert su._scan_reme_procs() == [(5, su.REME_DEFAULT_HOST, su.REME_DEFAULT_PORT)] + + +def test_running_app_config_preserves_plugins(monkeypatch): + """Process replay exposes the full config while the compatibility helper returns service only.""" + monkeypatch.setattr(su, "_reme_start_argv", lambda: [["config=example"]]) + monkeypatch.setattr("reme.config.parse_args", lambda *_args: ("start", {"config": "example"})) + monkeypatch.setattr( + "reme.config.resolve_app_config", + lambda **_kwargs: { + "plugins": ["example"], + "service": {"backend": "plugin-client", "port": 9911}, + }, + ) + + assert su.running_app_config() == { + "plugins": ["example"], + "service": {"backend": "plugin-client", "port": 9911}, + } + assert su.running_service_config() == {"backend": "plugin-client", "port": 9911}