feat: add entry-point plugin system and extract Auto Fin (#459)

* feat: add entry-point plugin system

* fix: harden plugin config and client loading

* docs(workflow): add detailed manual for publishing reme-auto-fin to PyPI

- Provide step-by-step instructions for updating project.version and merging branches
- Explain dependency verification for reme-ai on PyPI during build
- Specify requirements for GitHub Actions secret configuration and version uniqueness
- Describe manual workflow triggering and input of version number
- Recommend publishing order for related projects
- Clarify that only manual dispatch triggers publishing, no automatic triggers on push or tag

* feat: support plugin-defined component types

* refactor: simplify plugin configuration

* fix: isolate plugin loading and defer client fallback

* refactor: freeze built-in component registry

* fix: isolate config entry point loading

* fix: complete auto-fin package metadata
This commit is contained in:
jinliyl 2026-08-19 17:23:23 +08:00 committed by GitHub
parent d3aee1adf5
commit 618e8cec66
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
61 changed files with 1474 additions and 275 deletions

138
.github/workflows/auto-fin-publish.yml vendored Normal file
View file

@ -0,0 +1,138 @@
# 发布操作手册:
# 1. 先将 plugin/auto-fin/pyproject.toml 中的 project.version 更新为待发布版本并合入目标分支。
# 2. 确认插件依赖的 reme-ai 版本已经发布到 PyPI本工作流会在构建阶段验证该依赖可下载。
# 3. 确认仓库 Actions Secret 已配置 PYPI_API_TOKEN且 PyPI 上不存在相同版本。
# 4. 在 GitHub 仓库的 Actions 页面选择“Publish reme-auto-fin to PyPI”点击“Run workflow”。
# 5. 输入与 project.version 完全一致的版本号(例如 0.1.0)后运行;版本也可以带 v 前缀。
#
# 推荐发布顺序reme-ai -> reme-auto-fin -> QwenPaw 更新依赖并通过 plugins: [auto-fin] 启用。
# 当前仅支持 workflow_dispatch 手动触发,不会因 push、tag 或 release 自动发布。
name: Publish reme-auto-fin to PyPI
run-name: Publish reme-auto-fin ${{ inputs.version }}
on:
workflow_dispatch:
inputs:
version:
description: Version from plugin/auto-fin/pyproject.toml (for example, 0.1.0)
required: true
type: string
permissions:
contents: read
concurrency:
group: publish-reme-auto-fin
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
env:
RELEASE_VERSION: ${{ inputs.version }}
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install test and build dependencies
run: |
python -m pip install --upgrade pip
python -m pip install build packaging pytest pytest-asyncio twine
python -m pip install -e ".[core]"
python -m pip install --no-deps -e plugin/auto-fin
- name: Validate package name and release version
id: package
run: |
python - "${RELEASE_VERSION}" <<'PY'
import os
import sys
import tomllib
from pathlib import Path
from packaging.requirements import Requirement
from packaging.version import Version
project = tomllib.loads(Path("plugin/auto-fin/pyproject.toml").read_text(encoding="utf-8"))["project"]
expected = Version(sys.argv[1].removeprefix("v"))
actual = Version(project["version"])
if project["name"] != "reme-auto-fin":
raise SystemExit(f"Expected project name 'reme-auto-fin', found {project['name']!r}")
if actual != expected:
raise SystemExit(f"Package version is {actual}, but workflow input is {expected}")
requirements = [requirement for requirement in project["dependencies"] if requirement.startswith("reme-ai")]
if len(requirements) != 1:
raise SystemExit(f"Expected one reme-ai dependency, found {requirements!r}")
reme_requirement = Requirement(requirements[0])
if reme_requirement.name != "reme-ai" or set(reme_requirement.extras) != {"core"}:
raise SystemExit(f"Expected a reme-ai[core] dependency, found {requirements[0]!r}")
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
print(f"reme_requirement={reme_requirement}", file=output)
print(f"Publishing {project['name']} {actual}")
PY
- name: Run Auto Fin tests
run: python -m pytest plugin/auto-fin -q
- name: Require the plugin-enabled ReMe release on PyPI
run: |
python -m pip download --no-deps \
--dest "${RUNNER_TEMP}/reme-auto-fin-core" \
"${{ steps.package.outputs.reme_requirement }}"
- name: Build and check distributions
run: |
mkdir -p dist/auto-fin
python -m build plugin/auto-fin --outdir dist/auto-fin
python -m twine check dist/auto-fin/*
- name: Verify distributions and isolated installation
run: |
AUTO_FIN_WHEEL="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.whl)"
AUTO_FIN_SDIST="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.tar.gz)"
python -m zipfile -l "${AUTO_FIN_WHEEL}" | grep 'dist-info/licenses/LICENSE'
python -m tarfile -l "${AUTO_FIN_SDIST}" | grep '/LICENSE'
python -m venv "${RUNNER_TEMP}/reme-auto-fin-smoke"
"${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" -m pip install "${AUTO_FIN_WHEEL}"
cd "${RUNNER_TEMP}"
"${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" - <<'PY'
from importlib.metadata import distribution
package = distribution("reme-auto-fin")
plugins = {entry.name: entry for entry in package.entry_points if entry.group == "reme.plugins"}
configs = {entry.name: entry for entry in package.entry_points if entry.group == "reme.configs"}
assert plugins["auto-fin"].load().name == "auto-fin"
assert configs["auto-fin"].load().is_file()
PY
- name: Upload distributions
uses: actions/upload-artifact@v4
with:
name: reme-auto-fin-${{ inputs.version }}
path: dist/auto-fin/
if-no-files-found: error
publish:
needs: build
runs-on: ubuntu-latest
steps:
- name: Download distributions
uses: actions/download-artifact@v4
with:
name: reme-auto-fin-${{ inputs.version }}
path: dist/auto-fin
- name: Publish reme-auto-fin
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
packages-dir: dist/auto-fin

View file

@ -12,7 +12,7 @@ on:
- 'github-pages/**'
- 'website/README*.md'
- 'website/public/og.jpg'
- 'cookbook/*/README*.md'
- 'plugin/*/README*.md'
- 'benchmark/*/README*.md'
- 'skills/reme_memory/SKILL.md'
pull_request:
@ -26,7 +26,7 @@ on:
- 'github-pages/**'
- 'website/README*.md'
- 'website/public/og.jpg'
- 'cookbook/*/README*.md'
- 'plugin/*/README*.md'
- 'benchmark/*/README*.md'
- 'skills/reme_memory/SKILL.md'
workflow_dispatch:

View file

@ -10,7 +10,7 @@ on:
- "README_ZH.md"
- "website/README*.md"
- "website/public/og.jpg"
- "cookbook/*/README*.md"
- "plugin/*/README*.md"
- "benchmark/*/README*.md"
- "skills/reme_memory/SKILL.md"
- "AGENTS.md"

View file

@ -33,11 +33,12 @@ jobs:
run: |
python -m pip install --upgrade pip setuptools wheel
pip install -e packages/reme_ai_studio -e ".[dev,core]"
pip install --no-deps -e plugin/auto-fin
pip install coverage
- name: Run unit tests
run: |
coverage run -m pytest tests/unit \
coverage run -m pytest tests/unit plugin/auto-fin \
-v \
--tb=long \
-s \

View file

@ -41,7 +41,7 @@ and concise documentation together.
- `reme/components/application_context.py`: application-wide wiring and in-memory shared state.
- `reme/components/runtime_context.py`: request-scoped data, response, streaming queue, and stop event.
- `reme/components/base_component.py`: component lifecycle, dependency binding, and workspace helpers.
- `reme/components/component_registry.py`: the process-wide `(component type, backend)` registry.
- `reme/components/component_registry.py`: the frozen built-in registry template and application-local registry factory.
- `reme/components/job/`: base, stream, background, and cron job implementations.
- `reme/components/service/`: local CLI, HTTP, and MCP service backends.
- `reme/components/`: agent wrappers, model adapters, stores, catalogs, graphs, indexes, clients, tokenizers, and
@ -56,7 +56,7 @@ and concise documentation together.
- `plugins/claude_code/` and `plugins/hermes_agent/`: agent integrations.
- `skills/`: standalone skills; `reme_memory` calls ReMe, while other skills may use separate tools or direct-file
conventions.
- `benchmark/` and `cookbook/`: runnable evaluation and example workflows.
- `benchmark/` and `plugin/`: runnable evaluations and external plugin examples.
- `docs/`: README-linked supporting pages and figures.
## Development Setup

View file

@ -200,13 +200,13 @@ These Markdown guides cover the main user workflows and the runtime contracts im
| [Framework](docs/en/framework.md) | Understand Application, Job, Step, Component, service, configuration, and lifecycle boundaries. |
| [ReMe Blog](https://agentscope-ai.github.io/ReMe/?doc=en-reme-blog) | Read the product story, design rationale, examples, and benchmark summary. |
## 🧑‍🍳 Cookbooks
## 🔌 Plugins
Cookbooks are optional, end-to-end workflows assembled from ReMe jobs and steps. They are not enabled by the default
configuration; select the cookbook's standalone configuration when starting ReMe. Each new cookbook will be added as
another row in this table.
Plugins are optional Python distributions that contribute Component, Step, or Job backends and configuration. They are
installed separately and enabled explicitly by configuration. Auto Fin is the complete external-plugin example; Daily
Paper remains an optional research workflow while it is migrated to the same packaging model.
| Cookbook | Capability |
| Plugin / workflow | Capability |
|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------|
| [Daily Paper](https://reme.agentscope.io/?doc=daily-paper-en) | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. |
| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-en) | Fetch topic-related CLS news, search ReMe history, and generate wikilink-backed Markdown reports. |

View file

@ -193,12 +193,12 @@ ReMe 会把 Agent 记忆保存为可读的 Markdown。
| [框架说明](docs/zh/framework.md) | 理解 Application、Job、Step、Component、service、配置和生命周期边界。 |
| [ReMe 博客](https://agentscope-ai.github.io/ReMe/?doc=zh-reme-blog) | 了解完整产品故事、设计动机、使用示例和评测摘要。 |
## 🧑‍🍳 Cookbooks
## 🔌 插件
Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们;启动 ReMe 时选择对应的独立配置即可启用。后续新增的
cookbook 会继续在表格中按行追加
插件是可选的独立 Python distribution可以贡献 Component、Step、Job backend 和配置并通过配置显式启用。Auto Fin
是完整的外部插件示例;每日论文在迁移到同一打包模型前仍作为可选研究工作流提供
| Cookbook | 能力 |
| 插件 / 工作流 | 能力 |
|-----------------------------------------------|--------------------------------------------------------------------------------|
| [每日论文](https://reme.agentscope.io/?doc=daily-paper-zh) | 发现并排序论文,使用 Agent 解读 PDF生成文件化论文笔记和五分钟简报。 |
| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-zh) | 拉取主题相关财联社新闻,搜索 ReMe 历史材料并生成带 wikilink 的 Markdown 报告。 |

View file

@ -55,11 +55,12 @@ Core layers:
reme/
reme.py # CLI entry point
application.py # Application assembly and lifecycle
plugin.py # installed plugin contract and entry-point loader
config/
default.yaml # default service / jobs / components
config_parser.py # config=, dot notation, and env placeholder parsing
components/
component_registry.py # global registry R
component_registry.py # backend registry and application-local copies
base_component.py # ComponentMixin / BaseComponent / bind dependency declarations
runtime_context.py # context for one Job execution
job/ # BaseJob / StreamJob / BackgroundJob / CronJob
@ -81,6 +82,8 @@ reme/
index/ # watch/init/update/search/traverse
evolve/ # auto_memory, auto_resource, auto_dream, proactive
transfer/ # upload/download
plugin/
auto-fin/ # independent example plugin distribution
```
The default workspace directories are defined by `ApplicationConfig`:
@ -216,14 +219,23 @@ The registry key is:
The same backend name can therefore exist under different component types. For example, `http` can be both a service
backend and a client backend.
### 4.2 Registration Through Module Imports
`ComponentEnum` provides the built-in identifiers, but installed plugins may declare a new type with a namespaced
string such as `example.reranker`. Custom identifiers use lowercase letters and numbers separated by `.`, `_`, or `-`.
They are configured under `components` and participate in the same dependency ordering and lifecycle as built-ins.
Registration happens when a module is imported. `reme/components/__init__.py` imports component packages, while
`reme/steps/__init__.py` imports `benchmark/common/cookbook/evolve/file_io/index/transfer`. Each package's `__init__.py`
then imports its concrete modules, causing `@R.register(...)` to execute.
### 4.2 Built-in and Plugin Registration
After adding a Step file, make sure the package's `__init__.py` imports it. Otherwise, the backend will not appear in
the registry.
Built-in implementations populate the built-in registry through package imports. ReMe freezes that template after
bootstrap, and each `Application` receives a mutable copy. Runtime code resolves backends through the application's
registry rather than changing the process-wide template. ReMe then loads only the installed plugins explicitly named by
`plugins` in the resolved configuration. A plugin
is exposed through the `reme.plugins` Python entry-point group and returns a declarative `reme.plugin.Plugin` containing
its named backend classes and default configuration. Plugin registration therefore stays local to one application;
duplicate `(component_type, backend)` providers fail during assembly instead of overwriting each other.
Installed plugins may also expose named configuration files through `reme.configs`. Configuration can use `extends` to
inherit another built-in, plugin, or file-based configuration. The [Auto Fin plugin](../../plugin/auto-fin/README.md)
is the complete packaging example.
### 4.3 Component.bind

View file

@ -50,11 +50,12 @@ flowchart LR
reme/
reme.py # CLI 入口
application.py # Application 装配与生命周期
plugin.py # 已安装插件契约与 entry-point loader
config/
default.yaml # 默认 service / jobs / components
config_parser.py # config=、dot notation、env 占位符解析
components/
component_registry.py # 全局注册表 R
component_registry.py # backend 注册表及 Application 局部副本
base_component.py # ComponentMixin / BaseComponent / bind 依赖声明
runtime_context.py # 单次 Job 执行上下文
job/ # BaseJob / StreamJob / BackgroundJob / CronJob
@ -76,6 +77,8 @@ reme/
index/ # watch/init/update/search/traverse
evolve/ # auto_memory、auto_resource、auto_dream、proactive
transfer/ # upload/download
plugin/
auto-fin/ # 独立发布的示例插件
```
默认 workspace 目录由 `ApplicationConfig` 定义:
@ -208,13 +211,19 @@ class VersionStep(BaseStep):
所以同名 backend 在不同 component type 下可以共存。例如 `http` 同时可以是 service backend 和 client backend。
### 4.2 模块导入触发注册
`ComponentEnum` 提供内置类型标识;已安装插件也可以用 `example.reranker` 这样的命名空间字符串声明新类型。自定义标识仅使用
小写字母和数字,并以 `.``_``-` 分隔。它们配置在 `components` 下,与内置组件参与相同的依赖排序和生命周期。
注册发生在模块 import 时。`reme/components/__init__.py` 会 import 各组件包,`reme/steps/__init__.py` 会 import
`benchmark/common/cookbook/evolve/file_io/index/transfer`。这些包的 `__init__.py` 再 import 具体模块,从而执行
`@R.register(...)`
### 4.2 内置注册与插件注册
新增 Step 文件后,必须保证它所在包的 `__init__.py` 会 import 该模块,否则注册表里找不到这个 backend。
内置实现通过 package import 填充内置注册表bootstrap 完成后 ReMe 会冻结这个模板,并为每个 `Application` 创建可写副本。
运行期代码通过当前 Application 的注册表解析 backend不能修改进程级模板。随后只加载最终配置中 `plugins` 明确启用的已安装插件。
插件通过 Python `reme.plugins` entry-point group 暴露,返回声明式 `reme.plugin.Plugin`,其中包含命名
backend class 和默认配置。插件注册因此只影响当前 Application两个插件提供相同 `(component_type, backend)` 时会在装配阶段失败,
不会互相覆盖。
插件还可以通过 `reme.configs` 暴露命名配置;配置的 `extends` 可以继承内置配置、插件配置或文件配置。完整打包示例见
[Auto Fin 插件](../../plugin/auto-fin/README_ZH.md)。
### 4.3 Component.bind

View file

@ -51,7 +51,7 @@ The build script reads the canonical repository files directly. Do not edit gene
- `docs/en/` and `docs/zh/`: English and Chinese guides
- `docs/figure/`: documentation images
- `website/README.md` and `website/README_ZH.md`: ReMe Studio guide
- `cookbook/*/README*.md`: research workflow guides
- `plugin/*/README*.md`: plugin and research workflow guides
- `benchmark/{beam,longmemeval,pibench,toolmemory}/README*.md`: benchmark guides and results
- `skills/reme_memory/SKILL.md`: ReMe Memory skill guide
- `AGENTS.md`: repository development guide

View file

@ -64,7 +64,7 @@ const productDocuments = [
},
{
slug: "daily-paper",
source: "cookbook/daily_paper",
source: "plugin/daily_paper",
titles: { zh: "每日论文", en: "Daily Paper" },
descriptions: {
zh: "发现论文、解析 PDF并生成阅读笔记与每日简报。",
@ -74,7 +74,7 @@ const productDocuments = [
},
{
slug: "auto-fin",
source: "cookbook/auto-fin",
source: "plugin/auto-fin",
titles: { zh: "财经研究", en: "Auto Fin" },
descriptions: {
zh: "结合最新财联社新闻与本地历史记忆生成研究报告。",

201
plugin/auto-fin/LICENSE Normal file
View file

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2025 Alibaba Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View file

@ -1,12 +1,12 @@
# Auto Fin Cookbook
# Auto Fin Plugin
[中文](README_ZH.md)
Auto Fin fetches a rolling window of CLS telegraph news (24 hours by default), selects items related to configured
topics, searches ReMe for useful historical context, and writes one Chinese Markdown report with validated wikilinks.
Current news and topic selection stay in runtime memory; only the final report becomes durable memory. The
implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/) and is assembled by
[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml).
Current news and topic selection stay in runtime memory; only the final report becomes durable memory. This directory
is an independent Python distribution. Its `reme.plugins` entry point contributes the three Step backends and their Job
configuration; its `reme.configs` entry point exposes the runnable `auto-fin` configuration.
> Auto Fin has no reliable market-price feed. It does not calculate returns, targets, or entry points and is not
> investment advice.
@ -14,11 +14,11 @@ implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookb
## Quick start
```bash
python -m pip install -e ".[core]"
python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin
export LLM_API_KEY="your-api-key"
export LLM_MODEL_NAME="qwen3.7-plus"
export LLM_BASE_URL="https://your-provider.example/v1"
reme start config=daily_cookbook job=auto_fin
reme start config=auto-fin job=auto_fin
```
`LLM_MODEL_NAME` defaults to `qwen3.7-plus`. There is no built-in `LLM_BASE_URL`, so set the OpenAI-compatible endpoint
@ -27,7 +27,7 @@ required by the selected provider.
The default topics are `黄金,机器人,半导体`. Override them per run:
```bash
reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片"
reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片"
```
An empty value also uses the defaults.
@ -76,23 +76,22 @@ refreshes the daily index. No JSONL, intermediate Markdown, or structured Agent
| `request_interval` | `10` | Minimum delay in seconds after every CLS request attempt; may be zero |
| `max_retries` | `3` | Maximum attempts for each CLS page request; must be at least one |
The built-in schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`.
The plugin-provided schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`.
## Output
```text
reme_workspace/daily/YYYY-MM-DD/auto_fin.md
.reme/daily/YYYY-MM-DD/auto_fin.md
```
The report includes a title, description, current CLS evidence, historical analysis, contextual wikilinks, and a fixed
non-investment disclaimer. Network errors and invalid Agent output fail explicitly; no relevant current news is a
successful skip. If `DINGTALK_CONVERSATION_IDS` is empty, delivery is a no-op. If it is set, the DingTalk credentials
described in the [Daily Paper cookbook](../daily_paper/README.md#6-dingtalk) are required.
successful skip.
## Validation
```bash
pytest tests/unit/test_auto_fin.py -v
python -m pytest plugin/auto-fin -v
```
Unit tests mock the CLS and Agent boundaries and do not contact external services.

View file

@ -1,22 +1,22 @@
# Auto Fin Cookbook
# Auto Fin 插件
[English](README.md)
Auto Fin 自动拉取一个滚动时间窗口内的财联社电报(默认 24 小时),按配置 topics 筛选相关新闻,搜索 ReMe 中有回顾价值的历史材料,最后写入一份带校验
wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中,只有最终报告成为持久记忆。实现位于
[`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/),并由
[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配
wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中,只有最终报告成为持久记忆。本目录是一个独立 Python
distribution`reme.plugins` entry point 贡献三个 Step backend 及其 Job 配置,`reme.configs` entry point 暴露可直接运行的
`auto-fin` 配置
> Auto Fin 没有可靠行情数据,不计算收益、目标价或买卖点,也不提供投资建议。
## 快速开始
```bash
python -m pip install -e ".[core]"
python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin
export LLM_API_KEY="your-api-key"
export LLM_MODEL_NAME="qwen3.7-plus"
export LLM_BASE_URL="https://your-provider.example/v1"
reme start config=daily_cookbook job=auto_fin
reme start config=auto-fin job=auto_fin
```
`LLM_MODEL_NAME` 默认是 `qwen3.7-plus`。代码没有内置 `LLM_BASE_URL`,请设置所选服务商提供的 OpenAI 兼容 endpoint。
@ -24,7 +24,7 @@ reme start config=daily_cookbook job=auto_fin
默认 topics 是 `黄金,机器人,半导体`。可在运行时覆盖:
```bash
reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片"
reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片"
```
传入空值也会使用默认 topics。
@ -69,22 +69,21 @@ workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠
| `request_interval` | `10` | 每次财联社请求尝试后的最小等待秒数,可设为 0 |
| `max_retries` | `3` | 每页财联社请求的最大尝试次数,至少为 1 |
内置定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。
插件提供的定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。
## 产物
```text
reme_workspace/daily/YYYY-MM-DD/auto_fin.md
.reme/daily/YYYY-MM-DD/auto_fin.md
```
报告包含标题、说明、当前 CLS 证据、历史分析、上下文 wikilink 和固定非投资建议声明。网络错误与无效 Agent 输出
会明确失败;没有相关当前新闻则成功跳过。`DINGTALK_CONVERSATION_IDS` 为空时发送步骤无副作用;设置该变量后,
还必须提供[每日论文 Cookbook](../daily_paper/README_ZH.md#6-dingtalk)中列出的钉钉凭据。
会明确失败;没有相关当前新闻则成功跳过。
## 验证
```bash
pytest tests/unit/test_auto_fin.py -v
python -m pytest plugin/auto-fin -v
```
单元测试 mock CLS 与 Agent 边界,不访问外部服务。

View file

@ -0,0 +1,34 @@
[project]
name = "reme-auto-fin"
version = "0.1.0"
description = "Auto Fin example plugin for ReMe."
readme = "README.md"
license = "Apache-2.0"
license-files = ["LICENSE"]
requires-python = ">=3.11"
dependencies = [
"reme-ai[core]>=0.4.1.8",
]
[project.entry-points."reme.plugins"]
auto-fin = "reme_auto_fin:plugin"
[project.entry-points."reme.configs"]
auto-fin = "reme_auto_fin:CONFIG_PATH"
[tool.setuptools]
package-dir = { "" = "src" }
packages = ["reme_auto_fin"]
include-package-data = true
[tool.setuptools.package-data]
reme_auto_fin = ["*.yaml"]
[tool.pytest.ini_options]
asyncio_default_fixture_loop_scope = "function"
pythonpath = ["src", "../.."]
testpaths = ["tests"]
[build-system]
requires = ["setuptools>=77", "wheel"]
build-backend = "setuptools.build_meta"

View file

@ -0,0 +1,21 @@
"""Auto Fin news research workflow."""
from pathlib import Path
from .data import AutoFinDataStep
from .merge import AutoFinMergeStep
from .plugin import plugin
from .schema import AutoFinReportOutput, AutoFinTopicOutput
from .topic import AutoFinTopicStep
CONFIG_PATH = Path(__file__).with_name("config.yaml")
__all__ = [
"AutoFinReportOutput",
"AutoFinDataStep",
"AutoFinMergeStep",
"AutoFinTopicOutput",
"AutoFinTopicStep",
"CONFIG_PATH",
"plugin",
]

View file

@ -12,7 +12,7 @@ from uuid import uuid4
from pydantic import BaseModel
from ...base_step import BaseStep
from reme.steps import BaseStep
AGENT_INPUT_LOG_LIMIT = 2000
AGENT_OUTPUT_LOG_LIMIT = 4000

View file

@ -0,0 +1,2 @@
extends: default
plugins: [auto-fin]

View file

@ -9,8 +9,7 @@ from typing import Any
import httpx
from ....components import R
from ._base import AutoFinStep, _plain_text
from .base import AutoFinStep, _plain_text
API_URL = "https://www.cls.cn/v1/roll/get_roll_list"
HEADERS = {
@ -23,7 +22,6 @@ DEFAULT_TOPICS = ("黄金", "机器人", "半导体")
WINDOW = timedelta(hours=24)
@R.register("auto_fin_data_step")
class AutoFinDataStep(AutoFinStep):
"""Fetch and normalize one rolling day of CLS news without writing files."""

View file

@ -0,0 +1,54 @@
jobs:
auto_fin:
backend: base
description: "Fetch and research recent topic-related CLS news."
parameters:
type: object
properties:
date:
type: string
description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai."
default: ""
now:
type: string
description: "Optional simulated current time in ISO 8601 format; empty means the real current time."
default: ""
topics:
type: string
description: "Comma-separated topics used to filter current CLS news."
default: "黄金,机器人,半导体"
window_hours:
type: number
exclusiveMinimum: 0
description: "Rolling number of hours of CLS news to fetch."
default: 24
request_interval:
type: number
minimum: 0
description: "Minimum delay in seconds after each CLS request attempt."
default: 10
max_retries:
type: integer
minimum: 1
description: "Maximum attempts for each CLS page request."
default: 3
steps: &auto_fin_steps
- backend: auto_fin_data_step
- backend: auto_fin_topic_step
- backend: auto_fin_merge_step
job_tools: [memory_search, read]
auto_fin_0930_cron:
backend: cron
cron: "30 9 * * *"
steps: *auto_fin_steps
auto_fin_1130_cron:
backend: cron
cron: "30 11 * * *"
steps: *auto_fin_steps
auto_fin_1800_cron:
backend: cron
cron: "0 18 * * *"
steps: *auto_fin_steps

View file

@ -8,10 +8,10 @@ from datetime import date, timedelta
from pathlib import Path
from types import SimpleNamespace
from ....components import R
from ....schema import AutoFinReportOutput
from ...file_io import refresh_day_index
from ._base import AutoFinStep, _write
from reme.steps.file_io import refresh_day_index
from .base import AutoFinStep, _write
from .schema import AutoFinReportOutput
_WIKILINK_RE = re.compile(r"\[\[([^\[\]\n]+)\]\]")
_HYBRID_WIKILINK_RE = re.compile(
@ -19,7 +19,6 @@ _HYBRID_WIKILINK_RE = re.compile(
)
@R.register("auto_fin_merge_step")
class AutoFinMergeStep(AutoFinStep):
"""Give one Agent read-only ReMe tools, then validate links in its Markdown."""

View file

@ -0,0 +1,31 @@
"""Auto Fin plugin declaration."""
from pathlib import Path
import yaml
from reme.plugin import Backend, Plugin
from .data import AutoFinDataStep
from .merge import AutoFinMergeStep
from .topic import AutoFinTopicStep
def _default_config() -> dict:
path = Path(__file__).with_name("defaults.yaml")
with path.open(encoding="utf-8") as stream:
config = yaml.safe_load(stream)
if not isinstance(config, dict):
raise ValueError(f"Auto Fin defaults must be a mapping: {path}")
return config
plugin = Plugin(
name="auto-fin",
backends=(
Backend("auto_fin_data_step", AutoFinDataStep),
Backend("auto_fin_topic_step", AutoFinTopicStep),
Backend("auto_fin_merge_step", AutoFinMergeStep),
),
config=_default_config(),
)

View file

@ -4,12 +4,10 @@ from __future__ import annotations
import json
from ....components import R
from ....schema import AutoFinTopicOutput
from ._base import AutoFinStep
from .base import AutoFinStep
from .schema import AutoFinTopicOutput
@R.register("auto_fin_topic_step")
class AutoFinTopicStep(AutoFinStep):
"""Filter current news in bounded Agent batches without writing files."""

View file

@ -8,14 +8,15 @@ from zoneinfo import ZoneInfo
import pytest
from reme_auto_fin.base import _plain_text, _write
from reme_auto_fin.data import AutoFinDataStep
from reme_auto_fin.merge import AutoFinMergeStep
from reme_auto_fin.plugin import plugin
from reme_auto_fin.schema import AutoFinReportOutput, AutoFinTopicOutput
from reme_auto_fin.topic import AutoFinTopicStep
from reme.components import ApplicationContext
from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper
from reme.components.runtime_context import RuntimeContext
from reme.schema import AutoFinReportOutput, AutoFinTopicOutput
from reme.steps.cookbook.auto_fin._base import _plain_text, _write
from reme.steps.cookbook.auto_fin.data import AutoFinDataStep
from reme.steps.cookbook.auto_fin.merge import AutoFinMergeStep
from reme.steps.cookbook.auto_fin.topic import AutoFinTopicStep
SHANGHAI = ZoneInfo("Asia/Shanghai")
@ -28,7 +29,7 @@ def test_atomic_write_preserves_existing_file_on_failure(tmp_path: Path, monkeyp
path = tmp_path / "result.md"
path.write_text("existing", encoding="utf-8")
monkeypatch.setattr(
"reme.steps.cookbook.auto_fin._base.os.replace",
"reme_auto_fin.base.os.replace",
lambda *_args: (_ for _ in ()).throw(OSError()),
)
@ -210,7 +211,7 @@ async def test_merge_writes_only_final_report_and_validates_historical_links(tmp
def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path: Path, monkeypatch):
import reme.steps.cookbook.auto_fin.merge as merge_module
import reme_auto_fin.merge as merge_module
step = AutoFinMergeStep(
app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"),
@ -237,11 +238,8 @@ def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path
assert step._normalize_hybrid_wikilinks(body) == body
def test_config_has_default_topics_and_no_intermediate_index_step():
from reme.config.config_parser import _load_config
config = _load_config("daily_cookbook")
job = config["jobs"]["auto_fin"]
def test_plugin_config_has_default_topics_and_no_intermediate_index_step():
job = plugin.config["jobs"]["auto_fin"]
assert job["parameters"]["properties"]["topics"]["default"] == "黄金,机器人,半导体"
assert job["parameters"]["properties"]["window_hours"]["default"] == 24
assert job["parameters"]["properties"]["request_interval"]["default"] == 10
@ -251,7 +249,6 @@ def test_config_has_default_topics_and_no_intermediate_index_step():
"auto_fin_data_step",
"auto_fin_topic_step",
"auto_fin_merge_step",
"dingtalk_markdown_send_step",
]
assert job["steps"][2]["job_tools"] == ["memory_search", "read"]
for name, schedule in {
@ -259,8 +256,8 @@ def test_config_has_default_topics_and_no_intermediate_index_step():
"auto_fin_1130_cron": "30 11 * * *",
"auto_fin_1800_cron": "0 18 * * *",
}.items():
assert config["jobs"][name]["cron"] == schedule
assert config["jobs"][name]["steps"] == job["steps"]
assert plugin.config["jobs"][name]["cron"] == schedule
assert plugin.config["jobs"][name]["steps"] == job["steps"]
def test_agent_schemas_are_small_and_required():

View file

@ -8,13 +8,20 @@ from . import enumeration
from . import schema
from . import steps
from . import utils
from .components import BaseComponent, R
from .application import Application
from .components import BaseComponent
from .plugin import Backend, Plugin
from .reme import ReMe
# Component and Step packages above have completed their decorator-driven
# bootstrap. Runtime code receives mutable copies of this immutable template.
R.freeze()
__all__ = [
"Application",
"BaseComponent",
"Backend",
"Plugin",
"ReMe",
# submodules
"config",

View file

@ -7,22 +7,27 @@ from pathlib import Path
from typing import AsyncGenerator, TypeVar
from . import __version__
from .components import BaseComponent, ApplicationContext
from .components import ApplicationContext, BaseComponent, create_application_registry
from .components.job import BackgroundJob, BaseJob, CronJob, StreamJob
from .components.service import BaseService
from .enumeration import ComponentEnum
from .enumeration import ComponentEnum, ComponentType, component_type_name
from .plugin import PluginManager
from .schema import ComponentConfig, Response, StreamChunk
from .utils import execute_stream_task, print_logo, get_logger
T = TypeVar("T", bound=BaseComponent)
_NodeKey = tuple[ComponentEnum, str]
_NodeKey = tuple[str, str]
class Application(BaseComponent):
"""Wires components from config and runs jobs against them."""
def __init__(self, **kwargs) -> None:
self.context = ApplicationContext(**kwargs)
plugin_manager = PluginManager.discover(kwargs.get("plugins") or ())
resolved = plugin_manager.merge_config(kwargs)
registry = create_application_registry()
plugin_manager.register(registry)
self.context = ApplicationContext(registry=registry, **resolved)
self._started_components: list[BaseComponent] = []
self._setup_workspace_directories()
@ -98,7 +103,7 @@ class Application(BaseComponent):
def _instantiate(
self,
ctype: ComponentEnum,
ctype: ComponentType,
cfg: ComponentConfig,
*,
label: str,
@ -109,16 +114,13 @@ class Application(BaseComponent):
`label` is the human-readable identifier used only in error messages.
`expected_type` narrows the return type and guards against a backend
registered under the wrong ComponentEnum.
registered under the wrong component type.
`name` is forwarded to the constructor for named components/jobs;
leave it None for the service, which is keyed solely by type.
"""
# Lazy import: the registry self-populates as component modules load.
from .components import R
if not cfg.backend:
raise ValueError(f"{label} is missing the required 'backend' field")
backend_cls = R.get(ctype, cfg.backend)
backend_cls = self.context.registry.get(ctype, cfg.backend)
if backend_cls is None:
raise ValueError(f"Unregistered backend '{cfg.backend}' for {label}")
@ -153,7 +155,7 @@ class Application(BaseComponent):
heapq.heappush(ready, downstream)
if len(ordered) != len(nodes):
unresolved = [f"{k[0].value}:{k[1]}" for k, d in in_degree.items() if d > 0]
unresolved = [f"{k[0]}:{k[1]}" for k, d in in_degree.items() if d > 0]
raise ValueError(f"Circular dependency detected among: {unresolved}")
return ordered
@ -172,7 +174,7 @@ class Application(BaseComponent):
in_degree[key] += 1
elif not dep.optional:
raise ValueError(
f"Component {key[0].value}:{key[1]} depends on unregistered {dep.ctype.value}:{dep.name}",
f"Component {key[0]}:{key[1]} depends on unregistered {dep.ctype}:{dep.name}",
)
return in_degree, dependents
@ -205,7 +207,7 @@ class Application(BaseComponent):
await c.start()
self._started_components.append(c)
except Exception as e:
self.logger.exception(f"Failed to start {c.component_type.value}:{c.name}: {e}")
self.logger.exception(f"Failed to start {component_type_name(c.component_type)}:{c.name}: {e}")
raise
async def _close(self) -> None:
@ -214,23 +216,23 @@ class Application(BaseComponent):
try:
await c.close()
except Exception as e:
self.logger.exception(f"Failed to close {c.component_type.value}:{c.name}: {e}")
self.logger.exception(f"Failed to close {component_type_name(c.component_type)}:{c.name}: {e}")
self._started_components.clear()
if self.context.thread_pool is not None:
self.context.thread_pool.shutdown(wait=True)
self.context.thread_pool = None
async def update_component(self, component_enum: ComponentEnum | str, name: str, /, **kwargs) -> BaseComponent:
async def update_component(self, component_enum: ComponentType, name: str, /, **kwargs) -> BaseComponent:
"""Update an existing component by type/name; never creates missing components."""
component_enum = ComponentEnum(component_enum)
group = self.context.components.get(component_enum)
component_type = component_type_name(component_enum)
group = self.context.components.get(component_type)
if not group or name not in group:
raise KeyError(f"Component '{name}' not found in {component_enum.value}")
raise KeyError(f"Component '{name}' not found in {component_type}")
component = group[name]
for key, value in kwargs.items():
if not hasattr(component, key):
raise AttributeError(f"Component {component_enum.value}:{name} has no attribute '{key}'")
raise AttributeError(f"Component {component_type}:{name} has no attribute '{key}'")
setattr(component, key, value)
return component

View file

@ -16,7 +16,7 @@ from . import service
from . import tokenizer
from .application_context import ApplicationContext
from .base_component import BaseComponent, ComponentMixin
from .component_registry import ComponentRegistry, R
from .component_registry import ComponentRegistry, R, create_application_registry
from .prompt_handler import PromptHandler
from .runtime_context import RuntimeContext
@ -26,6 +26,7 @@ __all__ = [
"ComponentMixin",
"ComponentRegistry",
"R",
"create_application_registry",
"PromptHandler",
"RuntimeContext",
# base components

View file

@ -3,8 +3,8 @@
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
from ..enumeration import ComponentEnum
from ..schema import ApplicationConfig
from .component_registry import ComponentRegistry, create_application_registry
if TYPE_CHECKING:
from .base_component import BaseComponent
@ -20,13 +20,14 @@ class ApplicationContext:
components, jobs, and the service can find each other at runtime.
"""
def __init__(self, **kwargs):
def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs):
# Parse raw kwargs into a typed, validated config object.
self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
self.registry = registry or create_application_registry()
# Populated by Application during initialization.
self.service: "BaseService | None" = None
self.components: dict[ComponentEnum, dict[str, "BaseComponent"]] = {}
self.components: dict[str, dict[str, "BaseComponent"]] = {}
self.jobs: dict[str, "BaseJob"] = {}
self.thread_pool: ThreadPoolExecutor | None = None

View file

@ -5,7 +5,7 @@ from abc import ABC
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
from ..enumeration import ComponentEnum
from ..enumeration import ComponentEnum, ComponentType, component_type_name
from ..utils import get_logger
if TYPE_CHECKING:
@ -61,24 +61,24 @@ class Dependency:
def __init__(
self,
ctype: ComponentEnum,
ctype: ComponentType,
name: str,
default_factory: Callable[[], Any] | None = None,
optional: bool = True,
) -> None:
self.ctype = ctype
self.ctype = component_type_name(ctype)
self.name = name
self.default_factory = default_factory
self.optional = optional
def __repr__(self) -> str:
suffix = "?" if self.optional else ""
return f"<unresolved {self.ctype.value}:{self.name}{suffix}>"
return f"<unresolved {self.ctype}:{self.name}{suffix}>"
def __getattr__(self, item: str) -> Any:
# Catches accidental use of the placeholder before start() resolves it.
raise RuntimeError(
f"Dependency {self.ctype.value}:{self.name} accessed before start() " f"(attribute '{item}')",
f"Dependency {self.ctype}:{self.name} accessed before start() " f"(attribute '{item}')",
)
@ -126,10 +126,14 @@ class BaseComponent(ComponentMixin, ABC):
if not name:
return None
ctype = getattr(base_cls, "component_type", None)
if not isinstance(ctype, ComponentEnum) or ctype is ComponentEnum.BASE:
try:
ctype = component_type_name(ctype)
except (TypeError, ValueError) as exc:
raise TypeError(
f"{base_cls.__name__} must declare a non-BASE ComponentEnum 'component_type'",
)
f"{base_cls.__name__} must declare a non-BASE string 'component_type'",
) from exc
if ctype == ComponentEnum.BASE.value:
raise TypeError(f"{base_cls.__name__} must declare a non-BASE 'component_type'")
return cast(T, Dependency(ctype, name, default_factory, optional))
@property
@ -172,7 +176,7 @@ class BaseComponent(ComponentMixin, ABC):
elif dep.optional:
setattr(self, attr, None)
else:
raise ValueError(f"{dep.ctype.value} '{dep.name}' not found.")
raise ValueError(f"{dep.ctype} '{dep.name}' not found.")
# ----- Workspace path helpers --------------------------------------------
@ -186,7 +190,7 @@ class BaseComponent(ComponentMixin, ABC):
@property
def component_metadata_path(self) -> Path:
"""Per-component metadata directory under the workspace."""
return self.workspace_metadata_path / self.component_type.value
return self.workspace_metadata_path / component_type_name(self.component_type)
# ----- Lifecycle hooks (override in subclasses) ----------------------

View file

@ -1,12 +1,14 @@
"""Global registry mapping ``(ComponentEnum, name) -> component class``."""
"""Registry mapping ``(component type, backend)`` to implementation classes."""
from collections.abc import Iterator
from contextlib import contextmanager
from threading import RLock
from typing import Callable, TypeVar, cast
from .base_component import BaseComponent
from ..enumeration import ComponentEnum
from ..utils import get_logger
from .base_component import ComponentMixin
from ..enumeration import ComponentType, component_type_name
T = TypeVar("T", bound=BaseComponent)
T = TypeVar("T", bound=ComponentMixin)
class ComponentRegistry:
@ -17,27 +19,46 @@ class ComponentRegistry:
"""
def __init__(self) -> None:
self._registry: dict[ComponentEnum, dict[str, type[BaseComponent]]] = {}
self.logger = get_logger(log_to_file=False)
self._registry: dict[str, dict[str, type[ComponentMixin]]] = {}
self._owners: dict[tuple[str, str], str] = {}
self._lock = RLock()
self._frozen = False
def _do_register(self, cls: type[T], name: str) -> type[T]:
"""Insert `cls` under its ``component_type`` group; warn on overwrite."""
component_type = getattr(cls, "component_type", None)
if not isinstance(component_type, ComponentEnum):
raise TypeError(
f"{cls.__name__} must have a ComponentEnum 'component_type' attribute",
)
def _ensure_mutable(self) -> None:
"""Reject changes after this registry becomes an immutable template."""
if self._frozen:
raise RuntimeError("Component registry is frozen")
def _do_register(self, cls: type[T], name: str, *, owner: str | None = None) -> type[T]:
"""Insert ``cls`` under its component type and reject ambiguous providers."""
try:
component_type = component_type_name(getattr(cls, "component_type", None))
except (TypeError, ValueError) as exc:
raise TypeError(f"{cls.__name__} must have a non-empty string 'component_type' attribute") from exc
if not name:
raise ValueError("Component name cannot be empty")
group = self._registry.setdefault(component_type, {})
if name in group:
self.logger.warning(
f"Component '{name}' already registered for {component_type}, overwriting",
)
group[name] = cls
with self._lock:
self._ensure_mutable()
group = self._registry.setdefault(component_type, {})
key = (component_type, name)
if name in group:
existing = group[name]
existing_owner = self._owners[key]
new_owner = owner or cls.__module__
if existing is cls and existing_owner == new_owner:
return cls
raise ValueError(
f"Backend '{component_type}:{name}' is provided by both " f"'{existing_owner}' and '{new_owner}'",
)
group[name] = cls
self._owners[key] = owner or cls.__module__
return cls
def add(self, name: str, cls: type[T], *, owner: str) -> type[T]:
"""Register one explicitly owned plugin contribution."""
return self._do_register(cls, name, owner=owner)
def register(
self,
cls_or_name: type[T] | str,
@ -60,25 +81,76 @@ class ComponentRegistry:
return decorator
def get(self, component_type: ComponentEnum, name: str) -> type[BaseComponent] | None:
def get(self, component_type: ComponentType, name: str) -> type[ComponentMixin] | None:
"""Look up a registered class; return None if not found."""
return self._registry.get(component_type, {}).get(name)
with self._lock:
return self._registry.get(component_type_name(component_type), {}).get(name)
def get_all(self, component_type: ComponentEnum) -> dict[str, type[BaseComponent]]:
def get_all(self, component_type: ComponentType) -> dict[str, type[ComponentMixin]]:
"""Return a shallow copy of all classes registered under `component_type`."""
return dict(self._registry.get(component_type, {}))
with self._lock:
return dict(self._registry.get(component_type_name(component_type), {}))
def unregister(self, component_type: ComponentEnum, name: str) -> bool:
def unregister(self, component_type: ComponentType, name: str) -> bool:
"""Remove an entry; return True if it existed, False otherwise."""
if (group := self._registry.get(component_type)) and name in group:
del group[name]
return True
return False
component_type = component_type_name(component_type)
with self._lock:
self._ensure_mutable()
if (group := self._registry.get(component_type)) and name in group:
del group[name]
self._owners.pop((component_type, name), None)
return True
return False
def clear(self) -> None:
"""Drop every registered entry."""
self._registry.clear()
with self._lock:
self._ensure_mutable()
self._registry.clear()
self._owners.clear()
def freeze(self) -> None:
"""Make this registry an immutable template for future copies."""
with self._lock:
self._frozen = True
@property
def frozen(self) -> bool:
"""Whether mutating operations are disabled."""
with self._lock:
return self._frozen
def copy(self) -> "ComponentRegistry":
"""Return an independent registry containing the same providers."""
copied = ComponentRegistry()
with self._lock:
for component_type, group in self._registry.items():
for name, implementation in group.items():
copied.add(name, implementation, owner=self._owners[(component_type, name)])
return copied
@contextmanager
def preserve(self, *, allow_mutation: bool = False) -> Iterator[None]:
"""Restore the registry after code that may register through import side effects."""
with self._lock:
registry = {component_type: dict(group) for component_type, group in self._registry.items()}
owners = dict(self._owners)
frozen = self._frozen
if allow_mutation:
self._frozen = False
try:
yield
finally:
self._registry = registry
self._owners = owners
self._frozen = frozen
# Process-wide singleton used throughout the codebase.
# Import-time registry for built-in implementations. Runtime code should use
# ``create_application_registry`` rather than mutate this template.
R = ComponentRegistry()
def create_application_registry() -> ComponentRegistry:
"""Return a mutable registry initialized from the frozen built-in template."""
return R.copy()

View file

@ -47,7 +47,7 @@ class BaseJob(BaseComponent):
config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
if not config.backend:
raise ValueError("Step is missing the required 'backend' field")
step_cls = R.get(ComponentEnum.STEP, config.backend)
step_cls = self.app_context.registry.get(ComponentEnum.STEP, config.backend)
if not step_cls:
raise ValueError(f"Unregistered backend '{config.backend}' of type '{ComponentEnum.STEP}'")
params = config.model_dump()

View file

@ -1,8 +1,10 @@
"""Config"""
from .config_parser import parse_args, resolve_app_config
from .config_parser import deep_merge_config, expand_env_vars, parse_args, resolve_app_config
__all__ = [
"deep_merge_config",
"expand_env_vars",
"parse_args",
"resolve_app_config",
]

View file

@ -3,15 +3,20 @@
import json
import os
import re
from collections.abc import Mapping
from importlib import metadata
from pathlib import Path
from typing import Any
import yaml
from ..entry_point import load_entry_point
# Config files are looked up relative to this module's directory
_CONFIG_DIR = Path(__file__).parent
# Extensions in priority order: yaml > yml > json when stems collide
_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
_CONFIG_ENTRY_POINT_GROUP = "reme.configs"
_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?}")
# Strings like "007" / "00501" must stay as strings, not be coerced to numbers
_LEADING_ZERO_RE = re.compile(r"^-?0\d")
@ -41,6 +46,11 @@ def _expand_env_vars(value: Any) -> Any:
return value
def expand_env_vars(value: Any) -> Any:
"""Expand environment placeholders in an arbitrary plugin config value."""
return _expand_env_vars(value)
def _discover_configs() -> dict[str, Path]:
"""Pre-scan config directory: maps file stem (name without ext) -> Path."""
discovered: dict[str, Path] = {}
@ -117,17 +127,43 @@ def _convert_value(value_str: str) -> Any:
return s
def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
"""Load a YAML or JSON config file.
def _external_config_entries(name: str) -> list[metadata.EntryPoint]:
"""Find installed config providers without importing their packages."""
return list(metadata.entry_points().select(group=_CONFIG_ENTRY_POINT_GROUP, name=name))
First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY).
If not, treat as a file path and load directly.
"""
# 1. Try pre-discovered configs first
if name_or_path in _CONFIG_REGISTRY:
return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding)
# 2. Treat as file path
def _external_config_path(name: str, entries: list[metadata.EntryPoint] | None = None) -> Path | None:
"""Resolve an installed plugin config exposed through ``reme.configs``."""
entries = _external_config_entries(name) if entries is None else entries
if not entries:
return None
if len(entries) > 1:
providers = ", ".join(sorted(entry.value for entry in entries))
raise ValueError(f"Config '{name}' has multiple installed providers: {providers}")
value = load_entry_point(entries[0], invoke=True)
path = Path(value)
if path.suffix not in _SUPPORTED_EXTS or not path.is_file():
raise ValueError(f"Config entry point '{name}' did not resolve to a YAML or JSON file")
return path
def _load_config(name_or_path: str, encoding: str = "utf-8", _stack: tuple[str, ...] = ()) -> dict:
"""Load a built-in, installed-plugin, or direct YAML/JSON config."""
if name_or_path in _stack:
chain = " -> ".join((*_stack, name_or_path))
raise ValueError(f"Circular config inheritance: {chain}")
built_in = _CONFIG_REGISTRY.get(name_or_path)
external_entries = _external_config_entries(name_or_path)
if built_in is not None and external_entries:
raise ValueError(f"Config '{name_or_path}' is provided by both ReMe and an installed distribution")
if built_in is not None:
return _load_config_path(built_in, name_or_path, encoding, _stack)
external = _external_config_path(name_or_path, external_entries)
if external is not None:
return _load_config_path(external, name_or_path, encoding, _stack)
p = Path(name_or_path)
if p.suffix in _SUPPORTED_EXTS:
candidates = [p]
@ -135,13 +171,31 @@ def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
candidates.append(_CONFIG_DIR / p)
for candidate in candidates:
if candidate.exists():
return _read_config_file(candidate, encoding)
identity = str(candidate.resolve())
return _load_config_path(candidate, identity, encoding, _stack)
raise FileNotFoundError(f"Config file not found: {p}")
known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none"
raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}")
def _load_config_path(path: Path, identity: str, encoding: str, stack: tuple[str, ...]) -> dict:
"""Load one config and merge its optional parents before its own values."""
config = _read_config_file(path, encoding)
raw_parents = config.pop("extends", ())
parents = [raw_parents] if isinstance(raw_parents, str) else list(raw_parents or ())
merged: dict = {}
for parent in parents:
if not isinstance(parent, str) or not parent:
raise ValueError(f"Config 'extends' entries must be non-empty strings: {path}")
parent_name = parent
relative = path.parent / parent
if Path(parent).suffix in _SUPPORTED_EXTS and relative.is_file():
parent_name = str(relative.resolve())
merged = deep_merge_config(merged, _load_config(parent_name, encoding, (*stack, identity)))
return deep_merge_config(merged, config)
def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
"""Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
with path.open(encoding=encoding) as f:
@ -156,12 +210,12 @@ def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
return _expand_env_vars(result)
def _deep_merge(base: dict, update: dict) -> dict:
"""Recursively merge dicts."""
result = base.copy()
def deep_merge_config(base: Mapping[str, Any], update: Mapping[str, Any]) -> dict[str, Any]:
"""Recursively merge configuration mappings without mutating either input."""
result = dict(base)
for k, v in update.items():
if k in result and isinstance(result[k], dict) and isinstance(v, dict):
result[k] = _deep_merge(result[k], v)
if k in result and isinstance(result[k], Mapping) and isinstance(v, Mapping):
result[k] = deep_merge_config(result[k], v)
else:
result[k] = v
return result
@ -230,6 +284,6 @@ def resolve_app_config(*, log_config: bool = True, **kwargs) -> dict:
merged: dict = {}
for cfg in configs:
merged = _deep_merge(merged, cfg)
merged = deep_merge_config(merged, cfg)
return merged

View file

@ -316,71 +316,6 @@ jobs:
steps:
- backend: edit_step
auto_fin:
backend: base
description: "Fetch and research the latest 24 hours of topic-related CLS news."
parameters:
type: object
properties:
date:
type: string
description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai."
default: ""
now:
type: string
description: "Optional simulated current time in ISO 8601 format; empty means the real current time."
default: ""
topics:
type: string
description: "Comma-separated topics used to filter current CLS news."
default: "黄金,机器人,半导体"
window_hours:
type: number
exclusiveMinimum: 0
description: "Rolling number of hours of CLS news to fetch."
default: 24
request_interval:
type: number
minimum: 0
description: "Minimum delay in seconds after each CLS request attempt."
default: 10
max_retries:
type: integer
minimum: 1
description: "Maximum attempts for each CLS page request."
default: 3
steps: &auto_fin_steps
- backend: auto_fin_data_step
- backend: auto_fin_topic_step
- backend: auto_fin_merge_step
job_tools: [memory_search, read]
- backend: dingtalk_markdown_send_step
input_mapping:
auto_fin_digest_path: markdown_path
app_key: ${DINGTALK_APP_KEY:-}
app_secret: ${DINGTALK_APP_SECRET:-}
robot_code: ${DINGTALK_ROBOT_CODE:-}
conversation_ids: ${DINGTALK_CONVERSATION_IDS:-}
title: ReMe Auto Fin
timeout: 15
# Three intraday runs (Asia/Shanghai). News remains useful outside market
# sessions. Each rerun refines the same day's report.
auto_fin_0930_cron:
backend: cron
cron: "30 9 * * *"
steps: *auto_fin_steps
auto_fin_1130_cron:
backend: cron
cron: "30 11 * * *"
steps: *auto_fin_steps
auto_fin_1800_cron:
backend: cron
cron: "0 18 * * *"
steps: *auto_fin_steps
daily_paper:
backend: base
description: "Build detailed readings and a five-minute brief from Hugging Face weekly/monthly papers."

14
reme/entry_point.py Normal file
View file

@ -0,0 +1,14 @@
"""Safe loading boundary for third-party Python entry points."""
from importlib import metadata
from typing import Any
def load_entry_point(entry: metadata.EntryPoint, *, invoke: bool = False) -> Any:
"""Load and optionally invoke an entry point without retaining registrations."""
# Import lazily so config parser imports do not pull in the component graph.
from .components.component_registry import R
with R.preserve(allow_mutation=True):
loaded = entry.load()
return loaded() if invoke and callable(loaded) else loaded

View file

@ -2,12 +2,15 @@
from .chunk_enum import ChunkEnum
from .component_enum import ComponentEnum
from .component_type import ComponentType, component_type_name
from .dream_bucket_enum import DreamBucketEnum
from .link_scope_enum import LinkScopeEnum
__all__ = [
"ChunkEnum",
"ComponentEnum",
"ComponentType",
"DreamBucketEnum",
"LinkScopeEnum",
"component_type_name",
]

View file

@ -0,0 +1,25 @@
"""Extensible component type identifiers."""
import re
from typing import TypeAlias
from .component_enum import ComponentEnum
ComponentType: TypeAlias = ComponentEnum | str
_COMPONENT_TYPE_PATTERN = re.compile(r"[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*")
def component_type_name(value: ComponentType) -> str:
"""Return the canonical string name for a built-in or plugin component type."""
if isinstance(value, ComponentEnum):
return value.value
if not isinstance(value, str):
raise TypeError(f"Component type must be a string, got {type(value).__name__}")
name = value.strip()
if not name:
raise ValueError("Component type cannot be empty")
if _COMPONENT_TYPE_PATTERN.fullmatch(name) is None:
raise ValueError(
f"Invalid component type {name!r}; use lowercase letters and numbers separated by '.', '_' or '-'",
)
return name

84
reme/plugin.py Normal file
View file

@ -0,0 +1,84 @@
"""Installed plugin discovery and application-local registration."""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from importlib import metadata
from typing import Any
from .components.base_component import ComponentMixin
from .components.component_registry import ComponentRegistry
from .config import deep_merge_config, expand_env_vars
from .entry_point import load_entry_point
PLUGIN_ENTRY_POINT_GROUP = "reme.plugins"
@dataclass(frozen=True)
class Backend:
"""One named component, step, or job backend contributed by a plugin."""
name: str
implementation: type[ComponentMixin]
@dataclass(frozen=True)
class Plugin:
"""Declarative plugin loaded from the ``reme.plugins`` entry-point group."""
name: str
backends: tuple[Backend, ...] = ()
config: Mapping[str, Any] = field(default_factory=dict)
def _entry_points(group: str, name: str) -> list[metadata.EntryPoint]:
"""Return matching entry points for one group and name."""
return list(metadata.entry_points().select(group=group, name=name))
class PluginManager:
"""Resolve enabled plugins and apply their contributions to one application."""
def __init__(self, plugins: Iterable[Plugin] = ()) -> None:
self.plugins = tuple(plugins)
@classmethod
def discover(cls, specs: Iterable[str]) -> "PluginManager":
"""Load explicitly enabled plugins by entry-point name."""
plugins: list[Plugin] = []
seen: set[str] = set()
for name in specs:
if not isinstance(name, str):
raise TypeError(f"Invalid plugin name: {name!r}")
if not name:
raise ValueError("Plugin name cannot be empty")
if name in seen:
raise ValueError(f"Plugin '{name}' is enabled more than once")
entries = _entry_points(PLUGIN_ENTRY_POINT_GROUP, name)
if not entries:
raise ValueError(f"Plugin '{name}' is not installed")
if len(entries) > 1:
providers = ", ".join(sorted(entry.value for entry in entries))
raise ValueError(f"Plugin '{name}' has multiple installed providers: {providers}")
plugin = load_entry_point(entries[0], invoke=True)
if not isinstance(plugin, Plugin):
raise TypeError(f"Plugin entry point '{name}' did not return reme.plugin.Plugin")
if plugin.name != name:
raise ValueError(f"Plugin entry point '{name}' returned plugin '{plugin.name}'")
plugins.append(plugin)
seen.add(name)
return cls(plugins)
def merge_config(self, application_config: Mapping[str, Any]) -> dict[str, Any]:
"""Place plugin defaults below the user's resolved application config."""
merged: dict[str, Any] = {}
for plugin in self.plugins:
merged = deep_merge_config(merged, expand_env_vars(plugin.config))
return deep_merge_config(merged, application_config)
def register(self, registry: ComponentRegistry) -> None:
"""Register every backend into an application-local registry."""
for plugin in self.plugins:
for backend in plugin.backends:
registry.add(backend.name, backend.implementation, owner=plugin.name)

View file

@ -4,11 +4,12 @@ import asyncio
import sys
from .application import Application
from .components import R
from .components import create_application_registry
from .components.service.cli_service import prepare_start_config, should_precheck_start
from .config import parse_args, resolve_app_config
from .enumeration import ComponentEnum
from .utils import cli_find_reme, load_env, precheck_start, running_service_config
from .plugin import PluginManager
from .utils import cli_find_reme, load_env, precheck_start, running_app_config
_CLIENT_KWARGS = {"host", "port", "timeout", "transport", "command", "args", "show_metadata"}
@ -36,10 +37,14 @@ async def call_server(action: str, **kwargs):
if isinstance(kwargs.get("service"), dict):
resolve_kwargs["service"] = kwargs.pop("service")
# Prefer the running server's real config; fall back to the local config file.
service = running_service_config()
if service is None:
service = resolve_app_config(log_config=False, **resolve_kwargs).get("service")
# Prefer the running server's complete config so its enabled client plugins
# are available even when the caller does not repeat config=<name>.
app_config = running_app_config()
if app_config is None:
app_config = resolve_app_config(log_config=False, **resolve_kwargs)
plugin_manager = PluginManager.discover(app_config.get("plugins") or ())
app_config = plugin_manager.merge_config(app_config)
service = app_config.get("service")
service = service if isinstance(service, dict) else {}
backend: str = kwargs.pop("backend", None) or service.get("backend", "http")
@ -50,7 +55,9 @@ async def call_server(action: str, **kwargs):
client_kwargs = {k: seed[k] for k in _CLIENT_KWARGS if k in seed}
client_kwargs.update({key: kwargs.pop(key) for key in list(kwargs) if key in _CLIENT_KWARGS})
client_cls = R.get(ComponentEnum.CLIENT, backend)
registry = create_application_registry()
plugin_manager.register(registry)
client_cls = registry.get(ComponentEnum.CLIENT, backend)
if client_cls is None:
raise ValueError(f"Unknown client backend: {backend!r}")
async with client_cls(**client_kwargs) as client:

View file

@ -1,7 +1,6 @@
"""Schema"""
from .application_config import ApplicationConfig, ComponentConfig, JobConfig
from .auto_fin import AutoFinReportOutput, AutoFinTopicOutput
from .daily_paper import (
AnalyzedPaper,
DailyPaperMarkdownOutput,
@ -32,8 +31,6 @@ from .traverse_graph import TraverseGraph, TraverseGraphEdge, TraverseGraphNode
__all__ = [
"ApplicationConfig",
"AutoFinReportOutput",
"AutoFinTopicOutput",
"ComponentConfig",
"AnalyzedPaper",
"DailyPaperMarkdownOutput",

View file

@ -6,7 +6,7 @@ from pathlib import PurePosixPath, PureWindowsPath
from pydantic import BaseModel, ConfigDict, Field, field_validator
from ..enumeration import ComponentEnum
from ..enumeration import component_type_name
class ComponentConfig(BaseModel):
@ -52,14 +52,25 @@ class ApplicationConfig(BaseModel):
log_to_console: bool = Field(default=True, description="Log to console")
log_to_file: bool = Field(default=True, description="Log to file")
mcp_servers: dict[str, dict] = Field(default_factory=dict, description="MCP server configs by name")
plugins: list[str] = Field(default_factory=list, description="Installed plugins enabled for this app")
service: ComponentConfig = Field(default_factory=ComponentConfig, description="Service endpoint config")
jobs: dict[str, JobConfig] = Field(default_factory=dict, description="Job definitions keyed by job name")
thread_pool_max_workers: int = Field(default=0, description="Max worker threads; 0 to disable")
components: dict[ComponentEnum, dict[str, ComponentConfig]] = Field(
components: dict[str, dict[str, ComponentConfig]] = Field(
default_factory=dict,
description="Component registry keyed by type then name",
)
@field_validator("components", mode="before")
@classmethod
def normalize_component_types(cls, value):
"""Canonicalize built-in enums and allow plugin-defined component type names."""
if value is None:
return {}
if not isinstance(value, dict):
return value
return {component_type_name(component_type): group for component_type, group in value.items()}
@field_validator("workspace_dir", mode="before")
@classmethod
def normalize_workspace_dir(cls, value) -> str:

View file

@ -201,7 +201,8 @@ class BaseStep(ComponentMixin, ABC):
backend = params.get("backend", "")
if not backend:
raise ValueError("Dispatch step is missing the required 'backend' field")
step_cls = R.get(ComponentEnum.STEP, backend)
registry = self.app_context.registry if self.app_context is not None else R
step_cls = registry.get(ComponentEnum.STEP, backend)
if step_cls is None:
raise RuntimeError(f"Unregistered step '{backend}'")
params["app_context"] = self.app_context

View file

@ -1,5 +1,5 @@
"""Optional, end-to-end cookbook workflows."""
from . import auto_fin, daily_paper, dingtalk
from . import daily_paper, dingtalk
__all__ = ["auto_fin", "daily_paper", "dingtalk"]
__all__ = ["daily_paper", "dingtalk"]

View file

@ -1,11 +0,0 @@
"""Auto Fin news research workflow."""
from .data import AutoFinDataStep
from .merge import AutoFinMergeStep
from .topic import AutoFinTopicStep
__all__ = [
"AutoFinDataStep",
"AutoFinMergeStep",
"AutoFinTopicStep",
]

View file

@ -12,7 +12,14 @@ from .link_expansion import expand_links, render_expansion_lines
from .line_anchor import format_line_anchor, parse_line_anchor
from .logger_utils import get_logger
from .logo_utils import print_logo
from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme, running_service_config
from .service_utils import (
cli_find_reme,
find_reme,
locate_reme,
precheck_start,
running_app_config,
running_service_config,
)
from .similarity_utils import cosine_similarity, batch_cosine_similarity
from .token_utils import estimate_token_count
from .web_static import REME_WEB_STATIC_DIR, resolve_web_static_dir
@ -43,6 +50,7 @@ __all__ = [
"locate_reme",
"precheck_start",
"cli_find_reme",
"running_app_config",
"running_service_config",
"cosine_similarity",
"batch_cosine_similarity",

View file

@ -88,13 +88,12 @@ def _reme_start_argv() -> list[list[str]]:
return argvs
def running_service_config() -> dict | None:
"""Resolve the ``service`` config of a running reme by replaying its start args.
def running_app_config() -> dict | None:
"""Resolve the full config of a running ReMe by replaying its start args.
Reads the live ``reme start ...`` process cmdline and re-runs the same
``resolve_app_config`` the server used, so the result matches the running
server's real backend/transport/host/port even when those were passed on the
command line and are absent from (or differ from) the on-disk config file.
``resolve_app_config`` the server used. Returning the full config preserves
enabled plugins as well as service connection settings for CLI clients.
Returns ``None`` when no running reme is found or its args can't be parsed.
"""
from ..config import parse_args, resolve_app_config
@ -102,14 +101,23 @@ def running_service_config() -> dict | None:
for argv in _reme_start_argv():
try:
_, kwargs = parse_args("start", *argv)
config = resolve_app_config(log_config=False, **kwargs)
except ValueError:
continue
service = resolve_app_config(log_config=False, **kwargs).get("service")
if isinstance(service, dict):
return service
if isinstance(config, dict):
return config
return None
def running_service_config() -> dict | None:
"""Return only the service section of the running ReMe configuration."""
config = running_app_config()
if config is None:
return None
service = config.get("service")
return service if isinstance(service, dict) else None
async def locate_reme() -> tuple[str, int, int | None] | None:
"""Find a running reme: try default port, then scanned processes."""
if await find_reme(REME_DEFAULT_HOST, REME_DEFAULT_PORT) == "reme":

View file

@ -26,6 +26,34 @@ def test_workspace_dir_expands_user_home(monkeypatch, tmp_path):
assert config.workspace_dir == str(tmp_path / ".copaw/workspaces/default")
def test_application_config_accepts_plugin_names_only():
"""Plugin configuration remains a simple list of installed entry-point names."""
config = ApplicationConfig(plugins=["example"])
assert config.plugins == ["example"]
with pytest.raises(ValidationError):
ApplicationConfig(plugins=[{"name": "example"}])
def test_application_config_accepts_plugin_defined_component_type():
"""Plugin component type names remain typed configuration buckets."""
config = ApplicationConfig(
components={
"example.reranker": {
"default": {"backend": "cross_encoder"},
},
},
)
assert config.components["example.reranker"]["default"].backend == "cross_encoder"
def test_application_config_rejects_unsafe_component_type():
"""Component type names cannot escape their workspace metadata directory."""
with pytest.raises(ValidationError, match="Invalid component type"):
ApplicationConfig(components={"../outside": {"default": {"backend": "unsafe"}}})
def test_dialog_dir_is_not_an_application_config_field():
"""The removed option is absent from schemas and ignored when supplied."""
custom = ApplicationConfig(session_dir="sessions/", dialog_dir="somewhere/else")

View file

@ -23,6 +23,7 @@ import numpy as np
import pytest
from watchfiles import Change
from reme.components import R
from reme.components.agent_wrapper import BaseAgentWrapper
from reme.components.file_chunker import DefaultFileChunker
from reme.components.file_catalog import LocalFileCatalog
@ -115,6 +116,7 @@ def _make_app_context(workspace_path: Path, daily_dir="daily", digest_dir="diges
ctx.app_config.resource_dir = resource_dir
ctx.app_config.session_dir = "session"
ctx.app_config.timezone = None
ctx.registry = R.copy()
return ctx

View file

@ -86,6 +86,16 @@ def test_bind_rejects_no_component_type():
BaseComponent.bind("x", NoType)
def test_bind_accepts_plugin_defined_component_type():
class PluginTarget(BaseComponent):
component_type = "example.reranker"
result = BaseComponent.bind("default", PluginTarget, optional=False)
assert isinstance(result, Dependency)
assert result.ctype == "example.reranker"
def test_bind_with_default_factory():
def factory():
return DepTarget(name="default")

View file

@ -5,7 +5,7 @@
import pytest
from reme.components.base_component import BaseComponent
from reme.components.component_registry import ComponentRegistry
from reme.components.component_registry import ComponentRegistry, R, create_application_registry
from reme.enumeration import ComponentEnum
@ -52,10 +52,32 @@ def test_register_decorator():
def test_register_rejects_missing_component_type():
reg = ComponentRegistry()
with pytest.raises(TypeError, match="ComponentEnum"):
with pytest.raises(TypeError, match="component_type"):
reg.register(_NoComponentType, "bad")
def test_register_plugin_defined_component_type():
reg = ComponentRegistry()
class PluginComponent(BaseComponent):
component_type = "example.reranker"
reg.register(PluginComponent, "cross_encoder")
assert reg.get("example.reranker", "cross_encoder") is PluginComponent
assert reg.get_all("example.reranker") == {"cross_encoder": PluginComponent}
def test_register_rejects_unsafe_plugin_component_type():
reg = ComponentRegistry()
class UnsafePluginComponent(BaseComponent):
component_type = "../outside"
with pytest.raises(TypeError, match="component_type"):
reg.register(UnsafePluginComponent, "unsafe")
def test_register_rejects_empty_name():
reg = ComponentRegistry()
with pytest.raises(ValueError, match="empty"):
@ -127,6 +149,21 @@ def test_clear():
assert not reg.get_all(ComponentEnum.KEYWORD_INDEX)
def test_builtin_registry_is_frozen_after_package_bootstrap():
assert R.frozen is True
with pytest.raises(RuntimeError, match="frozen"):
R.register(_DummyComponent, "runtime-mutation")
def test_application_registry_copy_remains_mutable():
reg = create_application_registry()
assert reg.frozen is False
reg.register(_DummyComponent, "runtime-component")
assert reg.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is _DummyComponent
assert R.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is None
if __name__ == "__main__":
print("\n=== ComponentRegistry Tests ===")
test_register_direct_with_explicit_name()

View file

@ -21,6 +21,37 @@ def test_load_builtin_config_by_filename_with_suffix():
assert cfg["service"]["backend"] == "http"
def test_builtin_and_external_config_name_collision_fails(monkeypatch):
"""An installed config cannot be silently shadowed by a built-in name."""
class FakeEntryPoint:
"""Installed config entry point with a built-in name."""
name = "default"
value = "example:CONFIG_PATH"
@staticmethod
def load():
"""The provider need not be imported to detect the collision."""
raise AssertionError("colliding provider should not be loaded")
class FakeEntryPoints(list):
"""Minimal selectable entry-point collection."""
def select(self, *, group, name):
"""Return entries matching the requested group and name."""
assert group == "reme.configs"
return [entry for entry in self if entry.name == name]
monkeypatch.setattr(
"reme.config.config_parser.metadata.entry_points",
lambda: FakeEntryPoints([FakeEntryPoint()]),
)
with pytest.raises(ValueError, match="provided by both ReMe and an installed distribution"):
_load_config("default")
def test_resolve_app_config_can_suppress_config_log(monkeypatch):
"""Client-side config resolution can avoid polluting command output."""
messages = []

View file

@ -47,7 +47,7 @@ def test_resolve_step_missing_backend():
def test_resolve_step_unregistered_backend():
job = BaseJob(name="j")
job.app_context = MagicMock()
job.app_context = SimpleNamespace(registry=ComponentRegistry())
with pytest.raises(ValueError, match="Unregistered backend"):
job._resolve_step(ComponentConfig(backend="nonexistent_step"))

View file

@ -5,6 +5,7 @@ from pathlib import Path
import tomllib
from types import ModuleType
from packaging.requirements import Requirement
import pytest
REPOSITORY = Path(__file__).resolve().parents[2]
@ -181,6 +182,23 @@ def test_studio_package_preparation_copies_license(monkeypatch, tmp_path: Path)
)
def test_auto_fin_license_matches_repository() -> None:
"""Keep the independently distributed Auto Fin license complete and current."""
assert (REPOSITORY / "plugin" / "auto-fin" / "LICENSE").read_text(encoding="utf-8") == (
REPOSITORY / "LICENSE"
).read_text(encoding="utf-8")
def test_auto_fin_requires_reme_core() -> None:
"""Install the optional runtime packages needed while loading Auto Fin's entry points."""
config = tomllib.loads((REPOSITORY / "plugin" / "auto-fin" / "pyproject.toml").read_text(encoding="utf-8"))
requirements = [Requirement(value) for value in config["project"]["dependencies"]]
reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"]
assert len(reme_requirements) == 1
assert set(reme_requirements[0].extras) == {"core"}
def test_studio_package_preparation_preserves_static_gitignore(monkeypatch, tmp_path: Path) -> None:
"""Keep generated static assets ignored after staging the Studio build."""
package_dir = tmp_path / "reme_ai_studio"

238
tests/unit/test_plugin.py Normal file
View file

@ -0,0 +1,238 @@
"""Tests for installed plugin discovery and application-local registration."""
# pylint: disable=missing-class-docstring,missing-function-docstring
from importlib.metadata import EntryPoint
from pathlib import Path
import pytest
from reme.application import Application
from reme.components.base_component import BaseComponent, ComponentMixin
from reme.components.component_registry import ComponentRegistry, R
from reme.config.config_parser import _load_config
from reme.enumeration import ComponentEnum
from reme.plugin import Backend, Plugin, PluginManager
class _PluginStep(ComponentMixin):
component_type = ComponentEnum.STEP
class _PluginComponent(BaseComponent):
component_type = "example.reranker"
def test_plugin_defaults_are_below_application_config():
manager = PluginManager([Plugin(name="example", config={"jobs": {"task": {"backend": "base", "value": 1}}})])
merged = manager.merge_config({"jobs": {"task": {"value": 2}}})
assert merged["jobs"]["task"] == {"backend": "base", "value": 2}
def test_plugin_defaults_expand_environment(monkeypatch):
monkeypatch.setenv("PLUGIN_LIMIT", "12")
manager = PluginManager([Plugin(name="example", config={"limit": "${PLUGIN_LIMIT}"})])
assert manager.merge_config({})["limit"] == 12
def test_plugin_registers_into_only_the_supplied_registry():
manager = PluginManager([Plugin(name="example", backends=(Backend("example_step", _PluginStep),))])
first = ComponentRegistry()
second = ComponentRegistry()
manager.register(first)
assert first.get(ComponentEnum.STEP, "example_step") is _PluginStep
assert second.get(ComponentEnum.STEP, "example_step") is None
@pytest.mark.asyncio
async def test_plugin_registers_and_runs_custom_component_type(monkeypatch, tmp_path):
manager = PluginManager(
[
Plugin(
name="example",
backends=(Backend("cross_encoder", _PluginComponent),),
config={
"components": {
"example.reranker": {
"default": {"backend": "cross_encoder"},
},
},
},
),
],
)
monkeypatch.setattr(PluginManager, "discover", classmethod(lambda cls, specs: manager))
app = Application(
plugins=["example"],
workspace_dir=str(tmp_path),
enable_logo=False,
log_to_console=False,
log_to_file=False,
service={"backend": "cli"},
)
component = app.context.components["example.reranker"]["default"]
assert isinstance(component, _PluginComponent)
assert app.context.registry.get("example.reranker", "cross_encoder") is _PluginComponent
await app.start()
assert component.is_started is True
await app.update_component("example.reranker", "default", backend="updated")
assert component.backend == "updated"
await app.close()
assert component.is_started is False
def test_plugin_backend_collision_fails_with_both_owners():
registry = ComponentRegistry()
registry.add("same", _PluginStep, owner="first")
class OtherStep(ComponentMixin):
component_type = ComponentEnum.STEP
with pytest.raises(ValueError, match="both 'first' and 'second'"):
registry.add("same", OtherStep, owner="second")
def test_plugin_manager_loads_explicit_entry_point(monkeypatch):
descriptor = Plugin(name="example", backends=(Backend("example_step", _PluginStep),))
class FakeEntryPoint:
name = "example"
value = "example:plugin"
@staticmethod
def load():
return descriptor
class FakeEntryPoints(list):
def select(self, *, group, name):
assert group == "reme.plugins"
return [entry for entry in self if entry.name == name]
monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()]))
manager = PluginManager.discover(["example"])
assert manager.plugins == (descriptor,)
def test_plugin_entry_point_import_side_effect_does_not_leak(monkeypatch, tmp_path):
class UndeclaredClient(ComponentMixin):
component_type = ComponentEnum.CLIENT
descriptor = Plugin(name="example")
class FakeEntryPoint:
name = "example"
value = "example:plugin"
@staticmethod
def load():
R.register(UndeclaredClient, "undeclared-client")
return descriptor
class FakeEntryPoints(list):
def select(self, *, group, name):
assert group == "reme.plugins"
return [entry for entry in self if entry.name == name]
monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()]))
app = Application(
plugins=["example"],
workspace_dir=str(tmp_path),
enable_logo=False,
log_to_console=False,
log_to_file=False,
service={"backend": "cli"},
)
assert R.get(ComponentEnum.CLIENT, "undeclared-client") is None
assert app.context.registry.get(ComponentEnum.CLIENT, "undeclared-client") is None
def test_plugin_manager_rejects_non_string_name():
with pytest.raises(TypeError, match="Invalid plugin name"):
PluginManager.discover([{"name": "example"}])
def test_config_can_extend_another_config(tmp_path: Path):
parent = tmp_path / "parent.yaml"
child = tmp_path / "child.yaml"
parent.write_text("service:\n backend: http\n port: 8000\n", encoding="utf-8")
child.write_text("extends: parent.yaml\nservice:\n port: 9000\n", encoding="utf-8")
assert _load_config(str(child))["service"] == {"backend": "http", "port": 9000}
def test_config_can_come_from_installed_entry_point(tmp_path: Path, monkeypatch):
config = tmp_path / "example.yaml"
config.write_text("plugins: [example]\n", encoding="utf-8")
entry = EntryPoint(name="example", value="pathlib:Path", group="reme.configs")
class LoadedEntryPoint:
name = entry.name
value = entry.value
@staticmethod
def load():
return config
class FakeEntryPoints(list):
def select(self, *, group, name):
assert group == "reme.configs"
return [item for item in self if item.name == name]
monkeypatch.setattr(
"reme.config.config_parser.metadata.entry_points",
lambda: FakeEntryPoints([LoadedEntryPoint()]),
)
assert _load_config("example") == {"plugins": ["example"]}
def test_config_entry_point_import_side_effect_does_not_leak(tmp_path: Path, monkeypatch):
config = tmp_path / "side-effect.yaml"
config.write_text("service:\n backend: cli\n", encoding="utf-8")
class UndeclaredClient(ComponentMixin):
component_type = ComponentEnum.CLIENT
class LoadedEntryPoint:
name = "side-effect"
value = "example:CONFIG_PATH"
@staticmethod
def load():
R.register(UndeclaredClient, "config-side-effect-client")
return config
class FakeEntryPoints(list):
def select(self, *, group, name):
assert group == "reme.configs"
return [item for item in self if item.name == name]
monkeypatch.setattr(
"reme.config.config_parser.metadata.entry_points",
lambda: FakeEntryPoints([LoadedEntryPoint()]),
)
loaded = _load_config("side-effect")
app = Application(
**loaded,
workspace_dir=str(tmp_path / "workspace"),
enable_logo=False,
log_to_console=False,
log_to_file=False,
)
assert loaded == {"service": {"backend": "cli"}}
assert R.get(ComponentEnum.CLIENT, "config-side-effect-client") is None
assert app.context.registry.get(ComponentEnum.CLIENT, "config-side-effect-client") is None

View file

@ -11,6 +11,9 @@ import pytest
from reme.components.service import cli_service
from reme.components.service.cli_service import CliService
from reme import reme as reme_module
from reme.components.base_component import ComponentMixin
from reme.enumeration import ComponentEnum
from reme.plugin import Backend, Plugin, PluginManager
def test_package_import_does_not_load_optional_core_dependencies():
@ -257,8 +260,12 @@ def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys):
seen["payload"] = kwargs
yield "ok"
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
monkeypatch.setattr(
reme_module,
"create_application_registry",
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
)
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
async def run():
await reme_module.call_server(
@ -299,8 +306,12 @@ def test_call_server_treats_show_metadata_as_client_kwarg(monkeypatch, capsys):
seen["payload"] = kwargs
yield "ok"
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
monkeypatch.setattr(
reme_module,
"create_application_registry",
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
)
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
async def run():
await reme_module.call_server("version", backend="http", show_metadata=True)
@ -334,8 +345,12 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys):
seen["payload"] = kwargs
yield "ok"
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
monkeypatch.setattr(
reme_module,
"create_application_registry",
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
)
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
async def run():
await reme_module.call_server("shell", backend="http", cmd="ls", shell_timeout=5)
@ -345,3 +360,86 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys):
assert seen["action"] == "shell"
assert seen["payload"] == {"cmd": "ls", "shell_timeout": 5}
assert capsys.readouterr().out == "ok\n"
def test_call_server_uses_running_plugins_and_their_service_defaults(monkeypatch, capsys):
"""A bare client call can load the Client backend enabled by the running app."""
seen = {}
class PluginClient(ComponentMixin):
"""Client backend supplied by an enabled plugin."""
component_type = ComponentEnum.CLIENT
def __init__(self, **kwargs):
super().__init__(**kwargs)
seen["client_kwargs"] = kwargs
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
return None
async def __call__(self, action: str, **kwargs):
seen["action"] = action
seen["payload"] = kwargs
yield "plugin-ok"
manager = PluginManager(
[
Plugin(
name="example",
backends=(Backend("plugin-client", PluginClient),),
config={"service": {"backend": "plugin-client", "host": "127.0.0.9", "port": 9911}},
),
],
)
monkeypatch.setattr(reme_module, "resolve_app_config", lambda **_kwargs: {"service": {"backend": "http"}})
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"plugins": ["example"]})
monkeypatch.setattr(reme_module.PluginManager, "discover", lambda _specs: manager)
asyncio.run(reme_module.call_server("search", query="hello"))
assert seen["client_kwargs"]["host"] == "127.0.0.9"
assert seen["client_kwargs"]["port"] == 9911
assert seen["action"] == "search"
assert seen["payload"] == {"query": "hello"}
assert capsys.readouterr().out == "plugin-ok\n"
def test_call_server_skips_local_fallback_when_server_is_running(monkeypatch, capsys):
"""A usable running config prevents eager parsing of the local fallback."""
class FakeClient:
"""Minimal running-service client."""
def __init__(self, **_kwargs):
pass
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
return None
async def __call__(self, action: str, **kwargs):
assert action == "version"
assert not kwargs
yield "ok"
monkeypatch.setattr(
reme_module,
"create_application_registry",
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
)
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"service": {"backend": "http"}})
def fail_local_resolution(**_kwargs):
raise AssertionError("local fallback should not be resolved")
monkeypatch.setattr(reme_module, "resolve_app_config", fail_local_resolution)
asyncio.run(reme_module.call_server("version"))
assert capsys.readouterr().out == "ok\n"

View file

@ -156,3 +156,22 @@ def test_scan_reme_procs_skips_access_denied(monkeypatch):
]
_patch_iter(monkeypatch, procs)
assert su._scan_reme_procs() == [(5, su.REME_DEFAULT_HOST, su.REME_DEFAULT_PORT)]
def test_running_app_config_preserves_plugins(monkeypatch):
"""Process replay exposes the full config while the compatibility helper returns service only."""
monkeypatch.setattr(su, "_reme_start_argv", lambda: [["config=example"]])
monkeypatch.setattr("reme.config.parse_args", lambda *_args: ("start", {"config": "example"}))
monkeypatch.setattr(
"reme.config.resolve_app_config",
lambda **_kwargs: {
"plugins": ["example"],
"service": {"backend": "plugin-client", "port": 9911},
},
)
assert su.running_app_config() == {
"plugins": ["example"],
"service": {"backend": "plugin-client", "port": 9911},
}
assert su.running_service_config() == {"backend": "plugin-client", "port": 9911}