mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
feat: add entry-point plugin system and extract Auto Fin (#459)
* feat: add entry-point plugin system * fix: harden plugin config and client loading * docs(workflow): add detailed manual for publishing reme-auto-fin to PyPI - Provide step-by-step instructions for updating project.version and merging branches - Explain dependency verification for reme-ai on PyPI during build - Specify requirements for GitHub Actions secret configuration and version uniqueness - Describe manual workflow triggering and input of version number - Recommend publishing order for related projects - Clarify that only manual dispatch triggers publishing, no automatic triggers on push or tag * feat: support plugin-defined component types * refactor: simplify plugin configuration * fix: isolate plugin loading and defer client fallback * refactor: freeze built-in component registry * fix: isolate config entry point loading * fix: complete auto-fin package metadata
This commit is contained in:
parent
d3aee1adf5
commit
618e8cec66
61 changed files with 1474 additions and 275 deletions
138
.github/workflows/auto-fin-publish.yml
vendored
Normal file
138
.github/workflows/auto-fin-publish.yml
vendored
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
# 发布操作手册:
|
||||
# 1. 先将 plugin/auto-fin/pyproject.toml 中的 project.version 更新为待发布版本并合入目标分支。
|
||||
# 2. 确认插件依赖的 reme-ai 版本已经发布到 PyPI;本工作流会在构建阶段验证该依赖可下载。
|
||||
# 3. 确认仓库 Actions Secret 已配置 PYPI_API_TOKEN,且 PyPI 上不存在相同版本。
|
||||
# 4. 在 GitHub 仓库的 Actions 页面选择“Publish reme-auto-fin to PyPI”,点击“Run workflow”。
|
||||
# 5. 输入与 project.version 完全一致的版本号(例如 0.1.0)后运行;版本也可以带 v 前缀。
|
||||
#
|
||||
# 推荐发布顺序:reme-ai -> reme-auto-fin -> QwenPaw 更新依赖并通过 plugins: [auto-fin] 启用。
|
||||
# 当前仅支持 workflow_dispatch 手动触发,不会因 push、tag 或 release 自动发布。
|
||||
|
||||
name: Publish reme-auto-fin to PyPI
|
||||
|
||||
run-name: Publish reme-auto-fin ${{ inputs.version }}
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Version from plugin/auto-fin/pyproject.toml (for example, 0.1.0)
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: publish-reme-auto-fin
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
RELEASE_VERSION: ${{ inputs.version }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install test and build dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install build packaging pytest pytest-asyncio twine
|
||||
python -m pip install -e ".[core]"
|
||||
python -m pip install --no-deps -e plugin/auto-fin
|
||||
|
||||
- name: Validate package name and release version
|
||||
id: package
|
||||
run: |
|
||||
python - "${RELEASE_VERSION}" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.version import Version
|
||||
|
||||
project = tomllib.loads(Path("plugin/auto-fin/pyproject.toml").read_text(encoding="utf-8"))["project"]
|
||||
expected = Version(sys.argv[1].removeprefix("v"))
|
||||
actual = Version(project["version"])
|
||||
if project["name"] != "reme-auto-fin":
|
||||
raise SystemExit(f"Expected project name 'reme-auto-fin', found {project['name']!r}")
|
||||
if actual != expected:
|
||||
raise SystemExit(f"Package version is {actual}, but workflow input is {expected}")
|
||||
requirements = [requirement for requirement in project["dependencies"] if requirement.startswith("reme-ai")]
|
||||
if len(requirements) != 1:
|
||||
raise SystemExit(f"Expected one reme-ai dependency, found {requirements!r}")
|
||||
reme_requirement = Requirement(requirements[0])
|
||||
if reme_requirement.name != "reme-ai" or set(reme_requirement.extras) != {"core"}:
|
||||
raise SystemExit(f"Expected a reme-ai[core] dependency, found {requirements[0]!r}")
|
||||
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
|
||||
print(f"reme_requirement={reme_requirement}", file=output)
|
||||
print(f"Publishing {project['name']} {actual}")
|
||||
PY
|
||||
|
||||
- name: Run Auto Fin tests
|
||||
run: python -m pytest plugin/auto-fin -q
|
||||
|
||||
- name: Require the plugin-enabled ReMe release on PyPI
|
||||
run: |
|
||||
python -m pip download --no-deps \
|
||||
--dest "${RUNNER_TEMP}/reme-auto-fin-core" \
|
||||
"${{ steps.package.outputs.reme_requirement }}"
|
||||
|
||||
- name: Build and check distributions
|
||||
run: |
|
||||
mkdir -p dist/auto-fin
|
||||
python -m build plugin/auto-fin --outdir dist/auto-fin
|
||||
python -m twine check dist/auto-fin/*
|
||||
|
||||
- name: Verify distributions and isolated installation
|
||||
run: |
|
||||
AUTO_FIN_WHEEL="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.whl)"
|
||||
AUTO_FIN_SDIST="$(pwd)/$(ls dist/auto-fin/reme_auto_fin-*.tar.gz)"
|
||||
python -m zipfile -l "${AUTO_FIN_WHEEL}" | grep 'dist-info/licenses/LICENSE'
|
||||
python -m tarfile -l "${AUTO_FIN_SDIST}" | grep '/LICENSE'
|
||||
python -m venv "${RUNNER_TEMP}/reme-auto-fin-smoke"
|
||||
"${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" -m pip install "${AUTO_FIN_WHEEL}"
|
||||
cd "${RUNNER_TEMP}"
|
||||
"${RUNNER_TEMP}/reme-auto-fin-smoke/bin/python" - <<'PY'
|
||||
from importlib.metadata import distribution
|
||||
|
||||
package = distribution("reme-auto-fin")
|
||||
plugins = {entry.name: entry for entry in package.entry_points if entry.group == "reme.plugins"}
|
||||
configs = {entry.name: entry for entry in package.entry_points if entry.group == "reme.configs"}
|
||||
assert plugins["auto-fin"].load().name == "auto-fin"
|
||||
assert configs["auto-fin"].load().is_file()
|
||||
PY
|
||||
|
||||
- name: Upload distributions
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: reme-auto-fin-${{ inputs.version }}
|
||||
path: dist/auto-fin/
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Download distributions
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: reme-auto-fin-${{ inputs.version }}
|
||||
path: dist/auto-fin
|
||||
|
||||
- name: Publish reme-auto-fin
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
user: __token__
|
||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||
packages-dir: dist/auto-fin
|
||||
4
.github/workflows/github-pages-check.yml
vendored
4
.github/workflows/github-pages-check.yml
vendored
|
|
@ -12,7 +12,7 @@ on:
|
|||
- 'github-pages/**'
|
||||
- 'website/README*.md'
|
||||
- 'website/public/og.jpg'
|
||||
- 'cookbook/*/README*.md'
|
||||
- 'plugin/*/README*.md'
|
||||
- 'benchmark/*/README*.md'
|
||||
- 'skills/reme_memory/SKILL.md'
|
||||
pull_request:
|
||||
|
|
@ -26,7 +26,7 @@ on:
|
|||
- 'github-pages/**'
|
||||
- 'website/README*.md'
|
||||
- 'website/public/og.jpg'
|
||||
- 'cookbook/*/README*.md'
|
||||
- 'plugin/*/README*.md'
|
||||
- 'benchmark/*/README*.md'
|
||||
- 'skills/reme_memory/SKILL.md'
|
||||
workflow_dispatch:
|
||||
|
|
|
|||
2
.github/workflows/pages.yml
vendored
2
.github/workflows/pages.yml
vendored
|
|
@ -10,7 +10,7 @@ on:
|
|||
- "README_ZH.md"
|
||||
- "website/README*.md"
|
||||
- "website/public/og.jpg"
|
||||
- "cookbook/*/README*.md"
|
||||
- "plugin/*/README*.md"
|
||||
- "benchmark/*/README*.md"
|
||||
- "skills/reme_memory/SKILL.md"
|
||||
- "AGENTS.md"
|
||||
|
|
|
|||
3
.github/workflows/unittest.yml
vendored
3
.github/workflows/unittest.yml
vendored
|
|
@ -33,11 +33,12 @@ jobs:
|
|||
run: |
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
pip install -e packages/reme_ai_studio -e ".[dev,core]"
|
||||
pip install --no-deps -e plugin/auto-fin
|
||||
pip install coverage
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
coverage run -m pytest tests/unit \
|
||||
coverage run -m pytest tests/unit plugin/auto-fin \
|
||||
-v \
|
||||
--tb=long \
|
||||
-s \
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ and concise documentation together.
|
|||
- `reme/components/application_context.py`: application-wide wiring and in-memory shared state.
|
||||
- `reme/components/runtime_context.py`: request-scoped data, response, streaming queue, and stop event.
|
||||
- `reme/components/base_component.py`: component lifecycle, dependency binding, and workspace helpers.
|
||||
- `reme/components/component_registry.py`: the process-wide `(component type, backend)` registry.
|
||||
- `reme/components/component_registry.py`: the frozen built-in registry template and application-local registry factory.
|
||||
- `reme/components/job/`: base, stream, background, and cron job implementations.
|
||||
- `reme/components/service/`: local CLI, HTTP, and MCP service backends.
|
||||
- `reme/components/`: agent wrappers, model adapters, stores, catalogs, graphs, indexes, clients, tokenizers, and
|
||||
|
|
@ -56,7 +56,7 @@ and concise documentation together.
|
|||
- `plugins/claude_code/` and `plugins/hermes_agent/`: agent integrations.
|
||||
- `skills/`: standalone skills; `reme_memory` calls ReMe, while other skills may use separate tools or direct-file
|
||||
conventions.
|
||||
- `benchmark/` and `cookbook/`: runnable evaluation and example workflows.
|
||||
- `benchmark/` and `plugin/`: runnable evaluations and external plugin examples.
|
||||
- `docs/`: README-linked supporting pages and figures.
|
||||
|
||||
## Development Setup
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -200,13 +200,13 @@ These Markdown guides cover the main user workflows and the runtime contracts im
|
|||
| [Framework](docs/en/framework.md) | Understand Application, Job, Step, Component, service, configuration, and lifecycle boundaries. |
|
||||
| [ReMe Blog](https://agentscope-ai.github.io/ReMe/?doc=en-reme-blog) | Read the product story, design rationale, examples, and benchmark summary. |
|
||||
|
||||
## 🧑🍳 Cookbooks
|
||||
## 🔌 Plugins
|
||||
|
||||
Cookbooks are optional, end-to-end workflows assembled from ReMe jobs and steps. They are not enabled by the default
|
||||
configuration; select the cookbook's standalone configuration when starting ReMe. Each new cookbook will be added as
|
||||
another row in this table.
|
||||
Plugins are optional Python distributions that contribute Component, Step, or Job backends and configuration. They are
|
||||
installed separately and enabled explicitly by configuration. Auto Fin is the complete external-plugin example; Daily
|
||||
Paper remains an optional research workflow while it is migrated to the same packaging model.
|
||||
|
||||
| Cookbook | Capability |
|
||||
| Plugin / workflow | Capability |
|
||||
|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------|
|
||||
| [Daily Paper](https://reme.agentscope.io/?doc=daily-paper-en) | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. |
|
||||
| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-en) | Fetch topic-related CLS news, search ReMe history, and generate wikilink-backed Markdown reports. |
|
||||
|
|
|
|||
|
|
@ -193,12 +193,12 @@ ReMe 会把 Agent 记忆保存为可读的 Markdown。
|
|||
| [框架说明](docs/zh/framework.md) | 理解 Application、Job、Step、Component、service、配置和生命周期边界。 |
|
||||
| [ReMe 博客](https://agentscope-ai.github.io/ReMe/?doc=zh-reme-blog) | 了解完整产品故事、设计动机、使用示例和评测摘要。 |
|
||||
|
||||
## 🧑🍳 Cookbooks
|
||||
## 🔌 插件
|
||||
|
||||
Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们;启动 ReMe 时选择对应的独立配置即可启用。后续新增的
|
||||
cookbook 会继续在表格中按行追加。
|
||||
插件是可选的独立 Python distribution,可以贡献 Component、Step、Job backend 和配置,并通过配置显式启用。Auto Fin
|
||||
是完整的外部插件示例;每日论文在迁移到同一打包模型前仍作为可选研究工作流提供。
|
||||
|
||||
| Cookbook | 能力 |
|
||||
| 插件 / 工作流 | 能力 |
|
||||
|-----------------------------------------------|--------------------------------------------------------------------------------|
|
||||
| [每日论文](https://reme.agentscope.io/?doc=daily-paper-zh) | 发现并排序论文,使用 Agent 解读 PDF,生成文件化论文笔记和五分钟简报。 |
|
||||
| [Auto Fin](https://reme.agentscope.io/?doc=auto-fin-zh) | 拉取主题相关财联社新闻,搜索 ReMe 历史材料并生成带 wikilink 的 Markdown 报告。 |
|
||||
|
|
|
|||
|
|
@ -55,11 +55,12 @@ Core layers:
|
|||
reme/
|
||||
reme.py # CLI entry point
|
||||
application.py # Application assembly and lifecycle
|
||||
plugin.py # installed plugin contract and entry-point loader
|
||||
config/
|
||||
default.yaml # default service / jobs / components
|
||||
config_parser.py # config=, dot notation, and env placeholder parsing
|
||||
components/
|
||||
component_registry.py # global registry R
|
||||
component_registry.py # backend registry and application-local copies
|
||||
base_component.py # ComponentMixin / BaseComponent / bind dependency declarations
|
||||
runtime_context.py # context for one Job execution
|
||||
job/ # BaseJob / StreamJob / BackgroundJob / CronJob
|
||||
|
|
@ -81,6 +82,8 @@ reme/
|
|||
index/ # watch/init/update/search/traverse
|
||||
evolve/ # auto_memory, auto_resource, auto_dream, proactive
|
||||
transfer/ # upload/download
|
||||
plugin/
|
||||
auto-fin/ # independent example plugin distribution
|
||||
```
|
||||
|
||||
The default workspace directories are defined by `ApplicationConfig`:
|
||||
|
|
@ -216,14 +219,23 @@ The registry key is:
|
|||
The same backend name can therefore exist under different component types. For example, `http` can be both a service
|
||||
backend and a client backend.
|
||||
|
||||
### 4.2 Registration Through Module Imports
|
||||
`ComponentEnum` provides the built-in identifiers, but installed plugins may declare a new type with a namespaced
|
||||
string such as `example.reranker`. Custom identifiers use lowercase letters and numbers separated by `.`, `_`, or `-`.
|
||||
They are configured under `components` and participate in the same dependency ordering and lifecycle as built-ins.
|
||||
|
||||
Registration happens when a module is imported. `reme/components/__init__.py` imports component packages, while
|
||||
`reme/steps/__init__.py` imports `benchmark/common/cookbook/evolve/file_io/index/transfer`. Each package's `__init__.py`
|
||||
then imports its concrete modules, causing `@R.register(...)` to execute.
|
||||
### 4.2 Built-in and Plugin Registration
|
||||
|
||||
After adding a Step file, make sure the package's `__init__.py` imports it. Otherwise, the backend will not appear in
|
||||
the registry.
|
||||
Built-in implementations populate the built-in registry through package imports. ReMe freezes that template after
|
||||
bootstrap, and each `Application` receives a mutable copy. Runtime code resolves backends through the application's
|
||||
registry rather than changing the process-wide template. ReMe then loads only the installed plugins explicitly named by
|
||||
`plugins` in the resolved configuration. A plugin
|
||||
is exposed through the `reme.plugins` Python entry-point group and returns a declarative `reme.plugin.Plugin` containing
|
||||
its named backend classes and default configuration. Plugin registration therefore stays local to one application;
|
||||
duplicate `(component_type, backend)` providers fail during assembly instead of overwriting each other.
|
||||
|
||||
Installed plugins may also expose named configuration files through `reme.configs`. Configuration can use `extends` to
|
||||
inherit another built-in, plugin, or file-based configuration. The [Auto Fin plugin](../../plugin/auto-fin/README.md)
|
||||
is the complete packaging example.
|
||||
|
||||
### 4.3 Component.bind
|
||||
|
||||
|
|
|
|||
|
|
@ -50,11 +50,12 @@ flowchart LR
|
|||
reme/
|
||||
reme.py # CLI 入口
|
||||
application.py # Application 装配与生命周期
|
||||
plugin.py # 已安装插件契约与 entry-point loader
|
||||
config/
|
||||
default.yaml # 默认 service / jobs / components
|
||||
config_parser.py # config=、dot notation、env 占位符解析
|
||||
components/
|
||||
component_registry.py # 全局注册表 R
|
||||
component_registry.py # backend 注册表及 Application 局部副本
|
||||
base_component.py # ComponentMixin / BaseComponent / bind 依赖声明
|
||||
runtime_context.py # 单次 Job 执行上下文
|
||||
job/ # BaseJob / StreamJob / BackgroundJob / CronJob
|
||||
|
|
@ -76,6 +77,8 @@ reme/
|
|||
index/ # watch/init/update/search/traverse
|
||||
evolve/ # auto_memory、auto_resource、auto_dream、proactive
|
||||
transfer/ # upload/download
|
||||
plugin/
|
||||
auto-fin/ # 独立发布的示例插件
|
||||
```
|
||||
|
||||
默认 workspace 目录由 `ApplicationConfig` 定义:
|
||||
|
|
@ -208,13 +211,19 @@ class VersionStep(BaseStep):
|
|||
|
||||
所以同名 backend 在不同 component type 下可以共存。例如 `http` 同时可以是 service backend 和 client backend。
|
||||
|
||||
### 4.2 模块导入触发注册
|
||||
`ComponentEnum` 提供内置类型标识;已安装插件也可以用 `example.reranker` 这样的命名空间字符串声明新类型。自定义标识仅使用
|
||||
小写字母和数字,并以 `.`、`_` 或 `-` 分隔。它们配置在 `components` 下,与内置组件参与相同的依赖排序和生命周期。
|
||||
|
||||
注册发生在模块 import 时。`reme/components/__init__.py` 会 import 各组件包,`reme/steps/__init__.py` 会 import
|
||||
`benchmark/common/cookbook/evolve/file_io/index/transfer`。这些包的 `__init__.py` 再 import 具体模块,从而执行
|
||||
`@R.register(...)`。
|
||||
### 4.2 内置注册与插件注册
|
||||
|
||||
新增 Step 文件后,必须保证它所在包的 `__init__.py` 会 import 该模块,否则注册表里找不到这个 backend。
|
||||
内置实现通过 package import 填充内置注册表;bootstrap 完成后 ReMe 会冻结这个模板,并为每个 `Application` 创建可写副本。
|
||||
运行期代码通过当前 Application 的注册表解析 backend,不能修改进程级模板。随后只加载最终配置中 `plugins` 明确启用的已安装插件。
|
||||
插件通过 Python `reme.plugins` entry-point group 暴露,返回声明式 `reme.plugin.Plugin`,其中包含命名
|
||||
backend class 和默认配置。插件注册因此只影响当前 Application;两个插件提供相同 `(component_type, backend)` 时会在装配阶段失败,
|
||||
不会互相覆盖。
|
||||
|
||||
插件还可以通过 `reme.configs` 暴露命名配置;配置的 `extends` 可以继承内置配置、插件配置或文件配置。完整打包示例见
|
||||
[Auto Fin 插件](../../plugin/auto-fin/README_ZH.md)。
|
||||
|
||||
### 4.3 Component.bind
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ The build script reads the canonical repository files directly. Do not edit gene
|
|||
- `docs/en/` and `docs/zh/`: English and Chinese guides
|
||||
- `docs/figure/`: documentation images
|
||||
- `website/README.md` and `website/README_ZH.md`: ReMe Studio guide
|
||||
- `cookbook/*/README*.md`: research workflow guides
|
||||
- `plugin/*/README*.md`: plugin and research workflow guides
|
||||
- `benchmark/{beam,longmemeval,pibench,toolmemory}/README*.md`: benchmark guides and results
|
||||
- `skills/reme_memory/SKILL.md`: ReMe Memory skill guide
|
||||
- `AGENTS.md`: repository development guide
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ const productDocuments = [
|
|||
},
|
||||
{
|
||||
slug: "daily-paper",
|
||||
source: "cookbook/daily_paper",
|
||||
source: "plugin/daily_paper",
|
||||
titles: { zh: "每日论文", en: "Daily Paper" },
|
||||
descriptions: {
|
||||
zh: "发现论文、解析 PDF,并生成阅读笔记与每日简报。",
|
||||
|
|
@ -74,7 +74,7 @@ const productDocuments = [
|
|||
},
|
||||
{
|
||||
slug: "auto-fin",
|
||||
source: "cookbook/auto-fin",
|
||||
source: "plugin/auto-fin",
|
||||
titles: { zh: "财经研究", en: "Auto Fin" },
|
||||
descriptions: {
|
||||
zh: "结合最新财联社新闻与本地历史记忆生成研究报告。",
|
||||
|
|
|
|||
201
plugin/auto-fin/LICENSE
Normal file
201
plugin/auto-fin/LICENSE
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2025 Alibaba Group
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
# Auto Fin Cookbook
|
||||
# Auto Fin Plugin
|
||||
|
||||
[中文](README_ZH.md)
|
||||
|
||||
Auto Fin fetches a rolling window of CLS telegraph news (24 hours by default), selects items related to configured
|
||||
topics, searches ReMe for useful historical context, and writes one Chinese Markdown report with validated wikilinks.
|
||||
Current news and topic selection stay in runtime memory; only the final report becomes durable memory. The
|
||||
implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/) and is assembled by
|
||||
[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml).
|
||||
Current news and topic selection stay in runtime memory; only the final report becomes durable memory. This directory
|
||||
is an independent Python distribution. Its `reme.plugins` entry point contributes the three Step backends and their Job
|
||||
configuration; its `reme.configs` entry point exposes the runnable `auto-fin` configuration.
|
||||
|
||||
> Auto Fin has no reliable market-price feed. It does not calculate returns, targets, or entry points and is not
|
||||
> investment advice.
|
||||
|
|
@ -14,11 +14,11 @@ implementation lives in [`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookb
|
|||
## Quick start
|
||||
|
||||
```bash
|
||||
python -m pip install -e ".[core]"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin
|
||||
export LLM_API_KEY="your-api-key"
|
||||
export LLM_MODEL_NAME="qwen3.7-plus"
|
||||
export LLM_BASE_URL="https://your-provider.example/v1"
|
||||
reme start config=daily_cookbook job=auto_fin
|
||||
reme start config=auto-fin job=auto_fin
|
||||
```
|
||||
|
||||
`LLM_MODEL_NAME` defaults to `qwen3.7-plus`. There is no built-in `LLM_BASE_URL`, so set the OpenAI-compatible endpoint
|
||||
|
|
@ -27,7 +27,7 @@ required by the selected provider.
|
|||
The default topics are `黄金,机器人,半导体`. Override them per run:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片"
|
||||
reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片"
|
||||
```
|
||||
|
||||
An empty value also uses the defaults.
|
||||
|
|
@ -76,23 +76,22 @@ refreshes the daily index. No JSONL, intermediate Markdown, or structured Agent
|
|||
| `request_interval` | `10` | Minimum delay in seconds after every CLS request attempt; may be zero |
|
||||
| `max_retries` | `3` | Maximum attempts for each CLS page request; must be at least one |
|
||||
|
||||
The built-in schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`.
|
||||
The plugin-provided schedules run daily at 09:30, 11:30, and 18:00 in `Asia/Shanghai`.
|
||||
|
||||
## Output
|
||||
|
||||
```text
|
||||
reme_workspace/daily/YYYY-MM-DD/auto_fin.md
|
||||
.reme/daily/YYYY-MM-DD/auto_fin.md
|
||||
```
|
||||
|
||||
The report includes a title, description, current CLS evidence, historical analysis, contextual wikilinks, and a fixed
|
||||
non-investment disclaimer. Network errors and invalid Agent output fail explicitly; no relevant current news is a
|
||||
successful skip. If `DINGTALK_CONVERSATION_IDS` is empty, delivery is a no-op. If it is set, the DingTalk credentials
|
||||
described in the [Daily Paper cookbook](../daily_paper/README.md#6-dingtalk) are required.
|
||||
successful skip.
|
||||
|
||||
## Validation
|
||||
|
||||
```bash
|
||||
pytest tests/unit/test_auto_fin.py -v
|
||||
python -m pytest plugin/auto-fin -v
|
||||
```
|
||||
|
||||
Unit tests mock the CLS and Agent boundaries and do not contact external services.
|
||||
|
|
@ -1,22 +1,22 @@
|
|||
# Auto Fin Cookbook
|
||||
# Auto Fin 插件
|
||||
|
||||
[English](README.md)
|
||||
|
||||
Auto Fin 自动拉取一个滚动时间窗口内的财联社电报(默认 24 小时),按配置 topics 筛选相关新闻,搜索 ReMe 中有回顾价值的历史材料,最后写入一份带校验
|
||||
wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中,只有最终报告成为持久记忆。实现位于
|
||||
[`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/),并由
|
||||
[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配。
|
||||
wikilink 的中文 Markdown 报告。当前新闻和筛选结果只存在于本次运行内存中,只有最终报告成为持久记忆。本目录是一个独立 Python
|
||||
distribution:`reme.plugins` entry point 贡献三个 Step backend 及其 Job 配置,`reme.configs` entry point 暴露可直接运行的
|
||||
`auto-fin` 配置。
|
||||
|
||||
> Auto Fin 没有可靠行情数据,不计算收益、目标价或买卖点,也不提供投资建议。
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
python -m pip install -e ".[core]"
|
||||
python -m pip install "reme-ai[core]>=0.4.1.8" reme-auto-fin
|
||||
export LLM_API_KEY="your-api-key"
|
||||
export LLM_MODEL_NAME="qwen3.7-plus"
|
||||
export LLM_BASE_URL="https://your-provider.example/v1"
|
||||
reme start config=daily_cookbook job=auto_fin
|
||||
reme start config=auto-fin job=auto_fin
|
||||
```
|
||||
|
||||
`LLM_MODEL_NAME` 默认是 `qwen3.7-plus`。代码没有内置 `LLM_BASE_URL`,请设置所选服务商提供的 OpenAI 兼容 endpoint。
|
||||
|
|
@ -24,7 +24,7 @@ reme start config=daily_cookbook job=auto_fin
|
|||
默认 topics 是 `黄金,机器人,半导体`。可在运行时覆盖:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=auto_fin topics="黄金,AI,存储芯片"
|
||||
reme start config=auto-fin job=auto_fin topics="黄金,AI,存储芯片"
|
||||
```
|
||||
|
||||
传入空值也会使用默认 topics。
|
||||
|
|
@ -69,22 +69,21 @@ workspace 的 Markdown 目标。不存在、绝对路径、越界、带反斜杠
|
|||
| `request_interval` | `10` | 每次财联社请求尝试后的最小等待秒数,可设为 0 |
|
||||
| `max_retries` | `3` | 每页财联社请求的最大尝试次数,至少为 1 |
|
||||
|
||||
内置定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。
|
||||
插件提供的定时任务每天按 `Asia/Shanghai` 在 09:30、11:30 和 18:00 运行。
|
||||
|
||||
## 产物
|
||||
|
||||
```text
|
||||
reme_workspace/daily/YYYY-MM-DD/auto_fin.md
|
||||
.reme/daily/YYYY-MM-DD/auto_fin.md
|
||||
```
|
||||
|
||||
报告包含标题、说明、当前 CLS 证据、历史分析、上下文 wikilink 和固定非投资建议声明。网络错误与无效 Agent 输出
|
||||
会明确失败;没有相关当前新闻则成功跳过。`DINGTALK_CONVERSATION_IDS` 为空时发送步骤无副作用;设置该变量后,
|
||||
还必须提供[每日论文 Cookbook](../daily_paper/README_ZH.md#6-dingtalk)中列出的钉钉凭据。
|
||||
会明确失败;没有相关当前新闻则成功跳过。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
pytest tests/unit/test_auto_fin.py -v
|
||||
python -m pytest plugin/auto-fin -v
|
||||
```
|
||||
|
||||
单元测试 mock CLS 与 Agent 边界,不访问外部服务。
|
||||
34
plugin/auto-fin/pyproject.toml
Normal file
34
plugin/auto-fin/pyproject.toml
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
[project]
|
||||
name = "reme-auto-fin"
|
||||
version = "0.1.0"
|
||||
description = "Auto Fin example plugin for ReMe."
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
license-files = ["LICENSE"]
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"reme-ai[core]>=0.4.1.8",
|
||||
]
|
||||
|
||||
[project.entry-points."reme.plugins"]
|
||||
auto-fin = "reme_auto_fin:plugin"
|
||||
|
||||
[project.entry-points."reme.configs"]
|
||||
auto-fin = "reme_auto_fin:CONFIG_PATH"
|
||||
|
||||
[tool.setuptools]
|
||||
package-dir = { "" = "src" }
|
||||
packages = ["reme_auto_fin"]
|
||||
include-package-data = true
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
reme_auto_fin = ["*.yaml"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
pythonpath = ["src", "../.."]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=77", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
21
plugin/auto-fin/src/reme_auto_fin/__init__.py
Normal file
21
plugin/auto-fin/src/reme_auto_fin/__init__.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Auto Fin news research workflow."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .data import AutoFinDataStep
|
||||
from .merge import AutoFinMergeStep
|
||||
from .plugin import plugin
|
||||
from .schema import AutoFinReportOutput, AutoFinTopicOutput
|
||||
from .topic import AutoFinTopicStep
|
||||
|
||||
CONFIG_PATH = Path(__file__).with_name("config.yaml")
|
||||
|
||||
__all__ = [
|
||||
"AutoFinReportOutput",
|
||||
"AutoFinDataStep",
|
||||
"AutoFinMergeStep",
|
||||
"AutoFinTopicOutput",
|
||||
"AutoFinTopicStep",
|
||||
"CONFIG_PATH",
|
||||
"plugin",
|
||||
]
|
||||
|
|
@ -12,7 +12,7 @@ from uuid import uuid4
|
|||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...base_step import BaseStep
|
||||
from reme.steps import BaseStep
|
||||
|
||||
AGENT_INPUT_LOG_LIMIT = 2000
|
||||
AGENT_OUTPUT_LOG_LIMIT = 4000
|
||||
2
plugin/auto-fin/src/reme_auto_fin/config.yaml
Normal file
2
plugin/auto-fin/src/reme_auto_fin/config.yaml
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
extends: default
|
||||
plugins: [auto-fin]
|
||||
|
|
@ -9,8 +9,7 @@ from typing import Any
|
|||
|
||||
import httpx
|
||||
|
||||
from ....components import R
|
||||
from ._base import AutoFinStep, _plain_text
|
||||
from .base import AutoFinStep, _plain_text
|
||||
|
||||
API_URL = "https://www.cls.cn/v1/roll/get_roll_list"
|
||||
HEADERS = {
|
||||
|
|
@ -23,7 +22,6 @@ DEFAULT_TOPICS = ("黄金", "机器人", "半导体")
|
|||
WINDOW = timedelta(hours=24)
|
||||
|
||||
|
||||
@R.register("auto_fin_data_step")
|
||||
class AutoFinDataStep(AutoFinStep):
|
||||
"""Fetch and normalize one rolling day of CLS news without writing files."""
|
||||
|
||||
54
plugin/auto-fin/src/reme_auto_fin/defaults.yaml
Normal file
54
plugin/auto-fin/src/reme_auto_fin/defaults.yaml
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
jobs:
|
||||
auto_fin:
|
||||
backend: base
|
||||
description: "Fetch and research recent topic-related CLS news."
|
||||
parameters:
|
||||
type: object
|
||||
properties:
|
||||
date:
|
||||
type: string
|
||||
description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai."
|
||||
default: ""
|
||||
now:
|
||||
type: string
|
||||
description: "Optional simulated current time in ISO 8601 format; empty means the real current time."
|
||||
default: ""
|
||||
topics:
|
||||
type: string
|
||||
description: "Comma-separated topics used to filter current CLS news."
|
||||
default: "黄金,机器人,半导体"
|
||||
window_hours:
|
||||
type: number
|
||||
exclusiveMinimum: 0
|
||||
description: "Rolling number of hours of CLS news to fetch."
|
||||
default: 24
|
||||
request_interval:
|
||||
type: number
|
||||
minimum: 0
|
||||
description: "Minimum delay in seconds after each CLS request attempt."
|
||||
default: 10
|
||||
max_retries:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: "Maximum attempts for each CLS page request."
|
||||
default: 3
|
||||
steps: &auto_fin_steps
|
||||
- backend: auto_fin_data_step
|
||||
- backend: auto_fin_topic_step
|
||||
- backend: auto_fin_merge_step
|
||||
job_tools: [memory_search, read]
|
||||
|
||||
auto_fin_0930_cron:
|
||||
backend: cron
|
||||
cron: "30 9 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
||||
auto_fin_1130_cron:
|
||||
backend: cron
|
||||
cron: "30 11 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
||||
auto_fin_1800_cron:
|
||||
backend: cron
|
||||
cron: "0 18 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
|
@ -8,10 +8,10 @@ from datetime import date, timedelta
|
|||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from ....components import R
|
||||
from ....schema import AutoFinReportOutput
|
||||
from ...file_io import refresh_day_index
|
||||
from ._base import AutoFinStep, _write
|
||||
from reme.steps.file_io import refresh_day_index
|
||||
|
||||
from .base import AutoFinStep, _write
|
||||
from .schema import AutoFinReportOutput
|
||||
|
||||
_WIKILINK_RE = re.compile(r"\[\[([^\[\]\n]+)\]\]")
|
||||
_HYBRID_WIKILINK_RE = re.compile(
|
||||
|
|
@ -19,7 +19,6 @@ _HYBRID_WIKILINK_RE = re.compile(
|
|||
)
|
||||
|
||||
|
||||
@R.register("auto_fin_merge_step")
|
||||
class AutoFinMergeStep(AutoFinStep):
|
||||
"""Give one Agent read-only ReMe tools, then validate links in its Markdown."""
|
||||
|
||||
31
plugin/auto-fin/src/reme_auto_fin/plugin.py
Normal file
31
plugin/auto-fin/src/reme_auto_fin/plugin.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Auto Fin plugin declaration."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from reme.plugin import Backend, Plugin
|
||||
|
||||
from .data import AutoFinDataStep
|
||||
from .merge import AutoFinMergeStep
|
||||
from .topic import AutoFinTopicStep
|
||||
|
||||
|
||||
def _default_config() -> dict:
|
||||
path = Path(__file__).with_name("defaults.yaml")
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
config = yaml.safe_load(stream)
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError(f"Auto Fin defaults must be a mapping: {path}")
|
||||
return config
|
||||
|
||||
|
||||
plugin = Plugin(
|
||||
name="auto-fin",
|
||||
backends=(
|
||||
Backend("auto_fin_data_step", AutoFinDataStep),
|
||||
Backend("auto_fin_topic_step", AutoFinTopicStep),
|
||||
Backend("auto_fin_merge_step", AutoFinMergeStep),
|
||||
),
|
||||
config=_default_config(),
|
||||
)
|
||||
|
|
@ -4,12 +4,10 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
|
||||
from ....components import R
|
||||
from ....schema import AutoFinTopicOutput
|
||||
from ._base import AutoFinStep
|
||||
from .base import AutoFinStep
|
||||
from .schema import AutoFinTopicOutput
|
||||
|
||||
|
||||
@R.register("auto_fin_topic_step")
|
||||
class AutoFinTopicStep(AutoFinStep):
|
||||
"""Filter current news in bounded Agent batches without writing files."""
|
||||
|
||||
|
|
@ -8,14 +8,15 @@ from zoneinfo import ZoneInfo
|
|||
|
||||
import pytest
|
||||
|
||||
from reme_auto_fin.base import _plain_text, _write
|
||||
from reme_auto_fin.data import AutoFinDataStep
|
||||
from reme_auto_fin.merge import AutoFinMergeStep
|
||||
from reme_auto_fin.plugin import plugin
|
||||
from reme_auto_fin.schema import AutoFinReportOutput, AutoFinTopicOutput
|
||||
from reme_auto_fin.topic import AutoFinTopicStep
|
||||
from reme.components import ApplicationContext
|
||||
from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper
|
||||
from reme.components.runtime_context import RuntimeContext
|
||||
from reme.schema import AutoFinReportOutput, AutoFinTopicOutput
|
||||
from reme.steps.cookbook.auto_fin._base import _plain_text, _write
|
||||
from reme.steps.cookbook.auto_fin.data import AutoFinDataStep
|
||||
from reme.steps.cookbook.auto_fin.merge import AutoFinMergeStep
|
||||
from reme.steps.cookbook.auto_fin.topic import AutoFinTopicStep
|
||||
|
||||
SHANGHAI = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
|
@ -28,7 +29,7 @@ def test_atomic_write_preserves_existing_file_on_failure(tmp_path: Path, monkeyp
|
|||
path = tmp_path / "result.md"
|
||||
path.write_text("existing", encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
"reme.steps.cookbook.auto_fin._base.os.replace",
|
||||
"reme_auto_fin.base.os.replace",
|
||||
lambda *_args: (_ for _ in ()).throw(OSError()),
|
||||
)
|
||||
|
||||
|
|
@ -210,7 +211,7 @@ async def test_merge_writes_only_final_report_and_validates_historical_links(tmp
|
|||
|
||||
|
||||
def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path: Path, monkeypatch):
|
||||
import reme.steps.cookbook.auto_fin.merge as merge_module
|
||||
import reme_auto_fin.merge as merge_module
|
||||
|
||||
step = AutoFinMergeStep(
|
||||
app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"),
|
||||
|
|
@ -237,11 +238,8 @@ def test_hybrid_wikilink_normalization_is_conservative_and_failure_safe(tmp_path
|
|||
assert step._normalize_hybrid_wikilinks(body) == body
|
||||
|
||||
|
||||
def test_config_has_default_topics_and_no_intermediate_index_step():
|
||||
from reme.config.config_parser import _load_config
|
||||
|
||||
config = _load_config("daily_cookbook")
|
||||
job = config["jobs"]["auto_fin"]
|
||||
def test_plugin_config_has_default_topics_and_no_intermediate_index_step():
|
||||
job = plugin.config["jobs"]["auto_fin"]
|
||||
assert job["parameters"]["properties"]["topics"]["default"] == "黄金,机器人,半导体"
|
||||
assert job["parameters"]["properties"]["window_hours"]["default"] == 24
|
||||
assert job["parameters"]["properties"]["request_interval"]["default"] == 10
|
||||
|
|
@ -251,7 +249,6 @@ def test_config_has_default_topics_and_no_intermediate_index_step():
|
|||
"auto_fin_data_step",
|
||||
"auto_fin_topic_step",
|
||||
"auto_fin_merge_step",
|
||||
"dingtalk_markdown_send_step",
|
||||
]
|
||||
assert job["steps"][2]["job_tools"] == ["memory_search", "read"]
|
||||
for name, schedule in {
|
||||
|
|
@ -259,8 +256,8 @@ def test_config_has_default_topics_and_no_intermediate_index_step():
|
|||
"auto_fin_1130_cron": "30 11 * * *",
|
||||
"auto_fin_1800_cron": "0 18 * * *",
|
||||
}.items():
|
||||
assert config["jobs"][name]["cron"] == schedule
|
||||
assert config["jobs"][name]["steps"] == job["steps"]
|
||||
assert plugin.config["jobs"][name]["cron"] == schedule
|
||||
assert plugin.config["jobs"][name]["steps"] == job["steps"]
|
||||
|
||||
|
||||
def test_agent_schemas_are_small_and_required():
|
||||
|
|
@ -8,13 +8,20 @@ from . import enumeration
|
|||
from . import schema
|
||||
from . import steps
|
||||
from . import utils
|
||||
from .components import BaseComponent, R
|
||||
from .application import Application
|
||||
from .components import BaseComponent
|
||||
from .plugin import Backend, Plugin
|
||||
from .reme import ReMe
|
||||
|
||||
# Component and Step packages above have completed their decorator-driven
|
||||
# bootstrap. Runtime code receives mutable copies of this immutable template.
|
||||
R.freeze()
|
||||
|
||||
__all__ = [
|
||||
"Application",
|
||||
"BaseComponent",
|
||||
"Backend",
|
||||
"Plugin",
|
||||
"ReMe",
|
||||
# submodules
|
||||
"config",
|
||||
|
|
|
|||
|
|
@ -7,22 +7,27 @@ from pathlib import Path
|
|||
from typing import AsyncGenerator, TypeVar
|
||||
|
||||
from . import __version__
|
||||
from .components import BaseComponent, ApplicationContext
|
||||
from .components import ApplicationContext, BaseComponent, create_application_registry
|
||||
from .components.job import BackgroundJob, BaseJob, CronJob, StreamJob
|
||||
from .components.service import BaseService
|
||||
from .enumeration import ComponentEnum
|
||||
from .enumeration import ComponentEnum, ComponentType, component_type_name
|
||||
from .plugin import PluginManager
|
||||
from .schema import ComponentConfig, Response, StreamChunk
|
||||
from .utils import execute_stream_task, print_logo, get_logger
|
||||
|
||||
T = TypeVar("T", bound=BaseComponent)
|
||||
_NodeKey = tuple[ComponentEnum, str]
|
||||
_NodeKey = tuple[str, str]
|
||||
|
||||
|
||||
class Application(BaseComponent):
|
||||
"""Wires components from config and runs jobs against them."""
|
||||
|
||||
def __init__(self, **kwargs) -> None:
|
||||
self.context = ApplicationContext(**kwargs)
|
||||
plugin_manager = PluginManager.discover(kwargs.get("plugins") or ())
|
||||
resolved = plugin_manager.merge_config(kwargs)
|
||||
registry = create_application_registry()
|
||||
plugin_manager.register(registry)
|
||||
self.context = ApplicationContext(registry=registry, **resolved)
|
||||
self._started_components: list[BaseComponent] = []
|
||||
|
||||
self._setup_workspace_directories()
|
||||
|
|
@ -98,7 +103,7 @@ class Application(BaseComponent):
|
|||
|
||||
def _instantiate(
|
||||
self,
|
||||
ctype: ComponentEnum,
|
||||
ctype: ComponentType,
|
||||
cfg: ComponentConfig,
|
||||
*,
|
||||
label: str,
|
||||
|
|
@ -109,16 +114,13 @@ class Application(BaseComponent):
|
|||
|
||||
`label` is the human-readable identifier used only in error messages.
|
||||
`expected_type` narrows the return type and guards against a backend
|
||||
registered under the wrong ComponentEnum.
|
||||
registered under the wrong component type.
|
||||
`name` is forwarded to the constructor for named components/jobs;
|
||||
leave it None for the service, which is keyed solely by type.
|
||||
"""
|
||||
# Lazy import: the registry self-populates as component modules load.
|
||||
from .components import R
|
||||
|
||||
if not cfg.backend:
|
||||
raise ValueError(f"{label} is missing the required 'backend' field")
|
||||
backend_cls = R.get(ctype, cfg.backend)
|
||||
backend_cls = self.context.registry.get(ctype, cfg.backend)
|
||||
if backend_cls is None:
|
||||
raise ValueError(f"Unregistered backend '{cfg.backend}' for {label}")
|
||||
|
||||
|
|
@ -153,7 +155,7 @@ class Application(BaseComponent):
|
|||
heapq.heappush(ready, downstream)
|
||||
|
||||
if len(ordered) != len(nodes):
|
||||
unresolved = [f"{k[0].value}:{k[1]}" for k, d in in_degree.items() if d > 0]
|
||||
unresolved = [f"{k[0]}:{k[1]}" for k, d in in_degree.items() if d > 0]
|
||||
raise ValueError(f"Circular dependency detected among: {unresolved}")
|
||||
return ordered
|
||||
|
||||
|
|
@ -172,7 +174,7 @@ class Application(BaseComponent):
|
|||
in_degree[key] += 1
|
||||
elif not dep.optional:
|
||||
raise ValueError(
|
||||
f"Component {key[0].value}:{key[1]} depends on unregistered {dep.ctype.value}:{dep.name}",
|
||||
f"Component {key[0]}:{key[1]} depends on unregistered {dep.ctype}:{dep.name}",
|
||||
)
|
||||
return in_degree, dependents
|
||||
|
||||
|
|
@ -205,7 +207,7 @@ class Application(BaseComponent):
|
|||
await c.start()
|
||||
self._started_components.append(c)
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to start {c.component_type.value}:{c.name}: {e}")
|
||||
self.logger.exception(f"Failed to start {component_type_name(c.component_type)}:{c.name}: {e}")
|
||||
raise
|
||||
|
||||
async def _close(self) -> None:
|
||||
|
|
@ -214,23 +216,23 @@ class Application(BaseComponent):
|
|||
try:
|
||||
await c.close()
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to close {c.component_type.value}:{c.name}: {e}")
|
||||
self.logger.exception(f"Failed to close {component_type_name(c.component_type)}:{c.name}: {e}")
|
||||
self._started_components.clear()
|
||||
if self.context.thread_pool is not None:
|
||||
self.context.thread_pool.shutdown(wait=True)
|
||||
self.context.thread_pool = None
|
||||
|
||||
async def update_component(self, component_enum: ComponentEnum | str, name: str, /, **kwargs) -> BaseComponent:
|
||||
async def update_component(self, component_enum: ComponentType, name: str, /, **kwargs) -> BaseComponent:
|
||||
"""Update an existing component by type/name; never creates missing components."""
|
||||
component_enum = ComponentEnum(component_enum)
|
||||
group = self.context.components.get(component_enum)
|
||||
component_type = component_type_name(component_enum)
|
||||
group = self.context.components.get(component_type)
|
||||
if not group or name not in group:
|
||||
raise KeyError(f"Component '{name}' not found in {component_enum.value}")
|
||||
raise KeyError(f"Component '{name}' not found in {component_type}")
|
||||
|
||||
component = group[name]
|
||||
for key, value in kwargs.items():
|
||||
if not hasattr(component, key):
|
||||
raise AttributeError(f"Component {component_enum.value}:{name} has no attribute '{key}'")
|
||||
raise AttributeError(f"Component {component_type}:{name} has no attribute '{key}'")
|
||||
setattr(component, key, value)
|
||||
return component
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from . import service
|
|||
from . import tokenizer
|
||||
from .application_context import ApplicationContext
|
||||
from .base_component import BaseComponent, ComponentMixin
|
||||
from .component_registry import ComponentRegistry, R
|
||||
from .component_registry import ComponentRegistry, R, create_application_registry
|
||||
from .prompt_handler import PromptHandler
|
||||
from .runtime_context import RuntimeContext
|
||||
|
||||
|
|
@ -26,6 +26,7 @@ __all__ = [
|
|||
"ComponentMixin",
|
||||
"ComponentRegistry",
|
||||
"R",
|
||||
"create_application_registry",
|
||||
"PromptHandler",
|
||||
"RuntimeContext",
|
||||
# base components
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from ..enumeration import ComponentEnum
|
||||
from ..schema import ApplicationConfig
|
||||
from .component_registry import ComponentRegistry, create_application_registry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base_component import BaseComponent
|
||||
|
|
@ -20,13 +20,14 @@ class ApplicationContext:
|
|||
components, jobs, and the service can find each other at runtime.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
def __init__(self, *, registry: ComponentRegistry | None = None, **kwargs):
|
||||
# Parse raw kwargs into a typed, validated config object.
|
||||
self.app_config: ApplicationConfig = ApplicationConfig(**kwargs)
|
||||
self.registry = registry or create_application_registry()
|
||||
|
||||
# Populated by Application during initialization.
|
||||
self.service: "BaseService | None" = None
|
||||
self.components: dict[ComponentEnum, dict[str, "BaseComponent"]] = {}
|
||||
self.components: dict[str, dict[str, "BaseComponent"]] = {}
|
||||
self.jobs: dict[str, "BaseJob"] = {}
|
||||
self.thread_pool: ThreadPoolExecutor | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from abc import ABC
|
|||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
|
||||
|
||||
from ..enumeration import ComponentEnum
|
||||
from ..enumeration import ComponentEnum, ComponentType, component_type_name
|
||||
from ..utils import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -61,24 +61,24 @@ class Dependency:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
ctype: ComponentEnum,
|
||||
ctype: ComponentType,
|
||||
name: str,
|
||||
default_factory: Callable[[], Any] | None = None,
|
||||
optional: bool = True,
|
||||
) -> None:
|
||||
self.ctype = ctype
|
||||
self.ctype = component_type_name(ctype)
|
||||
self.name = name
|
||||
self.default_factory = default_factory
|
||||
self.optional = optional
|
||||
|
||||
def __repr__(self) -> str:
|
||||
suffix = "?" if self.optional else ""
|
||||
return f"<unresolved {self.ctype.value}:{self.name}{suffix}>"
|
||||
return f"<unresolved {self.ctype}:{self.name}{suffix}>"
|
||||
|
||||
def __getattr__(self, item: str) -> Any:
|
||||
# Catches accidental use of the placeholder before start() resolves it.
|
||||
raise RuntimeError(
|
||||
f"Dependency {self.ctype.value}:{self.name} accessed before start() " f"(attribute '{item}')",
|
||||
f"Dependency {self.ctype}:{self.name} accessed before start() " f"(attribute '{item}')",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -126,10 +126,14 @@ class BaseComponent(ComponentMixin, ABC):
|
|||
if not name:
|
||||
return None
|
||||
ctype = getattr(base_cls, "component_type", None)
|
||||
if not isinstance(ctype, ComponentEnum) or ctype is ComponentEnum.BASE:
|
||||
try:
|
||||
ctype = component_type_name(ctype)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TypeError(
|
||||
f"{base_cls.__name__} must declare a non-BASE ComponentEnum 'component_type'",
|
||||
)
|
||||
f"{base_cls.__name__} must declare a non-BASE string 'component_type'",
|
||||
) from exc
|
||||
if ctype == ComponentEnum.BASE.value:
|
||||
raise TypeError(f"{base_cls.__name__} must declare a non-BASE 'component_type'")
|
||||
return cast(T, Dependency(ctype, name, default_factory, optional))
|
||||
|
||||
@property
|
||||
|
|
@ -172,7 +176,7 @@ class BaseComponent(ComponentMixin, ABC):
|
|||
elif dep.optional:
|
||||
setattr(self, attr, None)
|
||||
else:
|
||||
raise ValueError(f"{dep.ctype.value} '{dep.name}' not found.")
|
||||
raise ValueError(f"{dep.ctype} '{dep.name}' not found.")
|
||||
|
||||
# ----- Workspace path helpers --------------------------------------------
|
||||
|
||||
|
|
@ -186,7 +190,7 @@ class BaseComponent(ComponentMixin, ABC):
|
|||
@property
|
||||
def component_metadata_path(self) -> Path:
|
||||
"""Per-component metadata directory under the workspace."""
|
||||
return self.workspace_metadata_path / self.component_type.value
|
||||
return self.workspace_metadata_path / component_type_name(self.component_type)
|
||||
|
||||
# ----- Lifecycle hooks (override in subclasses) ----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
"""Global registry mapping ``(ComponentEnum, name) -> component class``."""
|
||||
"""Registry mapping ``(component type, backend)`` to implementation classes."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import RLock
|
||||
from typing import Callable, TypeVar, cast
|
||||
|
||||
from .base_component import BaseComponent
|
||||
from ..enumeration import ComponentEnum
|
||||
from ..utils import get_logger
|
||||
from .base_component import ComponentMixin
|
||||
from ..enumeration import ComponentType, component_type_name
|
||||
|
||||
T = TypeVar("T", bound=BaseComponent)
|
||||
T = TypeVar("T", bound=ComponentMixin)
|
||||
|
||||
|
||||
class ComponentRegistry:
|
||||
|
|
@ -17,27 +19,46 @@ class ComponentRegistry:
|
|||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._registry: dict[ComponentEnum, dict[str, type[BaseComponent]]] = {}
|
||||
self.logger = get_logger(log_to_file=False)
|
||||
self._registry: dict[str, dict[str, type[ComponentMixin]]] = {}
|
||||
self._owners: dict[tuple[str, str], str] = {}
|
||||
self._lock = RLock()
|
||||
self._frozen = False
|
||||
|
||||
def _do_register(self, cls: type[T], name: str) -> type[T]:
|
||||
"""Insert `cls` under its ``component_type`` group; warn on overwrite."""
|
||||
component_type = getattr(cls, "component_type", None)
|
||||
if not isinstance(component_type, ComponentEnum):
|
||||
raise TypeError(
|
||||
f"{cls.__name__} must have a ComponentEnum 'component_type' attribute",
|
||||
)
|
||||
def _ensure_mutable(self) -> None:
|
||||
"""Reject changes after this registry becomes an immutable template."""
|
||||
if self._frozen:
|
||||
raise RuntimeError("Component registry is frozen")
|
||||
|
||||
def _do_register(self, cls: type[T], name: str, *, owner: str | None = None) -> type[T]:
|
||||
"""Insert ``cls`` under its component type and reject ambiguous providers."""
|
||||
try:
|
||||
component_type = component_type_name(getattr(cls, "component_type", None))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise TypeError(f"{cls.__name__} must have a non-empty string 'component_type' attribute") from exc
|
||||
if not name:
|
||||
raise ValueError("Component name cannot be empty")
|
||||
|
||||
group = self._registry.setdefault(component_type, {})
|
||||
if name in group:
|
||||
self.logger.warning(
|
||||
f"Component '{name}' already registered for {component_type}, overwriting",
|
||||
)
|
||||
group[name] = cls
|
||||
with self._lock:
|
||||
self._ensure_mutable()
|
||||
group = self._registry.setdefault(component_type, {})
|
||||
key = (component_type, name)
|
||||
if name in group:
|
||||
existing = group[name]
|
||||
existing_owner = self._owners[key]
|
||||
new_owner = owner or cls.__module__
|
||||
if existing is cls and existing_owner == new_owner:
|
||||
return cls
|
||||
raise ValueError(
|
||||
f"Backend '{component_type}:{name}' is provided by both " f"'{existing_owner}' and '{new_owner}'",
|
||||
)
|
||||
group[name] = cls
|
||||
self._owners[key] = owner or cls.__module__
|
||||
return cls
|
||||
|
||||
def add(self, name: str, cls: type[T], *, owner: str) -> type[T]:
|
||||
"""Register one explicitly owned plugin contribution."""
|
||||
return self._do_register(cls, name, owner=owner)
|
||||
|
||||
def register(
|
||||
self,
|
||||
cls_or_name: type[T] | str,
|
||||
|
|
@ -60,25 +81,76 @@ class ComponentRegistry:
|
|||
|
||||
return decorator
|
||||
|
||||
def get(self, component_type: ComponentEnum, name: str) -> type[BaseComponent] | None:
|
||||
def get(self, component_type: ComponentType, name: str) -> type[ComponentMixin] | None:
|
||||
"""Look up a registered class; return None if not found."""
|
||||
return self._registry.get(component_type, {}).get(name)
|
||||
with self._lock:
|
||||
return self._registry.get(component_type_name(component_type), {}).get(name)
|
||||
|
||||
def get_all(self, component_type: ComponentEnum) -> dict[str, type[BaseComponent]]:
|
||||
def get_all(self, component_type: ComponentType) -> dict[str, type[ComponentMixin]]:
|
||||
"""Return a shallow copy of all classes registered under `component_type`."""
|
||||
return dict(self._registry.get(component_type, {}))
|
||||
with self._lock:
|
||||
return dict(self._registry.get(component_type_name(component_type), {}))
|
||||
|
||||
def unregister(self, component_type: ComponentEnum, name: str) -> bool:
|
||||
def unregister(self, component_type: ComponentType, name: str) -> bool:
|
||||
"""Remove an entry; return True if it existed, False otherwise."""
|
||||
if (group := self._registry.get(component_type)) and name in group:
|
||||
del group[name]
|
||||
return True
|
||||
return False
|
||||
component_type = component_type_name(component_type)
|
||||
with self._lock:
|
||||
self._ensure_mutable()
|
||||
if (group := self._registry.get(component_type)) and name in group:
|
||||
del group[name]
|
||||
self._owners.pop((component_type, name), None)
|
||||
return True
|
||||
return False
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Drop every registered entry."""
|
||||
self._registry.clear()
|
||||
with self._lock:
|
||||
self._ensure_mutable()
|
||||
self._registry.clear()
|
||||
self._owners.clear()
|
||||
|
||||
def freeze(self) -> None:
|
||||
"""Make this registry an immutable template for future copies."""
|
||||
with self._lock:
|
||||
self._frozen = True
|
||||
|
||||
@property
|
||||
def frozen(self) -> bool:
|
||||
"""Whether mutating operations are disabled."""
|
||||
with self._lock:
|
||||
return self._frozen
|
||||
|
||||
def copy(self) -> "ComponentRegistry":
|
||||
"""Return an independent registry containing the same providers."""
|
||||
copied = ComponentRegistry()
|
||||
with self._lock:
|
||||
for component_type, group in self._registry.items():
|
||||
for name, implementation in group.items():
|
||||
copied.add(name, implementation, owner=self._owners[(component_type, name)])
|
||||
return copied
|
||||
|
||||
@contextmanager
|
||||
def preserve(self, *, allow_mutation: bool = False) -> Iterator[None]:
|
||||
"""Restore the registry after code that may register through import side effects."""
|
||||
with self._lock:
|
||||
registry = {component_type: dict(group) for component_type, group in self._registry.items()}
|
||||
owners = dict(self._owners)
|
||||
frozen = self._frozen
|
||||
if allow_mutation:
|
||||
self._frozen = False
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._registry = registry
|
||||
self._owners = owners
|
||||
self._frozen = frozen
|
||||
|
||||
|
||||
# Process-wide singleton used throughout the codebase.
|
||||
# Import-time registry for built-in implementations. Runtime code should use
|
||||
# ``create_application_registry`` rather than mutate this template.
|
||||
R = ComponentRegistry()
|
||||
|
||||
|
||||
def create_application_registry() -> ComponentRegistry:
|
||||
"""Return a mutable registry initialized from the frozen built-in template."""
|
||||
return R.copy()
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class BaseJob(BaseComponent):
|
|||
config = raw if isinstance(raw, ComponentConfig) else ComponentConfig(**raw)
|
||||
if not config.backend:
|
||||
raise ValueError("Step is missing the required 'backend' field")
|
||||
step_cls = R.get(ComponentEnum.STEP, config.backend)
|
||||
step_cls = self.app_context.registry.get(ComponentEnum.STEP, config.backend)
|
||||
if not step_cls:
|
||||
raise ValueError(f"Unregistered backend '{config.backend}' of type '{ComponentEnum.STEP}'")
|
||||
params = config.model_dump()
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
"""Config"""
|
||||
|
||||
from .config_parser import parse_args, resolve_app_config
|
||||
from .config_parser import deep_merge_config, expand_env_vars, parse_args, resolve_app_config
|
||||
|
||||
__all__ = [
|
||||
"deep_merge_config",
|
||||
"expand_env_vars",
|
||||
"parse_args",
|
||||
"resolve_app_config",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,15 +3,20 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from ..entry_point import load_entry_point
|
||||
|
||||
# Config files are looked up relative to this module's directory
|
||||
_CONFIG_DIR = Path(__file__).parent
|
||||
# Extensions in priority order: yaml > yml > json when stems collide
|
||||
_SUPPORTED_EXTS = (".yaml", ".yml", ".json")
|
||||
_CONFIG_ENTRY_POINT_GROUP = "reme.configs"
|
||||
_ENV_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?}")
|
||||
# Strings like "007" / "00501" must stay as strings, not be coerced to numbers
|
||||
_LEADING_ZERO_RE = re.compile(r"^-?0\d")
|
||||
|
|
@ -41,6 +46,11 @@ def _expand_env_vars(value: Any) -> Any:
|
|||
return value
|
||||
|
||||
|
||||
def expand_env_vars(value: Any) -> Any:
|
||||
"""Expand environment placeholders in an arbitrary plugin config value."""
|
||||
return _expand_env_vars(value)
|
||||
|
||||
|
||||
def _discover_configs() -> dict[str, Path]:
|
||||
"""Pre-scan config directory: maps file stem (name without ext) -> Path."""
|
||||
discovered: dict[str, Path] = {}
|
||||
|
|
@ -117,17 +127,43 @@ def _convert_value(value_str: str) -> Any:
|
|||
return s
|
||||
|
||||
|
||||
def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
|
||||
"""Load a YAML or JSON config file.
|
||||
def _external_config_entries(name: str) -> list[metadata.EntryPoint]:
|
||||
"""Find installed config providers without importing their packages."""
|
||||
return list(metadata.entry_points().select(group=_CONFIG_ENTRY_POINT_GROUP, name=name))
|
||||
|
||||
First check if name_or_path matches a pre-discovered config (key in _CONFIG_REGISTRY).
|
||||
If not, treat as a file path and load directly.
|
||||
"""
|
||||
# 1. Try pre-discovered configs first
|
||||
if name_or_path in _CONFIG_REGISTRY:
|
||||
return _read_config_file(_CONFIG_REGISTRY[name_or_path], encoding)
|
||||
|
||||
# 2. Treat as file path
|
||||
def _external_config_path(name: str, entries: list[metadata.EntryPoint] | None = None) -> Path | None:
|
||||
"""Resolve an installed plugin config exposed through ``reme.configs``."""
|
||||
entries = _external_config_entries(name) if entries is None else entries
|
||||
if not entries:
|
||||
return None
|
||||
if len(entries) > 1:
|
||||
providers = ", ".join(sorted(entry.value for entry in entries))
|
||||
raise ValueError(f"Config '{name}' has multiple installed providers: {providers}")
|
||||
value = load_entry_point(entries[0], invoke=True)
|
||||
path = Path(value)
|
||||
if path.suffix not in _SUPPORTED_EXTS or not path.is_file():
|
||||
raise ValueError(f"Config entry point '{name}' did not resolve to a YAML or JSON file")
|
||||
return path
|
||||
|
||||
|
||||
def _load_config(name_or_path: str, encoding: str = "utf-8", _stack: tuple[str, ...] = ()) -> dict:
|
||||
"""Load a built-in, installed-plugin, or direct YAML/JSON config."""
|
||||
if name_or_path in _stack:
|
||||
chain = " -> ".join((*_stack, name_or_path))
|
||||
raise ValueError(f"Circular config inheritance: {chain}")
|
||||
|
||||
built_in = _CONFIG_REGISTRY.get(name_or_path)
|
||||
external_entries = _external_config_entries(name_or_path)
|
||||
if built_in is not None and external_entries:
|
||||
raise ValueError(f"Config '{name_or_path}' is provided by both ReMe and an installed distribution")
|
||||
if built_in is not None:
|
||||
return _load_config_path(built_in, name_or_path, encoding, _stack)
|
||||
|
||||
external = _external_config_path(name_or_path, external_entries)
|
||||
if external is not None:
|
||||
return _load_config_path(external, name_or_path, encoding, _stack)
|
||||
|
||||
p = Path(name_or_path)
|
||||
if p.suffix in _SUPPORTED_EXTS:
|
||||
candidates = [p]
|
||||
|
|
@ -135,13 +171,31 @@ def _load_config(name_or_path: str, encoding: str = "utf-8") -> dict:
|
|||
candidates.append(_CONFIG_DIR / p)
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
return _read_config_file(candidate, encoding)
|
||||
identity = str(candidate.resolve())
|
||||
return _load_config_path(candidate, identity, encoding, _stack)
|
||||
raise FileNotFoundError(f"Config file not found: {p}")
|
||||
|
||||
known = ", ".join(sorted(_CONFIG_REGISTRY)) if _CONFIG_REGISTRY else "none"
|
||||
raise FileNotFoundError(f"Config file not found: {name_or_path}. Available: {known}")
|
||||
|
||||
|
||||
def _load_config_path(path: Path, identity: str, encoding: str, stack: tuple[str, ...]) -> dict:
|
||||
"""Load one config and merge its optional parents before its own values."""
|
||||
config = _read_config_file(path, encoding)
|
||||
raw_parents = config.pop("extends", ())
|
||||
parents = [raw_parents] if isinstance(raw_parents, str) else list(raw_parents or ())
|
||||
merged: dict = {}
|
||||
for parent in parents:
|
||||
if not isinstance(parent, str) or not parent:
|
||||
raise ValueError(f"Config 'extends' entries must be non-empty strings: {path}")
|
||||
parent_name = parent
|
||||
relative = path.parent / parent
|
||||
if Path(parent).suffix in _SUPPORTED_EXTS and relative.is_file():
|
||||
parent_name = str(relative.resolve())
|
||||
merged = deep_merge_config(merged, _load_config(parent_name, encoding, (*stack, identity)))
|
||||
return deep_merge_config(merged, config)
|
||||
|
||||
|
||||
def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
|
||||
"""Read YAML or JSON file based on extension. Expands ${ENV_VAR}."""
|
||||
with path.open(encoding=encoding) as f:
|
||||
|
|
@ -156,12 +210,12 @@ def _read_config_file(path: Path, encoding: str = "utf-8") -> dict:
|
|||
return _expand_env_vars(result)
|
||||
|
||||
|
||||
def _deep_merge(base: dict, update: dict) -> dict:
|
||||
"""Recursively merge dicts."""
|
||||
result = base.copy()
|
||||
def deep_merge_config(base: Mapping[str, Any], update: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Recursively merge configuration mappings without mutating either input."""
|
||||
result = dict(base)
|
||||
for k, v in update.items():
|
||||
if k in result and isinstance(result[k], dict) and isinstance(v, dict):
|
||||
result[k] = _deep_merge(result[k], v)
|
||||
if k in result and isinstance(result[k], Mapping) and isinstance(v, Mapping):
|
||||
result[k] = deep_merge_config(result[k], v)
|
||||
else:
|
||||
result[k] = v
|
||||
return result
|
||||
|
|
@ -230,6 +284,6 @@ def resolve_app_config(*, log_config: bool = True, **kwargs) -> dict:
|
|||
|
||||
merged: dict = {}
|
||||
for cfg in configs:
|
||||
merged = _deep_merge(merged, cfg)
|
||||
merged = deep_merge_config(merged, cfg)
|
||||
|
||||
return merged
|
||||
|
|
|
|||
|
|
@ -316,71 +316,6 @@ jobs:
|
|||
steps:
|
||||
- backend: edit_step
|
||||
|
||||
auto_fin:
|
||||
backend: base
|
||||
description: "Fetch and research the latest 24 hours of topic-related CLS news."
|
||||
parameters:
|
||||
type: object
|
||||
properties:
|
||||
date:
|
||||
type: string
|
||||
description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai."
|
||||
default: ""
|
||||
now:
|
||||
type: string
|
||||
description: "Optional simulated current time in ISO 8601 format; empty means the real current time."
|
||||
default: ""
|
||||
topics:
|
||||
type: string
|
||||
description: "Comma-separated topics used to filter current CLS news."
|
||||
default: "黄金,机器人,半导体"
|
||||
window_hours:
|
||||
type: number
|
||||
exclusiveMinimum: 0
|
||||
description: "Rolling number of hours of CLS news to fetch."
|
||||
default: 24
|
||||
request_interval:
|
||||
type: number
|
||||
minimum: 0
|
||||
description: "Minimum delay in seconds after each CLS request attempt."
|
||||
default: 10
|
||||
max_retries:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: "Maximum attempts for each CLS page request."
|
||||
default: 3
|
||||
steps: &auto_fin_steps
|
||||
- backend: auto_fin_data_step
|
||||
- backend: auto_fin_topic_step
|
||||
- backend: auto_fin_merge_step
|
||||
job_tools: [memory_search, read]
|
||||
- backend: dingtalk_markdown_send_step
|
||||
input_mapping:
|
||||
auto_fin_digest_path: markdown_path
|
||||
app_key: ${DINGTALK_APP_KEY:-}
|
||||
app_secret: ${DINGTALK_APP_SECRET:-}
|
||||
robot_code: ${DINGTALK_ROBOT_CODE:-}
|
||||
conversation_ids: ${DINGTALK_CONVERSATION_IDS:-}
|
||||
title: ReMe Auto Fin
|
||||
timeout: 15
|
||||
|
||||
# Three intraday runs (Asia/Shanghai). News remains useful outside market
|
||||
# sessions. Each rerun refines the same day's report.
|
||||
auto_fin_0930_cron:
|
||||
backend: cron
|
||||
cron: "30 9 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
||||
auto_fin_1130_cron:
|
||||
backend: cron
|
||||
cron: "30 11 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
||||
auto_fin_1800_cron:
|
||||
backend: cron
|
||||
cron: "0 18 * * *"
|
||||
steps: *auto_fin_steps
|
||||
|
||||
daily_paper:
|
||||
backend: base
|
||||
description: "Build detailed readings and a five-minute brief from Hugging Face weekly/monthly papers."
|
||||
|
|
|
|||
14
reme/entry_point.py
Normal file
14
reme/entry_point.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
"""Safe loading boundary for third-party Python entry points."""
|
||||
|
||||
from importlib import metadata
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_entry_point(entry: metadata.EntryPoint, *, invoke: bool = False) -> Any:
|
||||
"""Load and optionally invoke an entry point without retaining registrations."""
|
||||
# Import lazily so config parser imports do not pull in the component graph.
|
||||
from .components.component_registry import R
|
||||
|
||||
with R.preserve(allow_mutation=True):
|
||||
loaded = entry.load()
|
||||
return loaded() if invoke and callable(loaded) else loaded
|
||||
|
|
@ -2,12 +2,15 @@
|
|||
|
||||
from .chunk_enum import ChunkEnum
|
||||
from .component_enum import ComponentEnum
|
||||
from .component_type import ComponentType, component_type_name
|
||||
from .dream_bucket_enum import DreamBucketEnum
|
||||
from .link_scope_enum import LinkScopeEnum
|
||||
|
||||
__all__ = [
|
||||
"ChunkEnum",
|
||||
"ComponentEnum",
|
||||
"ComponentType",
|
||||
"DreamBucketEnum",
|
||||
"LinkScopeEnum",
|
||||
"component_type_name",
|
||||
]
|
||||
|
|
|
|||
25
reme/enumeration/component_type.py
Normal file
25
reme/enumeration/component_type.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
"""Extensible component type identifiers."""
|
||||
|
||||
import re
|
||||
from typing import TypeAlias
|
||||
|
||||
from .component_enum import ComponentEnum
|
||||
|
||||
ComponentType: TypeAlias = ComponentEnum | str
|
||||
_COMPONENT_TYPE_PATTERN = re.compile(r"[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*")
|
||||
|
||||
|
||||
def component_type_name(value: ComponentType) -> str:
|
||||
"""Return the canonical string name for a built-in or plugin component type."""
|
||||
if isinstance(value, ComponentEnum):
|
||||
return value.value
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f"Component type must be a string, got {type(value).__name__}")
|
||||
name = value.strip()
|
||||
if not name:
|
||||
raise ValueError("Component type cannot be empty")
|
||||
if _COMPONENT_TYPE_PATTERN.fullmatch(name) is None:
|
||||
raise ValueError(
|
||||
f"Invalid component type {name!r}; use lowercase letters and numbers separated by '.', '_' or '-'",
|
||||
)
|
||||
return name
|
||||
84
reme/plugin.py
Normal file
84
reme/plugin.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Installed plugin discovery and application-local registration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from importlib import metadata
|
||||
from typing import Any
|
||||
|
||||
from .components.base_component import ComponentMixin
|
||||
from .components.component_registry import ComponentRegistry
|
||||
from .config import deep_merge_config, expand_env_vars
|
||||
from .entry_point import load_entry_point
|
||||
|
||||
PLUGIN_ENTRY_POINT_GROUP = "reme.plugins"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Backend:
|
||||
"""One named component, step, or job backend contributed by a plugin."""
|
||||
|
||||
name: str
|
||||
implementation: type[ComponentMixin]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Plugin:
|
||||
"""Declarative plugin loaded from the ``reme.plugins`` entry-point group."""
|
||||
|
||||
name: str
|
||||
backends: tuple[Backend, ...] = ()
|
||||
config: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _entry_points(group: str, name: str) -> list[metadata.EntryPoint]:
|
||||
"""Return matching entry points for one group and name."""
|
||||
return list(metadata.entry_points().select(group=group, name=name))
|
||||
|
||||
|
||||
class PluginManager:
|
||||
"""Resolve enabled plugins and apply their contributions to one application."""
|
||||
|
||||
def __init__(self, plugins: Iterable[Plugin] = ()) -> None:
|
||||
self.plugins = tuple(plugins)
|
||||
|
||||
@classmethod
|
||||
def discover(cls, specs: Iterable[str]) -> "PluginManager":
|
||||
"""Load explicitly enabled plugins by entry-point name."""
|
||||
plugins: list[Plugin] = []
|
||||
seen: set[str] = set()
|
||||
for name in specs:
|
||||
if not isinstance(name, str):
|
||||
raise TypeError(f"Invalid plugin name: {name!r}")
|
||||
if not name:
|
||||
raise ValueError("Plugin name cannot be empty")
|
||||
if name in seen:
|
||||
raise ValueError(f"Plugin '{name}' is enabled more than once")
|
||||
entries = _entry_points(PLUGIN_ENTRY_POINT_GROUP, name)
|
||||
if not entries:
|
||||
raise ValueError(f"Plugin '{name}' is not installed")
|
||||
if len(entries) > 1:
|
||||
providers = ", ".join(sorted(entry.value for entry in entries))
|
||||
raise ValueError(f"Plugin '{name}' has multiple installed providers: {providers}")
|
||||
plugin = load_entry_point(entries[0], invoke=True)
|
||||
if not isinstance(plugin, Plugin):
|
||||
raise TypeError(f"Plugin entry point '{name}' did not return reme.plugin.Plugin")
|
||||
if plugin.name != name:
|
||||
raise ValueError(f"Plugin entry point '{name}' returned plugin '{plugin.name}'")
|
||||
plugins.append(plugin)
|
||||
seen.add(name)
|
||||
return cls(plugins)
|
||||
|
||||
def merge_config(self, application_config: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Place plugin defaults below the user's resolved application config."""
|
||||
merged: dict[str, Any] = {}
|
||||
for plugin in self.plugins:
|
||||
merged = deep_merge_config(merged, expand_env_vars(plugin.config))
|
||||
return deep_merge_config(merged, application_config)
|
||||
|
||||
def register(self, registry: ComponentRegistry) -> None:
|
||||
"""Register every backend into an application-local registry."""
|
||||
for plugin in self.plugins:
|
||||
for backend in plugin.backends:
|
||||
registry.add(backend.name, backend.implementation, owner=plugin.name)
|
||||
21
reme/reme.py
21
reme/reme.py
|
|
@ -4,11 +4,12 @@ import asyncio
|
|||
import sys
|
||||
|
||||
from .application import Application
|
||||
from .components import R
|
||||
from .components import create_application_registry
|
||||
from .components.service.cli_service import prepare_start_config, should_precheck_start
|
||||
from .config import parse_args, resolve_app_config
|
||||
from .enumeration import ComponentEnum
|
||||
from .utils import cli_find_reme, load_env, precheck_start, running_service_config
|
||||
from .plugin import PluginManager
|
||||
from .utils import cli_find_reme, load_env, precheck_start, running_app_config
|
||||
|
||||
_CLIENT_KWARGS = {"host", "port", "timeout", "transport", "command", "args", "show_metadata"}
|
||||
|
||||
|
|
@ -36,10 +37,14 @@ async def call_server(action: str, **kwargs):
|
|||
if isinstance(kwargs.get("service"), dict):
|
||||
resolve_kwargs["service"] = kwargs.pop("service")
|
||||
|
||||
# Prefer the running server's real config; fall back to the local config file.
|
||||
service = running_service_config()
|
||||
if service is None:
|
||||
service = resolve_app_config(log_config=False, **resolve_kwargs).get("service")
|
||||
# Prefer the running server's complete config so its enabled client plugins
|
||||
# are available even when the caller does not repeat config=<name>.
|
||||
app_config = running_app_config()
|
||||
if app_config is None:
|
||||
app_config = resolve_app_config(log_config=False, **resolve_kwargs)
|
||||
plugin_manager = PluginManager.discover(app_config.get("plugins") or ())
|
||||
app_config = plugin_manager.merge_config(app_config)
|
||||
service = app_config.get("service")
|
||||
service = service if isinstance(service, dict) else {}
|
||||
|
||||
backend: str = kwargs.pop("backend", None) or service.get("backend", "http")
|
||||
|
|
@ -50,7 +55,9 @@ async def call_server(action: str, **kwargs):
|
|||
client_kwargs = {k: seed[k] for k in _CLIENT_KWARGS if k in seed}
|
||||
client_kwargs.update({key: kwargs.pop(key) for key in list(kwargs) if key in _CLIENT_KWARGS})
|
||||
|
||||
client_cls = R.get(ComponentEnum.CLIENT, backend)
|
||||
registry = create_application_registry()
|
||||
plugin_manager.register(registry)
|
||||
client_cls = registry.get(ComponentEnum.CLIENT, backend)
|
||||
if client_cls is None:
|
||||
raise ValueError(f"Unknown client backend: {backend!r}")
|
||||
async with client_cls(**client_kwargs) as client:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Schema"""
|
||||
|
||||
from .application_config import ApplicationConfig, ComponentConfig, JobConfig
|
||||
from .auto_fin import AutoFinReportOutput, AutoFinTopicOutput
|
||||
from .daily_paper import (
|
||||
AnalyzedPaper,
|
||||
DailyPaperMarkdownOutput,
|
||||
|
|
@ -32,8 +31,6 @@ from .traverse_graph import TraverseGraph, TraverseGraphEdge, TraverseGraphNode
|
|||
|
||||
__all__ = [
|
||||
"ApplicationConfig",
|
||||
"AutoFinReportOutput",
|
||||
"AutoFinTopicOutput",
|
||||
"ComponentConfig",
|
||||
"AnalyzedPaper",
|
||||
"DailyPaperMarkdownOutput",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from pathlib import PurePosixPath, PureWindowsPath
|
|||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from ..enumeration import ComponentEnum
|
||||
from ..enumeration import component_type_name
|
||||
|
||||
|
||||
class ComponentConfig(BaseModel):
|
||||
|
|
@ -52,14 +52,25 @@ class ApplicationConfig(BaseModel):
|
|||
log_to_console: bool = Field(default=True, description="Log to console")
|
||||
log_to_file: bool = Field(default=True, description="Log to file")
|
||||
mcp_servers: dict[str, dict] = Field(default_factory=dict, description="MCP server configs by name")
|
||||
plugins: list[str] = Field(default_factory=list, description="Installed plugins enabled for this app")
|
||||
service: ComponentConfig = Field(default_factory=ComponentConfig, description="Service endpoint config")
|
||||
jobs: dict[str, JobConfig] = Field(default_factory=dict, description="Job definitions keyed by job name")
|
||||
thread_pool_max_workers: int = Field(default=0, description="Max worker threads; 0 to disable")
|
||||
components: dict[ComponentEnum, dict[str, ComponentConfig]] = Field(
|
||||
components: dict[str, dict[str, ComponentConfig]] = Field(
|
||||
default_factory=dict,
|
||||
description="Component registry keyed by type then name",
|
||||
)
|
||||
|
||||
@field_validator("components", mode="before")
|
||||
@classmethod
|
||||
def normalize_component_types(cls, value):
|
||||
"""Canonicalize built-in enums and allow plugin-defined component type names."""
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
return {component_type_name(component_type): group for component_type, group in value.items()}
|
||||
|
||||
@field_validator("workspace_dir", mode="before")
|
||||
@classmethod
|
||||
def normalize_workspace_dir(cls, value) -> str:
|
||||
|
|
|
|||
|
|
@ -201,7 +201,8 @@ class BaseStep(ComponentMixin, ABC):
|
|||
backend = params.get("backend", "")
|
||||
if not backend:
|
||||
raise ValueError("Dispatch step is missing the required 'backend' field")
|
||||
step_cls = R.get(ComponentEnum.STEP, backend)
|
||||
registry = self.app_context.registry if self.app_context is not None else R
|
||||
step_cls = registry.get(ComponentEnum.STEP, backend)
|
||||
if step_cls is None:
|
||||
raise RuntimeError(f"Unregistered step '{backend}'")
|
||||
params["app_context"] = self.app_context
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""Optional, end-to-end cookbook workflows."""
|
||||
|
||||
from . import auto_fin, daily_paper, dingtalk
|
||||
from . import daily_paper, dingtalk
|
||||
|
||||
__all__ = ["auto_fin", "daily_paper", "dingtalk"]
|
||||
__all__ = ["daily_paper", "dingtalk"]
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
"""Auto Fin news research workflow."""
|
||||
|
||||
from .data import AutoFinDataStep
|
||||
from .merge import AutoFinMergeStep
|
||||
from .topic import AutoFinTopicStep
|
||||
|
||||
__all__ = [
|
||||
"AutoFinDataStep",
|
||||
"AutoFinMergeStep",
|
||||
"AutoFinTopicStep",
|
||||
]
|
||||
|
|
@ -12,7 +12,14 @@ from .link_expansion import expand_links, render_expansion_lines
|
|||
from .line_anchor import format_line_anchor, parse_line_anchor
|
||||
from .logger_utils import get_logger
|
||||
from .logo_utils import print_logo
|
||||
from .service_utils import find_reme, locate_reme, precheck_start, cli_find_reme, running_service_config
|
||||
from .service_utils import (
|
||||
cli_find_reme,
|
||||
find_reme,
|
||||
locate_reme,
|
||||
precheck_start,
|
||||
running_app_config,
|
||||
running_service_config,
|
||||
)
|
||||
from .similarity_utils import cosine_similarity, batch_cosine_similarity
|
||||
from .token_utils import estimate_token_count
|
||||
from .web_static import REME_WEB_STATIC_DIR, resolve_web_static_dir
|
||||
|
|
@ -43,6 +50,7 @@ __all__ = [
|
|||
"locate_reme",
|
||||
"precheck_start",
|
||||
"cli_find_reme",
|
||||
"running_app_config",
|
||||
"running_service_config",
|
||||
"cosine_similarity",
|
||||
"batch_cosine_similarity",
|
||||
|
|
|
|||
|
|
@ -88,13 +88,12 @@ def _reme_start_argv() -> list[list[str]]:
|
|||
return argvs
|
||||
|
||||
|
||||
def running_service_config() -> dict | None:
|
||||
"""Resolve the ``service`` config of a running reme by replaying its start args.
|
||||
def running_app_config() -> dict | None:
|
||||
"""Resolve the full config of a running ReMe by replaying its start args.
|
||||
|
||||
Reads the live ``reme start ...`` process cmdline and re-runs the same
|
||||
``resolve_app_config`` the server used, so the result matches the running
|
||||
server's real backend/transport/host/port even when those were passed on the
|
||||
command line and are absent from (or differ from) the on-disk config file.
|
||||
``resolve_app_config`` the server used. Returning the full config preserves
|
||||
enabled plugins as well as service connection settings for CLI clients.
|
||||
Returns ``None`` when no running reme is found or its args can't be parsed.
|
||||
"""
|
||||
from ..config import parse_args, resolve_app_config
|
||||
|
|
@ -102,14 +101,23 @@ def running_service_config() -> dict | None:
|
|||
for argv in _reme_start_argv():
|
||||
try:
|
||||
_, kwargs = parse_args("start", *argv)
|
||||
config = resolve_app_config(log_config=False, **kwargs)
|
||||
except ValueError:
|
||||
continue
|
||||
service = resolve_app_config(log_config=False, **kwargs).get("service")
|
||||
if isinstance(service, dict):
|
||||
return service
|
||||
if isinstance(config, dict):
|
||||
return config
|
||||
return None
|
||||
|
||||
|
||||
def running_service_config() -> dict | None:
|
||||
"""Return only the service section of the running ReMe configuration."""
|
||||
config = running_app_config()
|
||||
if config is None:
|
||||
return None
|
||||
service = config.get("service")
|
||||
return service if isinstance(service, dict) else None
|
||||
|
||||
|
||||
async def locate_reme() -> tuple[str, int, int | None] | None:
|
||||
"""Find a running reme: try default port, then scanned processes."""
|
||||
if await find_reme(REME_DEFAULT_HOST, REME_DEFAULT_PORT) == "reme":
|
||||
|
|
|
|||
|
|
@ -26,6 +26,34 @@ def test_workspace_dir_expands_user_home(monkeypatch, tmp_path):
|
|||
assert config.workspace_dir == str(tmp_path / ".copaw/workspaces/default")
|
||||
|
||||
|
||||
def test_application_config_accepts_plugin_names_only():
|
||||
"""Plugin configuration remains a simple list of installed entry-point names."""
|
||||
config = ApplicationConfig(plugins=["example"])
|
||||
|
||||
assert config.plugins == ["example"]
|
||||
with pytest.raises(ValidationError):
|
||||
ApplicationConfig(plugins=[{"name": "example"}])
|
||||
|
||||
|
||||
def test_application_config_accepts_plugin_defined_component_type():
|
||||
"""Plugin component type names remain typed configuration buckets."""
|
||||
config = ApplicationConfig(
|
||||
components={
|
||||
"example.reranker": {
|
||||
"default": {"backend": "cross_encoder"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
assert config.components["example.reranker"]["default"].backend == "cross_encoder"
|
||||
|
||||
|
||||
def test_application_config_rejects_unsafe_component_type():
|
||||
"""Component type names cannot escape their workspace metadata directory."""
|
||||
with pytest.raises(ValidationError, match="Invalid component type"):
|
||||
ApplicationConfig(components={"../outside": {"default": {"backend": "unsafe"}}})
|
||||
|
||||
|
||||
def test_dialog_dir_is_not_an_application_config_field():
|
||||
"""The removed option is absent from schemas and ignored when supplied."""
|
||||
custom = ApplicationConfig(session_dir="sessions/", dialog_dir="somewhere/else")
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import numpy as np
|
|||
import pytest
|
||||
from watchfiles import Change
|
||||
|
||||
from reme.components import R
|
||||
from reme.components.agent_wrapper import BaseAgentWrapper
|
||||
from reme.components.file_chunker import DefaultFileChunker
|
||||
from reme.components.file_catalog import LocalFileCatalog
|
||||
|
|
@ -115,6 +116,7 @@ def _make_app_context(workspace_path: Path, daily_dir="daily", digest_dir="diges
|
|||
ctx.app_config.resource_dir = resource_dir
|
||||
ctx.app_config.session_dir = "session"
|
||||
ctx.app_config.timezone = None
|
||||
ctx.registry = R.copy()
|
||||
return ctx
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -86,6 +86,16 @@ def test_bind_rejects_no_component_type():
|
|||
BaseComponent.bind("x", NoType)
|
||||
|
||||
|
||||
def test_bind_accepts_plugin_defined_component_type():
|
||||
class PluginTarget(BaseComponent):
|
||||
component_type = "example.reranker"
|
||||
|
||||
result = BaseComponent.bind("default", PluginTarget, optional=False)
|
||||
|
||||
assert isinstance(result, Dependency)
|
||||
assert result.ctype == "example.reranker"
|
||||
|
||||
|
||||
def test_bind_with_default_factory():
|
||||
def factory():
|
||||
return DepTarget(name="default")
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
import pytest
|
||||
|
||||
from reme.components.base_component import BaseComponent
|
||||
from reme.components.component_registry import ComponentRegistry
|
||||
from reme.components.component_registry import ComponentRegistry, R, create_application_registry
|
||||
from reme.enumeration import ComponentEnum
|
||||
|
||||
|
||||
|
|
@ -52,10 +52,32 @@ def test_register_decorator():
|
|||
|
||||
def test_register_rejects_missing_component_type():
|
||||
reg = ComponentRegistry()
|
||||
with pytest.raises(TypeError, match="ComponentEnum"):
|
||||
with pytest.raises(TypeError, match="component_type"):
|
||||
reg.register(_NoComponentType, "bad")
|
||||
|
||||
|
||||
def test_register_plugin_defined_component_type():
|
||||
reg = ComponentRegistry()
|
||||
|
||||
class PluginComponent(BaseComponent):
|
||||
component_type = "example.reranker"
|
||||
|
||||
reg.register(PluginComponent, "cross_encoder")
|
||||
|
||||
assert reg.get("example.reranker", "cross_encoder") is PluginComponent
|
||||
assert reg.get_all("example.reranker") == {"cross_encoder": PluginComponent}
|
||||
|
||||
|
||||
def test_register_rejects_unsafe_plugin_component_type():
|
||||
reg = ComponentRegistry()
|
||||
|
||||
class UnsafePluginComponent(BaseComponent):
|
||||
component_type = "../outside"
|
||||
|
||||
with pytest.raises(TypeError, match="component_type"):
|
||||
reg.register(UnsafePluginComponent, "unsafe")
|
||||
|
||||
|
||||
def test_register_rejects_empty_name():
|
||||
reg = ComponentRegistry()
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
|
|
@ -127,6 +149,21 @@ def test_clear():
|
|||
assert not reg.get_all(ComponentEnum.KEYWORD_INDEX)
|
||||
|
||||
|
||||
def test_builtin_registry_is_frozen_after_package_bootstrap():
|
||||
assert R.frozen is True
|
||||
with pytest.raises(RuntimeError, match="frozen"):
|
||||
R.register(_DummyComponent, "runtime-mutation")
|
||||
|
||||
|
||||
def test_application_registry_copy_remains_mutable():
|
||||
reg = create_application_registry()
|
||||
|
||||
assert reg.frozen is False
|
||||
reg.register(_DummyComponent, "runtime-component")
|
||||
assert reg.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is _DummyComponent
|
||||
assert R.get(ComponentEnum.FILE_CHUNKER, "runtime-component") is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n=== ComponentRegistry Tests ===")
|
||||
test_register_direct_with_explicit_name()
|
||||
|
|
|
|||
|
|
@ -21,6 +21,37 @@ def test_load_builtin_config_by_filename_with_suffix():
|
|||
assert cfg["service"]["backend"] == "http"
|
||||
|
||||
|
||||
def test_builtin_and_external_config_name_collision_fails(monkeypatch):
|
||||
"""An installed config cannot be silently shadowed by a built-in name."""
|
||||
|
||||
class FakeEntryPoint:
|
||||
"""Installed config entry point with a built-in name."""
|
||||
|
||||
name = "default"
|
||||
value = "example:CONFIG_PATH"
|
||||
|
||||
@staticmethod
|
||||
def load():
|
||||
"""The provider need not be imported to detect the collision."""
|
||||
raise AssertionError("colliding provider should not be loaded")
|
||||
|
||||
class FakeEntryPoints(list):
|
||||
"""Minimal selectable entry-point collection."""
|
||||
|
||||
def select(self, *, group, name):
|
||||
"""Return entries matching the requested group and name."""
|
||||
assert group == "reme.configs"
|
||||
return [entry for entry in self if entry.name == name]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"reme.config.config_parser.metadata.entry_points",
|
||||
lambda: FakeEntryPoints([FakeEntryPoint()]),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="provided by both ReMe and an installed distribution"):
|
||||
_load_config("default")
|
||||
|
||||
|
||||
def test_resolve_app_config_can_suppress_config_log(monkeypatch):
|
||||
"""Client-side config resolution can avoid polluting command output."""
|
||||
messages = []
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def test_resolve_step_missing_backend():
|
|||
|
||||
def test_resolve_step_unregistered_backend():
|
||||
job = BaseJob(name="j")
|
||||
job.app_context = MagicMock()
|
||||
job.app_context = SimpleNamespace(registry=ComponentRegistry())
|
||||
with pytest.raises(ValueError, match="Unregistered backend"):
|
||||
job._resolve_step(ComponentConfig(backend="nonexistent_step"))
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from pathlib import Path
|
|||
import tomllib
|
||||
from types import ModuleType
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
import pytest
|
||||
|
||||
REPOSITORY = Path(__file__).resolve().parents[2]
|
||||
|
|
@ -181,6 +182,23 @@ def test_studio_package_preparation_copies_license(monkeypatch, tmp_path: Path)
|
|||
)
|
||||
|
||||
|
||||
def test_auto_fin_license_matches_repository() -> None:
|
||||
"""Keep the independently distributed Auto Fin license complete and current."""
|
||||
assert (REPOSITORY / "plugin" / "auto-fin" / "LICENSE").read_text(encoding="utf-8") == (
|
||||
REPOSITORY / "LICENSE"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_auto_fin_requires_reme_core() -> None:
|
||||
"""Install the optional runtime packages needed while loading Auto Fin's entry points."""
|
||||
config = tomllib.loads((REPOSITORY / "plugin" / "auto-fin" / "pyproject.toml").read_text(encoding="utf-8"))
|
||||
requirements = [Requirement(value) for value in config["project"]["dependencies"]]
|
||||
reme_requirements = [requirement for requirement in requirements if requirement.name == "reme-ai"]
|
||||
|
||||
assert len(reme_requirements) == 1
|
||||
assert set(reme_requirements[0].extras) == {"core"}
|
||||
|
||||
|
||||
def test_studio_package_preparation_preserves_static_gitignore(monkeypatch, tmp_path: Path) -> None:
|
||||
"""Keep generated static assets ignored after staging the Studio build."""
|
||||
package_dir = tmp_path / "reme_ai_studio"
|
||||
|
|
|
|||
238
tests/unit/test_plugin.py
Normal file
238
tests/unit/test_plugin.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Tests for installed plugin discovery and application-local registration."""
|
||||
|
||||
# pylint: disable=missing-class-docstring,missing-function-docstring
|
||||
|
||||
from importlib.metadata import EntryPoint
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from reme.application import Application
|
||||
from reme.components.base_component import BaseComponent, ComponentMixin
|
||||
from reme.components.component_registry import ComponentRegistry, R
|
||||
from reme.config.config_parser import _load_config
|
||||
from reme.enumeration import ComponentEnum
|
||||
from reme.plugin import Backend, Plugin, PluginManager
|
||||
|
||||
|
||||
class _PluginStep(ComponentMixin):
|
||||
component_type = ComponentEnum.STEP
|
||||
|
||||
|
||||
class _PluginComponent(BaseComponent):
|
||||
component_type = "example.reranker"
|
||||
|
||||
|
||||
def test_plugin_defaults_are_below_application_config():
|
||||
manager = PluginManager([Plugin(name="example", config={"jobs": {"task": {"backend": "base", "value": 1}}})])
|
||||
|
||||
merged = manager.merge_config({"jobs": {"task": {"value": 2}}})
|
||||
|
||||
assert merged["jobs"]["task"] == {"backend": "base", "value": 2}
|
||||
|
||||
|
||||
def test_plugin_defaults_expand_environment(monkeypatch):
|
||||
monkeypatch.setenv("PLUGIN_LIMIT", "12")
|
||||
manager = PluginManager([Plugin(name="example", config={"limit": "${PLUGIN_LIMIT}"})])
|
||||
|
||||
assert manager.merge_config({})["limit"] == 12
|
||||
|
||||
|
||||
def test_plugin_registers_into_only_the_supplied_registry():
|
||||
manager = PluginManager([Plugin(name="example", backends=(Backend("example_step", _PluginStep),))])
|
||||
first = ComponentRegistry()
|
||||
second = ComponentRegistry()
|
||||
|
||||
manager.register(first)
|
||||
|
||||
assert first.get(ComponentEnum.STEP, "example_step") is _PluginStep
|
||||
assert second.get(ComponentEnum.STEP, "example_step") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_registers_and_runs_custom_component_type(monkeypatch, tmp_path):
|
||||
manager = PluginManager(
|
||||
[
|
||||
Plugin(
|
||||
name="example",
|
||||
backends=(Backend("cross_encoder", _PluginComponent),),
|
||||
config={
|
||||
"components": {
|
||||
"example.reranker": {
|
||||
"default": {"backend": "cross_encoder"},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(PluginManager, "discover", classmethod(lambda cls, specs: manager))
|
||||
|
||||
app = Application(
|
||||
plugins=["example"],
|
||||
workspace_dir=str(tmp_path),
|
||||
enable_logo=False,
|
||||
log_to_console=False,
|
||||
log_to_file=False,
|
||||
service={"backend": "cli"},
|
||||
)
|
||||
|
||||
component = app.context.components["example.reranker"]["default"]
|
||||
assert isinstance(component, _PluginComponent)
|
||||
assert app.context.registry.get("example.reranker", "cross_encoder") is _PluginComponent
|
||||
|
||||
await app.start()
|
||||
assert component.is_started is True
|
||||
await app.update_component("example.reranker", "default", backend="updated")
|
||||
assert component.backend == "updated"
|
||||
await app.close()
|
||||
assert component.is_started is False
|
||||
|
||||
|
||||
def test_plugin_backend_collision_fails_with_both_owners():
|
||||
registry = ComponentRegistry()
|
||||
registry.add("same", _PluginStep, owner="first")
|
||||
|
||||
class OtherStep(ComponentMixin):
|
||||
component_type = ComponentEnum.STEP
|
||||
|
||||
with pytest.raises(ValueError, match="both 'first' and 'second'"):
|
||||
registry.add("same", OtherStep, owner="second")
|
||||
|
||||
|
||||
def test_plugin_manager_loads_explicit_entry_point(monkeypatch):
|
||||
descriptor = Plugin(name="example", backends=(Backend("example_step", _PluginStep),))
|
||||
|
||||
class FakeEntryPoint:
|
||||
name = "example"
|
||||
value = "example:plugin"
|
||||
|
||||
@staticmethod
|
||||
def load():
|
||||
return descriptor
|
||||
|
||||
class FakeEntryPoints(list):
|
||||
def select(self, *, group, name):
|
||||
assert group == "reme.plugins"
|
||||
return [entry for entry in self if entry.name == name]
|
||||
|
||||
monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()]))
|
||||
|
||||
manager = PluginManager.discover(["example"])
|
||||
|
||||
assert manager.plugins == (descriptor,)
|
||||
|
||||
|
||||
def test_plugin_entry_point_import_side_effect_does_not_leak(monkeypatch, tmp_path):
|
||||
class UndeclaredClient(ComponentMixin):
|
||||
component_type = ComponentEnum.CLIENT
|
||||
|
||||
descriptor = Plugin(name="example")
|
||||
|
||||
class FakeEntryPoint:
|
||||
name = "example"
|
||||
value = "example:plugin"
|
||||
|
||||
@staticmethod
|
||||
def load():
|
||||
R.register(UndeclaredClient, "undeclared-client")
|
||||
return descriptor
|
||||
|
||||
class FakeEntryPoints(list):
|
||||
def select(self, *, group, name):
|
||||
assert group == "reme.plugins"
|
||||
return [entry for entry in self if entry.name == name]
|
||||
|
||||
monkeypatch.setattr("reme.plugin.metadata.entry_points", lambda: FakeEntryPoints([FakeEntryPoint()]))
|
||||
|
||||
app = Application(
|
||||
plugins=["example"],
|
||||
workspace_dir=str(tmp_path),
|
||||
enable_logo=False,
|
||||
log_to_console=False,
|
||||
log_to_file=False,
|
||||
service={"backend": "cli"},
|
||||
)
|
||||
|
||||
assert R.get(ComponentEnum.CLIENT, "undeclared-client") is None
|
||||
assert app.context.registry.get(ComponentEnum.CLIENT, "undeclared-client") is None
|
||||
|
||||
|
||||
def test_plugin_manager_rejects_non_string_name():
|
||||
with pytest.raises(TypeError, match="Invalid plugin name"):
|
||||
PluginManager.discover([{"name": "example"}])
|
||||
|
||||
|
||||
def test_config_can_extend_another_config(tmp_path: Path):
|
||||
parent = tmp_path / "parent.yaml"
|
||||
child = tmp_path / "child.yaml"
|
||||
parent.write_text("service:\n backend: http\n port: 8000\n", encoding="utf-8")
|
||||
child.write_text("extends: parent.yaml\nservice:\n port: 9000\n", encoding="utf-8")
|
||||
|
||||
assert _load_config(str(child))["service"] == {"backend": "http", "port": 9000}
|
||||
|
||||
|
||||
def test_config_can_come_from_installed_entry_point(tmp_path: Path, monkeypatch):
|
||||
config = tmp_path / "example.yaml"
|
||||
config.write_text("plugins: [example]\n", encoding="utf-8")
|
||||
entry = EntryPoint(name="example", value="pathlib:Path", group="reme.configs")
|
||||
|
||||
class LoadedEntryPoint:
|
||||
name = entry.name
|
||||
value = entry.value
|
||||
|
||||
@staticmethod
|
||||
def load():
|
||||
return config
|
||||
|
||||
class FakeEntryPoints(list):
|
||||
def select(self, *, group, name):
|
||||
assert group == "reme.configs"
|
||||
return [item for item in self if item.name == name]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"reme.config.config_parser.metadata.entry_points",
|
||||
lambda: FakeEntryPoints([LoadedEntryPoint()]),
|
||||
)
|
||||
|
||||
assert _load_config("example") == {"plugins": ["example"]}
|
||||
|
||||
|
||||
def test_config_entry_point_import_side_effect_does_not_leak(tmp_path: Path, monkeypatch):
|
||||
config = tmp_path / "side-effect.yaml"
|
||||
config.write_text("service:\n backend: cli\n", encoding="utf-8")
|
||||
|
||||
class UndeclaredClient(ComponentMixin):
|
||||
component_type = ComponentEnum.CLIENT
|
||||
|
||||
class LoadedEntryPoint:
|
||||
name = "side-effect"
|
||||
value = "example:CONFIG_PATH"
|
||||
|
||||
@staticmethod
|
||||
def load():
|
||||
R.register(UndeclaredClient, "config-side-effect-client")
|
||||
return config
|
||||
|
||||
class FakeEntryPoints(list):
|
||||
def select(self, *, group, name):
|
||||
assert group == "reme.configs"
|
||||
return [item for item in self if item.name == name]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"reme.config.config_parser.metadata.entry_points",
|
||||
lambda: FakeEntryPoints([LoadedEntryPoint()]),
|
||||
)
|
||||
|
||||
loaded = _load_config("side-effect")
|
||||
app = Application(
|
||||
**loaded,
|
||||
workspace_dir=str(tmp_path / "workspace"),
|
||||
enable_logo=False,
|
||||
log_to_console=False,
|
||||
log_to_file=False,
|
||||
)
|
||||
|
||||
assert loaded == {"service": {"backend": "cli"}}
|
||||
assert R.get(ComponentEnum.CLIENT, "config-side-effect-client") is None
|
||||
assert app.context.registry.get(ComponentEnum.CLIENT, "config-side-effect-client") is None
|
||||
|
|
@ -11,6 +11,9 @@ import pytest
|
|||
from reme.components.service import cli_service
|
||||
from reme.components.service.cli_service import CliService
|
||||
from reme import reme as reme_module
|
||||
from reme.components.base_component import ComponentMixin
|
||||
from reme.enumeration import ComponentEnum
|
||||
from reme.plugin import Backend, Plugin, PluginManager
|
||||
|
||||
|
||||
def test_package_import_does_not_load_optional_core_dependencies():
|
||||
|
|
@ -257,8 +260,12 @@ def test_call_server_passes_client_kwargs_to_client(monkeypatch, capsys):
|
|||
seen["payload"] = kwargs
|
||||
yield "ok"
|
||||
|
||||
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
|
||||
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
reme_module,
|
||||
"create_application_registry",
|
||||
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
|
||||
)
|
||||
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
||||
|
||||
async def run():
|
||||
await reme_module.call_server(
|
||||
|
|
@ -299,8 +306,12 @@ def test_call_server_treats_show_metadata_as_client_kwarg(monkeypatch, capsys):
|
|||
seen["payload"] = kwargs
|
||||
yield "ok"
|
||||
|
||||
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
|
||||
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
reme_module,
|
||||
"create_application_registry",
|
||||
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
|
||||
)
|
||||
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
||||
|
||||
async def run():
|
||||
await reme_module.call_server("version", backend="http", show_metadata=True)
|
||||
|
|
@ -334,8 +345,12 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys):
|
|||
seen["payload"] = kwargs
|
||||
yield "ok"
|
||||
|
||||
monkeypatch.setattr(reme_module.R, "get", lambda component_type, backend: FakeClient)
|
||||
monkeypatch.setattr(reme_module, "running_service_config", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
reme_module,
|
||||
"create_application_registry",
|
||||
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
|
||||
)
|
||||
monkeypatch.setattr(reme_module, "running_app_config", lambda: None)
|
||||
|
||||
async def run():
|
||||
await reme_module.call_server("shell", backend="http", cmd="ls", shell_timeout=5)
|
||||
|
|
@ -345,3 +360,86 @@ def test_call_server_passes_shell_parameters_as_payload(monkeypatch, capsys):
|
|||
assert seen["action"] == "shell"
|
||||
assert seen["payload"] == {"cmd": "ls", "shell_timeout": 5}
|
||||
assert capsys.readouterr().out == "ok\n"
|
||||
|
||||
|
||||
def test_call_server_uses_running_plugins_and_their_service_defaults(monkeypatch, capsys):
|
||||
"""A bare client call can load the Client backend enabled by the running app."""
|
||||
seen = {}
|
||||
|
||||
class PluginClient(ComponentMixin):
|
||||
"""Client backend supplied by an enabled plugin."""
|
||||
|
||||
component_type = ComponentEnum.CLIENT
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
seen["client_kwargs"] = kwargs
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
async def __call__(self, action: str, **kwargs):
|
||||
seen["action"] = action
|
||||
seen["payload"] = kwargs
|
||||
yield "plugin-ok"
|
||||
|
||||
manager = PluginManager(
|
||||
[
|
||||
Plugin(
|
||||
name="example",
|
||||
backends=(Backend("plugin-client", PluginClient),),
|
||||
config={"service": {"backend": "plugin-client", "host": "127.0.0.9", "port": 9911}},
|
||||
),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(reme_module, "resolve_app_config", lambda **_kwargs: {"service": {"backend": "http"}})
|
||||
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"plugins": ["example"]})
|
||||
monkeypatch.setattr(reme_module.PluginManager, "discover", lambda _specs: manager)
|
||||
|
||||
asyncio.run(reme_module.call_server("search", query="hello"))
|
||||
|
||||
assert seen["client_kwargs"]["host"] == "127.0.0.9"
|
||||
assert seen["client_kwargs"]["port"] == 9911
|
||||
assert seen["action"] == "search"
|
||||
assert seen["payload"] == {"query": "hello"}
|
||||
assert capsys.readouterr().out == "plugin-ok\n"
|
||||
|
||||
|
||||
def test_call_server_skips_local_fallback_when_server_is_running(monkeypatch, capsys):
|
||||
"""A usable running config prevents eager parsing of the local fallback."""
|
||||
|
||||
class FakeClient:
|
||||
"""Minimal running-service client."""
|
||||
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
async def __call__(self, action: str, **kwargs):
|
||||
assert action == "version"
|
||||
assert not kwargs
|
||||
yield "ok"
|
||||
|
||||
monkeypatch.setattr(
|
||||
reme_module,
|
||||
"create_application_registry",
|
||||
lambda: SimpleNamespace(get=lambda component_type, backend: FakeClient),
|
||||
)
|
||||
monkeypatch.setattr(reme_module, "running_app_config", lambda: {"service": {"backend": "http"}})
|
||||
|
||||
def fail_local_resolution(**_kwargs):
|
||||
raise AssertionError("local fallback should not be resolved")
|
||||
|
||||
monkeypatch.setattr(reme_module, "resolve_app_config", fail_local_resolution)
|
||||
|
||||
asyncio.run(reme_module.call_server("version"))
|
||||
|
||||
assert capsys.readouterr().out == "ok\n"
|
||||
|
|
|
|||
|
|
@ -156,3 +156,22 @@ def test_scan_reme_procs_skips_access_denied(monkeypatch):
|
|||
]
|
||||
_patch_iter(monkeypatch, procs)
|
||||
assert su._scan_reme_procs() == [(5, su.REME_DEFAULT_HOST, su.REME_DEFAULT_PORT)]
|
||||
|
||||
|
||||
def test_running_app_config_preserves_plugins(monkeypatch):
|
||||
"""Process replay exposes the full config while the compatibility helper returns service only."""
|
||||
monkeypatch.setattr(su, "_reme_start_argv", lambda: [["config=example"]])
|
||||
monkeypatch.setattr("reme.config.parse_args", lambda *_args: ("start", {"config": "example"}))
|
||||
monkeypatch.setattr(
|
||||
"reme.config.resolve_app_config",
|
||||
lambda **_kwargs: {
|
||||
"plugins": ["example"],
|
||||
"service": {"backend": "plugin-client", "port": 9911},
|
||||
},
|
||||
)
|
||||
|
||||
assert su.running_app_config() == {
|
||||
"plugins": ["example"],
|
||||
"service": {"backend": "plugin-client", "port": 9911},
|
||||
}
|
||||
assert su.running_service_config() == {"backend": "plugin-client", "port": 9911}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue