mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-08-28 05:25:04 +00:00
feat: add daily paper cookbook and DingTalk agent integration (#385)
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
Tests ReMe / Unit Tests - py3.11 (push) Has been cancelled
Tests ReMe / Unit Tests - py3.12 (push) Has been cancelled
Tests ReMe / Unit Tests - py3.13 (push) Has been cancelled
Windows Smoke / CLI smoke - py3.11 (push) Has been cancelled
Some checks failed
Pre-commit / run (ubuntu-latest) (push) Has been cancelled
Tests ReMe / Unit Tests - py3.11 (push) Has been cancelled
Tests ReMe / Unit Tests - py3.12 (push) Has been cancelled
Tests ReMe / Unit Tests - py3.13 (push) Has been cancelled
Windows Smoke / CLI smoke - py3.11 (push) Has been cancelled
* feat(daily-paper): add daily paper cookbook workflow with schema and tests - Introduce daily paper schema types (DailyBriefOutput, PaperInfo, PaperNoteOutput, etc.) - Create daily paper cookbook module with analyze, collect, digest, rank, and select steps - Add cookbook entry point and integrate into main steps module - Replace job config export with daily brief output in schema exports - Add comprehensive unit tests covering pipeline, filtering, and output generation - Update dependencies including openai-codex and pypdf packages - Configure standalone daily paper cron job with proper scheduling and routing * test(daily_paper): update tests to use Claude Code wrapper exclusively - Add test to verify web search is disallowed by default in Claude Code - Update imports to include DailyBriefOutput, PaperNoteOutput, and PaperSelection schemas - Change test name from standalone_config_has_backend_split to reflect Claude Code only usage - Remove default agent wrapper and configure all steps to use Claude Code wrapper - Rename select_wrapper to cc_wrapper for clarity and consistency - Remove duplicate Claude Code wrapper initialization - Update test assertions to verify output schema usage matches expected sequence - Remove unused as_llm component from standalone configuration test * refactor(agent-wrapper): simplify skill resolution logic across all wrappers - Replace duplicate skill resolution code with centralized _resolve_project_skills method - Add project_path property with configurable relative path resolution - Introduce proper validation for skill names and directory existence - Change Codex wrapper to use project_path instead of workspace_path for skills - Add SKILL.md requirement validation for project skills - Remove redundant skill processing logic from individual wrappers * feat(daily_paper): add daily paper workflow with PDF analysis and brief generation - Implement shared state management and file helpers for daily-paper steps - Add PDF download and text extraction capabilities with arXiv integration - Create paper collection step with Hugging Face weekly/monthly rankings - Build ranking system using reciprocal-rank fusion with memory keyword scoring - Add Claude Code integration for paper analysis and detailed note generation - Implement digest step to create final five-minute brief from detailed notes - Add configuration for standalone daily cookbook application with cron scheduling - Create typed schema for paper information, selection, and output formats - Add atomic file writing with temporary file safety mechanisms - Implement exclusion logic for previously recommended papers and daily filters * feat(daily_paper): add DingTalk notification integration and enhance logging - Integrate DingTalk markdown send step to notify groups about daily paper briefs - Add comprehensive logging throughout daily paper workflow including start/finish events - Update daily paper analysis prompt to include code repository context requirement - Configure DingTalk notification in daily_cookbook.yaml with app credentials - Add dingtalk-stream dependency for proactive message API integration - Enhance daily paper README with DingTalk notification section and updated flow chart - Implement detailed logging for each step including paper processing and agent calls - Add test coverage for DingTalk markdown sending functionality and configuration - Update pre-commit config to exclude skills directory from checks - Add .claude/skills to gitignore for local development environment * refactor(dingtalk): move dingtalk_stream import to local scope and improve code safety - Moved global dingtalk_stream import to local scope in send.py to avoid eager loading - Added dynamic import with error handling for optional dependency cases - Updated test suite to verify lazy loading behavior works correctly - Fixed markdown title generation by using safe variable naming in wait.py - Enhanced test coverage for arxiv PDF download caching functionality - Updated application context initialization with proper resource directory configuration - Modified paper metadata to include source PDF path reference in output files * refactor(daily_paper): remove manifest system and store selection metadata in digest files - Remove JSON manifest creation and storage functionality - Store selection data directly in digest file frontmatter instead of separate manifest files - Add load_saved_selection method to rebuild selection from digest and paper-note metadata - Update README documentation to reflect new cookbook workflow architecture - Modify test cases to verify selection metadata in digest files instead of manifest JSON - Remove unused json import from multiple daily paper modules - Integrate PaperSelection schema for proper data validation in stored metadata * docs(daily_paper): add bilingual cookbook guides
This commit is contained in:
parent
630f26b119
commit
46adb5ae1e
40 changed files with 6399 additions and 60 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -54,6 +54,12 @@ vault/
|
|||
docs/_build/
|
||||
site/
|
||||
|
||||
evaluation/
|
||||
datasets/
|
||||
|
||||
# Claude Code skills (local only)
|
||||
.claude/skills/
|
||||
|
||||
# Memory workspaces (keep dirs, ignore contents)
|
||||
benchmark/memory_workspaces/
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
exclude: ^skills/
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
|
|
|
|||
13
README.md
13
README.md
|
|
@ -58,6 +58,9 @@ memory, then continuously indexes, links, and consolidates that memory for futur
|
|||
|
||||
## 📰 News
|
||||
|
||||
- [2026.07] - Introduced optional Cookbook workflows, starting with
|
||||
[Daily Paper](cookbook/daily_paper/README.md) for scheduled paper discovery, agent-assisted PDF analysis, reusable
|
||||
Markdown notes, and five-minute briefs.
|
||||
- [2026.07] - Our
|
||||
paper [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/)
|
||||
has been accepted to Findings of ACL 2026.
|
||||
|
|
@ -161,6 +164,16 @@ ReMe stores agent memory as readable Markdown.
|
|||
Related: [[digest/wiki/memory-as-file.md]]
|
||||
```
|
||||
|
||||
## 🧑🍳 Cookbooks
|
||||
|
||||
Cookbooks are optional, end-to-end workflows assembled from ReMe jobs and steps. They are not enabled by the default
|
||||
configuration; select the cookbook's standalone configuration when starting ReMe. Each new cookbook will be added as
|
||||
another row in this table.
|
||||
|
||||
| Cookbook | Capability | Introduction |
|
||||
|-------------|---------------------------------------------------------------------------------------------------------------|------------------------------------------|
|
||||
| Daily Paper | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. | [README](cookbook/daily_paper/README.md) |
|
||||
|
||||
## 📁 Memory System
|
||||
|
||||
> Memory as File, File as Memory.
|
||||
|
|
|
|||
13
README_ZH.md
13
README_ZH.md
|
|
@ -51,6 +51,8 @@ Agent 能够可靠召回。
|
|||
|
||||
## 📰 新闻
|
||||
|
||||
- [2026.07] - 新增可选的 Cookbook 工作流,首个能力为 [每日论文](cookbook/daily_paper/README_ZH.md),支持定时发现论文、
|
||||
Agent 辅助解析 PDF、沉淀可复用的 Markdown 笔记并生成五分钟简报。
|
||||
- [2026.07] -
|
||||
我们的论文 [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/)
|
||||
已被 Findings of ACL 2026 接收。
|
||||
|
|
@ -154,6 +156,15 @@ ReMe 会把 Agent 记忆保存为可读的 Markdown。
|
|||
相关链接:[[digest/wiki/memory-as-file.md]]
|
||||
```
|
||||
|
||||
## 🧑🍳 Cookbooks
|
||||
|
||||
Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们;启动 ReMe 时选择对应的
|
||||
独立配置即可启用。后续新增的 cookbook 会继续在表格中按行追加。
|
||||
|
||||
| Cookbook | 能力 | 介绍 |
|
||||
|----------|------------------------------------------|-------------------------------------------|
|
||||
| 每日论文 | 发现并排序论文,使用 Agent 解读 PDF,生成文件化论文笔记和五分钟简报。 | [使用说明](cookbook/daily_paper/README_ZH.md) |
|
||||
|
||||
## 📁 记忆系统
|
||||
|
||||
> Memory as File, File as Memory.
|
||||
|
|
@ -277,7 +288,7 @@ frontmatter 和文件操作接口主要用于维护、调试或高级集成。
|
|||
|-------------------------------------------|---------------------------------------------|
|
||||
| `reme start` | 启动本地 ReMe 服务。 |
|
||||
| `reme version` / `reme health_check` | 检查包版本和组件状态。 |
|
||||
| `reme status` | 查看有状态数据组件的内存估算及进程 RSS。 |
|
||||
| `reme status` | 查看有状态数据组件的内存估算及进程 RSS。 |
|
||||
| [`reme search`](docs/zh/memory_search.md) | 默认使用 BM25 和 wikilink 检索,启用后增加向量检索。 |
|
||||
| `reme read` / `reme write` / `reme edit` | 检查和维护 Markdown 记忆文件。 |
|
||||
| `reme auto_memory` | 将对话 messages 转为 daily 记忆卡片;需要 LLM 凭证。 |
|
||||
|
|
|
|||
297
cookbook/daily_paper/README.md
Normal file
297
cookbook/daily_paper/README.md
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
# Daily Paper Cookbook
|
||||
|
||||
[中文](README_ZH.md)
|
||||
|
||||
Daily Paper is a local-first, file-native workflow for turning research feeds into a daily reading package. It collects
|
||||
papers from the Hugging Face weekly and monthly rankings, removes yesterday's papers and recent recommendations, ranks
|
||||
the remaining candidates, and uses Claude Code to produce detailed Chinese paper notes and a five-minute Chinese brief.
|
||||
|
||||
The workflow is assembled by [`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml). Its schemas live in
|
||||
[`reme/schema/daily_paper.py`](../../reme/schema/daily_paper.py), and its steps live in
|
||||
[`reme/steps/cookbook/daily_paper/`](../../reme/steps/cookbook/daily_paper/).
|
||||
|
||||
## Quick start
|
||||
|
||||
Daily Paper requires Python 3.11 or later, the `core` dependencies, network access to Hugging Face and arXiv, and
|
||||
credentials for the configured Claude Code endpoint.
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
python -m pip install -e ".[dev,core]"
|
||||
export CLAUDE_CODE_API_KEY="your-api-key"
|
||||
reme start config=daily_cookbook job=daily_paper
|
||||
```
|
||||
|
||||
The built-in configuration uses `qwen3.7-max` through DashScope's Anthropic-compatible endpoint. Override
|
||||
`CLAUDE_CODE_MODEL_NAME` and `CLAUDE_CODE_BASE_URL` when using another compatible model or provider.
|
||||
|
||||
By default, outputs are written under `.reme/` in the directory where ReMe starts.
|
||||
|
||||
## What it creates
|
||||
|
||||
A successful run writes ordinary PDFs and Markdown files beneath `workspace_dir`:
|
||||
|
||||
```text
|
||||
.reme/
|
||||
├── daily/
|
||||
│ ├── YYYY-MM-DD.md
|
||||
│ └── YYYY-MM-DD/
|
||||
│ ├── daily-paper-brief.md
|
||||
│ ├── paper-<arxiv-id>.md
|
||||
│ └── ...
|
||||
├── resource/
|
||||
│ └── papers/
|
||||
│ ├── <arxiv-id>.pdf
|
||||
│ └── ...
|
||||
└── mem_session/
|
||||
└── claude_config/
|
||||
```
|
||||
|
||||
- `paper-<arxiv-id>.md` is a detailed Chinese reading note with YAML frontmatter linking back to the source PDF and
|
||||
paper pages.
|
||||
- `daily-paper-brief.md` is a roughly five-minute Chinese digest with wikilinks to every selected paper note.
|
||||
- `daily/YYYY-MM-DD.md` is a derived day index rebuilt from the Markdown files for that date.
|
||||
- `resource/papers/` holds reusable source PDFs.
|
||||
|
||||
The paper notes are the source of truth for recommendation history: their frontmatter contains the `arxiv_id` values
|
||||
used for future deduplication. The day index is derived and can be rebuilt. The workflow does not currently write a
|
||||
separate run manifest.
|
||||
|
||||
## How the workflow works
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
HF[Hugging Face<br/>weekly + monthly] --> C[1. Collect]
|
||||
Y[Yesterday's papers] --> C
|
||||
H[Recent local notes] --> C
|
||||
C --> R[2. Rank]
|
||||
R --> S[3. Select]
|
||||
S --> A[4. Analyze PDFs]
|
||||
A --> D[5. Build digest]
|
||||
D --> N[6. Notify DingTalk]
|
||||
A --> P[PDFs + paper notes]
|
||||
D --> B[Brief + day index]
|
||||
```
|
||||
|
||||
### 1. Collect and deduplicate
|
||||
|
||||
The Collect step fetches the weekly ranking for the run date's ISO week, the monthly ranking for its calendar month,
|
||||
and the Hugging Face Daily Papers IDs for exactly the previous calendar day. It merges weekly and monthly metadata by
|
||||
arXiv ID and preserves each list's display rank.
|
||||
|
||||
It then scans `daily/<prior-date>/paper-*.md` over the configured history window and excludes IDs found in note
|
||||
frontmatter. The job fails clearly if no eligible papers remain.
|
||||
|
||||
### 2. Rank candidates
|
||||
|
||||
The Rank step uses reciprocal-rank fusion:
|
||||
|
||||
```text
|
||||
score = 1 / (rrf_k + monthly_rank)
|
||||
+ weekly_weight / (rrf_k + weekly_rank)
|
||||
```
|
||||
|
||||
A missing rank contributes zero. Candidates are ordered by fused score, upvotes, and arXiv ID. The bounded candidate
|
||||
pool also reserves several positions for papers whose titles or summaries match memory-related terms such as agent
|
||||
memory, memory retrieval, continual learning, context compression, knowledge graphs, and RAG. This reserve is a simple
|
||||
keyword heuristic, not a semantic classifier.
|
||||
|
||||
### 3. Select papers
|
||||
|
||||
Claude Code receives the bounded candidate pool and returns a structured `PaperSelection`. The implementation requires
|
||||
exactly `top_k` unique in-pool IDs with consecutive ranks. Invalid output is returned to the agent once as validation
|
||||
feedback; a second invalid response fails the job.
|
||||
|
||||
### 4. Download and analyze PDFs
|
||||
|
||||
Selected papers are processed sequentially. For each paper, the workflow:
|
||||
|
||||
1. validates the modern arXiv ID format;
|
||||
2. downloads and validates the PDF, or reuses an existing file with a valid `%PDF-` header;
|
||||
3. extracts text with `pypdf`, adding page markers and applying page and character limits;
|
||||
4. asks Claude Code for a structured detailed reading; and
|
||||
5. writes normalized frontmatter plus the generated Markdown body.
|
||||
|
||||
The current extractor requires a usable PDF text layer. Scanned or image-only PDFs fail because there is no OCR
|
||||
fallback. If extraction exceeds a configured limit, the note records that the input was truncated.
|
||||
|
||||
### 5. Build the brief and index
|
||||
|
||||
Claude Code reads every detailed note and produces the daily brief. The code verifies that each source-note wikilink is
|
||||
present and appends any missing links before writing the file. It then rebuilds `daily/YYYY-MM-DD.md` from that day's
|
||||
Markdown frontmatter.
|
||||
|
||||
### 6. Optionally notify DingTalk
|
||||
|
||||
The final step sends the brief body, without YAML frontmatter, to each configured DingTalk group in order. With no
|
||||
conversation IDs it is a no-op. If one group fails, the step still attempts the remaining groups and reports the
|
||||
combined failure afterward.
|
||||
|
||||
## Dates, reruns, and idempotency
|
||||
|
||||
- `date` must be an exact `YYYY-MM-DD` value. When omitted, the job uses today in the application timezone, which is
|
||||
`Asia/Shanghai` in the built-in configuration.
|
||||
- “Yesterday” means `date - 1 day`, not the previous 24 hours.
|
||||
- `history_days` considers prior dated note directories only; the current run date is never part of its history scan.
|
||||
- If `daily/<date>/daily-paper-brief.md` already exists and `force=false`, collection, ranking, model calls, PDF work,
|
||||
and digest generation are skipped. The existing brief remains available to the DingTalk notification step.
|
||||
- `force=true` regenerates the notes and brief. Existing valid PDFs are still reused.
|
||||
|
||||
Each PDF, detailed note, and final brief uses a temporary file followed by replacement so callers do not see a
|
||||
partially written file. The complete multi-file workflow is not transactional, and there is no global lock for two
|
||||
concurrent runs of the same date.
|
||||
|
||||
## Running the cookbook
|
||||
|
||||
The standalone configuration defines three jobs:
|
||||
|
||||
| Job | Behavior |
|
||||
|--------------------|---------------------------------------------------------|
|
||||
| `daily_paper` | On-demand generation through the CLI or HTTP service |
|
||||
| `daily_paper_cron` | The same pipeline every day at 08:00 in `Asia/Shanghai` |
|
||||
| `dingtalk_wait` | A supervised background DingTalk agent |
|
||||
|
||||
### One-time runs
|
||||
|
||||
Generate today's brief:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=daily_paper
|
||||
```
|
||||
|
||||
Generate a specific date with selected overrides:
|
||||
|
||||
```bash
|
||||
reme start \
|
||||
config=daily_cookbook \
|
||||
job=daily_paper \
|
||||
date=2026-07-21 \
|
||||
top_k=3 \
|
||||
history_days=30
|
||||
```
|
||||
|
||||
Regenerate a date whose brief already exists:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=daily_paper date=2026-07-21 force=true
|
||||
```
|
||||
|
||||
Add `service.show_metadata=true` to a one-time command when the response metadata is useful for diagnostics.
|
||||
|
||||
### Long-running service and cron
|
||||
|
||||
Start the standalone HTTP service and its scheduled/background jobs:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook
|
||||
```
|
||||
|
||||
It listens on `127.0.0.1:8001` by default, so it can run beside the default ReMe service. Call the on-demand job from
|
||||
another terminal with either the ReMe client or HTTP:
|
||||
|
||||
```bash
|
||||
reme daily_paper host=127.0.0.1 port=8001
|
||||
```
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8001/daily_paper \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"date":"2026-07-21","top_k":3,"force":false}'
|
||||
```
|
||||
|
||||
Service and schedule settings can be overridden at startup:
|
||||
|
||||
```bash
|
||||
reme start \
|
||||
config=daily_cookbook \
|
||||
service.host=0.0.0.0 \
|
||||
service.port=8101 \
|
||||
jobs.daily_paper_cron.cron="30 7 * * *"
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The most useful job settings are:
|
||||
|
||||
| Setting | Default | Purpose |
|
||||
|-------------------|-------------:|--------------------------------------------------------------|
|
||||
| `candidate_limit` | `20` | Maximum number of papers sent to selection |
|
||||
| `memory_reserve` | `5` | Candidate positions reserved by the memory-keyword heuristic |
|
||||
| `top_k` | `3` | Number of papers selected and analyzed |
|
||||
| `rrf_k` | `60` | Reciprocal-rank fusion constant |
|
||||
| `weekly_weight` | `0.7` | Weight of the weekly ranking in fusion |
|
||||
| `history_days` | `30` | Prior recommendation window excluded by arXiv ID |
|
||||
| `hf_timeout` | `30` seconds | Hugging Face request timeout |
|
||||
| `hf_max_retries` | `3` | Maximum Hugging Face request attempts |
|
||||
| `pdf_timeout` | `90` seconds | arXiv download timeout |
|
||||
| `max_pdf_bytes` | `52428800` | Maximum PDF size (50 MiB) |
|
||||
| `max_pdf_pages` | `80` | Maximum pages extracted for analysis |
|
||||
| `max_pdf_chars` | `240000` | Maximum extracted characters sent for one paper |
|
||||
|
||||
The public job parameters are `date`, `force`, `top_k`, `weekly_weight`, and `history_days`. Explicit invocation values
|
||||
take precedence over the job defaults.
|
||||
|
||||
The standalone application also accepts these environment variables:
|
||||
|
||||
| Variable | Purpose |
|
||||
|-----------------------------------------|-------------------------------------------------|
|
||||
| `DAILY_PAPER_WORKSPACE_DIR` | Overrides the default `.reme` workspace |
|
||||
| `DAILY_PAPER_PROJECT_PATH` | Repository/project path visible to Claude Code |
|
||||
| `DAILY_PAPER_HOST` / `DAILY_PAPER_PORT` | HTTP bind address |
|
||||
| `CLAUDE_CODE_API_KEY` | API key for the configured Claude Code endpoint |
|
||||
| `CLAUDE_CODE_MODEL_NAME` | Model override |
|
||||
| `CLAUDE_CODE_BASE_URL` | Anthropic-compatible endpoint override |
|
||||
|
||||
`DAILY_PAPER_PROJECT_PATH` defaults to `..` relative to the workspace. With the default `.reme` workspace, starting
|
||||
from the repository root resolves it back to the repository. If the workspace lives elsewhere, set both paths
|
||||
explicitly.
|
||||
|
||||
ReMe loads an uncommitted `.env` file found from the current directory upward, so the same values may be placed there
|
||||
instead of exported in the shell.
|
||||
|
||||
## DingTalk configuration
|
||||
|
||||
DingTalk is optional. Configure it only when brief delivery or the background DingTalk agent is needed:
|
||||
|
||||
```dotenv
|
||||
DINGTALK_APP_KEY=your-app-key
|
||||
DINGTALK_APP_SECRET=your-app-secret
|
||||
DINGTALK_ROBOT_CODE=your-robot-code
|
||||
DINGTALK_CONVERSATION_IDS=cid-group-one,cid-group-two
|
||||
```
|
||||
|
||||
`DINGTALK_CONVERSATION_IDS` is required only for proactive brief delivery. The background `dingtalk_wait` job uses the
|
||||
first three credentials but not the conversation list.
|
||||
|
||||
## Failure recovery and boundaries
|
||||
|
||||
| Situation | Behavior |
|
||||
|--------------------------------------|----------------------------------------------------------------------|
|
||||
| Temporary Hugging Face failure | Retries with exponential delay up to `hf_max_retries` attempts |
|
||||
| No eligible papers | Fails before ranking |
|
||||
| Invalid `top_k` or selection output | Fails after validation; selection output gets one retry |
|
||||
| Oversized, invalid, or textless PDF | Stops during analysis |
|
||||
| PDF exceeds page or character limits | Continues with truncated text and records the truncation |
|
||||
| One paper analysis fails | Stops the job; earlier PDFs and notes remain on disk |
|
||||
| Brief misses a source-note link | Appends the missing wikilink before writing |
|
||||
| Existing final brief | Skips generation unless `force=true`; notification can still send it |
|
||||
| Concurrent runs for one date | No pipeline-level lock; later writes may replace earlier results |
|
||||
|
||||
To recover, inspect the date's notes and PDFs, fix the network, credential, model, or PDF issue, then rerun the same
|
||||
date with `force=true`. Valid cached PDFs will be reused.
|
||||
|
||||
The built-in Claude Code component runs with `permission_mode: bypassPermissions`. ReMe disables Claude Code's
|
||||
`WebSearch` tool, and the analysis/digest prompts constrain what the agent should read, but these steps do not set a
|
||||
strict per-call tool allowlist or an operating-system sandbox. Run the cookbook only with a trusted project and
|
||||
workspace, and tighten the agent configuration before shared or production use.
|
||||
|
||||
## Tests
|
||||
|
||||
The focused unit suite mocks Hugging Face, arXiv, Claude Code, and DingTalk boundaries:
|
||||
|
||||
```bash
|
||||
pytest tests/unit/test_daily_paper.py -v
|
||||
```
|
||||
|
||||
Real runs access external services and may incur model costs; they should not be used as ordinary unit tests.
|
||||
282
cookbook/daily_paper/README_ZH.md
Normal file
282
cookbook/daily_paper/README_ZH.md
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
# 每日论文 Cookbook
|
||||
|
||||
[English](README.md)
|
||||
|
||||
每日论文是一个本地优先、文件原生的研究资讯工作流。它从 Hugging Face 周榜和月榜采集论文,排除昨日论文和近期
|
||||
已经推荐过的论文,对剩余候选进行排序,再使用 Claude Code 生成中文详细论文笔记和约五分钟可读完的中文简报。
|
||||
|
||||
工作流由 [`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配,公共 schema 位于
|
||||
[`reme/schema/daily_paper.py`](../../reme/schema/daily_paper.py),各步骤位于
|
||||
[`reme/steps/cookbook/daily_paper/`](../../reme/steps/cookbook/daily_paper/)。
|
||||
|
||||
## 快速开始
|
||||
|
||||
每日论文要求 Python 3.11 或更高版本、`core` 依赖、可访问 Hugging Face 和 arXiv 的网络,以及所配置
|
||||
Claude Code endpoint 的凭据。
|
||||
|
||||
在仓库根目录运行:
|
||||
|
||||
```bash
|
||||
python -m pip install -e ".[dev,core]"
|
||||
export CLAUDE_CODE_API_KEY="your-api-key"
|
||||
reme start config=daily_cookbook job=daily_paper
|
||||
```
|
||||
|
||||
内置配置默认通过 DashScope 的 Anthropic 兼容 endpoint 使用 `qwen3.7-max`。如需使用其他兼容模型或服务商,
|
||||
请覆盖 `CLAUDE_CODE_MODEL_NAME` 和 `CLAUDE_CODE_BASE_URL`。
|
||||
|
||||
默认情况下,产物写入 ReMe 启动目录下的 `.reme/`。
|
||||
|
||||
## 文件产物
|
||||
|
||||
一次成功运行会在 `workspace_dir` 下写入普通 PDF 和 Markdown 文件:
|
||||
|
||||
```text
|
||||
.reme/
|
||||
├── daily/
|
||||
│ ├── YYYY-MM-DD.md
|
||||
│ └── YYYY-MM-DD/
|
||||
│ ├── daily-paper-brief.md
|
||||
│ ├── paper-<arxiv-id>.md
|
||||
│ └── ...
|
||||
├── resource/
|
||||
│ └── papers/
|
||||
│ ├── <arxiv-id>.pdf
|
||||
│ └── ...
|
||||
└── mem_session/
|
||||
└── claude_config/
|
||||
```
|
||||
|
||||
- `paper-<arxiv-id>.md` 是中文详细论文解读,其 YAML frontmatter 会链接原始 PDF 和论文页面。
|
||||
- `daily-paper-brief.md` 是约五分钟可读完的中文简报,并包含每篇入选论文笔记的 wikilink。
|
||||
- `daily/YYYY-MM-DD.md` 是从当日 Markdown 文件重建的派生日索引。
|
||||
- `resource/papers/` 保存可复用的原始 PDF。
|
||||
|
||||
论文笔记是推荐历史的事实来源:后续排重会读取其 frontmatter 中的 `arxiv_id`。日索引属于可重建的派生文件。
|
||||
当前工作流不会另外写入运行 manifest。
|
||||
|
||||
## 工作流程
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
HF[Hugging Face<br/>周榜 + 月榜] --> C[1. Collect]
|
||||
Y[昨日论文] --> C
|
||||
H[近期本地笔记] --> C
|
||||
C --> R[2. Rank]
|
||||
R --> S[3. Select]
|
||||
S --> A[4. Analyze PDFs]
|
||||
A --> D[5. Build digest]
|
||||
D --> N[6. Notify DingTalk]
|
||||
A --> P[PDF + 论文笔记]
|
||||
D --> B[简报 + 日索引]
|
||||
```
|
||||
|
||||
### 1. 采集与排重
|
||||
|
||||
Collect 会获取运行日所在 ISO week 的周榜、所在自然月的月榜,以及严格前一个自然日的 Hugging Face Daily Papers
|
||||
ID。周榜和月榜元数据按 arXiv ID 合并,同时保留两个榜单各自的展示排名。
|
||||
|
||||
随后,它会在配置的历史窗口内扫描 `daily/<prior-date>/paper-*.md`,排除笔记 frontmatter 中已有的 ID。如果排重后
|
||||
没有任何可选论文,Job 会明确失败。
|
||||
|
||||
### 2. 候选排序
|
||||
|
||||
Rank 使用 reciprocal-rank fusion(RRF):
|
||||
|
||||
```text
|
||||
score = 1 / (rrf_k + monthly_rank)
|
||||
+ weekly_weight / (rrf_k + weekly_rank)
|
||||
```
|
||||
|
||||
论文缺少某个榜单排名时,该项贡献为零。候选按融合分、upvotes 和 arXiv ID 排序。有界候选池还会为标题或摘要命中
|
||||
Agent memory、memory retrieval、continual learning、context compression、knowledge graph、RAG 等记忆相关关键词的
|
||||
论文保留若干位置。这个保留策略只是关键词启发式,不是语义分类器。
|
||||
|
||||
### 3. 精选论文
|
||||
|
||||
Claude Code 接收有界候选池,并返回结构化的 `PaperSelection`。实现要求恰好选择 `top_k` 个候选池内的唯一 ID,
|
||||
且 rank 必须连续。输出不合法时,校验错误会反馈给 Agent 并重试一次;第二次仍不合法则 Job 失败。
|
||||
|
||||
### 4. 下载并解读 PDF
|
||||
|
||||
入选论文按顺序逐篇处理。每篇论文都会经过:
|
||||
|
||||
1. 校验当前支持的新版 arXiv ID 格式;
|
||||
2. 下载并校验 PDF,或复用文件头为 `%PDF-` 的已有文件;
|
||||
3. 使用 `pypdf` 提取文本、插入页码标记,并应用页数和字符数限制;
|
||||
4. 请求 Claude Code 返回结构化的详细解读;
|
||||
5. 写入规范化 frontmatter 和生成的 Markdown 正文。
|
||||
|
||||
当前提取器依赖可用的 PDF 文本层。扫描版或纯图片 PDF 会失败,因为没有 OCR fallback。提取内容超过配置限制时,
|
||||
笔记会记录输入已被截断。
|
||||
|
||||
### 5. 生成简报与索引
|
||||
|
||||
Claude Code 会读取全部详细笔记并生成当日简报。代码会检查每篇源笔记的 wikilink;如有遗漏,会在写入前自动补齐。
|
||||
随后,工作流根据当日 Markdown frontmatter 重建 `daily/YYYY-MM-DD.md`。
|
||||
|
||||
### 6. 可选的钉钉通知
|
||||
|
||||
最后一步会去掉 YAML frontmatter,把简报正文按顺序发送到每个已配置的钉钉群。未配置群会话 ID 时,该步骤无副作用
|
||||
跳过。某个群发送失败不会阻止继续尝试其他群,所有发送完成后再汇总报告失败。
|
||||
|
||||
## 日期、重跑与幂等
|
||||
|
||||
- `date` 必须严格符合 `YYYY-MM-DD`。省略时使用应用配置时区中的今天;内置配置为 `Asia/Shanghai`。
|
||||
- “昨日”表示 `date - 1 day`,不是模糊的最近 24 小时。
|
||||
- `history_days` 只扫描此前的日期目录,不会把本次运行日纳入历史窗口。
|
||||
- 如果 `daily/<date>/daily-paper-brief.md` 已存在且 `force=false`,采集、排序、模型调用、PDF 处理和简报生成都会
|
||||
跳过;已有简报仍会交给钉钉通知步骤。
|
||||
- `force=true` 会重新生成笔记和简报,但仍会复用已有且有效的 PDF。
|
||||
|
||||
每个 PDF、详细笔记和最终简报都会先写临时文件再替换,避免读取方看到半写状态。整个多文件工作流不是事务,
|
||||
同一日期的并发运行也没有全局锁。
|
||||
|
||||
## 运行方式
|
||||
|
||||
独立配置定义了三个 Job:
|
||||
|
||||
| Job | 行为 |
|
||||
|--------------------|----------------------------------------|
|
||||
| `daily_paper` | 通过 CLI 或 HTTP 服务按需生成 |
|
||||
| `daily_paper_cron` | 每天 08:00(`Asia/Shanghai`)执行相同 pipeline |
|
||||
| `dingtalk_wait` | 由 supervisor 管理的后台钉钉 Agent |
|
||||
|
||||
### 一次性运行
|
||||
|
||||
生成今天的简报:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=daily_paper
|
||||
```
|
||||
|
||||
生成指定日期,并覆盖部分参数:
|
||||
|
||||
```bash
|
||||
reme start \
|
||||
config=daily_cookbook \
|
||||
job=daily_paper \
|
||||
date=2026-07-21 \
|
||||
top_k=3 \
|
||||
history_days=30
|
||||
```
|
||||
|
||||
重新生成已有简报的日期:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook job=daily_paper date=2026-07-21 force=true
|
||||
```
|
||||
|
||||
需要查看响应 metadata 进行诊断时,可在一次性命令中加入 `service.show_metadata=true`。
|
||||
|
||||
### 常驻服务与 cron
|
||||
|
||||
启动独立 HTTP 服务以及定时、后台 Job:
|
||||
|
||||
```bash
|
||||
reme start config=daily_cookbook
|
||||
```
|
||||
|
||||
服务默认监听 `127.0.0.1:8001`,因此可以和默认 ReMe 服务并行运行。在另一个终端中通过 ReMe client 或 HTTP
|
||||
调用按需任务:
|
||||
|
||||
```bash
|
||||
reme daily_paper host=127.0.0.1 port=8001
|
||||
```
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8001/daily_paper \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"date":"2026-07-21","top_k":3,"force":false}'
|
||||
```
|
||||
|
||||
监听地址和调度时间可以在启动时覆盖:
|
||||
|
||||
```bash
|
||||
reme start \
|
||||
config=daily_cookbook \
|
||||
service.host=0.0.0.0 \
|
||||
service.port=8101 \
|
||||
jobs.daily_paper_cron.cron="30 7 * * *"
|
||||
```
|
||||
|
||||
## 配置
|
||||
|
||||
最常用的 Job 配置如下:
|
||||
|
||||
| 配置项 | 默认值 | 用途 |
|
||||
|-------------------|-----------:|-------------------------|
|
||||
| `candidate_limit` | `20` | 送入精选阶段的最大论文数 |
|
||||
| `memory_reserve` | `5` | 记忆关键词启发式保留的候选位置数 |
|
||||
| `top_k` | `3` | 最终精选和解读的论文数 |
|
||||
| `rrf_k` | `60` | RRF 常数 |
|
||||
| `weekly_weight` | `0.7` | 周榜在融合排序中的权重 |
|
||||
| `history_days` | `30` | 按 arXiv ID 排除近期推荐的时间窗口 |
|
||||
| `hf_timeout` | `30` 秒 | Hugging Face 请求 timeout |
|
||||
| `hf_max_retries` | `3` | Hugging Face 请求最多尝试次数 |
|
||||
| `pdf_timeout` | `90` 秒 | arXiv 下载 timeout |
|
||||
| `max_pdf_bytes` | `52428800` | PDF 大小上限(50 MiB) |
|
||||
| `max_pdf_pages` | `80` | 最多提取的 PDF 页数 |
|
||||
| `max_pdf_chars` | `240000` | 单篇论文送入模型的最大提取字符数 |
|
||||
|
||||
公开 Job 参数为 `date`、`force`、`top_k`、`weekly_weight` 和 `history_days`。调用时显式传入的值优先于 Job 默认值。
|
||||
|
||||
独立应用还支持以下环境变量:
|
||||
|
||||
| 环境变量 | 用途 |
|
||||
|-----------------------------------------|------------------------------------|
|
||||
| `DAILY_PAPER_WORKSPACE_DIR` | 覆盖默认 `.reme` workspace |
|
||||
| `DAILY_PAPER_PROJECT_PATH` | Claude Code 可见的仓库或项目路径 |
|
||||
| `DAILY_PAPER_HOST` / `DAILY_PAPER_PORT` | HTTP 监听地址 |
|
||||
| `CLAUDE_CODE_API_KEY` | 所配置 Claude Code endpoint 的 API key |
|
||||
| `CLAUDE_CODE_MODEL_NAME` | 覆盖模型名称 |
|
||||
| `CLAUDE_CODE_BASE_URL` | 覆盖 Anthropic 兼容 endpoint |
|
||||
|
||||
`DAILY_PAPER_PROJECT_PATH` 默认是相对于 workspace 的 `..`。使用默认 `.reme` workspace 并从仓库根目录启动时,
|
||||
它会解析回仓库根目录。如果 workspace 位于其他位置,请显式设置这两个路径。
|
||||
|
||||
ReMe 会从当前目录向上查找未提交的 `.env`,因此也可以把相同变量放在其中,而不是在 shell 中导出。
|
||||
|
||||
## 钉钉配置
|
||||
|
||||
钉钉是可选能力。仅在需要投递简报或运行后台钉钉 Agent 时配置:
|
||||
|
||||
```dotenv
|
||||
DINGTALK_APP_KEY=your-app-key
|
||||
DINGTALK_APP_SECRET=your-app-secret
|
||||
DINGTALK_ROBOT_CODE=your-robot-code
|
||||
DINGTALK_CONVERSATION_IDS=cid-group-one,cid-group-two
|
||||
```
|
||||
|
||||
只有主动投递简报需要 `DINGTALK_CONVERSATION_IDS`。后台 `dingtalk_wait` Job 使用前三项凭据,不使用群会话列表。
|
||||
|
||||
## 故障恢复与边界
|
||||
|
||||
| 场景 | 当前行为 |
|
||||
|-------------------|---------------------------------|
|
||||
| Hugging Face 暂时失败 | 按指数间隔重试,最多尝试 `hf_max_retries` 次 |
|
||||
| 没有 eligible 论文 | 在排序前失败 |
|
||||
| `top_k` 或精选结果不合法 | 校验后失败;精选结果可重试一次 |
|
||||
| PDF 太大、无效或没有文本层 | 在解读阶段停止 |
|
||||
| PDF 超过页数或字符数限制 | 使用截断文本继续,并记录截断状态 |
|
||||
| 某篇论文解读失败 | Job 停止;此前写入的 PDF 和笔记保留 |
|
||||
| 简报遗漏源笔记链接 | 写入前自动补齐 wikilink |
|
||||
| 最终简报已存在 | 默认跳过生成,通知仍可发送;`force=true` 可重跑 |
|
||||
| 同一日期并发运行 | 没有 pipeline 级锁,后写入结果可能替换先前结果 |
|
||||
|
||||
恢复时,先检查该日期已有的笔记和 PDF,修复网络、凭据、模型或 PDF 问题,再使用相同日期和 `force=true` 重跑。
|
||||
有效的缓存 PDF 会被复用。
|
||||
|
||||
内置 Claude Code 组件使用 `permission_mode: bypassPermissions`。ReMe 会禁用 Claude Code 的 `WebSearch` 工具,
|
||||
Analyze 和 Digest prompt 也会限制 Agent 应读取的内容,但这些步骤没有设置严格的逐次调用工具 allowlist,也不是
|
||||
操作系统级沙箱。请只在可信的项目和 workspace 中运行;用于共享或生产环境前,应进一步收紧 Agent 配置。
|
||||
|
||||
## 测试
|
||||
|
||||
聚焦的单元测试会 mock Hugging Face、arXiv、Claude Code 和钉钉边界:
|
||||
|
||||
```bash
|
||||
pytest tests/unit/test_daily_paper.py -v
|
||||
```
|
||||
|
||||
真实运行会访问外部服务并可能产生模型费用,不应把它当作普通单元测试执行。
|
||||
|
|
@ -44,12 +44,14 @@ dependencies = [
|
|||
core = [
|
||||
"agentscope==2.0.4.post1",
|
||||
"claude-agent-sdk>=0.2.123",
|
||||
"dingtalk-stream>=0.24.3",
|
||||
"openai-codex>=0.144.4",
|
||||
"faiss-cpu>=1.13.2",
|
||||
"jieba>=0.42.1",
|
||||
"rjieba>=0.2.1",
|
||||
"neo4j>=6.2.0",
|
||||
"networkx>=3.4.2",
|
||||
"openai-codex>=0.144.4",
|
||||
"pypdf>=5.0.0",
|
||||
]
|
||||
dev = [
|
||||
"pre-commit",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
"""AgentScope backend for the unified agent wrapper."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
|
@ -39,6 +42,7 @@ from agentscope.state import AgentState
|
|||
from agentscope.tool import (
|
||||
Bash,
|
||||
Edit,
|
||||
ExecResult,
|
||||
FunctionTool,
|
||||
Glob,
|
||||
Grep,
|
||||
|
|
@ -68,22 +72,56 @@ _UUID_RE = re.compile(
|
|||
|
||||
|
||||
class WorkspaceBackend(LocalBackend):
|
||||
"""LocalBackend whose reported cwd is the configured agent workspace.
|
||||
"""Local backend pinned to the agent cwd and configured environment.
|
||||
|
||||
Some AgentScope builtin tools use ``backend.getcwd()`` for default search
|
||||
paths or safety checks. Pinning it here keeps those operations aligned with
|
||||
the cwd passed to Bash. Tools that require absolute file paths still keep
|
||||
their own validation behavior.
|
||||
their own validation behavior. Subprocesses receive the startup environment
|
||||
captured in ``ApplicationConfig.environment`` in addition to the parent
|
||||
process environment.
|
||||
"""
|
||||
|
||||
def __init__(self, cwd: str) -> None:
|
||||
def __init__(self, cwd: str, environment: dict[str, str] | None = None) -> None:
|
||||
super().__init__()
|
||||
self._workspace_cwd = cwd
|
||||
self._environment = {**os.environ, **(environment or {})}
|
||||
|
||||
async def getcwd(self) -> str:
|
||||
"""Return the configured workspace directory."""
|
||||
return self._workspace_cwd
|
||||
|
||||
async def exec_shell(
|
||||
self,
|
||||
command: list[str],
|
||||
*,
|
||||
cwd: str | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
"""Run a local subprocess with the configured agent environment."""
|
||||
kwargs: dict[str, Any] = {
|
||||
"env": self._environment,
|
||||
"stderr": asyncio.subprocess.PIPE,
|
||||
"stdout": asyncio.subprocess.PIPE,
|
||||
}
|
||||
if cwd is not None:
|
||||
kwargs["cwd"] = cwd
|
||||
if os.name == "nt":
|
||||
kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(*command, **kwargs)
|
||||
except (FileNotFoundError, NotADirectoryError, OSError) as exc:
|
||||
return ExecResult(exit_code=127, stdout=b"", stderr=str(exc).encode("utf-8"))
|
||||
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.communicate()
|
||||
return ExecResult(exit_code=-1, stdout=b"", stderr=b"timed out")
|
||||
return ExecResult(exit_code=process.returncode or 0, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
class BypassAnalysisBash(Bash):
|
||||
"""Bash variant that delegates permission decisions to PermissionEngine.
|
||||
|
|
@ -140,7 +178,7 @@ class AsAgentWrapper(BaseAgentWrapper):
|
|||
) -> list[ToolBase]:
|
||||
"""Return selected AgentScope built-in tools rooted at ``self.cwd``."""
|
||||
cwd = str(self.cwd)
|
||||
backend = WorkspaceBackend(cwd)
|
||||
backend = WorkspaceBackend(cwd, self.subprocess_environment)
|
||||
factories = {
|
||||
"bash": lambda: BypassAnalysisBash(cwd=cwd, backend=backend),
|
||||
"edit": lambda: Edit(backend=backend),
|
||||
|
|
@ -249,13 +287,7 @@ class AsAgentWrapper(BaseAgentWrapper):
|
|||
|
||||
def _resolve_skills(self, skills: list[str] | str | None) -> list[str]:
|
||||
"""Resolve configured skill names to AgentScope local skill directories."""
|
||||
if skills is None:
|
||||
return []
|
||||
if skills == "all":
|
||||
return [str(self.project_skills_root)]
|
||||
if isinstance(skills, str):
|
||||
skills = [skills]
|
||||
return [str(self.project_skills_root / skill) for skill in skills]
|
||||
return [str(path) for path in self._resolve_project_skills(skills).values()]
|
||||
|
||||
async def _build_agent(self, inputs: Any, **kwargs) -> tuple[Agent, Any]:
|
||||
"""Build an Agent instance from kwargs. Returns (agent, processed_inputs)."""
|
||||
|
|
|
|||
|
|
@ -22,9 +22,15 @@ class BaseAgentWrapper(BaseComponent):
|
|||
component_type = ComponentEnum.AGENT_WRAPPER
|
||||
SDK_PACKAGE: ClassVar[str | None] = None
|
||||
|
||||
def __init__(self, cwd: str | Path | None = None, **kwargs) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
cwd: str | Path | None = None,
|
||||
project_path: str | Path | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._cwd = cwd
|
||||
self._project_path = project_path
|
||||
if self.SDK_PACKAGE:
|
||||
try:
|
||||
sdk_version = metadata.version(self.SDK_PACKAGE)
|
||||
|
|
@ -36,8 +42,7 @@ class BaseAgentWrapper(BaseComponent):
|
|||
def cwd(self) -> Path:
|
||||
"""Working directory shared by the agent's shell and file tools.
|
||||
|
||||
Defaults to the project root (the workspace) — the same directory
|
||||
Claude Code has always used. Override via the ``cwd`` init argument;
|
||||
Defaults to the project root. Override via the ``cwd`` init argument;
|
||||
a relative value resolves against the workspace root.
|
||||
"""
|
||||
if not self._cwd:
|
||||
|
|
@ -62,14 +67,53 @@ class BaseAgentWrapper(BaseComponent):
|
|||
|
||||
@property
|
||||
def project_path(self) -> Path:
|
||||
"""Project root that contains shared assets such as skills."""
|
||||
return self.workspace_path
|
||||
"""Project root containing shared assets such as skills.
|
||||
|
||||
A relative configured path resolves from the workspace so applications
|
||||
can keep runtime data in a subdirectory such as ``.reme`` while loading
|
||||
project assets from its parent. The workspace remains the default for
|
||||
backward compatibility.
|
||||
"""
|
||||
if not self._project_path:
|
||||
return self.workspace_path
|
||||
project_path = Path(self._project_path).expanduser()
|
||||
if not project_path.is_absolute():
|
||||
project_path = self.workspace_path / project_path
|
||||
return project_path.resolve(strict=False)
|
||||
|
||||
@property
|
||||
def project_skills_root(self) -> Path:
|
||||
"""Project-level skills directory shared by agent backends."""
|
||||
return self.project_path / "skills"
|
||||
|
||||
def _resolve_project_skills(self, skills: list[str] | str | None) -> dict[str, Path]:
|
||||
"""Resolve selected skill names to validated project directories."""
|
||||
if skills is None:
|
||||
return {}
|
||||
if skills == "all":
|
||||
if not self.project_skills_root.is_dir():
|
||||
raise FileNotFoundError(f"Project skills directory not found: {self.project_skills_root}")
|
||||
names = sorted(
|
||||
path.name
|
||||
for path in self.project_skills_root.iterdir()
|
||||
if path.is_dir() and (path / "SKILL.md").is_file()
|
||||
)
|
||||
else:
|
||||
names = [skills] if isinstance(skills, str) else list(skills)
|
||||
names = list(dict.fromkeys(names))
|
||||
|
||||
sources: dict[str, Path] = {}
|
||||
for name in names:
|
||||
if not name or Path(name).name != name or name in {".", ".."}:
|
||||
raise ValueError(f"Invalid skill name: {name!r}")
|
||||
source = self.project_skills_root / name
|
||||
if not source.is_dir():
|
||||
raise FileNotFoundError(f"Skill directory not found: {source}")
|
||||
if not (source / "SKILL.md").is_file():
|
||||
raise FileNotFoundError(f"Skill '{name}' is missing SKILL.md: {source}")
|
||||
sources[name] = source
|
||||
return sources
|
||||
|
||||
@property
|
||||
def subprocess_environment(self) -> dict[str, str]:
|
||||
"""Configured environment variables for child agent processes."""
|
||||
|
|
|
|||
|
|
@ -44,24 +44,7 @@ class CcAgentWrapper(BaseAgentWrapper):
|
|||
|
||||
def _ensure_claude_skill_dir(self, config_dir: Path, skills: list[str] | str) -> None:
|
||||
"""Add selected project skills to Claude Code discovery locations."""
|
||||
project_skills = self.project_skills_root
|
||||
if not project_skills.is_dir():
|
||||
return
|
||||
|
||||
if skills == "all":
|
||||
skill_names = sorted(path.name for path in project_skills.iterdir() if path.is_dir())
|
||||
else:
|
||||
skill_names = list(dict.fromkeys(skills))
|
||||
|
||||
for skill_name in skill_names:
|
||||
if not skill_name or Path(skill_name).name != skill_name or skill_name in {".", ".."}:
|
||||
raise ValueError(f"Invalid skill name: {skill_name!r}")
|
||||
|
||||
sources = {
|
||||
skill_name: project_skills / skill_name
|
||||
for skill_name in skill_names
|
||||
if (project_skills / skill_name).is_dir()
|
||||
}
|
||||
sources = self._resolve_project_skills(skills)
|
||||
if not sources:
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -130,31 +130,13 @@ class CodexAgentWrapper(BaseAgentWrapper):
|
|||
|
||||
def _ensure_skills(self, skills: list[str] | str | None) -> None:
|
||||
"""Expose selected project skills through Codex's repo-level directory."""
|
||||
if skills is None:
|
||||
sources = self._resolve_project_skills(skills)
|
||||
if not sources:
|
||||
return
|
||||
if skills == "all":
|
||||
if not self.project_skills_root.is_dir():
|
||||
raise FileNotFoundError(f"Project skills directory not found: {self.project_skills_root}")
|
||||
names = sorted(
|
||||
path.name
|
||||
for path in self.project_skills_root.iterdir()
|
||||
if path.is_dir() and (path / "SKILL.md").is_file()
|
||||
)
|
||||
else:
|
||||
names = [skills] if isinstance(skills, str) else list(skills)
|
||||
names = list(dict.fromkeys(names))
|
||||
|
||||
target_root = self.workspace_path / ".agents" / "skills"
|
||||
target_root = self.project_path / ".agents" / "skills"
|
||||
target_root.mkdir(parents=True, exist_ok=True)
|
||||
for name in names:
|
||||
if not name or Path(name).name != name or name in {".", ".."}:
|
||||
raise ValueError(f"Invalid skill name: {name!r}")
|
||||
source = self.project_skills_root / name
|
||||
if not source.is_dir():
|
||||
raise FileNotFoundError(f"Skill directory not found: {source}")
|
||||
if not (source / "SKILL.md").is_file():
|
||||
raise FileNotFoundError(f"Skill '{name}' is missing SKILL.md: {source}")
|
||||
|
||||
for name, source in sources.items():
|
||||
target = target_root / name
|
||||
if target.is_symlink():
|
||||
if target.resolve() == source.resolve():
|
||||
|
|
|
|||
109
reme/config/daily_cookbook.yaml
Normal file
109
reme/config/daily_cookbook.yaml
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
app_name: ReMe Daily Cookbook
|
||||
workspace_dir: ${DAILY_PAPER_WORKSPACE_DIR:-.reme}
|
||||
timezone: Asia/Shanghai
|
||||
language: zh
|
||||
|
||||
# This is a standalone application config. It intentionally does not inherit
|
||||
# default.yaml and listens on a separate port so it can run beside ReMe.
|
||||
service:
|
||||
backend: http
|
||||
host: ${DAILY_PAPER_HOST:-127.0.0.1}
|
||||
port: ${DAILY_PAPER_PORT:-8001}
|
||||
|
||||
jobs:
|
||||
daily_paper:
|
||||
backend: base
|
||||
description: "Build detailed readings and a five-minute brief from Hugging Face weekly/monthly papers."
|
||||
candidate_limit: &candidate_limit 20
|
||||
memory_reserve: &memory_reserve 5
|
||||
top_k: &top_k 3
|
||||
rrf_k: &rrf_k 60
|
||||
weekly_weight: &weekly_weight 0.7
|
||||
history_days: &history_days 30
|
||||
hf_timeout: &hf_timeout 30
|
||||
hf_max_retries: &hf_max_retries 3
|
||||
pdf_timeout: &pdf_timeout 90
|
||||
max_pdf_bytes: &max_pdf_bytes 52428800
|
||||
max_pdf_pages: &max_pdf_pages 80
|
||||
max_pdf_chars: &max_pdf_chars 240000
|
||||
parameters:
|
||||
type: object
|
||||
properties:
|
||||
date:
|
||||
type: string
|
||||
description: "Run date in YYYY-MM-DD; empty means today in Asia/Shanghai."
|
||||
default: ""
|
||||
force:
|
||||
type: boolean
|
||||
description: "Regenerate even when that day's final brief already exists."
|
||||
default: false
|
||||
top_k:
|
||||
type: integer
|
||||
description: "Number of papers selected by Claude Code."
|
||||
default: 3
|
||||
weekly_weight:
|
||||
type: number
|
||||
description: "Weekly contribution in reciprocal-rank fusion."
|
||||
default: 0.7
|
||||
history_days:
|
||||
type: integer
|
||||
description: "Prior recommendation window excluded by arXiv ID."
|
||||
default: 30
|
||||
steps: &daily_paper_steps
|
||||
- backend: daily_paper_collect_step
|
||||
- backend: daily_paper_rank_step
|
||||
- backend: daily_paper_select_step
|
||||
agent_wrapper: claude_code
|
||||
- backend: daily_paper_analyze_step
|
||||
agent_wrapper: claude_code
|
||||
- backend: daily_paper_digest_step
|
||||
agent_wrapper: claude_code
|
||||
- backend: dingtalk_markdown_send_step
|
||||
input_mapping:
|
||||
daily_paper_digest_path: markdown_path
|
||||
app_key: ${DINGTALK_APP_KEY:-}
|
||||
app_secret: ${DINGTALK_APP_SECRET:-}
|
||||
robot_code: ${DINGTALK_ROBOT_CODE:-}
|
||||
conversation_ids: ${DINGTALK_CONVERSATION_IDS:-}
|
||||
title: ReMe Daily Paper
|
||||
timeout: 15
|
||||
|
||||
daily_paper_cron:
|
||||
backend: cron
|
||||
cron: "0 8 * * *"
|
||||
candidate_limit: *candidate_limit
|
||||
memory_reserve: *memory_reserve
|
||||
top_k: *top_k
|
||||
rrf_k: *rrf_k
|
||||
weekly_weight: *weekly_weight
|
||||
history_days: *history_days
|
||||
hf_timeout: *hf_timeout
|
||||
hf_max_retries: *hf_max_retries
|
||||
pdf_timeout: *pdf_timeout
|
||||
max_pdf_bytes: *max_pdf_bytes
|
||||
max_pdf_pages: *max_pdf_pages
|
||||
max_pdf_chars: *max_pdf_chars
|
||||
steps: *daily_paper_steps
|
||||
|
||||
dingtalk_wait:
|
||||
backend: background
|
||||
supervisor: true
|
||||
close_timeout: 10
|
||||
steps:
|
||||
- backend: dingtalk_wait_step
|
||||
agent_wrapper: claude_code
|
||||
app_key: ${DINGTALK_APP_KEY:-}
|
||||
app_secret: ${DINGTALK_APP_SECRET:-}
|
||||
robot_code: ${DINGTALK_ROBOT_CODE:-}
|
||||
card_update_interval: 1.0
|
||||
worker_count: 4
|
||||
|
||||
components:
|
||||
agent_wrapper:
|
||||
claude_code:
|
||||
backend: claude_code
|
||||
project_path: ${DAILY_PAPER_PROJECT_PATH:-..}
|
||||
model: ${CLAUDE_CODE_MODEL_NAME:-qwen3.7-max}
|
||||
api_key: ${CLAUDE_CODE_API_KEY:-}
|
||||
base_url: ${CLAUDE_CODE_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic}
|
||||
permission_mode: bypassPermissions
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
"""Schema"""
|
||||
|
||||
from .application_config import ApplicationConfig, ComponentConfig, JobConfig
|
||||
from .daily_paper import DailyBriefOutput, PaperInfo, PaperNoteOutput, PaperSelection, SelectedPaper
|
||||
from .dream import (
|
||||
DreamExtractOutput,
|
||||
DreamState,
|
||||
|
|
@ -22,7 +23,7 @@ from .stream_chunk import StreamChunk
|
|||
__all__ = [
|
||||
"ApplicationConfig",
|
||||
"ComponentConfig",
|
||||
"JobConfig",
|
||||
"DailyBriefOutput",
|
||||
"DreamExtractOutput",
|
||||
"DreamState",
|
||||
"DreamTopic",
|
||||
|
|
@ -33,9 +34,14 @@ __all__ = [
|
|||
"FileLink",
|
||||
"FileNode",
|
||||
"IntegrateOutcome",
|
||||
"JobConfig",
|
||||
"PaperInfo",
|
||||
"PaperNoteOutput",
|
||||
"PaperSelection",
|
||||
"ProactiveResult",
|
||||
"Request",
|
||||
"Response",
|
||||
"SelectedPaper",
|
||||
"StreamChunk",
|
||||
"TopicSelectionOutput",
|
||||
]
|
||||
|
|
|
|||
71
reme/schema/daily_paper.py
Normal file
71
reme/schema/daily_paper.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Typed contracts for the daily-paper cookbook workflow."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class PaperInfo(BaseModel):
|
||||
"""Normalized Hugging Face paper metadata plus local ranking fields."""
|
||||
|
||||
arxiv_id: str
|
||||
title: str = ""
|
||||
summary: str = ""
|
||||
authors: list[str] = Field(default_factory=list)
|
||||
published_at: str | None = None
|
||||
submitted_on_daily_at: str | None = None
|
||||
upvotes: int = 0
|
||||
organization: str | None = None
|
||||
github_repo: str | None = None
|
||||
github_stars: int | None = None
|
||||
project_page: str | None = None
|
||||
thumbnail: str | None = None
|
||||
monthly_rank: int | None = None
|
||||
weekly_rank: int | None = None
|
||||
fused_score: float = 0.0
|
||||
|
||||
@property
|
||||
def hf_url(self) -> str:
|
||||
"""Return the canonical Hugging Face paper-page URL."""
|
||||
return f"https://huggingface.co/papers/{self.arxiv_id}"
|
||||
|
||||
@property
|
||||
def arxiv_url(self) -> str:
|
||||
"""Return the canonical arXiv abstract URL."""
|
||||
return f"https://arxiv.org/abs/{self.arxiv_id}"
|
||||
|
||||
@property
|
||||
def pdf_url(self) -> str:
|
||||
"""Return the canonical arXiv PDF download URL."""
|
||||
return f"https://arxiv.org/pdf/{self.arxiv_id}"
|
||||
|
||||
|
||||
class SelectedPaper(BaseModel):
|
||||
"""One agent-selected paper."""
|
||||
|
||||
arxiv_id: str
|
||||
rank: int
|
||||
reason: str
|
||||
memory_relevance: Literal["high", "medium", "low"]
|
||||
|
||||
|
||||
class PaperSelection(BaseModel):
|
||||
"""Structured paper selection result."""
|
||||
|
||||
selection_reasoning: str
|
||||
selected: list[SelectedPaper]
|
||||
alternates: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PaperNoteOutput(BaseModel):
|
||||
"""Structured Claude Code output for one detailed paper note."""
|
||||
|
||||
description: str
|
||||
body: str
|
||||
|
||||
|
||||
class DailyBriefOutput(BaseModel):
|
||||
"""Structured Claude Code output for the final five-minute brief."""
|
||||
|
||||
description: str
|
||||
body: str
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
"""steps"""
|
||||
|
||||
from . import benchmark, common, evolve, file_io, index, transfer
|
||||
from . import benchmark, common, cookbook, evolve, file_io, index, transfer
|
||||
from .base_step import BaseStep
|
||||
|
||||
__all__ = [
|
||||
"BaseStep",
|
||||
"benchmark",
|
||||
"common",
|
||||
"cookbook",
|
||||
"evolve",
|
||||
"file_io",
|
||||
"index",
|
||||
|
|
|
|||
5
reme/steps/cookbook/__init__.py
Normal file
5
reme/steps/cookbook/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""Optional, end-to-end cookbook workflows."""
|
||||
|
||||
from . import daily_paper, dingtalk
|
||||
|
||||
__all__ = ["daily_paper", "dingtalk"]
|
||||
15
reme/steps/cookbook/daily_paper/__init__.py
Normal file
15
reme/steps/cookbook/daily_paper/__init__.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""Daily-paper cookbook workflow."""
|
||||
|
||||
from .analyze import DailyPaperAnalyzeStep
|
||||
from .collect import DailyPaperCollectStep
|
||||
from .digest import DailyPaperDigestStep
|
||||
from .rank import DailyPaperRankStep
|
||||
from .select import DailyPaperSelectStep
|
||||
|
||||
__all__ = [
|
||||
"DailyPaperAnalyzeStep",
|
||||
"DailyPaperCollectStep",
|
||||
"DailyPaperDigestStep",
|
||||
"DailyPaperRankStep",
|
||||
"DailyPaperSelectStep",
|
||||
]
|
||||
77
reme/steps/cookbook/daily_paper/_common.py
Normal file
77
reme/steps/cookbook/daily_paper/_common.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
"""Shared state and file helpers for daily-paper steps."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
import aiofiles
|
||||
import frontmatter
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...base_step import BaseStep
|
||||
from ...file_io._file_io import get_path_lock
|
||||
|
||||
_STATE_PREFIX = "daily_paper_"
|
||||
_FRONTMATTER_PATTERN = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL)
|
||||
_OutputT = TypeVar("_OutputT", bound=BaseModel)
|
||||
|
||||
|
||||
def structured_output(result: dict[str, Any], model: type[_OutputT]) -> _OutputT:
|
||||
"""Validate an agent wrapper's structured output."""
|
||||
value = result.get("structured_output")
|
||||
return value if isinstance(value, model) else model.model_validate(value)
|
||||
|
||||
|
||||
def strip_frontmatter(body: str) -> str:
|
||||
"""Remove one model-generated YAML frontmatter block."""
|
||||
return _FRONTMATTER_PATTERN.sub("", body.strip(), count=1).strip()
|
||||
|
||||
|
||||
async def write_atomic(path: Path, content: str | bytes) -> None:
|
||||
"""Write through a sibling temporary file under the repository path lock."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock = await get_path_lock(path)
|
||||
async with lock:
|
||||
temp_path = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
|
||||
payload = content.encode("utf-8") if isinstance(content, str) else content
|
||||
try:
|
||||
async with aiofiles.open(temp_path, "wb") as stream:
|
||||
await stream.write(payload)
|
||||
os.replace(temp_path, path)
|
||||
finally:
|
||||
if temp_path.exists():
|
||||
temp_path.unlink()
|
||||
|
||||
|
||||
async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> None:
|
||||
"""Serialize a frontmatter Markdown document atomically."""
|
||||
rendered = frontmatter.dumps(frontmatter.Post(body.strip(), **metadata))
|
||||
await write_atomic(path, rendered if rendered.endswith("\n") else f"{rendered}\n")
|
||||
|
||||
|
||||
class DailyPaperStep(BaseStep):
|
||||
"""Shared helpers for steps in one daily-paper RuntimeContext."""
|
||||
|
||||
def _skip(self) -> bool:
|
||||
assert self.context is not None
|
||||
return bool(self.context.get(f"{_STATE_PREFIX}skip", False))
|
||||
|
||||
def _value(self, key: str, default: Any) -> Any:
|
||||
assert self.context is not None
|
||||
return self.context.get(key, self.kwargs.get(key, default))
|
||||
|
||||
def _state(self, key: str) -> Any:
|
||||
assert self.context is not None
|
||||
return self.context.get(f"{_STATE_PREFIX}{key}")
|
||||
|
||||
def _set_state(self, key: str, value: Any) -> None:
|
||||
assert self.context is not None
|
||||
self.context[f"{_STATE_PREFIX}{key}"] = value
|
||||
|
||||
def _run_day(self) -> str:
|
||||
value = self._state("run_date")
|
||||
if not value:
|
||||
raise RuntimeError("daily-paper run date is not initialized")
|
||||
return str(value)
|
||||
140
reme/steps/cookbook/daily_paper/analyze.py
Normal file
140
reme/steps/cookbook/daily_paper/analyze.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
"""Download and analyze selected daily-paper PDFs."""
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from ....components import R
|
||||
from ....schema import PaperInfo, PaperNoteOutput, PaperSelection, SelectedPaper
|
||||
from ....utils.arxiv import ArxivPdfClient
|
||||
from ._common import DailyPaperStep, strip_frontmatter, structured_output, write_markdown
|
||||
|
||||
|
||||
@R.register("daily_paper_analyze_step")
|
||||
class DailyPaperAnalyzeStep(DailyPaperStep):
|
||||
"""Download each selected PDF and use Claude Code for detailed reading."""
|
||||
|
||||
@staticmethod
|
||||
def _extract_pdf_text_sync(path: Path, max_pages: int, max_chars: int) -> tuple[str, int, bool]:
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError as exc: # pragma: no cover - dependency error has an explicit message
|
||||
raise RuntimeError("pypdf is required for the daily-paper workflow") from exc
|
||||
|
||||
reader = PdfReader(str(path))
|
||||
chunks: list[str] = []
|
||||
size = 0
|
||||
page_count = min(len(reader.pages), max_pages)
|
||||
truncated = len(reader.pages) > max_pages
|
||||
for page_number, page in enumerate(reader.pages[:page_count], start=1):
|
||||
block = f"\n\n--- PAGE {page_number} ---\n\n{(page.extract_text() or '').strip()}"
|
||||
if size + len(block) > max_chars:
|
||||
if (remaining := max_chars - size) > 0:
|
||||
chunks.append(block[:remaining])
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(block)
|
||||
size += len(block)
|
||||
content = "".join(chunks).strip()
|
||||
if not content:
|
||||
raise ValueError(f"No extractable text found in PDF: {path.name}")
|
||||
return content, len(reader.pages), truncated
|
||||
|
||||
async def _analyze_one(self, paper: PaperInfo, selected: SelectedPaper) -> tuple[str, str]:
|
||||
if self.agent_wrapper is None:
|
||||
raise RuntimeError("Claude Code agent_wrapper is required for paper analysis")
|
||||
day = self._run_day()
|
||||
daily_dir, resource_dir = (
|
||||
str(self.config_value("daily_dir")).strip("/"),
|
||||
str(self.config_value("resource_dir")).strip("/"),
|
||||
)
|
||||
pdf_rel, note_rel = (
|
||||
f"{resource_dir}/papers/{paper.arxiv_id}.pdf",
|
||||
f"{daily_dir}/{day}/paper-{paper.arxiv_id}.md",
|
||||
)
|
||||
pdf_path, note_path = self.workspace_path / pdf_rel, self.workspace_path / note_rel
|
||||
self.logger.info(f"[{self.name}] paper start arxiv_id={paper.arxiv_id}")
|
||||
|
||||
downloader = ArxivPdfClient(
|
||||
timeout=float(self._value("pdf_timeout", 90.0)),
|
||||
max_bytes=int(self._value("max_pdf_bytes", 50 * 1024 * 1024)),
|
||||
)
|
||||
await downloader.download(paper.arxiv_id, pdf_path)
|
||||
self.logger.info(f"[{self.name}] pdf ready arxiv_id={paper.arxiv_id} path={pdf_rel}")
|
||||
pdf_text, page_count, truncated = await asyncio.to_thread(
|
||||
self._extract_pdf_text_sync,
|
||||
pdf_path,
|
||||
int(self._value("max_pdf_pages", 80)),
|
||||
int(self._value("max_pdf_chars", 240_000)),
|
||||
)
|
||||
self.logger.info(
|
||||
f"[{self.name}] pdf extracted arxiv_id={paper.arxiv_id} pages={page_count} "
|
||||
f"chars={len(pdf_text)} truncated={truncated}",
|
||||
)
|
||||
self.logger.info(f"[{self.name}] agent start arxiv_id={paper.arxiv_id}")
|
||||
result = await self.agent_wrapper.reply(
|
||||
self.prompt_format(
|
||||
"analyze_user",
|
||||
paper_info=json.dumps(paper.model_dump(), ensure_ascii=False, indent=2),
|
||||
selection_reason=selected.reason,
|
||||
memory_relevance=selected.memory_relevance,
|
||||
page_count=page_count,
|
||||
truncated=str(truncated).lower(),
|
||||
pdf_text=pdf_text,
|
||||
),
|
||||
output_schema=PaperNoteOutput,
|
||||
)
|
||||
self.logger.info(f"[{self.name}] agent done arxiv_id={paper.arxiv_id}")
|
||||
output = structured_output(result, PaperNoteOutput)
|
||||
body = strip_frontmatter(output.body)
|
||||
if not output.description.strip() or not body:
|
||||
raise ValueError(f"Claude Code returned an empty paper note for {paper.arxiv_id}")
|
||||
await write_markdown(
|
||||
note_path,
|
||||
body,
|
||||
{
|
||||
"name": f"paper-{paper.arxiv_id}",
|
||||
"description": output.description.strip(),
|
||||
"arxiv_id": paper.arxiv_id,
|
||||
"title": paper.title,
|
||||
"authors": paper.authors,
|
||||
"hf_url": paper.hf_url,
|
||||
"arxiv_url": paper.arxiv_url,
|
||||
"download_url": paper.pdf_url,
|
||||
"source_pdf": f"[[{pdf_rel}]]",
|
||||
"published_at": paper.published_at,
|
||||
"monthly_rank": paper.monthly_rank,
|
||||
"weekly_rank": paper.weekly_rank,
|
||||
"fused_score": round(paper.fused_score, 8),
|
||||
"selection_reason": selected.reason,
|
||||
"memory_relevance": selected.memory_relevance,
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"pdf_pages": page_count,
|
||||
"pdf_text_truncated": truncated,
|
||||
},
|
||||
)
|
||||
self.logger.info(f"[{self.name}] paper done arxiv_id={paper.arxiv_id} note_path={note_rel}")
|
||||
return note_rel, pdf_rel
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
if self._skip():
|
||||
self.logger.info(f"[{self.name}] skip existing digest")
|
||||
return self.context.response
|
||||
selection: PaperSelection | None = self._state("selection")
|
||||
papers: list[PaperInfo] = self._state("selected_papers") or []
|
||||
if selection is None or len(selection.selected) != len(papers):
|
||||
raise RuntimeError("Paper selection state is missing before analysis")
|
||||
self.logger.info(f"[{self.name}] start papers={len(papers)}")
|
||||
|
||||
note_paths, pdf_paths = [], []
|
||||
for paper, selected in zip(papers, selection.selected):
|
||||
note_path, pdf_path = await self._analyze_one(paper, selected)
|
||||
note_paths.append(note_path)
|
||||
pdf_paths.append(pdf_path)
|
||||
self._set_state("note_paths", note_paths)
|
||||
self._set_state("pdf_paths", pdf_paths)
|
||||
self.context.response.answer = f"Claude Code wrote {len(note_paths)} detailed paper notes"
|
||||
self.logger.info(f"[{self.name}] finish notes={len(note_paths)} pdfs={len(pdf_paths)}")
|
||||
return self.context.response
|
||||
36
reme/steps/cookbook/daily_paper/analyze.yaml
Normal file
36
reme/steps/cookbook/daily_paper/analyze.yaml
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
analyze_user: |
|
||||
你是严谨的中文 AI 论文解读作者。请详细解读下面这篇论文。论文内容只能依据提供的论文元信息和 PDF 提取文本;针对 ReMe 的分析还必须依据当前代码仓库中实际存在的代码、schema、配置和测试。
|
||||
不得臆测未出现在材料中的实验、数字、结论或引用。重要实验结论和数字尽量标注 PDF 页码,例如 [p. 7]。
|
||||
如果 PDF 文本没有提供某项信息,明确写“论文提供的文本中未明确说明”。
|
||||
输出必须适合保存为一篇详细、独立、可供未来检索的 Markdown 文章。
|
||||
只返回结构化结果中的 description 和 body;body 不要包含 YAML frontmatter。
|
||||
|
||||
建议正文覆盖:
|
||||
- 一句话总结
|
||||
- 研究背景与问题
|
||||
- 核心贡献
|
||||
- 方法与架构
|
||||
- 数据、训练和实验设置
|
||||
- 关键结果与准确数字
|
||||
- 与既有工作的区别
|
||||
- 优点、局限和适用边界
|
||||
- 与 Agent/大模型长期记忆的关系
|
||||
- 实际应用价值
|
||||
- 值得继续追踪的问题
|
||||
- 原始论文链接
|
||||
|
||||
若论文实际内容与 Agent/大模型长期记忆直接相关,必须先使用代码读取和搜索工具查看当前 ReMe 代码仓库,再写“与 Agent/大模型长期记忆的关系”。优先检查与论文主题直接相关的 `reme/schema/`、`reme/components/`、`reme/steps/`、`reme/config/` 和相应测试,不要只根据 README 或项目概念推断实现。
|
||||
只有当论文的具体机制或实验结论能明确对应 ReMe 当前的代码或公开契约,且确实可能带来显著改进时,才增加“对 ReMe 演进的建议”小节。这应当是少数例外:一般情况下不要给建议,不要因为初筛标记为 high 就强行联系。如果增加该小节,每条建议都要说明论文依据、ReMe 现状、具体改进方向,并引用准确的仓库相对路径以及相关类或函数;不得编造实现现状。
|
||||
|
||||
选择理由:{selection_reason}
|
||||
长期记忆相关性初筛:{memory_relevance}
|
||||
PDF 总页数:{page_count}
|
||||
PDF 提取是否被截断:{truncated}
|
||||
|
||||
# 论文元信息
|
||||
|
||||
{paper_info}
|
||||
|
||||
# PDF 分页文本
|
||||
|
||||
{pdf_text}
|
||||
201
reme/steps/cookbook/daily_paper/collect.py
Normal file
201
reme/steps/cookbook/daily_paper/collect.py
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
"""Collect ranked Hugging Face papers for the daily-paper workflow."""
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
from pathlib import Path
|
||||
|
||||
import frontmatter
|
||||
|
||||
from ....components import R
|
||||
from ....schema import PaperInfo, PaperSelection
|
||||
from ....utils.arxiv import ARXIV_ID_PATTERN
|
||||
from ....utils.huggingface_papers import HuggingFacePapersClient
|
||||
from ...evolve import now
|
||||
from ._common import DailyPaperStep
|
||||
|
||||
|
||||
@R.register("daily_paper_collect_step")
|
||||
class DailyPaperCollectStep(DailyPaperStep):
|
||||
"""Collect current weekly/monthly rankings and strict-yesterday exclusions."""
|
||||
|
||||
@staticmethod
|
||||
def _strict_date(value: str) -> dt.date:
|
||||
text = str(value or "").strip()
|
||||
try:
|
||||
parsed = dt.date.fromisoformat(text)
|
||||
except ValueError as exc:
|
||||
raise ValueError("date must be YYYY-MM-DD") from exc
|
||||
if parsed.isoformat() != text:
|
||||
raise ValueError("date must be YYYY-MM-DD")
|
||||
return parsed
|
||||
|
||||
@staticmethod
|
||||
def _paper_scope_values(day: dt.date) -> tuple[str, str]:
|
||||
iso = day.isocalendar()
|
||||
return f"{iso.year}-W{iso.week:02d}", day.strftime("%Y-%m")
|
||||
|
||||
@staticmethod
|
||||
def load_historical_arxiv_ids(
|
||||
workspace: Path,
|
||||
run_date: dt.date,
|
||||
history_days: int,
|
||||
daily_dir: str,
|
||||
) -> set[str]:
|
||||
"""Read previously recommended paper ids from prior daily note frontmatter."""
|
||||
if history_days <= 0:
|
||||
return set()
|
||||
earliest, root = run_date - dt.timedelta(days=history_days), workspace / daily_dir
|
||||
if not root.is_dir():
|
||||
return set()
|
||||
|
||||
found: set[str] = set()
|
||||
for day_dir in root.iterdir():
|
||||
if not day_dir.is_dir():
|
||||
continue
|
||||
try:
|
||||
note_date = dt.date.fromisoformat(day_dir.name)
|
||||
except ValueError:
|
||||
continue
|
||||
if not earliest <= note_date < run_date:
|
||||
continue
|
||||
for note_path in day_dir.glob("paper-*.md"):
|
||||
try:
|
||||
metadata = frontmatter.load(note_path).metadata
|
||||
except (OSError, UnicodeError, ValueError):
|
||||
continue
|
||||
arxiv_id = str(metadata.get("arxiv_id") or "").strip()
|
||||
if ARXIV_ID_PATTERN.fullmatch(arxiv_id):
|
||||
found.add(arxiv_id)
|
||||
return found
|
||||
|
||||
@staticmethod
|
||||
def load_saved_selection(digest_path: Path) -> dict | None:
|
||||
"""Rebuild the saved selection from the digest and paper-note frontmatter."""
|
||||
try:
|
||||
digest_metadata = frontmatter.load(digest_path).metadata
|
||||
except (OSError, UnicodeError, ValueError):
|
||||
return None
|
||||
arxiv_ids = digest_metadata.get("arxiv_ids")
|
||||
if not isinstance(arxiv_ids, list) or not arxiv_ids:
|
||||
return None
|
||||
|
||||
selected = []
|
||||
for rank, value in enumerate(arxiv_ids, start=1):
|
||||
arxiv_id = str(value or "").strip()
|
||||
if not ARXIV_ID_PATTERN.fullmatch(arxiv_id):
|
||||
return None
|
||||
try:
|
||||
note_metadata = frontmatter.load(digest_path.parent / f"paper-{arxiv_id}.md").metadata
|
||||
except (OSError, UnicodeError, ValueError):
|
||||
return None
|
||||
selected.append(
|
||||
{
|
||||
"arxiv_id": arxiv_id,
|
||||
"rank": rank,
|
||||
"reason": str(note_metadata.get("selection_reason") or "").strip(),
|
||||
"memory_relevance": note_metadata.get("memory_relevance"),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
selection = PaperSelection.model_validate(
|
||||
{
|
||||
"selection_reasoning": str(digest_metadata.get("selection_reasoning") or "").strip(),
|
||||
"selected": selected,
|
||||
"alternates": digest_metadata.get("alternate_arxiv_ids") or [],
|
||||
},
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return selection.model_dump()
|
||||
|
||||
@staticmethod
|
||||
def _merge_paper(existing: PaperInfo | None, incoming: PaperInfo) -> PaperInfo:
|
||||
if existing is None:
|
||||
return incoming.model_copy(deep=True)
|
||||
values = existing.model_dump()
|
||||
for key, value in incoming.model_dump().items():
|
||||
if key == "upvotes":
|
||||
values[key] = max(int(values[key] or 0), int(value or 0))
|
||||
elif values.get(key) in (None, "", []):
|
||||
values[key] = value
|
||||
return PaperInfo.model_validate(values)
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
timezone = self.app_context.app_config.timezone if self.app_context is not None else None
|
||||
raw_date = str(self._value("date", "") or "").strip()
|
||||
run_date = self._strict_date(raw_date) if raw_date else now(timezone).date()
|
||||
day = run_date.isoformat()
|
||||
self._set_state("run_date", day)
|
||||
|
||||
daily_dir = str(self.config_value("daily_dir")).strip("/")
|
||||
digest_rel = f"{daily_dir}/{day}/daily-paper-brief.md"
|
||||
force = bool(self._value("force", False))
|
||||
self.logger.info(f"[{self.name}] start date={day} force={force}")
|
||||
digest_path = self.workspace_path / digest_rel
|
||||
if digest_path.is_file() and not force:
|
||||
self._set_state("skip", True)
|
||||
self._set_state("digest_path", digest_rel)
|
||||
self.context.response.success = True
|
||||
self.context.response.answer = f"Skipped: daily paper brief already exists at {digest_rel}"
|
||||
self.context.response.metadata.update({"date": day, "digest_path": digest_rel, "skipped": True})
|
||||
if selection := self.load_saved_selection(digest_path):
|
||||
self.context.response.metadata["selection"] = selection
|
||||
self.logger.info(f"[{self.name}] skip existing digest path={digest_rel}")
|
||||
return self.context.response
|
||||
|
||||
week, month = self._paper_scope_values(run_date)
|
||||
yesterday = (run_date - dt.timedelta(days=1)).isoformat()
|
||||
self.logger.info(
|
||||
f"[{self.name}] fetch start week={week} month={month} yesterday={yesterday}",
|
||||
)
|
||||
async with HuggingFacePapersClient(
|
||||
timeout=float(self._value("hf_timeout", 30.0)),
|
||||
max_retries=int(self._value("hf_max_retries", 3)),
|
||||
) as client:
|
||||
weekly, monthly, yesterday_ids = await asyncio.gather(
|
||||
client.fetch_scope("week", week),
|
||||
client.fetch_scope("month", month),
|
||||
client.fetch_daily_ids(yesterday),
|
||||
)
|
||||
self.logger.info(
|
||||
f"[{self.name}] fetch done weekly={len(weekly)} monthly={len(monthly)} " f"yesterday={len(yesterday_ids)}",
|
||||
)
|
||||
|
||||
merged: dict[str, PaperInfo] = {}
|
||||
for rank, paper in enumerate(monthly, start=1):
|
||||
merged[paper.arxiv_id] = self._merge_paper(merged.get(paper.arxiv_id), paper)
|
||||
merged[paper.arxiv_id].monthly_rank = rank
|
||||
for rank, paper in enumerate(weekly, start=1):
|
||||
merged[paper.arxiv_id] = self._merge_paper(merged.get(paper.arxiv_id), paper)
|
||||
merged[paper.arxiv_id].weekly_rank = rank
|
||||
|
||||
historical_ids = self.load_historical_arxiv_ids(
|
||||
self.workspace_path,
|
||||
run_date,
|
||||
int(self._value("history_days", 30)),
|
||||
daily_dir,
|
||||
)
|
||||
eligible = {key: paper for key, paper in merged.items() if key not in yesterday_ids | historical_ids}
|
||||
self.logger.info(
|
||||
f"[{self.name}] filter done merged={len(merged)} excluded_yesterday={len(yesterday_ids)} "
|
||||
f"excluded_history={len(historical_ids)} eligible={len(eligible)}",
|
||||
)
|
||||
if not eligible:
|
||||
raise RuntimeError("No eligible papers remain after yesterday and history exclusions")
|
||||
|
||||
for key, value in {
|
||||
"info": eligible,
|
||||
"week": week,
|
||||
"month": month,
|
||||
"yesterday": yesterday,
|
||||
"excluded_yesterday": sorted(yesterday_ids),
|
||||
"excluded_history": sorted(historical_ids),
|
||||
"source_counts": {"weekly": len(weekly), "monthly": len(monthly), "merged": len(merged)},
|
||||
}.items():
|
||||
self._set_state(key, value)
|
||||
self.context.response.success = True
|
||||
self.context.response.answer = f"Collected {len(eligible)} eligible papers"
|
||||
self.logger.info(f"[{self.name}] finish date={day} eligible={len(eligible)}")
|
||||
return self.context.response
|
||||
95
reme/steps/cookbook/daily_paper/digest.py
Normal file
95
reme/steps/cookbook/daily_paper/digest.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""Build the final daily-paper brief from detailed notes."""
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
from ....components import R
|
||||
from ....schema import DailyBriefOutput, PaperSelection
|
||||
from ...file_io import refresh_day_index
|
||||
from ._common import DailyPaperStep, strip_frontmatter, structured_output, write_markdown
|
||||
|
||||
|
||||
@R.register("daily_paper_digest_step")
|
||||
class DailyPaperDigestStep(DailyPaperStep):
|
||||
"""Use Claude Code to read the detailed notes and create the final brief."""
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
if self._skip():
|
||||
self.logger.info(f"[{self.name}] skip existing digest")
|
||||
return self.context.response
|
||||
if self.agent_wrapper is None:
|
||||
raise RuntimeError("Claude Code agent_wrapper is required for the daily brief")
|
||||
note_paths: list[str] = self._state("note_paths") or []
|
||||
selection: PaperSelection | None = self._state("selection")
|
||||
if selection is None or not note_paths:
|
||||
raise RuntimeError("Detailed paper notes are missing before digest generation")
|
||||
self.logger.info(f"[{self.name}] start notes={len(note_paths)}")
|
||||
|
||||
absolute_paths = [str((self.workspace_path / path).resolve()) for path in note_paths]
|
||||
wikilinks = [f"[[{path}]]" for path in note_paths]
|
||||
self.logger.info(f"[{self.name}] agent start notes={len(note_paths)}")
|
||||
result = await self.agent_wrapper.reply(
|
||||
self.prompt_format(
|
||||
"digest_user",
|
||||
top_k=len(note_paths),
|
||||
note_paths=json.dumps(absolute_paths, ensure_ascii=False, indent=2),
|
||||
wikilinks=json.dumps(wikilinks, ensure_ascii=False, indent=2),
|
||||
),
|
||||
output_schema=DailyBriefOutput,
|
||||
)
|
||||
self.logger.info(f"[{self.name}] agent done notes={len(note_paths)}")
|
||||
output = structured_output(result, DailyBriefOutput)
|
||||
body = strip_frontmatter(output.body)
|
||||
if not output.description.strip() or not body:
|
||||
raise ValueError("Claude Code returned an empty daily paper brief")
|
||||
missing_links = [link for link in wikilinks if link not in body]
|
||||
if missing_links:
|
||||
body += "\n\n## 详细文章\n\n" + "\n".join(f"- {link}" for link in missing_links)
|
||||
|
||||
day = self._run_day()
|
||||
daily_dir = str(self.config_value("daily_dir")).strip("/")
|
||||
digest_rel = f"{daily_dir}/{day}/daily-paper-brief.md"
|
||||
selected_ids = [item.arxiv_id for item in selection.selected]
|
||||
await write_markdown(
|
||||
self.workspace_path / digest_rel,
|
||||
body,
|
||||
{
|
||||
"name": "daily-paper-brief",
|
||||
"description": output.description.strip(),
|
||||
"date": day,
|
||||
"arxiv_ids": selected_ids,
|
||||
"selection_reasoning": selection.selection_reasoning,
|
||||
"alternate_arxiv_ids": selection.alternates,
|
||||
"source_notes": wikilinks,
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
self._set_state("digest_path", digest_rel)
|
||||
self.logger.info(f"[{self.name}] digest written path={digest_rel}")
|
||||
self.logger.info(f"[{self.name}] refresh index start date={day} daily_dir={daily_dir}")
|
||||
await refresh_day_index(SimpleNamespace(workspace_path=self.workspace_path), day, daily_dir)
|
||||
self.logger.info(f"[{self.name}] refresh index done date={day}")
|
||||
|
||||
self.context.response.success = True
|
||||
self.context.response.answer = f"Generated daily paper brief: {digest_rel}"
|
||||
self.context.response.metadata.update(
|
||||
{
|
||||
"date": day,
|
||||
"week": self._state("week"),
|
||||
"month": self._state("month"),
|
||||
"selection_reasoning": selection.selection_reasoning,
|
||||
"selected_arxiv_ids": selected_ids,
|
||||
"note_paths": note_paths,
|
||||
"pdf_paths": self._state("pdf_paths"),
|
||||
"digest_path": digest_rel,
|
||||
"source_counts": self._state("source_counts"),
|
||||
"excluded_yesterday_count": len(self._state("excluded_yesterday") or []),
|
||||
"excluded_history_count": len(self._state("excluded_history") or []),
|
||||
},
|
||||
)
|
||||
self.logger.info(
|
||||
f"[{self.name}] finish date={day} papers={len(selected_ids)} digest_path={digest_rel}",
|
||||
)
|
||||
return self.context.response
|
||||
19
reme/steps/cookbook/daily_paper/digest.yaml
Normal file
19
reme/steps/cookbook/daily_paper/digest.yaml
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
digest_user: |
|
||||
你是中文 AI 研究资讯主编。请依次调用 Read 阅读下面 {top_k} 个绝对路径,忠实阅读指定的详细论文 Markdown,再生成一篇普通读者五分钟可以读懂的每日论文速读。
|
||||
不得只根据文件名或标题写作,必须调用 Read 读取每一个指定文件。不要读取或修改其他文件。
|
||||
保留技术准确性,同时解释三篇论文为什么值得关注、它们之间有什么联系,以及哪些内容与大模型长期记忆有关。
|
||||
|
||||
# 必须读取的 Markdown 路径
|
||||
|
||||
{note_paths}
|
||||
|
||||
# 必须原样出现在正文中的 wikilink
|
||||
|
||||
{wikilinks}
|
||||
|
||||
输出要求:
|
||||
- 只返回结构化结果中的 description 和 body;body 不要包含 YAML frontmatter。
|
||||
- 总阅读时长约五分钟。
|
||||
- 包含“今日一句话”“每篇一分钟读懂”“三篇论文之间的联系”“今天最值得关注什么”等部分。
|
||||
- 每篇介绍必须附上对应的完整 wikilink,不能缩写或改写链接。
|
||||
- 对长期记忆相关论文,明确解释它对 Agent 记忆系统设计的启发。
|
||||
103
reme/steps/cookbook/daily_paper/rank.py
Normal file
103
reme/steps/cookbook/daily_paper/rank.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Rank collected papers for the daily-paper workflow."""
|
||||
|
||||
from ....components import R
|
||||
from ....schema import PaperInfo
|
||||
from ._common import DailyPaperStep
|
||||
|
||||
_MEMORY_KEYWORDS = (
|
||||
"long-term memory",
|
||||
"long term memory",
|
||||
"lifelong memory",
|
||||
"agent memory",
|
||||
"episodic memory",
|
||||
"memory consolidation",
|
||||
"memory retrieval",
|
||||
"self-evolving memory",
|
||||
"continual learning",
|
||||
"context compression",
|
||||
"personalization",
|
||||
"knowledge graph",
|
||||
"retrieval augmented",
|
||||
"rag",
|
||||
"长期记忆",
|
||||
"记忆整合",
|
||||
"记忆检索",
|
||||
)
|
||||
|
||||
|
||||
def rrf_score(
|
||||
monthly_rank: int | None,
|
||||
weekly_rank: int | None,
|
||||
*,
|
||||
rrf_k: int = 60,
|
||||
weekly_weight: float = 0.7,
|
||||
) -> float:
|
||||
"""Fuse optional monthly and weekly ranks with reciprocal-rank fusion."""
|
||||
if rrf_k < 0:
|
||||
raise ValueError("rrf_k must be non-negative")
|
||||
monthly_score = 0.0 if monthly_rank is None else 1.0 / (rrf_k + monthly_rank)
|
||||
weekly_score = 0.0 if weekly_rank is None else weekly_weight / (rrf_k + weekly_rank)
|
||||
return monthly_score + weekly_score
|
||||
|
||||
|
||||
def memory_keyword_score(paper: PaperInfo) -> int:
|
||||
"""Return a lightweight recall score used to reserve memory-related candidates."""
|
||||
text = f"{paper.title}\n{paper.summary}".lower()
|
||||
return sum(keyword in text for keyword in _MEMORY_KEYWORDS)
|
||||
|
||||
|
||||
def build_candidate_pool(papers: list[PaperInfo], *, limit: int = 20, memory_reserve: int = 5) -> list[PaperInfo]:
|
||||
"""Keep strong general papers while reserving room for memory-related work."""
|
||||
if limit <= 0:
|
||||
raise ValueError("candidate_limit must be positive")
|
||||
ranked = sorted(papers, key=lambda item: (-item.fused_score, -item.upvotes, item.arxiv_id))
|
||||
reserve = min(max(memory_reserve, 0), limit)
|
||||
selected = ranked[: max(0, limit - reserve)]
|
||||
selected_ids = {paper.arxiv_id for paper in selected}
|
||||
memory_candidates = [
|
||||
paper for paper in ranked if paper.arxiv_id not in selected_ids and memory_keyword_score(paper)
|
||||
]
|
||||
memory_candidates.sort(
|
||||
key=lambda item: (-memory_keyword_score(item), -item.fused_score, -item.upvotes, item.arxiv_id),
|
||||
)
|
||||
selected.extend(memory_candidates[:reserve])
|
||||
selected_ids = {paper.arxiv_id for paper in selected}
|
||||
selected.extend(paper for paper in ranked if paper.arxiv_id not in selected_ids and len(selected) < limit)
|
||||
return selected[:limit]
|
||||
|
||||
|
||||
@R.register("daily_paper_rank_step")
|
||||
class DailyPaperRankStep(DailyPaperStep):
|
||||
"""Apply RRF and produce the bounded selection pool."""
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
if self._skip():
|
||||
self.logger.info(f"[{self.name}] skip existing digest")
|
||||
return self.context.response
|
||||
papers_by_id: dict[str, PaperInfo] = self._state("info") or {}
|
||||
rrf_k, weekly_weight = int(self._value("rrf_k", 60)), float(self._value("weekly_weight", 0.7))
|
||||
candidate_limit = int(self._value("candidate_limit", 20))
|
||||
memory_reserve = int(self._value("memory_reserve", 5))
|
||||
self.logger.info(
|
||||
f"[{self.name}] start papers={len(papers_by_id)} rrf_k={rrf_k} weekly_weight={weekly_weight} "
|
||||
f"candidate_limit={candidate_limit} memory_reserve={memory_reserve}",
|
||||
)
|
||||
for paper in papers_by_id.values():
|
||||
paper.fused_score = rrf_score(
|
||||
paper.monthly_rank,
|
||||
paper.weekly_rank,
|
||||
rrf_k=rrf_k,
|
||||
weekly_weight=weekly_weight,
|
||||
)
|
||||
candidates = build_candidate_pool(
|
||||
list(papers_by_id.values()),
|
||||
limit=candidate_limit,
|
||||
memory_reserve=memory_reserve,
|
||||
)
|
||||
if not candidates:
|
||||
raise RuntimeError("RRF produced no paper candidates")
|
||||
self._set_state("candidates", candidates)
|
||||
self.context.response.answer = f"Ranked {len(candidates)} paper candidates with RRF"
|
||||
self.logger.info(f"[{self.name}] finish candidates={len(candidates)}")
|
||||
return self.context.response
|
||||
93
reme/steps/cookbook/daily_paper/select.py
Normal file
93
reme/steps/cookbook/daily_paper/select.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
"""Select final daily papers with Claude Code."""
|
||||
|
||||
import json
|
||||
|
||||
from ....components import R
|
||||
from ....schema import PaperInfo, PaperSelection
|
||||
from ._common import DailyPaperStep, structured_output
|
||||
from .rank import memory_keyword_score
|
||||
|
||||
|
||||
@R.register("daily_paper_select_step")
|
||||
class DailyPaperSelectStep(DailyPaperStep):
|
||||
"""Use Claude Code to select the final papers."""
|
||||
|
||||
@staticmethod
|
||||
def _validate_selection(selection: PaperSelection, candidates: list[PaperInfo], top_k: int) -> PaperSelection:
|
||||
candidate_ids = {paper.arxiv_id for paper in candidates}
|
||||
ordered = sorted(selection.selected, key=lambda item: item.rank)
|
||||
selected_ids = [item.arxiv_id for item in ordered]
|
||||
if len(ordered) != top_k:
|
||||
raise ValueError(f"Agent selected {len(ordered)} papers; expected {top_k}")
|
||||
if len(set(selected_ids)) != top_k or any(key not in candidate_ids for key in selected_ids):
|
||||
raise ValueError("Agent selection contains duplicate or out-of-pool ids")
|
||||
if [item.rank for item in ordered] != list(range(1, top_k + 1)):
|
||||
raise ValueError("Agent selection ranks must be consecutive starting at 1")
|
||||
alternates = [
|
||||
key for key in dict.fromkeys(selection.alternates) if key in candidate_ids and key not in selected_ids
|
||||
]
|
||||
return selection.model_copy(update={"selected": ordered, "alternates": alternates})
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
if self._skip():
|
||||
self.logger.info(f"[{self.name}] skip existing digest")
|
||||
return self.context.response
|
||||
if self.agent_wrapper is None:
|
||||
raise RuntimeError("Claude Code agent_wrapper is required for paper selection")
|
||||
candidates: list[PaperInfo] = self._state("candidates") or []
|
||||
top_k = int(self._value("top_k", 3))
|
||||
if top_k <= 0 or top_k > len(candidates):
|
||||
raise ValueError(f"top_k must be between 1 and {len(candidates)}")
|
||||
self.logger.info(f"[{self.name}] start candidates={len(candidates)} top_k={top_k}")
|
||||
|
||||
candidate_payload = [
|
||||
{
|
||||
"arxiv_id": paper.arxiv_id,
|
||||
"title": paper.title,
|
||||
"summary": paper.summary,
|
||||
"authors": paper.authors,
|
||||
"organization": paper.organization,
|
||||
"upvotes": paper.upvotes,
|
||||
"monthly_rank": paper.monthly_rank,
|
||||
"weekly_rank": paper.weekly_rank,
|
||||
"fused_score": round(paper.fused_score, 8),
|
||||
"github_repo": paper.github_repo,
|
||||
"github_stars": paper.github_stars,
|
||||
"memory_keyword_score": memory_keyword_score(paper),
|
||||
}
|
||||
for paper in candidates
|
||||
]
|
||||
feedback, selection = "", None
|
||||
for attempt in range(1, 3):
|
||||
self.logger.info(f"[{self.name}] agent start attempt={attempt}/2 candidates={len(candidates)}")
|
||||
result = await self.agent_wrapper.reply(
|
||||
self.prompt_format(
|
||||
"select_user",
|
||||
top_k=top_k,
|
||||
candidates=json.dumps(candidate_payload, ensure_ascii=False, indent=2),
|
||||
retry_feedback=feedback or "(none)",
|
||||
),
|
||||
output_schema=PaperSelection,
|
||||
)
|
||||
try:
|
||||
selection = self._validate_selection(structured_output(result, PaperSelection), candidates, top_k)
|
||||
self.logger.info(f"[{self.name}] agent done attempt={attempt}/2 valid=True")
|
||||
break
|
||||
except (ValueError, TypeError) as exc:
|
||||
feedback = str(exc)
|
||||
self.logger.warning(
|
||||
f"[{self.name}] agent done attempt={attempt}/2 valid=False error={feedback!r}",
|
||||
)
|
||||
if selection is None:
|
||||
raise RuntimeError(f"Claude Code paper selection failed validation: {feedback}")
|
||||
|
||||
candidate_map = {paper.arxiv_id: paper for paper in candidates}
|
||||
selected_papers = [candidate_map[item.arxiv_id] for item in selection.selected]
|
||||
self._set_state("selection", selection)
|
||||
self._set_state("selected_papers", selected_papers)
|
||||
self.context.response.answer = f"Selected {top_k} papers with Claude Code"
|
||||
self.logger.info(
|
||||
f"[{self.name}] finish selected={','.join(item.arxiv_id for item in selection.selected)}",
|
||||
)
|
||||
return self.context.response
|
||||
18
reme/steps/cookbook/daily_paper/select.yaml
Normal file
18
reme/steps/cookbook/daily_paper/select.yaml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
select_user: |
|
||||
你是 AI 研究论文编辑。请从以下候选中选择恰好 {top_k} 篇最值得深入阅读的论文,并另外给出至多 3 个候补 ID。
|
||||
选择应兼顾研究价值、技术新颖性、潜在影响和可读性。融合分与榜单排名是重要依据,但不是唯一依据。
|
||||
与大模型 Agent 长期记忆、记忆检索、记忆整合、持续学习、个性化和上下文管理直接相关的高质量论文应优先考虑。
|
||||
只能选择输入候选集合中的 arXiv ID,不得编造论文或事实。输出简洁、可核验的选择理由。
|
||||
|
||||
要求:
|
||||
1. selected 的 rank 必须从 1 到 {top_k} 连续排列。
|
||||
2. arxiv_id 必须逐字复制候选数据中的值,不能重复。
|
||||
3. reason 说明具体选择依据,不要只复述标题。
|
||||
4. memory_relevance 评价论文与大模型/Agent 长期记忆的相关程度。
|
||||
5. selection_reasoning 是面向读者的简短决策摘要,不要输出隐含的逐步思维过程。
|
||||
|
||||
上一次校验反馈:{retry_feedback}
|
||||
|
||||
# 候选论文
|
||||
|
||||
{candidates}
|
||||
6
reme/steps/cookbook/dingtalk/__init__.py
Normal file
6
reme/steps/cookbook/dingtalk/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""DingTalk cookbook integration."""
|
||||
|
||||
from .send import DingTalkMarkdownSendStep
|
||||
from .wait import DingTalkWaitStep
|
||||
|
||||
__all__ = ["DingTalkMarkdownSendStep", "DingTalkWaitStep"]
|
||||
123
reme/steps/cookbook/dingtalk/send.py
Normal file
123
reme/steps/cookbook/dingtalk/send.py
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
"""Send a workspace Markdown file to DingTalk group conversations."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import aiofiles
|
||||
import frontmatter
|
||||
import httpx
|
||||
|
||||
from ....components import R
|
||||
from ...base_step import BaseStep
|
||||
from ...file_io._path import gate_md, resolve_path
|
||||
|
||||
_GROUP_SEND_URL = "https://api.dingtalk.com/v1.0/robot/groupMessages/send"
|
||||
|
||||
|
||||
def _conversation_ids(value: str) -> list[str]:
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
|
||||
@R.register("dingtalk_markdown_send_step")
|
||||
class DingTalkMarkdownSendStep(BaseStep):
|
||||
"""Send one Markdown document serially to configured DingTalk groups."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app_key: str = "",
|
||||
app_secret: str = "",
|
||||
robot_code: str = "",
|
||||
conversation_ids: str = "",
|
||||
title: str = "",
|
||||
timeout: float = 15.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.app_key = app_key
|
||||
self.app_secret = app_secret
|
||||
self.robot_code = robot_code
|
||||
self.conversation_ids = _conversation_ids(conversation_ids)
|
||||
self.title = title
|
||||
self.timeout = timeout
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
recipients = self.conversation_ids
|
||||
self.context.response.metadata["dingtalk_configured_count"] = len(recipients)
|
||||
self.context.response.metadata["dingtalk_sent_count"] = 0
|
||||
if not recipients:
|
||||
self.logger.info(f"[{self.name}] skipped DingTalk Markdown delivery: no conversation IDs")
|
||||
return self.context.response
|
||||
if not all((self.app_key, self.app_secret, self.robot_code)):
|
||||
raise RuntimeError("DingTalk Markdown delivery requires app_key, app_secret, and robot_code")
|
||||
|
||||
raw_path = str(self.context.get("markdown_path") or "")
|
||||
if not raw_path:
|
||||
if self.context.response.metadata.get("skipped"):
|
||||
self.logger.info(f"[{self.name}] skipped DingTalk Markdown delivery: no markdown path")
|
||||
return self.context.response
|
||||
raise RuntimeError("DingTalk Markdown delivery requires markdown_path")
|
||||
|
||||
target, error = resolve_path(self.workspace_path, raw_path)
|
||||
if error:
|
||||
raise ValueError(f"Invalid DingTalk Markdown path: {error}")
|
||||
assert target is not None
|
||||
target, is_markdown = gate_md(target)
|
||||
if not is_markdown:
|
||||
raise ValueError("DingTalk Markdown delivery requires a .md file")
|
||||
if not target.is_file():
|
||||
raise FileNotFoundError(f"DingTalk Markdown file does not exist: {raw_path}")
|
||||
|
||||
async with aiofiles.open(target, encoding="utf-8") as stream:
|
||||
document = frontmatter.loads(await stream.read())
|
||||
markdown = document.content.strip()
|
||||
if not markdown:
|
||||
raise ValueError(f"DingTalk Markdown file is empty: {raw_path}")
|
||||
|
||||
import dingtalk_stream # pylint: disable=import-outside-toplevel
|
||||
|
||||
title = self.title or str(document.metadata.get("name") or target.stem)
|
||||
token_client = dingtalk_stream.DingTalkStreamClient(
|
||||
dingtalk_stream.Credential(self.app_key, self.app_secret),
|
||||
)
|
||||
access_token = await asyncio.to_thread(token_client.get_access_token)
|
||||
if not access_token:
|
||||
raise RuntimeError("Failed to obtain DingTalk access token")
|
||||
|
||||
self.logger.info(
|
||||
f"[{self.name}] sending DingTalk Markdown path={raw_path} recipients={len(recipients)} "
|
||||
f"chars={len(markdown)} timeout={self.timeout:.1f}s",
|
||||
)
|
||||
failures: list[str] = []
|
||||
headers = {"x-acs-dingtalk-access-token": access_token, "User-Agent": "ReMe DingTalk notifier"}
|
||||
transport = httpx.AsyncHTTPTransport(local_address="0.0.0.0")
|
||||
async with httpx.AsyncClient(timeout=self.timeout, headers=headers, transport=transport) as client:
|
||||
for index, conversation_id in enumerate(recipients, start=1):
|
||||
payload = {
|
||||
"robotCode": self.robot_code,
|
||||
"openConversationId": conversation_id,
|
||||
"msgKey": "sampleMarkdown",
|
||||
"msgParam": json.dumps({"title": title, "text": markdown}, ensure_ascii=False),
|
||||
}
|
||||
try:
|
||||
response = await client.post(_GROUP_SEND_URL, json=payload)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
if not isinstance(result, dict) or not result.get("processQueryKey"):
|
||||
raise ValueError("missing processQueryKey")
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
failures.append(f"recipient {index}: {type(exc).__name__}")
|
||||
self.logger.warning(
|
||||
f"[{self.name}] DingTalk delivery failed recipient={index}/{len(recipients)} "
|
||||
f"error_type={type(exc).__name__}",
|
||||
)
|
||||
continue
|
||||
self.context.response.metadata["dingtalk_sent_count"] += 1
|
||||
self.logger.info(f"[{self.name}] delivered DingTalk Markdown recipient={index}/{len(recipients)}")
|
||||
|
||||
sent_count = self.context.response.metadata["dingtalk_sent_count"]
|
||||
if failures:
|
||||
self.context.response.metadata["dingtalk_delivery_errors"] = failures
|
||||
raise RuntimeError(f"DingTalk Markdown delivery failed for {len(failures)} of {len(recipients)} recipients")
|
||||
self.logger.info(f"[{self.name}] DingTalk Markdown delivery complete sent={sent_count} total={len(recipients)}")
|
||||
return self.context.response
|
||||
426
reme/steps/cookbook/dingtalk/wait.py
Normal file
426
reme/steps/cookbook/dingtalk/wait.py
Normal file
|
|
@ -0,0 +1,426 @@
|
|||
"""Long-running DingTalk Stream bridge for the cookbook application."""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from ...base_step import BaseStep
|
||||
from ....components import R
|
||||
from ....enumeration import ChunkEnum
|
||||
|
||||
_VISIBLE_CHUNKS = {ChunkEnum.THINK, ChunkEnum.TOOL_CALL, ChunkEnum.TOOL_RESULT, ChunkEnum.CONTENT, ChunkEnum.ERROR}
|
||||
_CODE_CHUNKS = {ChunkEnum.TOOL_CALL, ChunkEnum.TOOL_RESULT}
|
||||
_BLOCK_CHAR_LIMIT = 100
|
||||
|
||||
|
||||
def _session_key(message: Any) -> str:
|
||||
"""Return the per-sender, per-conversation Claude session key."""
|
||||
parts = (
|
||||
message.conversation_type,
|
||||
message.conversation_id,
|
||||
message.sender_staff_id,
|
||||
)
|
||||
if not all(parts):
|
||||
raise ValueError("DingTalk message requires conversationType, conversationId, and senderStaffId")
|
||||
return ":".join(parts)
|
||||
|
||||
|
||||
def _payload_text(value: Any) -> str:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2)
|
||||
return text.replace("```", "'''")
|
||||
|
||||
|
||||
def _session_ref(key: str) -> str:
|
||||
"""Return a stable log correlation id without exposing DingTalk identifiers."""
|
||||
return hashlib.sha256(key.encode("utf-8")).hexdigest()[:12]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Block:
|
||||
chunk_type: ChunkEnum
|
||||
key: str
|
||||
name: str = ""
|
||||
parts: list[Any] = field(default_factory=list)
|
||||
chars: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
class _CardRenderer:
|
||||
"""Render append-only, size-limited DingTalk blocks."""
|
||||
|
||||
def __init__(self):
|
||||
self.started_at = time.monotonic()
|
||||
self.active: _Block | None = None
|
||||
self.has_output = False
|
||||
self.error = False
|
||||
|
||||
def feed(self, chunk) -> str:
|
||||
"""Buffer one block and return completed Markdown blocks."""
|
||||
if chunk.chunk_type not in _VISIBLE_CHUNKS:
|
||||
return ""
|
||||
if chunk.chunk_type == ChunkEnum.ERROR:
|
||||
self.error = True
|
||||
|
||||
key = chunk.block_id or chunk.tool_call_id or chunk.chunk_type.value
|
||||
current = (chunk.chunk_type, key)
|
||||
if not chunk.chunk:
|
||||
return self._complete_active() if self._active_key() == current else ""
|
||||
|
||||
delta = ""
|
||||
if self._active_key() != current:
|
||||
delta += self._complete_active()
|
||||
self.active = _Block(chunk.chunk_type, key, chunk.tool_call_name or "")
|
||||
assert self.active is not None
|
||||
self.has_output = True
|
||||
if chunk.chunk_type in _CODE_CHUNKS:
|
||||
self.active.parts.append(chunk.chunk)
|
||||
return delta
|
||||
|
||||
if self.active.chars == 0:
|
||||
delta += self._text_title(chunk.chunk_type)
|
||||
text = _payload_text(chunk.chunk)
|
||||
if chunk.chunk_type == ChunkEnum.CONTENT:
|
||||
self.active.chars += len(text)
|
||||
return delta + text
|
||||
remaining = _BLOCK_CHAR_LIMIT - self.active.chars
|
||||
delta += text[:remaining]
|
||||
self.active.chars += min(len(text), remaining)
|
||||
if len(text) > remaining and not self.active.truncated:
|
||||
delta += "..."
|
||||
self.active.truncated = True
|
||||
return delta
|
||||
|
||||
def fail(self, message: str) -> str:
|
||||
"""Return an error block for an unexpected local failure."""
|
||||
delta = self._complete_active()
|
||||
self.error = True
|
||||
self.has_output = True
|
||||
text = _payload_text(message)
|
||||
body = text if len(text) <= _BLOCK_CHAR_LIMIT else f"{text[:_BLOCK_CHAR_LIMIT]}..."
|
||||
return f"{delta}### ⚠️ Error\n\n{body}\n\n"
|
||||
|
||||
def _active_key(self) -> tuple[ChunkEnum, str] | None:
|
||||
if self.active is None:
|
||||
return None
|
||||
return self.active.chunk_type, self.active.key
|
||||
|
||||
def _complete_active(self) -> str:
|
||||
if self.active is None:
|
||||
return ""
|
||||
block, self.active = self.active, None
|
||||
if block.chunk_type not in _CODE_CHUNKS:
|
||||
return "\n\n"
|
||||
if block.chunk_type == ChunkEnum.TOOL_CALL and len(block.parts) == 1 and self._is_call_metadata(block.parts[0]):
|
||||
return ""
|
||||
|
||||
body = self._block_body(block)
|
||||
body = body if len(body) <= _BLOCK_CHAR_LIMIT else f"{body[:_BLOCK_CHAR_LIMIT]}..."
|
||||
if block.chunk_type == ChunkEnum.TOOL_CALL:
|
||||
name = block.name or self._metadata_name(block) or "tool"
|
||||
safe_name = name.replace("`", "'")
|
||||
title = f"### 🔧 Tool Call · `{safe_name}`"
|
||||
else:
|
||||
title = "### 📦 Tool Result"
|
||||
|
||||
body = "\n".join(f"> {self._visible_indent(line)}" for line in body.splitlines())
|
||||
return f"{title}\n\n{body}\n\n"
|
||||
|
||||
@staticmethod
|
||||
def _text_title(chunk_type: ChunkEnum) -> str:
|
||||
if chunk_type == ChunkEnum.THINK:
|
||||
return "### 🧠 Think\n\n"
|
||||
if chunk_type == ChunkEnum.ERROR:
|
||||
return "### ⚠️ Error\n\n"
|
||||
return "### 💬 Content\n\n"
|
||||
|
||||
def _block_body(self, block: _Block) -> str:
|
||||
if block.chunk_type not in _CODE_CHUNKS:
|
||||
return "".join(_payload_text(part) for part in block.parts)
|
||||
payload, is_json = self._tool_payload(block)
|
||||
if is_json:
|
||||
return json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
return "".join(_payload_text(part) for part in block.parts)
|
||||
|
||||
@staticmethod
|
||||
def _visible_indent(line: str) -> str:
|
||||
"""Keep JSON indentation after DingTalk collapses ordinary spaces."""
|
||||
spaces = len(line) - len(line.lstrip(" "))
|
||||
return " " * (spaces // 2) + line[spaces:]
|
||||
|
||||
def _tool_payload(self, block: _Block) -> tuple[Any, bool]:
|
||||
parts = block.parts
|
||||
if block.chunk_type == ChunkEnum.TOOL_CALL and len(parts) > 1 and self._is_call_metadata(parts[0]):
|
||||
parts = parts[1:]
|
||||
if len(parts) == 1 and not isinstance(parts[0], str):
|
||||
return self._expand_nested_json(parts[0]), True
|
||||
if parts and all(isinstance(part, str) for part in parts):
|
||||
try:
|
||||
value = json.loads("".join(parts))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
return self._expand_nested_json(value), True
|
||||
return None, False
|
||||
|
||||
def _metadata_name(self, block: _Block) -> str:
|
||||
if not block.parts or not self._is_call_metadata(block.parts[0]):
|
||||
return ""
|
||||
return json.loads(block.parts[0]).get("name") or ""
|
||||
|
||||
@staticmethod
|
||||
def _is_call_metadata(value: Any) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
return isinstance(parsed, dict) and bool(parsed) and set(parsed) <= {"id", "name"}
|
||||
|
||||
@classmethod
|
||||
def _expand_nested_json(cls, value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {key: cls._expand_nested_json(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [cls._expand_nested_json(item) for item in value]
|
||||
if isinstance(value, str) and value.lstrip().startswith(("{", "[")):
|
||||
try:
|
||||
return cls._expand_nested_json(json.loads(value))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return value
|
||||
|
||||
def finish(self) -> str:
|
||||
"""Close the active block and append elapsed time."""
|
||||
delta = self._complete_active()
|
||||
if not delta and self.has_output:
|
||||
delta = "\n\n"
|
||||
return f"{delta}{time.monotonic() - self.started_at:.1f}s"
|
||||
|
||||
|
||||
@R.register("dingtalk_wait_step")
|
||||
class DingTalkWaitStep(BaseStep):
|
||||
"""Receive DingTalk messages and stream Claude Code replies into AI cards."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app_key: str = "",
|
||||
app_secret: str = "",
|
||||
robot_code: str = "",
|
||||
card_update_interval: float = 1.0,
|
||||
worker_count: int = 4,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.app_key = app_key
|
||||
self.app_secret = app_secret
|
||||
self.robot_code = robot_code
|
||||
self.card_update_interval = max(0.05, card_update_interval)
|
||||
self.worker_count = max(1, worker_count)
|
||||
|
||||
async def execute(self):
|
||||
assert self.context is not None
|
||||
if self.context.stop_event is None or self.app_context is None:
|
||||
raise RuntimeError("dingtalk_wait_step requires an ApplicationContext and background stop_event")
|
||||
if self.agent_wrapper is None:
|
||||
raise RuntimeError("dingtalk_wait_step requires an agent_wrapper")
|
||||
if not self.app_key or not self.app_secret or not self.robot_code:
|
||||
raise RuntimeError("dingtalk_wait_step requires app_key, app_secret, and robot_code")
|
||||
|
||||
import dingtalk_stream # pylint: disable=import-outside-toplevel
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
handler = self._make_handler(dingtalk_stream, queue)
|
||||
client = dingtalk_stream.DingTalkStreamClient(
|
||||
dingtalk_stream.Credential(self.app_key, self.app_secret),
|
||||
)
|
||||
client.register_callback_handler(dingtalk_stream.ChatbotMessage.TOPIC, handler)
|
||||
sessions = self.app_context.metadata.setdefault("dingtalk_agent_sessions", {})
|
||||
locks: dict[str, asyncio.Lock] = {}
|
||||
self.logger.info(
|
||||
f"[{self.name}] starting DingTalk Stream bridge "
|
||||
f"workers={self.worker_count} card_update_interval={self.card_update_interval:.2f}s",
|
||||
)
|
||||
workers = [
|
||||
asyncio.create_task(self._worker(queue, locks, sessions, handler, dingtalk_stream))
|
||||
for _ in range(self.worker_count)
|
||||
]
|
||||
try:
|
||||
await self._run_client(client, self.context.stop_event)
|
||||
finally:
|
||||
for worker in workers:
|
||||
worker.cancel()
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
self.logger.info(f"[{self.name}] DingTalk Stream bridge stopped")
|
||||
return self.context.response
|
||||
|
||||
@staticmethod
|
||||
def _make_handler(dingtalk_stream, queue: asyncio.Queue):
|
||||
class QueueHandler(dingtalk_stream.ChatbotHandler):
|
||||
"""Acknowledge callbacks after placing them on the worker queue."""
|
||||
|
||||
async def process(self, callback):
|
||||
"""Enqueue one callback and immediately acknowledge it."""
|
||||
queue.put_nowait(dingtalk_stream.ChatbotMessage.from_dict(callback.data))
|
||||
return dingtalk_stream.AckMessage.STATUS_OK, "OK"
|
||||
|
||||
return QueueHandler()
|
||||
|
||||
async def _worker(self, queue, locks, sessions, handler, dingtalk_stream) -> None:
|
||||
while True:
|
||||
message = await queue.get()
|
||||
try:
|
||||
key = _session_key(message)
|
||||
async with locks.setdefault(key, asyncio.Lock()):
|
||||
await self._handle_message(message, key, sessions, handler, dingtalk_stream)
|
||||
except Exception as exc: # A bad message must not disconnect the Stream client.
|
||||
self.logger.exception(f"Failed to handle DingTalk message: {exc}")
|
||||
await asyncio.to_thread(handler.reply_text, f"处理失败:{exc}", message)
|
||||
finally:
|
||||
queue.task_done()
|
||||
|
||||
async def _handle_message(self, message, key, sessions, handler, dingtalk_stream) -> None:
|
||||
session_ref = _session_ref(key)
|
||||
self.logger.info(
|
||||
f"[{self.name}] handling DingTalk callback session={session_ref} "
|
||||
f"conversation_type={message.conversation_type!r} conversation_id={message.conversation_id!r} "
|
||||
f"sender_staff_id={message.sender_staff_id!r}",
|
||||
)
|
||||
if self.robot_code and getattr(message, "robot_code", "") != self.robot_code:
|
||||
self.logger.warning(
|
||||
f"[{self.name}] rejected DingTalk callback session={session_ref} reason=robot_code_mismatch",
|
||||
)
|
||||
raise ValueError("DingTalk callback robotCode does not match configured robot_code")
|
||||
text = (message.text.content if message.text else "").strip()
|
||||
if not text:
|
||||
self.logger.info(f"[{self.name}] ignored non-text DingTalk message session={session_ref}")
|
||||
await asyncio.to_thread(handler.reply_text, "暂时只支持文本消息。", message)
|
||||
return
|
||||
if text == "/clear":
|
||||
cleared = sessions.pop(key, None) is not None
|
||||
self.logger.info(f"[{self.name}] cleared DingTalk session session={session_ref} existed={cleared}")
|
||||
await asyncio.to_thread(
|
||||
handler.reply_text,
|
||||
"✅ Conversation cleared. The next message will start a new session.",
|
||||
message,
|
||||
)
|
||||
return
|
||||
|
||||
resumed = key in sessions
|
||||
self.logger.info(
|
||||
f"[{self.name}] received DingTalk text session={session_ref} chars={len(text)} resume={resumed}",
|
||||
)
|
||||
renderer = _CardRenderer()
|
||||
card = dingtalk_stream.AIMarkdownCardInstance(handler.dingtalk_client, message)
|
||||
card_id = await card.async_create_and_send_card(
|
||||
card.card_template_id,
|
||||
card.get_card_data(flow_status=dingtalk_stream.AICardStatus.PROCESSING),
|
||||
at_sender=True,
|
||||
)
|
||||
if not card_id:
|
||||
raise RuntimeError("创建钉钉 AI 卡片失败")
|
||||
self.logger.debug(f"[{self.name}] started DingTalk AI card session={session_ref}")
|
||||
|
||||
last_update = 0.0
|
||||
pending = ""
|
||||
chunk_count = 0
|
||||
update_count = 0
|
||||
streamed_chars = 0
|
||||
finalizing = False
|
||||
try:
|
||||
kwargs = {"resume": sessions[key]} if key in sessions else {}
|
||||
async for chunk in self.agent_wrapper.reply_stream(text, **kwargs):
|
||||
chunk_count += 1
|
||||
if chunk.session_id:
|
||||
sessions[key] = chunk.session_id
|
||||
pending += renderer.feed(chunk)
|
||||
if pending and time.monotonic() - last_update >= self.card_update_interval:
|
||||
delta = pending
|
||||
await card.async_streaming(
|
||||
card_id,
|
||||
"msgContent",
|
||||
delta,
|
||||
append=True,
|
||||
finished=False,
|
||||
failed=False,
|
||||
)
|
||||
pending = ""
|
||||
streamed_chars += len(delta)
|
||||
update_count += 1
|
||||
last_update = time.monotonic()
|
||||
|
||||
pending += renderer.finish()
|
||||
finalizing = True
|
||||
await card.async_streaming(
|
||||
card_id,
|
||||
"msgContent",
|
||||
pending,
|
||||
append=True,
|
||||
finished=True,
|
||||
failed=renderer.error,
|
||||
)
|
||||
streamed_chars += len(pending)
|
||||
pending = ""
|
||||
log = self.logger.warning if renderer.error else self.logger.info
|
||||
log(
|
||||
f"[{self.name}] completed DingTalk reply session={session_ref} success={not renderer.error} "
|
||||
f"chunks={chunk_count} card_updates={update_count} card_chars={streamed_chars} "
|
||||
f"elapsed={time.monotonic() - renderer.started_at:.2f}s",
|
||||
)
|
||||
except Exception:
|
||||
if not finalizing:
|
||||
if not renderer.error:
|
||||
pending += renderer.fail("Agent 执行失败")
|
||||
pending += renderer.finish()
|
||||
failure_delta, pending = pending, ""
|
||||
with contextlib.suppress(Exception):
|
||||
await card.async_streaming(
|
||||
card_id,
|
||||
"msgContent",
|
||||
failure_delta,
|
||||
append=True,
|
||||
finished=True,
|
||||
failed=True,
|
||||
)
|
||||
streamed_chars += len(failure_delta)
|
||||
self.logger.warning(
|
||||
f"[{self.name}] DingTalk reply failed session={session_ref} "
|
||||
f"chunks={chunk_count} card_updates={update_count} card_chars={streamed_chars} "
|
||||
f"elapsed={time.monotonic() - renderer.started_at:.2f}s",
|
||||
)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
async def _run_client(client, stop_event: asyncio.Event) -> None:
|
||||
"""Run one cancellable WebSocket connection; BackgroundJob owns retries."""
|
||||
import websockets # pylint: disable=import-outside-toplevel
|
||||
|
||||
client.pre_start()
|
||||
connection = await asyncio.to_thread(client.open_connection)
|
||||
if not connection:
|
||||
raise ConnectionError("DingTalk open connection failed")
|
||||
uri = f'{connection["endpoint"]}?ticket={quote_plus(connection["ticket"])}'
|
||||
async with websockets.connect(uri) as websocket:
|
||||
client.websocket = websocket
|
||||
keepalive = asyncio.create_task(client.keepalive(websocket))
|
||||
|
||||
async def close_when_stopped() -> None:
|
||||
await stop_event.wait()
|
||||
await websocket.close()
|
||||
|
||||
stopper = asyncio.create_task(close_when_stopped())
|
||||
try:
|
||||
async for raw_message in websocket:
|
||||
if await client.route_message(json.loads(raw_message)) == client.TAG_DISCONNECT:
|
||||
await websocket.close()
|
||||
finally:
|
||||
for task in (stopper, keepalive):
|
||||
task.cancel()
|
||||
await asyncio.gather(stopper, keepalive, return_exceptions=True)
|
||||
if not stop_event.is_set():
|
||||
raise ConnectionError("DingTalk WebSocket closed")
|
||||
59
reme/utils/arxiv.py
Normal file
59
reme/utils/arxiv.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""arXiv validation and PDF download helpers."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import aiofiles
|
||||
import httpx
|
||||
|
||||
ARXIV_ID_PATTERN = re.compile(r"^\d{4}\.\d{4,5}$")
|
||||
|
||||
|
||||
class ArxivPdfClient:
|
||||
"""Download validated arXiv PDFs to local files."""
|
||||
|
||||
def __init__(self, *, timeout: float = 90.0, max_bytes: int = 50 * 1024 * 1024) -> None:
|
||||
self.timeout, self.max_bytes = timeout, max_bytes
|
||||
|
||||
async def download(self, arxiv_id: str, target: Path) -> Path:
|
||||
"""Download one PDF atomically, reusing an existing valid target."""
|
||||
if not ARXIV_ID_PATTERN.fullmatch(arxiv_id):
|
||||
raise ValueError(f"Invalid arXiv id: {arxiv_id!r}")
|
||||
if target.is_file() and target.stat().st_size > 5:
|
||||
with target.open("rb") as existing:
|
||||
if existing.read(5) == b"%PDF-":
|
||||
return target
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
part_path = target.with_name(f".{target.name}.{uuid4().hex}.part")
|
||||
size = 0
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self.timeout,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": "ReMe arXiv client"},
|
||||
) as client:
|
||||
async with client.stream("GET", f"https://arxiv.org/pdf/{arxiv_id}") as response:
|
||||
response.raise_for_status()
|
||||
content_length = int(response.headers.get("content-length") or 0)
|
||||
if content_length and content_length > self.max_bytes:
|
||||
raise ValueError(f"PDF exceeds maximum size: {content_length} > {self.max_bytes}")
|
||||
async with aiofiles.open(part_path, "wb") as stream:
|
||||
async for chunk in response.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > self.max_bytes:
|
||||
raise ValueError(f"PDF exceeds maximum size: {size} > {self.max_bytes}")
|
||||
await stream.write(chunk)
|
||||
if size <= 5:
|
||||
raise ValueError(f"Downloaded PDF is empty for {arxiv_id}")
|
||||
async with aiofiles.open(part_path, "rb") as stream:
|
||||
header = await stream.read(5)
|
||||
if header != b"%PDF-":
|
||||
raise ValueError(f"Downloaded content is not a PDF for {arxiv_id}")
|
||||
os.replace(part_path, target)
|
||||
return target
|
||||
finally:
|
||||
if part_path.exists():
|
||||
part_path.unlink()
|
||||
156
reme/utils/huggingface_papers.py
Normal file
156
reme/utils/huggingface_papers.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
"""Client and response normalization for Hugging Face Papers."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from ..schema import PaperInfo
|
||||
from .arxiv import ARXIV_ID_PATTERN
|
||||
|
||||
HF_BASE_URL = "https://huggingface.co"
|
||||
_PAPER_LINK_PATTERN = re.compile(
|
||||
r"href=[\"'](?:https://huggingface\.co)?/papers/(\d{4}\.\d{4,5})(?:[^\"']*)?[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _organization_name(value: Any) -> str | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
name = value.get("fullname") or value.get("name")
|
||||
return str(name).strip() if name else None
|
||||
|
||||
|
||||
def paper_info_from_payload(item: dict[str, Any]) -> PaperInfo:
|
||||
"""Normalize either a daily-list item or a paper-detail response."""
|
||||
nested = item.get("paper")
|
||||
paper = nested if isinstance(nested, dict) else item
|
||||
arxiv_id = str(paper.get("id") or "").strip()
|
||||
if not ARXIV_ID_PATTERN.fullmatch(arxiv_id):
|
||||
raise ValueError(f"Invalid arXiv id from Hugging Face: {arxiv_id!r}")
|
||||
|
||||
authors = [
|
||||
str(author["name"]).strip()
|
||||
for author in paper.get("authors") or []
|
||||
if isinstance(author, dict) and author.get("name")
|
||||
]
|
||||
|
||||
organization = paper.get("organization") or item.get("organization")
|
||||
return PaperInfo(
|
||||
arxiv_id=arxiv_id,
|
||||
title=str(paper.get("title") or item.get("title") or "").strip(),
|
||||
summary=str(paper.get("summary") or item.get("summary") or "").strip(),
|
||||
authors=authors,
|
||||
published_at=paper.get("publishedAt") or item.get("publishedAt"),
|
||||
submitted_on_daily_at=paper.get("submittedOnDailyAt"),
|
||||
upvotes=int(paper.get("upvotes") or 0),
|
||||
organization=_organization_name(organization),
|
||||
github_repo=paper.get("githubRepo"),
|
||||
github_stars=paper.get("githubStars"),
|
||||
project_page=paper.get("projectPage"),
|
||||
thumbnail=item.get("thumbnail"),
|
||||
)
|
||||
|
||||
|
||||
def paper_ids_from_html(content: str) -> list[str]:
|
||||
"""Return unique paper ids in server-rendered display order."""
|
||||
return list(dict.fromkeys(_PAPER_LINK_PATTERN.findall(content)))
|
||||
|
||||
|
||||
class HuggingFacePapersClient:
|
||||
"""Fetch ranked paper pages and their accompanying JSON metadata."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: httpx.AsyncClient | None = None,
|
||||
timeout: float = 30.0,
|
||||
max_retries: int = 3,
|
||||
detail_concurrency: int = 5,
|
||||
) -> None:
|
||||
self._owns_client = client is None
|
||||
self.client = client or httpx.AsyncClient(
|
||||
base_url=HF_BASE_URL,
|
||||
timeout=timeout,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": "ReMe daily-paper cookbook"},
|
||||
)
|
||||
self.max_retries = max(1, int(max_retries))
|
||||
self.detail_concurrency = max(1, int(detail_concurrency))
|
||||
|
||||
async def __aenter__(self) -> "HuggingFacePapersClient":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args) -> None:
|
||||
if self._owns_client:
|
||||
await self.client.aclose()
|
||||
|
||||
async def _get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
response = await self.client.get(path, params=params)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = exc
|
||||
if attempt + 1 >= self.max_retries:
|
||||
break
|
||||
await asyncio.sleep(0.25 * (2**attempt))
|
||||
assert last_error is not None
|
||||
raise last_error
|
||||
|
||||
async def _json_list(self, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
response = await self._get("/api/daily_papers", params=params)
|
||||
payload = response.json()
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError(f"Unexpected Hugging Face daily_papers response: {type(payload).__name__}")
|
||||
return [item for item in payload if isinstance(item, dict)]
|
||||
|
||||
async def fetch_detail(self, arxiv_id: str) -> PaperInfo:
|
||||
"""Fetch and normalize one paper-detail response."""
|
||||
if not ARXIV_ID_PATTERN.fullmatch(arxiv_id):
|
||||
raise ValueError(f"Invalid arXiv id: {arxiv_id!r}")
|
||||
response = await self._get(f"/api/papers/{arxiv_id}")
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"Unexpected paper response for {arxiv_id}")
|
||||
return paper_info_from_payload(payload)
|
||||
|
||||
async def fetch_daily_ids(self, day: str) -> set[str]:
|
||||
"""Return paper ids for exactly one Hugging Face Daily Papers date."""
|
||||
items = await self._json_list({"date": day, "limit": 100})
|
||||
return {paper_info_from_payload(item).arxiv_id for item in items}
|
||||
|
||||
async def fetch_scope(self, scope: str, value: str) -> list[PaperInfo]:
|
||||
"""Fetch all paper cards shown on one weekly or monthly page."""
|
||||
if scope not in {"week", "month"}:
|
||||
raise ValueError("scope must be 'week' or 'month'")
|
||||
|
||||
page_response, api_items = await asyncio.gather(
|
||||
self._get(f"/papers/{scope}/{value}"),
|
||||
self._json_list({scope: value, "limit": 100}),
|
||||
)
|
||||
page_ids = paper_ids_from_html(page_response.text)
|
||||
api_infos = [paper_info_from_payload(item) for item in api_items]
|
||||
api_by_id = {paper.arxiv_id: paper for paper in api_infos}
|
||||
ranked_ids = page_ids or [paper.arxiv_id for paper in api_infos]
|
||||
|
||||
missing_ids = [arxiv_id for arxiv_id in ranked_ids if arxiv_id not in api_by_id]
|
||||
if missing_ids:
|
||||
details = await self._fetch_details_limited(missing_ids)
|
||||
api_by_id.update({paper.arxiv_id: paper for paper in details})
|
||||
|
||||
return [api_by_id[arxiv_id] for arxiv_id in ranked_ids if arxiv_id in api_by_id]
|
||||
|
||||
async def _fetch_details_limited(self, arxiv_ids: Iterable[str]) -> list[PaperInfo]:
|
||||
semaphore = asyncio.Semaphore(self.detail_concurrency)
|
||||
|
||||
async def fetch(arxiv_id: str) -> PaperInfo:
|
||||
async with semaphore:
|
||||
return await self.fetch_detail(arxiv_id)
|
||||
|
||||
return list(await asyncio.gather(*(fetch(arxiv_id) for arxiv_id in arxiv_ids)))
|
||||
985
skills/dingtalk-message/README.md
Normal file
985
skills/dingtalk-message/README.md
Normal file
|
|
@ -0,0 +1,985 @@
|
|||
# 钉钉消息发送技能 (dingtalk-message)
|
||||
|
||||
钉钉消息发送 CLI 工具,支持**企业内部机器人**和 **Webhook 自定义机器人**两种接入方式,支持文本、Markdown、链接、ActionCard、FeedCard、文件等多种消息类型。
|
||||
|
||||
> 官方文档:https://open.dingtalk.com/document/development/development-robot-overview
|
||||
|
||||
## 目录
|
||||
|
||||
- [API 接口文档](#api-接口文档)
|
||||
- [企业内部机器人](#一企业内部机器人-api)
|
||||
- [批量发送单聊消息](#1-批量发送人与机器人会话中机器人消息)
|
||||
- [发送群聊消息](#2-发送群聊消息)
|
||||
- [获取 Access Token](#3-获取-access-token)
|
||||
- [上传媒体文件](#4-上传媒体文件)
|
||||
- [Webhook 自定义机器人](#二webhook-自定义机器人-api)
|
||||
- [Webhook 概述](#webhook-概述)
|
||||
- [安全设置](#安全设置)
|
||||
- [Webhook 消息类型](#webhook-消息类型)
|
||||
- [CLI 使用指南](#cli-使用指南)
|
||||
- [配置说明](#配置说明)
|
||||
- [消息类型详解](#消息类型详解)
|
||||
- [Markdown 语法限制](#钉钉-markdown-语法限制)
|
||||
- [错误码参考](#错误码参考)
|
||||
- [注意事项与限制](#注意事项与限制)
|
||||
|
||||
---
|
||||
|
||||
## API 接口文档
|
||||
|
||||
### 一、企业内部机器人 API
|
||||
|
||||
### 1. 批量发送人与机器人会话中机器人消息
|
||||
|
||||
> 官方文档:https://open.dingtalk.com/document/development/chatbots-send-one-on-one-chat-messages-in-batches
|
||||
|
||||
#### 接口概述
|
||||
|
||||
调用本接口批量发送人与机器人会话(即人与机器人的单聊)中的机器人消息。通过该接口,企业内部应用的机器人可以向多个用户同时发送单聊消息。
|
||||
|
||||
#### 请求方式
|
||||
|
||||
- **HTTP 方法**:`POST`
|
||||
- **URL**:`https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend`
|
||||
|
||||
#### 请求头
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `x-acs-dingtalk-access-token` | String | 是 | 调用服务端接口的授权凭证(access_token) |
|
||||
| `Content-Type` | String | 是 | 固定值:`application/json` |
|
||||
|
||||
#### 请求体参数(Body)
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `robotCode` | String | 是 | 机器人的编码(robotCode),可在钉钉开发者后台的机器人管理页面获取 |
|
||||
| `userIds` | List\<String\> | 是 | 接收消息的用户 userId 列表,最多支持 **100** 个 |
|
||||
| `msgKey` | String | 是 | 消息模板 Key,用于指定消息类型,见下方[消息类型 msgKey 映射表](#消息类型-msgkey-映射表) |
|
||||
| `msgParam` | String | 是 | 消息模板参数,JSON 字符串格式,内容结构取决于 `msgKey` 的类型 |
|
||||
|
||||
#### 消息类型 msgKey 映射表
|
||||
|
||||
| 消息类型 | msgKey 值 | 说明 |
|
||||
|----------|-----------|------|
|
||||
| 文本消息 | `sampleText` | 纯文本消息,支持 @用户 |
|
||||
| 图片消息 | `sampleImage` | 需先上传获取 media_id |
|
||||
| 语音消息 | `sampleVoice` | 需先上传获取 media_id |
|
||||
| 文件消息 | `sampleFile` | 需先上传获取 media_id |
|
||||
| 链接消息 | `sampleLink` | 带缩略图的链接卡片 |
|
||||
| Markdown 消息 | `sampleMarkdown` | 支持有限 Markdown 语法 |
|
||||
| ActionCard(单按钮) | `sampleActionCard` | 单按钮交互卡片 |
|
||||
| ActionCard(多按钮) | `sampleMultiActionCard` | 多按钮交互卡片 |
|
||||
|
||||
|
||||
#### 各消息类型 msgParam 结构
|
||||
|
||||
**文本消息 (`sampleText`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"content": "消息内容",
|
||||
"atUserIds": ["userId1", "userId2"] // 可选,@指定用户
|
||||
}
|
||||
```
|
||||
|
||||
**Markdown 消息 (`sampleMarkdown`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "消息标题",
|
||||
"text": "#### 标题\n> 引用内容\n正文"
|
||||
}
|
||||
```
|
||||
|
||||
**链接消息 (`sampleLink`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "链接标题",
|
||||
"text": "链接描述",
|
||||
"messageUrl": "https://example.com",
|
||||
"picUrl": "https://example.com/image.png" // 可选,缩略图
|
||||
}
|
||||
```
|
||||
|
||||
**图片消息 (`sampleImage`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"mediaId": "@lADPxxxxxxxx",
|
||||
"caption": "图片描述" // 可选
|
||||
}
|
||||
```
|
||||
|
||||
**文件消息 (`sampleFile`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"mediaId": "@lADPxxxxxxxx",
|
||||
"fileName": "报告.pdf",
|
||||
"fileSize": "1024", // 可选,单位:字节
|
||||
"fileType": "pdf" // 可选
|
||||
}
|
||||
```
|
||||
|
||||
**语音消息 (`sampleVoice`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"mediaId": "@lADPxxxxxxxx",
|
||||
"duration": "10", // 语音时长,单位:秒
|
||||
"fileSize": "2048" // 可选,单位:字节
|
||||
}
|
||||
```
|
||||
|
||||
**ActionCard 单按钮 (`sampleActionCard`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "卡片标题",
|
||||
"markdown": "#### 内容标题\n正文",
|
||||
"singleTitle": "查看详情",
|
||||
"singleUrl": "https://example.com"
|
||||
}
|
||||
```
|
||||
|
||||
**ActionCard 多按钮 (`sampleMultiActionCard`)**
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "卡片标题",
|
||||
"markdown": "#### 内容标题\n正文",
|
||||
"btnOrientation": "0", // "0"=竖排,"1"=横排
|
||||
"btns": [
|
||||
{"title": "同意", "url": "https://example.com/approve"},
|
||||
{"title": "拒绝", "url": "https://example.com/reject"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### 响应参数
|
||||
|
||||
| 参数名 | 类型 | 说明 |
|
||||
|--------|------|------|
|
||||
| `processQueryKey` | String | 消息发送任务的查询 Key,可用于查询发送结果 |
|
||||
|
||||
**成功响应示例:**
|
||||
|
||||
```json
|
||||
{
|
||||
"processQueryKey": "msgTaskId_xxx"
|
||||
}
|
||||
```
|
||||
|
||||
**错误响应示例:**
|
||||
|
||||
```json
|
||||
{
|
||||
"code": "InvalidParameter.RobotCode",
|
||||
"message": "robotCode is invalid",
|
||||
"requestid": "xxxx-xxxx-xxxx"
|
||||
}
|
||||
```
|
||||
|
||||
#### 请求示例(cURL)
|
||||
|
||||
```bash
|
||||
curl -X POST 'https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend' \
|
||||
-H 'x-acs-dingtalk-access-token: YOUR_ACCESS_TOKEN' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"robotCode": "dingxxxxxxxx",
|
||||
"userIds": ["user001", "user002"],
|
||||
"msgKey": "sampleText",
|
||||
"msgParam": "{\"content\": \"Hello, 这是一条测试消息!\"}"
|
||||
}'
|
||||
```
|
||||
|
||||
#### 请求示例(Python)
|
||||
|
||||
```python
|
||||
import requests
|
||||
import json
|
||||
|
||||
url = "https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend"
|
||||
headers = {
|
||||
"x-acs-dingtalk-access-token": "YOUR_ACCESS_TOKEN",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
payload = {
|
||||
"robotCode": "dingxxxxxxxx",
|
||||
"userIds": ["user001", "user002"],
|
||||
"msgKey": "sampleText",
|
||||
"msgParam": json.dumps({"content": "Hello, 这是一条测试消息!"})
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload)
|
||||
print(response.json())
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 发送群聊消息
|
||||
|
||||
> 官方文档:https://open.dingtalk.com/document/orgapp/the-robot-sends-a-group-message
|
||||
|
||||
#### 接口概述
|
||||
|
||||
调用本接口向指定群聊发送机器人消息。机器人需要先加入群聊才能发送消息。
|
||||
|
||||
#### 请求方式
|
||||
|
||||
- **HTTP 方法**:`POST`
|
||||
- **URL**:`https://api.dingtalk.com/v1.0/robot/groupMessages/send`
|
||||
|
||||
#### 请求头
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `x-acs-dingtalk-access-token` | String | 是 | access_token |
|
||||
| `Content-Type` | String | 是 | 固定值:`application/json` |
|
||||
|
||||
#### 请求体参数(Body)
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `robotCode` | String | 是 | 机器人编码 |
|
||||
| `openConversationId` | String | 是 | 群聊会话 ID |
|
||||
| `msgKey` | String | 是 | 消息模板 Key |
|
||||
| `msgParam` | String | 是 | 消息模板参数,JSON 字符串 |
|
||||
|
||||
#### 响应参数
|
||||
|
||||
| 参数名 | 类型 | 说明 |
|
||||
|--------|------|------|
|
||||
| `processQueryKey` | String | 消息发送任务的查询 Key |
|
||||
|
||||
---
|
||||
|
||||
### 3. 获取 Access Token
|
||||
|
||||
> 官方文档:https://open.dingtalk.com/document/orgapp/obtain-the-access_token-of-an-internal-app
|
||||
|
||||
#### 请求方式
|
||||
|
||||
- **HTTP 方法**:`GET`
|
||||
- **URL**:`https://oapi.dingtalk.com/gettoken`
|
||||
|
||||
#### 查询参数(Query)
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `appkey` | String | 是 | 应用的 AppKey |
|
||||
| `appsecret` | String | 是 | 应用的 AppSecret |
|
||||
|
||||
#### 响应参数
|
||||
|
||||
| 参数名 | 类型 | 说明 |
|
||||
|--------|------|------|
|
||||
| `errcode` | Number | 错误码,0 表示成功 |
|
||||
| `errmsg` | String | 错误信息 |
|
||||
| `access_token` | String | 访问凭证 |
|
||||
| `expires_in` | Number | 有效期(秒),通常为 7200 |
|
||||
|
||||
#### 响应示例
|
||||
|
||||
```json
|
||||
{
|
||||
"errcode": 0,
|
||||
"errmsg": "ok",
|
||||
"access_token": "xxxxxx",
|
||||
"expires_in": 7200
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. 上传媒体文件
|
||||
|
||||
> 用于发送图片、语音、文件类消息前的文件上传。
|
||||
|
||||
#### 请求方式
|
||||
|
||||
- **HTTP 方法**:`POST`
|
||||
- **URL**:`https://oapi.dingtalk.com/media/upload`
|
||||
|
||||
#### 查询参数(Query)
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `access_token` | String | 是 | 访问凭证 |
|
||||
|
||||
#### 请求体参数(multipart/form-data)
|
||||
|
||||
| 参数名 | 类型 | 必填 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `type` | String | 是 | 文件类型:`image`/`voice`/`file` |
|
||||
| `media` | File | 是 | 上传的文件 |
|
||||
|
||||
#### 响应参数
|
||||
|
||||
| 参数名 | 类型 | 说明 |
|
||||
|--------|------|------|
|
||||
| `errcode` | Number | 错误码 |
|
||||
| `errmsg` | String | 错误信息 |
|
||||
| `media_id` | String | 媒体文件 ID,后续发送消息时使用 |
|
||||
| `type` | String | 文件类型 |
|
||||
| `created_at` | Number | 创建时间戳(毫秒) |
|
||||
|
||||
---
|
||||
|
||||
### 二、Webhook 自定义机器人 API
|
||||
|
||||
> 官方文档:https://open.dingtalk.com/document/orgapp/custom-bot-to-send-group-chat-messages
|
||||
|
||||
#### Webhook 概述
|
||||
|
||||
自定义机器人是一种可以直接添加到钉钉群聊的机器人,通过 Webhook URL 推送消息到群聊。与企业内部机器人不同,不需要创建应用、不需要 OAuth Token 流程,只需 POST JSON 到 Webhook URL 即可。
|
||||
|
||||
**适用场景:**
|
||||
- 监控告警通知
|
||||
- CI/CD 构建通知
|
||||
- 定时数据报告推送
|
||||
- 简单的群聊消息推送
|
||||
|
||||
**Webhook URL 格式:**
|
||||
|
||||
```
|
||||
https://oapi.dingtalk.com/robot/send?access_token=XXXXXX
|
||||
```
|
||||
|
||||
#### 请求方式
|
||||
|
||||
- **HTTP 方法**:`POST`
|
||||
- **Content-Type**:`application/json; charset=utf-8`
|
||||
- **频率限制**:每个机器人每分钟最多发送 **20** 条消息
|
||||
|
||||
#### 安全设置
|
||||
|
||||
创建自定义机器人时,必须至少选择以下三种安全方式之一:
|
||||
|
||||
**方式一:自定义关键词**
|
||||
|
||||
- 最多设置 10 个关键词
|
||||
- 消息内容必须包含至少一个关键词,否则被拒绝
|
||||
|
||||
**方式二:IP 地址白名单**
|
||||
|
||||
- 配置允许的 IP 地址或 CIDR 段
|
||||
- 仅允许白名单内的 IP 发送消息
|
||||
|
||||
**方式三:加签(推荐)**
|
||||
|
||||
启用加签后,每次请求需要在 URL 中附加 `timestamp` 和 `sign` 参数。
|
||||
|
||||
**签名算法:**
|
||||
|
||||
```
|
||||
timestamp = 当前毫秒时间戳
|
||||
string_to_sign = timestamp + "\n" + secret
|
||||
sign = URL_Encode(Base64(HMAC-SHA256(secret, string_to_sign)))
|
||||
```
|
||||
|
||||
**签名后的 URL:**
|
||||
|
||||
```
|
||||
https://oapi.dingtalk.com/robot/send?access_token=XXXXXX×tamp=1609459200000&sign=YYYY
|
||||
```
|
||||
|
||||
> 时间戳与服务器时间差不能超过 1 小时,否则请求被拒绝。
|
||||
> 脚本已内置加签逻辑,只需在配置文件中设置 `webhook_secret` 即可自动签名。
|
||||
|
||||
**Python 签名示例:**
|
||||
|
||||
```python
|
||||
import time, hmac, hashlib, base64, urllib.parse
|
||||
|
||||
def generate_sign(secret: str):
|
||||
timestamp = str(round(time.time() * 1000))
|
||||
string_to_sign = f'{timestamp}\n{secret}'
|
||||
hmac_code = hmac.new(
|
||||
secret.encode('utf-8'),
|
||||
string_to_sign.encode('utf-8'),
|
||||
digestmod=hashlib.sha256
|
||||
).digest()
|
||||
sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
|
||||
return timestamp, sign
|
||||
```
|
||||
|
||||
#### Webhook 消息类型
|
||||
|
||||
##### 文本消息 (text)
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": "消息内容"
|
||||
},
|
||||
"at": {
|
||||
"atMobiles": ["13800138000"],
|
||||
"atUserIds": ["user123"],
|
||||
"isAtAll": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### Markdown 消息 (markdown)
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"title": "标题(通知栏显示)",
|
||||
"text": "#### 标题\n> 引用内容\n正文"
|
||||
},
|
||||
"at": {
|
||||
"atMobiles": ["13800138000"],
|
||||
"isAtAll": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### 链接消息 (link)
|
||||
|
||||
> 不支持 @功能
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "link",
|
||||
"link": {
|
||||
"title": "链接标题",
|
||||
"text": "链接描述",
|
||||
"messageUrl": "https://example.com",
|
||||
"picUrl": "https://example.com/image.png"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### ActionCard 单按钮 (actionCard)
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "actionCard",
|
||||
"actionCard": {
|
||||
"title": "卡片标题",
|
||||
"text": "#### 内容\n正文(支持 Markdown)",
|
||||
"btnOrientation": "0",
|
||||
"singleTitle": "阅读全文",
|
||||
"singleURL": "https://example.com/"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
##### ActionCard 多按钮 (actionCard)
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "actionCard",
|
||||
"actionCard": {
|
||||
"title": "卡片标题",
|
||||
"text": "#### 内容\n正文",
|
||||
"btnOrientation": "0",
|
||||
"btns": [
|
||||
{"title": "同意", "actionURL": "https://example.com/approve"},
|
||||
{"title": "拒绝", "actionURL": "https://example.com/reject"}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> 注意:多按钮时使用 `btns` + `actionURL`,不要同时设置 `singleTitle`/`singleURL`
|
||||
|
||||
##### FeedCard 消息 (feedCard)
|
||||
|
||||
> 不支持 @功能
|
||||
|
||||
```json
|
||||
{
|
||||
"msgtype": "feedCard",
|
||||
"feedCard": {
|
||||
"links": [
|
||||
{
|
||||
"title": "链接标题1",
|
||||
"messageURL": "https://example.com/1",
|
||||
"picURL": "https://example.com/pic1.png"
|
||||
},
|
||||
{
|
||||
"title": "链接标题2",
|
||||
"messageURL": "https://example.com/2",
|
||||
"picURL": "https://example.com/pic2.png"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Webhook 响应
|
||||
|
||||
**成功:**
|
||||
|
||||
```json
|
||||
{
|
||||
"errcode": 0,
|
||||
"errmsg": "ok"
|
||||
}
|
||||
```
|
||||
|
||||
**错误示例:**
|
||||
|
||||
```json
|
||||
{
|
||||
"errcode": 310000,
|
||||
"errmsg": "keywords not in content"
|
||||
}
|
||||
```
|
||||
|
||||
#### 请求示例(cURL)
|
||||
|
||||
```bash
|
||||
curl -X POST 'https://oapi.dingtalk.com/robot/send?access_token=XXXXXX' \
|
||||
-H 'Content-Type: application/json; charset=utf-8' \
|
||||
-d '{
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": "Hello, 这是一条 Webhook 测试消息!"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
#### Webhook 与企业内部机器人对比
|
||||
|
||||
| 维度 | Webhook 自定义机器人 | 企业内部机器人 |
|
||||
|------|---------------------|---------------|
|
||||
| **接入方式** | 群聊添加自定义机器人 | 钉钉开放平台创建应用 |
|
||||
| **API 域名** | `oapi.dingtalk.com/robot/send` | `api.dingtalk.com/v1.0/robot/` |
|
||||
| **认证方式** | URL 中的 access_token + 可选签名 | Header 中的 OAuth access_token |
|
||||
| **配置项** | webhook_url, webhook_secret | app_key, app_secret, robot_code |
|
||||
| **消息范围** | 仅所在群聊 | 任意用户/群聊 |
|
||||
| **消息格式** | `msgtype` + 类型对象 | `msgKey` + `msgParam` (JSON 字符串) |
|
||||
| **频率限制** | 20 条/分钟 | 20 次/秒 |
|
||||
| **Token 管理** | 无需刷新(URL 固定) | Token 每 2 小时过期 |
|
||||
|
||||
---
|
||||
|
||||
## CLI 使用指南
|
||||
|
||||
### 安装依赖
|
||||
|
||||
```bash
|
||||
pip install requests
|
||||
```
|
||||
|
||||
### 命令行格式
|
||||
|
||||
```bash
|
||||
# 企业内部机器人
|
||||
python scripts/dingtalk.py <消息类型> [选项参数] "<消息内容>" --users <用户ID>
|
||||
|
||||
# Webhook 自定义机器人(命令以 webhook- 前缀开头)
|
||||
python scripts/dingtalk.py webhook-<消息类型> [选项参数] "<消息内容>"
|
||||
```
|
||||
|
||||
### 企业内部机器人命令
|
||||
|
||||
#### 发送文本消息
|
||||
|
||||
```bash
|
||||
# 单聊
|
||||
python scripts/dingtalk.py text "Hello, 这是一条测试消息!" --users user001,user002
|
||||
|
||||
# @指定用户
|
||||
python scripts/dingtalk.py text "Hello, @user001" --users user001,user002 --at-users user001
|
||||
|
||||
# 群聊
|
||||
python scripts/dingtalk.py text "大家好!" \
|
||||
--mode group \
|
||||
--conversation-id chatxxxxxxxxxxxxxxxx \
|
||||
--at-mobiles 13800138000,13900139000
|
||||
```
|
||||
|
||||
#### 发送 Markdown 消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py markdown \
|
||||
--title "天气提醒" \
|
||||
--users user001 \
|
||||
"#### 杭州天气\n> 9度,西北风1级"
|
||||
```
|
||||
|
||||
#### 发送链接消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py link \
|
||||
--title "时代在进步" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
--pic-url "https://example.com/image.png" \
|
||||
--users user001 \
|
||||
"点击查看详情"
|
||||
```
|
||||
|
||||
#### 发送 ActionCard 消息
|
||||
|
||||
```bash
|
||||
# 单按钮
|
||||
python scripts/dingtalk.py action-card \
|
||||
--title "审批通知" \
|
||||
--single-title "查看详情" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
--users user001 \
|
||||
"#### 请假申请\n请审批"
|
||||
|
||||
# 多按钮
|
||||
python scripts/dingtalk.py action-card \
|
||||
--title "审批通知" \
|
||||
--buttons "同意,https://approve.com/yes;拒绝,https://approve.com/no" \
|
||||
--btn-orientation 0 \
|
||||
--users user001 \
|
||||
"#### 请假申请"
|
||||
```
|
||||
|
||||
#### 发送文件
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py file \
|
||||
--users user001,user002 \
|
||||
--file /path/to/report.pdf \
|
||||
--file-name "月度报告.pdf"
|
||||
```
|
||||
|
||||
### Webhook 自定义机器人命令
|
||||
|
||||
> 所有 Webhook 命令以 `webhook-` 前缀开头。
|
||||
> Webhook URL 可通过 `--webhook-url` 参数传入,或在配置文件中设置 `webhook_url`。
|
||||
|
||||
#### Webhook 发送文本消息
|
||||
|
||||
```bash
|
||||
# 使用配置文件中的 webhook_url
|
||||
python scripts/dingtalk.py webhook-text "Hello, 这是一条 Webhook 消息!"
|
||||
|
||||
# 通过参数指定 webhook URL
|
||||
python scripts/dingtalk.py webhook-text \
|
||||
--webhook-url "https://oapi.dingtalk.com/robot/send?access_token=xxx" \
|
||||
"Hello, 测试消息"
|
||||
|
||||
# @指定手机号
|
||||
python scripts/dingtalk.py webhook-text \
|
||||
--at-mobiles 13800138000,13900139000 \
|
||||
"通知内容 @13800138000"
|
||||
|
||||
# @所有人
|
||||
python scripts/dingtalk.py webhook-text --at-all "全员通知"
|
||||
```
|
||||
|
||||
#### Webhook 发送 Markdown 消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-markdown \
|
||||
--title "天气提醒" \
|
||||
"#### 杭州天气\n> 9度,西北风1级"
|
||||
```
|
||||
|
||||
#### Webhook 发送链接消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-link \
|
||||
--title "时代在进步" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
--pic-url "https://example.com/image.png" \
|
||||
"点击查看详情"
|
||||
```
|
||||
|
||||
#### Webhook 发送 ActionCard 消息
|
||||
|
||||
```bash
|
||||
# 单按钮
|
||||
python scripts/dingtalk.py webhook-action-card \
|
||||
--title "审批通知" \
|
||||
--single-title "查看详情" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
"#### 请假申请\n请审批"
|
||||
|
||||
# 多按钮
|
||||
python scripts/dingtalk.py webhook-action-card \
|
||||
--title "审批通知" \
|
||||
--buttons "同意,https://approve.com/yes;拒绝,https://approve.com/no" \
|
||||
--btn-orientation 0 \
|
||||
"#### 请假申请"
|
||||
```
|
||||
|
||||
#### Webhook 发送 FeedCard 消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-feed-card \
|
||||
--links "新闻标题1,https://news1.com,https://img1.com/pic.png;新闻标题2,https://news2.com,https://img2.com/pic.png"
|
||||
```
|
||||
|
||||
#### Webhook 使用加签
|
||||
|
||||
```bash
|
||||
# 通过命令行参数
|
||||
python scripts/dingtalk.py webhook-text \
|
||||
--webhook-url "https://oapi.dingtalk.com/robot/send?access_token=xxx" \
|
||||
--webhook-secret "SECxxxxxxxxxxxxxxxxxxxxxxxxxx" \
|
||||
"带签名的消息"
|
||||
|
||||
# 或在配置文件中设置(推荐)
|
||||
python scripts/dingtalk.py config --set webhook_secret=SECxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
```
|
||||
|
||||
### 查看帮助
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py --help
|
||||
python scripts/dingtalk.py text --help
|
||||
python scripts/dingtalk.py webhook-text --help
|
||||
python scripts/dingtalk.py webhook-markdown --help
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 配置说明
|
||||
|
||||
### 配置文件
|
||||
|
||||
配置统一存储在系统配置目录,所有 AI agent 共享,无需重复配置:
|
||||
|
||||
| 平台 | 配置文件 | 状态文件 |
|
||||
|------|---------|---------|
|
||||
| macOS / Linux | `~/.config/dingtalk/config.json` | `~/.config/dingtalk/state.json` |
|
||||
| Windows | `%APPDATA%\dingtalk\config.json` | `%APPDATA%\dingtalk\state.json` |
|
||||
|
||||
```json
|
||||
{
|
||||
"default_robot": "技术告警群",
|
||||
"robots": [
|
||||
{
|
||||
"name": "技术告警群",
|
||||
"type": "webhook",
|
||||
"description": "发送技术告警到后端技术群",
|
||||
"webhook_token": "YOUR_ACCESS_TOKEN",
|
||||
"webhook_secret": ""
|
||||
},
|
||||
{
|
||||
"name": "内部通知机器人",
|
||||
"type": "app",
|
||||
"description": "企业内部机器人,支持单聊和群聊",
|
||||
"app_key": "YOUR_APP_KEY",
|
||||
"app_secret": "YOUR_APP_SECRET",
|
||||
"robot_code": "YOUR_ROBOT_CODE",
|
||||
"agent_id": "YOUR_AGENT_ID"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 参数获取说明
|
||||
|
||||
#### 企业内部机器人
|
||||
|
||||
| 参数 | 说明 | 获取方式 | 必填 |
|
||||
|------|------|----------|------|
|
||||
| `app_key` | 应用 AppKey | 钉钉开放平台 > 应用详情 > 凭证与基础信息 | 是 |
|
||||
| `app_secret` | 应用 AppSecret | 钉钉开放平台 > 应用详情 > 凭证与基础信息 | 是 |
|
||||
| `robot_code` | 机器人编号 | 钉钉开放平台 > 应用详情 > 机器人与消息推送 | 是 |
|
||||
| `agent_id` | 应用 AgentID | 钉钉开放平台 > 应用详情 > 凭证与基础信息 | 否 |
|
||||
|
||||
#### Webhook 自定义机器人
|
||||
|
||||
| 参数 | 说明 | 获取方式 | 必填 |
|
||||
|------|------|----------|------|
|
||||
| `webhook_url` | Webhook 地址 | 钉钉群 > 群设置 > 智能群助手 > 添加自定义机器人 > 复制 Webhook | 是 |
|
||||
| `webhook_secret` | 加签密钥 | 创建机器人时选择"加签"安全方式,复制 SEC 开头的密钥 | 否* |
|
||||
|
||||
> *如果安全设置选择了"加签"方式,则 webhook_secret 必填。
|
||||
> 两种机器人的配置可以同时存在于同一个配置文件中,互不影响。
|
||||
|
||||
### 配置管理命令
|
||||
|
||||
```bash
|
||||
# 初始化配置文件
|
||||
python scripts/dingtalk.py config --init
|
||||
|
||||
# 查看当前配置
|
||||
python scripts/dingtalk.py config --show
|
||||
|
||||
# 设置单个配置项
|
||||
python scripts/dingtalk.py config --set app_key=dingxxxxxxxxxxxx
|
||||
|
||||
# 添加机器人
|
||||
python scripts/dingtalk.py robot-add --name "技术告警群" --type webhook --webhook-token "token_xxx"
|
||||
python scripts/dingtalk.py robot-add --name "内部通知" --type app --app-key dingxxx --app-secret xxx --robot-code robot-xxx
|
||||
|
||||
# 强制覆盖已存在的配置
|
||||
python scripts/dingtalk.py config --init --force
|
||||
```
|
||||
|
||||
### 命令行参数覆盖
|
||||
|
||||
配置文件中的值可以通过命令行参数临时覆盖:
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py text "测试消息" \
|
||||
--users user001 \
|
||||
--app-key dingxxxxxxxxxxxx \
|
||||
--app-secret your-secret-key \
|
||||
--robot-code robot-xxxxxx
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 消息类型详解
|
||||
|
||||
| 类型 | CLI 指令 | 模式 | 说明 |
|
||||
|------|----------|------|------|
|
||||
| 文本消息 | `text` | 单聊/群聊 | 纯文本,支持 @用户 |
|
||||
| Markdown | `markdown` | 单聊/群聊 | 有限 Markdown 语法的富文本 |
|
||||
| 链接消息 | `link` | 单聊/群聊 | 带缩略图的链接卡片 |
|
||||
| ActionCard | `action-card` | 单聊/群聊 | 带按钮的交互卡片(单/多按钮) |
|
||||
| 文件消息 | `file` | 仅单聊 | 发送文件,自动上传 |
|
||||
| 图片消息 | `image` | 仅单聊 | 发送图片 |
|
||||
| 语音消息 | `voice` | 仅单聊 | 发送语音 |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 钉钉 Markdown 语法限制
|
||||
|
||||
钉钉的 Markdown 渲染器只支持标准 Markdown 的有限子集。
|
||||
|
||||
### 支持的语法
|
||||
|
||||
| 语法 | 写法 | 说明 |
|
||||
|------|------|------|
|
||||
| 标题 | `# 一级标题` ~ `###### 六级标题` | 正常支持 |
|
||||
| 加粗 | `**粗体文字**` | 正常支持 |
|
||||
| 链接 | `[链接文字](url)` | 正常支持 |
|
||||
| 图片 | `` | 正常支持 |
|
||||
| 无序列表 | `- 列表项` | 正常支持 |
|
||||
| 有序列表 | `1. 列表项` | 正常支持 |
|
||||
| 引用 | `> 引用文字` | 正常支持 |
|
||||
|
||||
### 不支持的语法(禁止使用)
|
||||
|
||||
以下语法在钉钉中**不会被渲染**,甚至可能导致显示异常:
|
||||
|
||||
- 水平分隔线:`---`、`***`、`___`
|
||||
- 表格:`| col1 | col2 |`
|
||||
- 代码块:`` ``` ``
|
||||
- 行内代码:`` `code` ``
|
||||
- 删除线:`~~text~~`
|
||||
- 任务列表:`- [ ] item`
|
||||
- 斜体:`*italic*`(不稳定)
|
||||
- 嵌套列表:多级缩进列表(显示不稳定)
|
||||
|
||||
### 编写建议
|
||||
|
||||
1. 用 `#` ~ `####` 标题组织结构
|
||||
2. 用 `- item` 无序列表展示条目
|
||||
3. 用 `**重点**` 加粗强调关键信息
|
||||
4. 用 `> 备注` 添加补充说明
|
||||
5. 不要用分隔线,用标题或空行替代
|
||||
6. 不要用表格,用列表格式替代
|
||||
7. 在 CLI 中使用 `\n` 表示换行
|
||||
|
||||
---
|
||||
|
||||
## 错误码参考
|
||||
|
||||
### 企业内部机器人
|
||||
|
||||
| 错误码 | 说明 |
|
||||
|--------|------|
|
||||
| 0 | 成功 |
|
||||
| -1 | 系统繁忙,请稍后重试 |
|
||||
| 40001 | access_token 不存在或已过期 |
|
||||
| 40002 | access_token 不合法 |
|
||||
| 40004 | 无效的机器人(robotCode 错误) |
|
||||
| 40007 | 无效的 openConversationId |
|
||||
| 40008 | 无效的消息内容(msgParam 格式错误) |
|
||||
| 40009 | 消息发送失败,用户不在该机器人可见范围 |
|
||||
| 40010 | 消息发送失败,用户未与机器人建立会话 |
|
||||
| 40014 | 无效的 userId |
|
||||
| 40037 | 发送消息过于频繁,已触发限流 |
|
||||
| 40056 | 无效的 agentId |
|
||||
|
||||
### Webhook 自定义机器人
|
||||
|
||||
| 错误码 | 说明 |
|
||||
|--------|------|
|
||||
| 0 | 成功 |
|
||||
| 300001 | 无效的 token / 机器人不存在 |
|
||||
| 310000 | 安全校验失败(关键词不匹配 / IP 不在白名单 / 签名错误 / 时间戳过期) |
|
||||
| 302503 | 频率限制,每分钟超过 20 条 |
|
||||
| 400013 | JSON 格式无效 |
|
||||
|
||||
---
|
||||
|
||||
## 注意事项与限制
|
||||
|
||||
### 企业内部机器人
|
||||
|
||||
#### 频率限制
|
||||
|
||||
- 每个应用每秒钟最多调用 **20 次** 消息发送接口
|
||||
- 超出限制会返回错误码 `40037`
|
||||
|
||||
#### 用户限制
|
||||
|
||||
- 批量发送单聊消息时,`userIds` 每次最多 **100 个**
|
||||
- 超过 100 个需分批发送
|
||||
|
||||
#### 会话前置条件
|
||||
|
||||
- 发送单聊消息前,用户需要先与机器人**建立会话**(用户主动给机器人发送过消息)
|
||||
- 未建立会话的用户会收到错误码 `40010`
|
||||
|
||||
#### 文件上传
|
||||
|
||||
- 发送文件/图片/语音消息前,需要先通过 `/media/upload` 接口上传文件获取 `media_id`
|
||||
- CLI 工具的 `file` 命令已自动集成上传流程
|
||||
|
||||
#### Token 管理
|
||||
|
||||
- `access_token` 有效期为 **7200 秒**(2 小时)
|
||||
- 工具内置缓存机制,过期前 5 分钟自动刷新
|
||||
- 不建议频繁调用 gettoken 接口
|
||||
|
||||
### Webhook 自定义机器人
|
||||
|
||||
#### 频率限制
|
||||
|
||||
- 每个机器人每分钟最多发送 **20 条** 消息
|
||||
- 超出限制返回错误码 `302503`
|
||||
|
||||
#### 安全设置
|
||||
|
||||
- 创建机器人时必须至少选择一种安全方式:自定义关键词、IP 白名单、加签
|
||||
- 使用加签时,时间戳与服务器时间差不能超过 **1 小时**
|
||||
- 脚本内置加签逻辑,配置 `webhook_secret` 后自动签名
|
||||
|
||||
#### 消息范围
|
||||
|
||||
- Webhook 机器人只能在添加到的群聊中发送消息,**不支持单聊**
|
||||
- `link` 和 `feedCard` 消息类型**不支持** @功能
|
||||
|
||||
#### Token 管理
|
||||
|
||||
- Webhook URL 中的 `access_token` 是固定的,无需刷新
|
||||
- 只要机器人未被删除,URL 持续有效
|
||||
|
||||
### 通用
|
||||
|
||||
#### 消息内容
|
||||
|
||||
- Markdown 消息仅支持有限语法子集,见 [Markdown 语法限制](#钉钉-markdown-语法限制)
|
||||
- 消息内容不宜过长,过长建议拆分或使用文件发送
|
||||
- 企业内部机器人的 `msgParam` 需要序列化为 JSON 字符串后传入(脚本已自动处理)
|
||||
|
||||
#### API 域名
|
||||
|
||||
- 企业内部机器人(消息发送):`https://api.dingtalk.com`
|
||||
- 企业内部机器人(gettoken、文件上传):`https://oapi.dingtalk.com`
|
||||
- Webhook 自定义机器人:`https://oapi.dingtalk.com/robot/send`
|
||||
268
skills/dingtalk-message/SKILL.md
Normal file
268
skills/dingtalk-message/SKILL.md
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
---
|
||||
name: dingtalk-message
|
||||
version: 0.3.0
|
||||
description: 钉钉消息发送技能。支持企业内部机器人(批量单聊/群聊)和 Webhook 自定义机器人两种接入方式,支持多机器人管理,支持文本、Markdown、链接、ActionCard、FeedCard等多种消息类型。
|
||||
---
|
||||
|
||||
# 钉钉消息发送技能
|
||||
|
||||
## 概述
|
||||
|
||||
支持两种接入方式:
|
||||
- **Webhook 自定义机器人**:通过 access_token 向群聊发送消息,接入简单
|
||||
- **企业内部机器人**:通过 app_key/app_secret 发送单聊、群聊消息,功能更全
|
||||
|
||||
支持多机器人管理,只有一个时自动使用,多个时按优先级自动选择。
|
||||
|
||||
## 环境要求
|
||||
|
||||
- Python 3.7+
|
||||
- `pip install requests`
|
||||
|
||||
## 首次配置
|
||||
|
||||
### 引导流程
|
||||
|
||||
首次使用此技能时,必须按以下流程引导用户完成配置:
|
||||
|
||||
1. **询问用户的机器人类型和凭证信息**:
|
||||
- Webhook 机器人:需要 `access_token`(和可选的加签密钥 `secret`)
|
||||
- 企业内部机器人:需要 `app_key`、`app_secret`、`robot_code`(和可选的 `agent_id`)
|
||||
|
||||
2. **执行配置命令**:`python scripts/dingtalk.py robot-add --name "机器人名" --type webhook ...`
|
||||
|
||||
3. **验证配置**:`python scripts/dingtalk.py config --show`
|
||||
|
||||
### 配置文件路径
|
||||
|
||||
配置统一存储在系统配置目录,所有 AI agent 共享,无需重复配置:
|
||||
|
||||
| 平台 | 配置文件 | 状态文件 |
|
||||
|------|---------|---------|
|
||||
| macOS / Linux | `~/.config/dingtalk/config.json` | `~/.config/dingtalk/state.json` |
|
||||
| Windows | `%APPDATA%\dingtalk\config.json` | `%APPDATA%\dingtalk\state.json` |
|
||||
|
||||
### 手动编辑配置文件
|
||||
|
||||
也可直接编辑配置文件,每个机器人用 `name` 标识(建议用群名、用途等有意义的名称),`description` 描述用途,方便智能匹配:
|
||||
|
||||
```json
|
||||
{
|
||||
"default_robot": "技术告警群",
|
||||
"robots": [
|
||||
{
|
||||
"name": "技术告警群",
|
||||
"type": "webhook",
|
||||
"description": "发送技术告警到后端技术群",
|
||||
"webhook_token": "你的access_token",
|
||||
"webhook_secret": ""
|
||||
},
|
||||
{
|
||||
"name": "内部通知机器人",
|
||||
"type": "app",
|
||||
"description": "企业内部机器人,支持单聊和群聊",
|
||||
"app_key": "你的AppKey",
|
||||
"app_secret": "你的AppSecret",
|
||||
"robot_code": "你的机器人编号",
|
||||
"agent_id": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
> `webhook_token` 填 access_token 即可,脚本自动拼接完整 URL。
|
||||
|
||||
### 通过命令行添加
|
||||
|
||||
```bash
|
||||
# 添加 Webhook 机器人
|
||||
python scripts/dingtalk.py robot-add --name "技术告警群" --type webhook --webhook-token "access_token_xxx" --desc "发送告警到后端技术群"
|
||||
|
||||
# 添加企业内部机器人
|
||||
python scripts/dingtalk.py robot-add --name "内部通知" --type app --app-key dingxxx --app-secret xxx --robot-code robot-xxx --desc "支持单聊群聊"
|
||||
```
|
||||
|
||||
### 验证配置
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py config --show
|
||||
```
|
||||
|
||||
## 机器人管理
|
||||
|
||||
### 选择逻辑
|
||||
|
||||
- 只配置一个机器人时,自动使用
|
||||
- 多个机器人时:`--robot` 指定 > `default_robot` > 最近使用过的 > 第一个可用的
|
||||
- 用户未明确指定时,可根据机器人的 `description` 和最近消息记录智能匹配,或询问用户
|
||||
|
||||
### 管理命令
|
||||
|
||||
```bash
|
||||
# 添加(--desc 描述用途,便于记忆和智能选择)
|
||||
python scripts/dingtalk.py robot-add --name "技术告警群" --type webhook --webhook-token "access_token_xxx" --desc "发送告警到后端技术群"
|
||||
python scripts/dingtalk.py robot-add --name "内部通知" --type app --app-key dingxxx --app-secret xxx --robot-code robot-xxx --desc "企业内部机器人,支持单聊群聊"
|
||||
|
||||
# 查看所有机器人(含描述、使用次数、最近消息)
|
||||
python scripts/dingtalk.py robot-list
|
||||
|
||||
# 更新描述 / 重命名
|
||||
python scripts/dingtalk.py robot-update --name "技术告警群" --desc "后端+SRE告警群"
|
||||
python scripts/dingtalk.py robot-update --name "alert-bot" --rename "技术告警群"
|
||||
|
||||
# 指定机器人发送
|
||||
python scripts/dingtalk.py webhook-text --robot "技术告警群" "告警消息"
|
||||
|
||||
# 设置默认 / 启用禁用 / 删除
|
||||
python scripts/dingtalk.py robot-default --name "产品日报群"
|
||||
python scripts/dingtalk.py robot-enable --name "技术告警群" --disable
|
||||
python scripts/dingtalk.py robot-remove --name "技术告警群"
|
||||
```
|
||||
|
||||
### 使用记录
|
||||
|
||||
每次发送消息会自动记录摘要到状态文件,包括:
|
||||
- 使用次数、最近使用时间、最近状态
|
||||
- 最近10条消息摘要(消息类型 + 内容前60字)
|
||||
|
||||
通过 `robot-list` 可查看,也用于智能选择机器人。
|
||||
|
||||
## Webhook 消息
|
||||
|
||||
> Webhook 命令以 `webhook-` 前缀开头。
|
||||
> Webhook URL 获取:钉钉群 > 群设置 > 智能群助手 > 添加自定义机器人 > 复制 Webhook 地址
|
||||
|
||||
### 文本消息
|
||||
|
||||
```bash
|
||||
# 已配置时直接发送
|
||||
python scripts/dingtalk.py webhook-text "消息内容"
|
||||
|
||||
# 临时指定 token(无需配置)
|
||||
python scripts/dingtalk.py webhook-text --webhook-token "access_token_xxx" "消息内容"
|
||||
|
||||
# @用户 / @所有人
|
||||
python scripts/dingtalk.py webhook-text --at-mobiles 13800138000 "消息 @13800138000"
|
||||
python scripts/dingtalk.py webhook-text --at-all "全员通知"
|
||||
```
|
||||
|
||||
### Markdown 消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-markdown \
|
||||
--title "天气提醒" \
|
||||
"#### 杭州天气\n> 9度,西北风1级"
|
||||
```
|
||||
|
||||
### 链接消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-link \
|
||||
--title "时代在进步" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
--pic-url "https://example.com/image.png" \
|
||||
"点击查看详情"
|
||||
```
|
||||
|
||||
### ActionCard 消息
|
||||
|
||||
```bash
|
||||
# 单按钮
|
||||
python scripts/dingtalk.py webhook-action-card \
|
||||
--title "审批通知" \
|
||||
--single-title "查看详情" \
|
||||
--url "https://www.dingtalk.com" \
|
||||
"#### 请假申请\n请审批"
|
||||
|
||||
# 多按钮
|
||||
python scripts/dingtalk.py webhook-action-card \
|
||||
--title "审批通知" \
|
||||
--buttons "同意,https://approve.com/yes;拒绝,https://approve.com/no" \
|
||||
--btn-orientation 0 \
|
||||
"#### 请假申请"
|
||||
```
|
||||
|
||||
### FeedCard 消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py webhook-feed-card \
|
||||
--links "新闻1,https://news1.com,https://img1.com/pic.png;新闻2,https://news2.com,https://img2.com/pic.png"
|
||||
```
|
||||
|
||||
### 加签安全
|
||||
|
||||
```bash
|
||||
# 配置加签密钥(一次设置)
|
||||
python scripts/dingtalk.py config --set webhook_secret=SECxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
|
||||
# 或通过参数临时指定
|
||||
python scripts/dingtalk.py webhook-text \
|
||||
--webhook-token "access_token_xxx" \
|
||||
--webhook-secret "SECxxxxxxxxxxxxxxxxxxxxxxxxxx" \
|
||||
"带加签的消息"
|
||||
```
|
||||
|
||||
## 企业内部机器人消息
|
||||
|
||||
### 单聊消息
|
||||
|
||||
```bash
|
||||
# 文本
|
||||
python scripts/dingtalk.py text "Hello!" --users user001,user002
|
||||
|
||||
# Markdown
|
||||
python scripts/dingtalk.py markdown --title "天气提醒" --users user001 \
|
||||
"#### 杭州天气\n> 9度,西北风1级"
|
||||
|
||||
# 链接
|
||||
python scripts/dingtalk.py link --title "时代在进步" --url "https://www.dingtalk.com" \
|
||||
--users user001 "点击查看详情"
|
||||
|
||||
# ActionCard
|
||||
python scripts/dingtalk.py action-card --title "审批通知" \
|
||||
--single-title "查看详情" --url "https://www.dingtalk.com" \
|
||||
--users user001 "#### 请假申请\n请审批"
|
||||
|
||||
# 文件
|
||||
python scripts/dingtalk.py file --users user001 --file /path/to/report.pdf --file-name "月度报告.pdf"
|
||||
```
|
||||
|
||||
### 群聊消息
|
||||
|
||||
```bash
|
||||
python scripts/dingtalk.py text "大家好!" \
|
||||
--mode group \
|
||||
--conversation-id chatxxxxxxxxxxxxxxxx \
|
||||
--at-mobiles 13800138000,13900139000
|
||||
```
|
||||
|
||||
## CLI 语法规则
|
||||
|
||||
1. `content` 是位置参数(无 `--` 前缀),放在可选参数之后
|
||||
2. 换行使用 `\n`,脚本自动处理转换
|
||||
3. `content` 必须用双引号包裹
|
||||
4. 可通过 `--app-key`、`--robot-code` 等参数临时覆盖配置文件
|
||||
|
||||
## 钉钉 Markdown 语法限制
|
||||
|
||||
钉钉 Markdown 只支持有限子集,构造内容时必须遵守:
|
||||
|
||||
**支持:** 标题(`#`)、加粗(`**`)、链接(`[](url)`)、图片(``)、无序列表(`-`)、有序列表(`1.`)、引用(`>`)
|
||||
|
||||
**不支持(禁止使用):** 分隔线(`---`)、表格、代码块、行内代码、删除线、任务列表、斜体、嵌套列表
|
||||
|
||||
> 脚本会自动移除分隔线 `---`,但其他不支持的语法需要手动避免。
|
||||
|
||||
## 常见错误码
|
||||
|
||||
**企业内部机器人:** 40001(token过期) / 40004(无效机器人) / 40009(用户不在可见范围) / 40010(未建立会话) / 40037(发送过频)
|
||||
|
||||
**Webhook:** 300001(无效token) / 310000(签名校验失败) / 302503(频率限制,每分钟20条)
|
||||
|
||||
## 注意事项
|
||||
|
||||
- **频率限制**:企业内部机器人每秒20次,Webhook每分钟20条
|
||||
- **Webhook 仅群聊**:不支持单聊,link/feedCard 不支持 @功能
|
||||
- **单聊前置条件**:用户需先主动给机器人发过消息
|
||||
- **批量限制**:单次最多发送100个用户
|
||||
- **消息长度**:过长内容建议拆分或使用文件发送
|
||||
11
skills/dingtalk-message/package.json
Normal file
11
skills/dingtalk-message/package.json
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"name": "dingtalk-message",
|
||||
"version": "0.3.0",
|
||||
"description": "钉钉消息发送技能。支持企业内部机器人(批量单聊/群聊)和 Webhook 自定义机器人两种接入方式,支持多机器人管理,支持文本、Markdown、链接、ActionCard、FeedCard等多种消息类型。",
|
||||
"publishConfig": {
|
||||
"registry": "https://contextlab.alibaba-inc.com/skill"
|
||||
},
|
||||
"aoneKit": {
|
||||
"generated": true
|
||||
}
|
||||
}
|
||||
1695
skills/dingtalk-message/scripts/dingtalk.py
Normal file
1695
skills/dingtalk-message/scripts/dingtalk.py
Normal file
File diff suppressed because it is too large
Load diff
45
skills/serper-search/SKILL.md
Normal file
45
skills/serper-search/SKILL.md
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
name: serper-search
|
||||
description: Search the public web through Serper instead of the built-in WebSearch tool. Use when an agent needs current information, general web sources, or URLs not available from local files and project APIs. SERPER_API_KEY is available in the environment.
|
||||
---
|
||||
|
||||
# Serper Search
|
||||
|
||||
Use Serper whenever external web search is needed. Prefer local files and project APIs when they already contain the required information.
|
||||
|
||||
## Rules
|
||||
|
||||
- Do not use the built-in WebSearch tool.
|
||||
- Do not expose the API key in answers, logs, committed fixtures, or browser/client code.
|
||||
- Prefer authoritative and primary sources in the returned results.
|
||||
- Refine the query when freshness, date, company, product, or source type matters.
|
||||
|
||||
## API
|
||||
|
||||
- Endpoint: `POST https://google.serper.dev/search`
|
||||
- Auth header: `X-API-KEY: $SERPER_API_KEY`
|
||||
- Content type: `application/json`
|
||||
- Required body: `{"q":"query text"}`
|
||||
|
||||
## Curl
|
||||
|
||||
```bash
|
||||
curl -s --location "https://google.serper.dev/search" \
|
||||
--header "X-API-KEY: $SERPER_API_KEY" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data '{"q":"apple inc"}'
|
||||
```
|
||||
|
||||
## Response Use
|
||||
|
||||
- Prefer `organic` results for normal search answers.
|
||||
- Use `searchParameters` to confirm the query actually sent.
|
||||
- Treat `credits` as usage accounting.
|
||||
- Verify important claims against the linked source rather than treating snippets as complete evidence.
|
||||
- Summarize results with source titles and links; do not copy large passages from snippets or pages.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
1. If DNS/network fails, report it as an environment/network issue and retry only when external network is available.
|
||||
2. If authentication fails, report that `SERPER_API_KEY` is missing or invalid without printing its value.
|
||||
3. If results are irrelevant, refine the query with names, dates, locations, or source domains.
|
||||
|
|
@ -1,11 +1,14 @@
|
|||
"""Tests for shared agent wrapper behavior."""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from reme.components.agent_wrapper import AsAgentWrapper, BaseAgentWrapper, CcAgentWrapper, CodexAgentWrapper
|
||||
from reme.components.agent_wrapper.as_agent_wrapper import WorkspaceBackend
|
||||
from reme.components.agent_wrapper import base_agent_wrapper
|
||||
from reme.components.application_context import ApplicationContext
|
||||
from reme.components import base_component
|
||||
|
||||
|
||||
|
|
@ -55,3 +58,50 @@ def test_init_logs_unknown_when_sdk_distribution_metadata_is_missing(monkeypatch
|
|||
def test_agent_wrappers_declare_sdk_package(wrapper_class, sdk_package):
|
||||
"""Each concrete backend identifies the distribution that provides its SDK."""
|
||||
assert wrapper_class.SDK_PACKAGE == sdk_package
|
||||
|
||||
|
||||
def test_project_path_is_independent_from_runtime_workspace(tmp_path):
|
||||
"""Project assets can live outside the runtime workspace."""
|
||||
workspace = tmp_path / "project" / ".reme"
|
||||
wrapper = _VersionedAgentWrapper(
|
||||
app_context=ApplicationContext(workspace_dir=str(workspace)),
|
||||
project_path="..",
|
||||
)
|
||||
|
||||
assert wrapper.workspace_path == workspace
|
||||
assert wrapper.project_path == tmp_path / "project"
|
||||
assert wrapper.cwd == tmp_path / "project"
|
||||
assert wrapper.project_skills_root == tmp_path / "project" / "skills"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapper_class", [AsAgentWrapper, CcAgentWrapper, CodexAgentWrapper])
|
||||
def test_agent_wrappers_share_project_skill_resolution(tmp_path, wrapper_class):
|
||||
"""Every backend resolves selected skills through the base project root."""
|
||||
workspace = tmp_path / "project" / ".reme"
|
||||
skill = tmp_path / "project" / "skills" / "one"
|
||||
skill.mkdir(parents=True)
|
||||
(skill / "SKILL.md").write_text("# one", encoding="utf-8")
|
||||
kwargs = {
|
||||
"app_context": ApplicationContext(workspace_dir=str(workspace)),
|
||||
"project_path": "..",
|
||||
}
|
||||
if wrapper_class is AsAgentWrapper:
|
||||
kwargs["as_llm"] = ""
|
||||
wrapper = wrapper_class(**kwargs)
|
||||
|
||||
assert wrapper._resolve_project_skills(["one", "one"]) == {"one": skill} # pylint: disable=protected-access
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agentscope_backend_passes_configured_environment_to_bash(tmp_path, monkeypatch):
|
||||
"""AgentScope subprocesses receive config environment values explicitly."""
|
||||
monkeypatch.setenv("REME_AGENT_ENV_TEST", "parent")
|
||||
backend = WorkspaceBackend(str(tmp_path), {"REME_AGENT_ENV_TEST": "configured"})
|
||||
|
||||
result = await backend.exec_shell(
|
||||
[sys.executable, "-c", "import os; print(os.environ['REME_AGENT_ENV_TEST'])"],
|
||||
cwd=str(tmp_path),
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert result.stdout == b"configured\n"
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ def test_ensure_claude_skill_dir_adds_selected_skills_without_replacing_existing
|
|||
project_skills = tmp_path / "skills"
|
||||
(project_skills / "one").mkdir(parents=True)
|
||||
(project_skills / "two").mkdir()
|
||||
for name in ("one", "two"):
|
||||
(project_skills / name / "SKILL.md").write_text(f"# {name}", encoding="utf-8")
|
||||
config_dir = tmp_path / "mem_session" / "claude_config"
|
||||
|
||||
for root in _skill_roots(tmp_path):
|
||||
|
|
@ -55,6 +57,8 @@ def test_ensure_claude_skill_dir_all_adds_each_project_skill(tmp_path):
|
|||
project_skills = tmp_path / "skills"
|
||||
(project_skills / "one").mkdir(parents=True)
|
||||
(project_skills / "two").mkdir()
|
||||
for name in ("one", "two"):
|
||||
(project_skills / name / "SKILL.md").write_text(f"# {name}", encoding="utf-8")
|
||||
config_dir = tmp_path / "mem_session" / "claude_config"
|
||||
|
||||
_wrapper(tmp_path)._ensure_claude_skill_dir(config_dir, "all")
|
||||
|
|
@ -69,6 +73,7 @@ def test_ensure_claude_skill_dir_preserves_existing_directory_link(tmp_path):
|
|||
"""An existing skills link is user-owned and remains untouched."""
|
||||
project_skills = tmp_path / "skills"
|
||||
(project_skills / "one").mkdir(parents=True)
|
||||
(project_skills / "one" / "SKILL.md").write_text("# one", encoding="utf-8")
|
||||
config_dir = tmp_path / "mem_session" / "claude_config"
|
||||
legacy_root = tmp_path / ".claude" / "skills"
|
||||
legacy_root.parent.mkdir(parents=True)
|
||||
|
|
@ -103,6 +108,8 @@ def test_configured_skills_use_latest_sdk_allowlist(tmp_path):
|
|||
project_skills = tmp_path / "skills"
|
||||
(project_skills / "one").mkdir(parents=True)
|
||||
(project_skills / "two").mkdir()
|
||||
for name in ("one", "two"):
|
||||
(project_skills / name / "SKILL.md").write_text(f"# {name}", encoding="utf-8")
|
||||
|
||||
opts = _wrapper(tmp_path)._build_options("hello", skills=["one"])
|
||||
|
||||
|
|
@ -112,6 +119,25 @@ def test_configured_skills_use_latest_sdk_allowlist(tmp_path):
|
|||
assert not (root / "two").exists()
|
||||
|
||||
|
||||
def test_configured_project_path_sources_skills_outside_workspace(tmp_path):
|
||||
"""Claude Code links project skills while keeping sessions in the workspace."""
|
||||
project = tmp_path / "project"
|
||||
workspace = project / ".reme"
|
||||
skill = project / "skills" / "serper-search"
|
||||
skill.mkdir(parents=True)
|
||||
(skill / "SKILL.md").write_text("# Serper Search", encoding="utf-8")
|
||||
wrapper = CcAgentWrapper(
|
||||
app_context=ApplicationContext(workspace_dir=str(workspace)),
|
||||
project_path="..",
|
||||
)
|
||||
|
||||
opts = wrapper._build_options("hello", skills=["serper-search"])
|
||||
|
||||
assert opts.cwd == project
|
||||
assert (project / ".claude" / "skills" / "serper-search").resolve() == skill
|
||||
assert (workspace / "mem_session" / "claude_config" / "skills" / "serper-search").resolve() == skill
|
||||
|
||||
|
||||
def test_sdk_native_system_prompt_preset_is_preserved(tmp_path):
|
||||
"""System prompt dictionaries pass directly to the latest SDK."""
|
||||
opts = _wrapper(tmp_path)._build_options(
|
||||
|
|
@ -130,6 +156,13 @@ def test_sdk_native_system_prompt_preset_is_preserved(tmp_path):
|
|||
}
|
||||
|
||||
|
||||
def test_web_search_is_disallowed_by_default(tmp_path):
|
||||
"""Claude Code keeps its normal tools except for web search."""
|
||||
opts = _wrapper(tmp_path)._build_options("hello")
|
||||
|
||||
assert opts.disallowed_tools == ["WebSearch"]
|
||||
|
||||
|
||||
def test_api_credentials_use_only_wrapper_config(tmp_path, monkeypatch):
|
||||
"""Claude Code credentials do not fall back to ambient or shared LLM configuration."""
|
||||
wrapper = _wrapper(tmp_path)
|
||||
|
|
|
|||
515
tests/unit/test_daily_paper.py
Normal file
515
tests/unit/test_daily_paper.py
Normal file
|
|
@ -0,0 +1,515 @@
|
|||
"""Focused tests for the daily-paper cookbook workflow."""
|
||||
|
||||
import datetime as dt
|
||||
import importlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import frontmatter
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from reme.components import ApplicationContext
|
||||
from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper
|
||||
from reme.components.runtime_context import RuntimeContext
|
||||
from reme.config.config_parser import _load_config
|
||||
from reme.schema import DailyBriefOutput, PaperInfo, PaperNoteOutput, PaperSelection
|
||||
from reme.steps.cookbook.daily_paper import (
|
||||
DailyPaperAnalyzeStep,
|
||||
DailyPaperCollectStep,
|
||||
DailyPaperDigestStep,
|
||||
DailyPaperRankStep,
|
||||
DailyPaperSelectStep,
|
||||
)
|
||||
from reme.steps.cookbook.daily_paper import analyze, collect
|
||||
from reme.steps.cookbook.daily_paper.rank import build_candidate_pool, rrf_score
|
||||
from reme.steps.cookbook.dingtalk import DingTalkMarkdownSendStep
|
||||
from reme.steps.cookbook.dingtalk import send as dingtalk_send
|
||||
from reme.utils import arxiv as arxiv_utils
|
||||
from reme.utils.huggingface_papers import paper_ids_from_html, paper_info_from_payload
|
||||
|
||||
|
||||
class _QueuedAgentWrapper(BaseAgentWrapper):
|
||||
"""Return queued structured responses without contacting an LLM."""
|
||||
|
||||
def __init__(self, outputs: list[dict], **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.outputs = list(outputs)
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def reply(self, inputs, **kwargs) -> dict:
|
||||
"""Record the request and pop the next structured fixture."""
|
||||
self.calls.append({"inputs": inputs, "kwargs": kwargs})
|
||||
return {"structured_output": self.outputs.pop(0), "result": "ok"}
|
||||
|
||||
|
||||
def _paper(arxiv_id: str, *, title: str = "Paper", upvotes: int = 10) -> PaperInfo:
|
||||
return PaperInfo(
|
||||
arxiv_id=arxiv_id,
|
||||
title=title,
|
||||
summary=f"Summary for {title}",
|
||||
authors=["A. Author"],
|
||||
upvotes=upvotes,
|
||||
)
|
||||
|
||||
|
||||
def test_hf_payload_and_html_normalization():
|
||||
"""HF list/detail shapes normalize and HTML rank order de-duplicates."""
|
||||
payload = {
|
||||
"paper": {
|
||||
"id": "2607.16051",
|
||||
"title": "Loop the Loopies!",
|
||||
"summary": "Abstract",
|
||||
"authors": [{"name": "Zitian Gao"}],
|
||||
"upvotes": 53,
|
||||
"githubRepo": "https://github.com/example/repo",
|
||||
},
|
||||
"organization": {"fullname": "IQuest"},
|
||||
}
|
||||
|
||||
paper = paper_info_from_payload(payload)
|
||||
|
||||
assert paper.arxiv_id == "2607.16051"
|
||||
assert paper.authors == ["Zitian Gao"]
|
||||
assert paper.organization == "IQuest"
|
||||
assert paper.github_repo == "https://github.com/example/repo"
|
||||
assert paper_ids_from_html(
|
||||
'<a href="/papers/2607.16051">one</a><a href="/papers/2607.16051">dup</a>'
|
||||
'<a href="/papers/2607.10001">two</a>',
|
||||
) == ["2607.16051", "2607.10001"]
|
||||
|
||||
|
||||
def test_rrf_and_memory_candidate_reserve():
|
||||
"""RRF is exact and the candidate pool preserves a memory-related slot."""
|
||||
general = _paper("2607.10001", title="General model", upvotes=100)
|
||||
memory = _paper("2607.10002", title="Long-term memory for agents", upvotes=1)
|
||||
general.fused_score = rrf_score(1, None, rrf_k=60, weekly_weight=0.7)
|
||||
memory.fused_score = rrf_score(100, None, rrf_k=60, weekly_weight=0.7)
|
||||
|
||||
candidates = build_candidate_pool([general, memory], limit=2, memory_reserve=1)
|
||||
|
||||
assert candidates == [general, memory]
|
||||
assert general.fused_score == pytest.approx(1 / 61)
|
||||
|
||||
|
||||
def test_history_exclusion_reads_prior_frontmatter_only(tmp_path: Path):
|
||||
"""Only prior dated paper notes contribute historical exclusions."""
|
||||
prior = tmp_path / "daily" / "2026-07-20"
|
||||
current = tmp_path / "daily" / "2026-07-21"
|
||||
prior.mkdir(parents=True)
|
||||
current.mkdir(parents=True)
|
||||
(prior / "paper-2607.10001.md").write_text(
|
||||
frontmatter.dumps(frontmatter.Post("body", arxiv_id="2607.10001")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(current / "paper-2607.10002.md").write_text(
|
||||
frontmatter.dumps(frontmatter.Post("body", arxiv_id="2607.10002")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
found = DailyPaperCollectStep.load_historical_arxiv_ids(
|
||||
tmp_path,
|
||||
dt.date(2026, 7, 21),
|
||||
30,
|
||||
"daily",
|
||||
)
|
||||
|
||||
assert found == {"2607.10001"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_pdf_downloads_missing_cache_once(tmp_path: Path, monkeypatch):
|
||||
"""A missing PDF is downloaded atomically and then reused on the next lookup."""
|
||||
requests: list[httpx.Request] = []
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
return httpx.Response(200, content=b"%PDF-downloaded")
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
async_client = httpx.AsyncClient
|
||||
monkeypatch.setattr(
|
||||
arxiv_utils.httpx,
|
||||
"AsyncClient",
|
||||
lambda **kwargs: async_client(transport=transport, **kwargs),
|
||||
)
|
||||
target = tmp_path / "resource" / "papers" / "2607.10001.pdf"
|
||||
client = arxiv_utils.ArxivPdfClient()
|
||||
|
||||
assert await client.download("2607.10001", target) == target
|
||||
assert await client.download("2607.10001", target) == target
|
||||
|
||||
assert target.read_bytes() == b"%PDF-downloaded"
|
||||
assert [str(request.url) for request in requests] == ["https://arxiv.org/pdf/2607.10001"]
|
||||
|
||||
|
||||
def test_standalone_config_uses_only_claude_code_and_eight_am_cron(monkeypatch):
|
||||
"""The standalone config schedules 08:00 and routes all agent work to CC."""
|
||||
for name in (
|
||||
"DINGTALK_APP_KEY",
|
||||
"DINGTALK_APP_SECRET",
|
||||
"DINGTALK_ROBOT_CODE",
|
||||
"DINGTALK_CONVERSATION_IDS",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
config = _load_config("daily_cookbook")
|
||||
|
||||
assert config.get("extends") is None
|
||||
assert config["jobs"]["daily_paper_cron"]["cron"] == "0 8 * * *"
|
||||
steps = config["jobs"]["daily_paper"]["steps"]
|
||||
assert config["jobs"]["daily_paper_cron"]["steps"] == steps
|
||||
agent_steps = [
|
||||
step
|
||||
for step in steps
|
||||
if step["backend"]
|
||||
in {
|
||||
"daily_paper_select_step",
|
||||
"daily_paper_analyze_step",
|
||||
"daily_paper_digest_step",
|
||||
}
|
||||
]
|
||||
assert {step.get("agent_wrapper") for step in agent_steps} == {"claude_code"}
|
||||
assert steps[-1] == {
|
||||
"backend": "dingtalk_markdown_send_step",
|
||||
"input_mapping": {"daily_paper_digest_path": "markdown_path"},
|
||||
"app_key": "",
|
||||
"app_secret": "",
|
||||
"robot_code": "",
|
||||
"conversation_ids": "",
|
||||
"title": "ReMe Daily Paper",
|
||||
"timeout": 15,
|
||||
}
|
||||
assert set(config["components"]["agent_wrapper"]) == {"claude_code"}
|
||||
assert "as_llm" not in config["components"]
|
||||
assert config["components"]["agent_wrapper"]["claude_code"]["project_path"] == ".."
|
||||
assert "skills" not in config["components"]["agent_wrapper"]["claude_code"]
|
||||
|
||||
|
||||
def test_daily_paper_config_passes_dingtalk_environment(monkeypatch):
|
||||
"""The notifier receives all proactive-message settings from the environment."""
|
||||
values = {
|
||||
"DINGTALK_APP_KEY": "app-key",
|
||||
"DINGTALK_APP_SECRET": "app-secret",
|
||||
"DINGTALK_ROBOT_CODE": "robot-code",
|
||||
"DINGTALK_CONVERSATION_IDS": "group-one,group-two",
|
||||
}
|
||||
for name, value in values.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
|
||||
step = _load_config("daily_cookbook")["jobs"]["daily_paper"]["steps"][-1]
|
||||
|
||||
assert {key: step[key] for key in ("app_key", "app_secret", "robot_code", "conversation_ids")} == {
|
||||
"app_key": "app-key",
|
||||
"app_secret": "app-secret",
|
||||
"robot_code": "robot-code",
|
||||
"conversation_ids": "group-one,group-two",
|
||||
}
|
||||
|
||||
|
||||
def test_reme_import_does_not_require_optional_dingtalk_stream():
|
||||
"""Importing ReMe must not eagerly load the core-only DingTalk dependency."""
|
||||
script = """
|
||||
import builtins
|
||||
|
||||
original_import = builtins.__import__
|
||||
|
||||
def guarded_import(name, *args, **kwargs):
|
||||
if name == "dingtalk_stream":
|
||||
raise ModuleNotFoundError("blocked optional dependency")
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
builtins.__import__ = guarded_import
|
||||
import reme
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=Path(__file__).parents[2],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_filters_strict_yesterday_and_writes_outputs(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""The complete mocked pipeline filters yesterday/history and writes linked notes."""
|
||||
papers = {
|
||||
"2607.10001": _paper("2607.10001", title="Best monthly paper", upvotes=100),
|
||||
"2607.10002": _paper("2607.10002", title="Yesterday paper", upvotes=90),
|
||||
"2607.10003": _paper("2607.10003", title="Previously recommended", upvotes=80),
|
||||
}
|
||||
|
||||
prior_dir = tmp_path / "daily" / "2026-07-19"
|
||||
prior_dir.mkdir(parents=True)
|
||||
(prior_dir / "paper-2607.10003.md").write_text(
|
||||
frontmatter.dumps(frontmatter.Post("old", arxiv_id="2607.10003")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
class _FakeHfClient:
|
||||
requested_daily: list[str] = []
|
||||
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
async def fetch_scope(self, scope: str, value: str):
|
||||
"""Return deterministic weekly/monthly fixtures."""
|
||||
if scope == "month":
|
||||
assert value == "2026-07"
|
||||
return list(papers.values())
|
||||
assert value == "2026-W30"
|
||||
return [papers["2607.10001"], papers["2607.10002"]]
|
||||
|
||||
async def fetch_daily_ids(self, day: str):
|
||||
"""Record and return the exact requested day."""
|
||||
self.requested_daily.append(day)
|
||||
return {"2607.10002"}
|
||||
|
||||
async def fake_download(_self, _arxiv_id: str, target: Path):
|
||||
"""Create a minimal cached-PDF fixture."""
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(b"%PDF-fake")
|
||||
return target
|
||||
|
||||
def fake_extract(_self, _path: Path, _max_pages: int, _max_chars: int):
|
||||
"""Return deterministic extracted text."""
|
||||
return "--- PAGE 1 ---\nPaper content", 1, False
|
||||
|
||||
monkeypatch.setattr(collect, "HuggingFacePapersClient", _FakeHfClient)
|
||||
monkeypatch.setattr(analyze.ArxivPdfClient, "download", fake_download)
|
||||
monkeypatch.setattr(analyze.DailyPaperAnalyzeStep, "_extract_pdf_text_sync", fake_extract)
|
||||
|
||||
cc_wrapper = _QueuedAgentWrapper(
|
||||
[
|
||||
{
|
||||
"selection_reasoning": "Best remaining ranked paper.",
|
||||
"selected": [
|
||||
{
|
||||
"arxiv_id": "2607.10001",
|
||||
"rank": 1,
|
||||
"reason": "Strong result",
|
||||
"memory_relevance": "low",
|
||||
},
|
||||
],
|
||||
"alternates": [],
|
||||
},
|
||||
{
|
||||
"description": "Detailed note",
|
||||
"body": "# Detailed reading\n\nEvidence [p. 1].",
|
||||
},
|
||||
{
|
||||
"description": "Five-minute brief",
|
||||
"body": "# 今日论文速读\n\n[[daily/2026-07-21/paper-2607.10001.md]]",
|
||||
},
|
||||
],
|
||||
)
|
||||
app_context = ApplicationContext(
|
||||
workspace_dir=str(tmp_path),
|
||||
resource_dir="external-assets",
|
||||
timezone="Asia/Shanghai",
|
||||
language="zh",
|
||||
)
|
||||
context = RuntimeContext(
|
||||
date="2026-07-21",
|
||||
top_k=1,
|
||||
candidate_limit=2,
|
||||
memory_reserve=0,
|
||||
)
|
||||
|
||||
await DailyPaperCollectStep(app_context=app_context)(context)
|
||||
await DailyPaperRankStep(app_context=app_context)(context)
|
||||
await DailyPaperSelectStep(app_context=app_context, agent_wrapper=cc_wrapper)(context)
|
||||
await DailyPaperAnalyzeStep(app_context=app_context, agent_wrapper=cc_wrapper)(context)
|
||||
await DailyPaperDigestStep(app_context=app_context, agent_wrapper=cc_wrapper)(context)
|
||||
|
||||
assert _FakeHfClient.requested_daily == ["2026-07-20"]
|
||||
assert context.response.metadata["selected_arxiv_ids"] == ["2607.10001"]
|
||||
assert context.response.metadata["excluded_yesterday_count"] == 1
|
||||
assert context.response.metadata["excluded_history_count"] == 1
|
||||
note_path = tmp_path / "daily" / "2026-07-21" / "paper-2607.10001.md"
|
||||
digest_path = tmp_path / "daily" / "2026-07-21" / "daily-paper-brief.md"
|
||||
note = frontmatter.load(note_path)
|
||||
assert note.metadata["arxiv_id"] == "2607.10001"
|
||||
assert note.metadata["source_pdf"] == "[[external-assets/papers/2607.10001.pdf]]"
|
||||
assert (tmp_path / "external-assets" / "papers" / "2607.10001.pdf").is_file()
|
||||
assert "[[daily/2026-07-21/paper-2607.10001.md]]" in digest_path.read_text(
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert not (tmp_path / "metadata" / "daily_paper" / "2026-07-21.json").exists()
|
||||
digest = frontmatter.load(digest_path)
|
||||
assert digest.metadata["selection_reasoning"] == "Best remaining ranked paper."
|
||||
assert digest.metadata["arxiv_ids"] == ["2607.10001"]
|
||||
assert all(set(call["kwargs"]) == {"output_schema"} for call in cc_wrapper.calls)
|
||||
assert [call["kwargs"]["output_schema"] for call in cc_wrapper.calls] == [
|
||||
PaperSelection,
|
||||
PaperNoteOutput,
|
||||
DailyBriefOutput,
|
||||
]
|
||||
analysis_prompt = cc_wrapper.calls[1]["inputs"]
|
||||
assert "长期记忆相关性初筛:low" in analysis_prompt
|
||||
assert "必须先使用代码读取和搜索工具查看当前 ReMe 代码仓库" in analysis_prompt
|
||||
assert "这应当是少数例外:一般情况下不要给建议" in analysis_prompt
|
||||
|
||||
rerun = RuntimeContext(date="2026-07-21")
|
||||
await DailyPaperCollectStep(app_context=app_context)(rerun)
|
||||
assert rerun.response.metadata["skipped"] is True
|
||||
assert rerun.response.metadata["selection"] == {
|
||||
"selection_reasoning": "Best remaining ranked paper.",
|
||||
"selected": [
|
||||
{
|
||||
"arxiv_id": "2607.10001",
|
||||
"rank": 1,
|
||||
"reason": "Strong result",
|
||||
"memory_relevance": "low",
|
||||
},
|
||||
],
|
||||
"alternates": [],
|
||||
}
|
||||
assert rerun.get("daily_paper_digest_path") == "daily/2026-07-21/daily-paper-brief.md"
|
||||
assert _FakeHfClient.requested_daily == ["2026-07-20"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dingtalk_markdown_sends_groups_serially_in_configured_order(tmp_path: Path, monkeypatch):
|
||||
"""The notifier gets one app token and posts once per group in list order."""
|
||||
digest_path = tmp_path / "daily" / "2026-07-21" / "daily-paper-brief.md"
|
||||
digest_path.parent.mkdir(parents=True)
|
||||
digest_path.write_text(
|
||||
frontmatter.dumps(frontmatter.Post("# 今日论文\n\n测试内容", name="daily-paper-brief")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
token_calls = 0
|
||||
seen_payloads: list[dict] = []
|
||||
|
||||
def get_access_token(client):
|
||||
nonlocal token_calls
|
||||
token_calls += 1
|
||||
assert client.credential.client_id == "app-key"
|
||||
assert client.credential.client_secret == "app-secret"
|
||||
return "app-access-token"
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/v1.0/robot/groupMessages/send"
|
||||
assert request.headers["x-acs-dingtalk-access-token"] == "app-access-token"
|
||||
seen_payloads.append(json.loads(request.content))
|
||||
return httpx.Response(200, json={"processQueryKey": f"query-{len(seen_payloads)}"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
transport_kwargs: dict = {}
|
||||
|
||||
def ipv4_transport(**kwargs):
|
||||
transport_kwargs.update(kwargs)
|
||||
return transport
|
||||
|
||||
dingtalk_stream = importlib.import_module("dingtalk_stream")
|
||||
monkeypatch.setattr(dingtalk_stream.DingTalkStreamClient, "get_access_token", get_access_token)
|
||||
monkeypatch.setattr(dingtalk_send.httpx, "AsyncHTTPTransport", ipv4_transport)
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
context = RuntimeContext(markdown_path="daily/2026-07-21/daily-paper-brief.md")
|
||||
|
||||
step = DingTalkMarkdownSendStep(
|
||||
app_context=app_context,
|
||||
app_key="app-key",
|
||||
app_secret="app-secret",
|
||||
robot_code="robot-code",
|
||||
conversation_ids=" group-one,group-two ",
|
||||
title="ReMe Daily Paper",
|
||||
)
|
||||
step.logger = MagicMock()
|
||||
response = await step(context)
|
||||
|
||||
assert token_calls == 1
|
||||
assert transport_kwargs == {"local_address": "0.0.0.0"}
|
||||
assert [payload["openConversationId"] for payload in seen_payloads] == ["group-one", "group-two"]
|
||||
assert all(payload["robotCode"] == "robot-code" for payload in seen_payloads)
|
||||
assert all(payload["msgKey"] == "sampleMarkdown" for payload in seen_payloads)
|
||||
assert [json.loads(payload["msgParam"]) for payload in seen_payloads] == [
|
||||
{"title": "ReMe Daily Paper", "text": "# 今日论文\n\n测试内容"},
|
||||
] * 2
|
||||
assert response.metadata["dingtalk_configured_count"] == 2
|
||||
assert response.metadata["dingtalk_sent_count"] == 2
|
||||
logs = "\n".join(call.args[0] for call in step.logger.info.call_args_list)
|
||||
assert "sending DingTalk Markdown" in logs
|
||||
assert "delivery complete sent=2 total=2" in logs
|
||||
assert all(value not in logs for value in ("app-key", "app-secret", "robot-code", "group-one", "group-two"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dingtalk_markdown_without_conversations_is_a_noop(tmp_path: Path):
|
||||
"""An empty conversation list keeps daily-paper generation usable without DingTalk."""
|
||||
context = RuntimeContext(markdown_path="missing.md")
|
||||
|
||||
response = await DingTalkMarkdownSendStep(app_context=ApplicationContext(workspace_dir=str(tmp_path)))(context)
|
||||
|
||||
assert response.success is True
|
||||
assert response.metadata["dingtalk_configured_count"] == 0
|
||||
assert response.metadata["dingtalk_sent_count"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_existing_daily_paper_is_reused_and_sent_to_dingtalk(tmp_path: Path, monkeypatch):
|
||||
"""An idempotent daily-paper run skips generation but still notifies DingTalk."""
|
||||
digest_path = tmp_path / "daily" / "2026-07-22" / "daily-paper-brief.md"
|
||||
digest_path.parent.mkdir(parents=True)
|
||||
digest_path.write_text(
|
||||
frontmatter.dumps(frontmatter.Post("# 已有日报\n\n复用正文", name="daily-paper-brief")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
seen_payloads: list[dict] = []
|
||||
|
||||
dingtalk_stream = importlib.import_module("dingtalk_stream")
|
||||
monkeypatch.setattr(
|
||||
dingtalk_stream.DingTalkStreamClient,
|
||||
"get_access_token",
|
||||
lambda _client: "app-access-token",
|
||||
)
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen_payloads.append(json.loads(request.content))
|
||||
return httpx.Response(200, json={"processQueryKey": "query-1"})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
|
||||
monkeypatch.setattr(dingtalk_send.httpx, "AsyncHTTPTransport", lambda **_kwargs: transport)
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
context = RuntimeContext(date="2026-07-22")
|
||||
|
||||
await DailyPaperCollectStep(app_context=app_context)(context)
|
||||
response = await DingTalkMarkdownSendStep(
|
||||
app_context=app_context,
|
||||
input_mapping={"daily_paper_digest_path": "markdown_path"},
|
||||
app_key="app-key",
|
||||
app_secret="app-secret",
|
||||
robot_code="robot-code",
|
||||
conversation_ids="existing-group",
|
||||
title="ReMe Daily Paper",
|
||||
)(context)
|
||||
|
||||
assert response.metadata["skipped"] is True
|
||||
assert response.metadata["dingtalk_sent_count"] == 1
|
||||
assert seen_payloads == [
|
||||
{
|
||||
"robotCode": "robot-code",
|
||||
"openConversationId": "existing-group",
|
||||
"msgKey": "sampleMarkdown",
|
||||
"msgParam": json.dumps(
|
||||
{"title": "ReMe Daily Paper", "text": "# 已有日报\n\n复用正文"},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
},
|
||||
]
|
||||
324
tests/unit/test_dingtalk_wait.py
Normal file
324
tests/unit/test_dingtalk_wait.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
"""Focused tests for the DingTalk background agent bridge."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,protected-access
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from reme.components import ApplicationContext, R
|
||||
from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper
|
||||
from reme.config.config_parser import _load_config
|
||||
from reme.enumeration import ChunkEnum, ComponentEnum
|
||||
from reme.schema import StreamChunk
|
||||
from reme.steps.cookbook.dingtalk.wait import DingTalkWaitStep, _CardRenderer, _session_key
|
||||
|
||||
|
||||
class _AgentWrapper(BaseAgentWrapper):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.calls = []
|
||||
|
||||
async def reply(self, inputs, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
async def reply_stream(self, inputs, **kwargs):
|
||||
self.calls.append((inputs, kwargs))
|
||||
session_id = kwargs.get("resume") or "session-1"
|
||||
yield StreamChunk(
|
||||
chunk_type=ChunkEnum.THINK,
|
||||
chunk="检查上下文",
|
||||
block_id="think-1",
|
||||
session_id=session_id,
|
||||
)
|
||||
yield StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1", session_id=session_id)
|
||||
yield StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="回答", session_id=session_id)
|
||||
|
||||
|
||||
class _Card:
|
||||
instances = []
|
||||
|
||||
def __init__(self, _client, _message):
|
||||
self.card_template_id = "template"
|
||||
self.title = None
|
||||
self.markdown = ""
|
||||
self.finished = False
|
||||
self.failed = False
|
||||
self.updates = []
|
||||
self.stream_flags = []
|
||||
self.content = ""
|
||||
self.at_sender = False
|
||||
self.__class__.instances.append(self)
|
||||
|
||||
def set_title_and_logo(self, title, _logo):
|
||||
self.title = title
|
||||
|
||||
def get_card_data(self, flow_status=None):
|
||||
data = {"msgContent": self.markdown}
|
||||
if flow_status is not None:
|
||||
data["flowStatus"] = flow_status
|
||||
return data
|
||||
|
||||
async def async_create_and_send_card(self, _template, _data, at_sender=False):
|
||||
self.at_sender = at_sender
|
||||
return "card-1"
|
||||
|
||||
async def async_streaming(self, _card_id, _key, content, append, finished, failed):
|
||||
self.updates.append(content)
|
||||
self.stream_flags.append((append, finished, failed))
|
||||
self.content = self.content + content if append else content
|
||||
if finished:
|
||||
self.finished = not failed
|
||||
self.failed = failed
|
||||
|
||||
|
||||
class _Handler:
|
||||
def __init__(self):
|
||||
self.dingtalk_client = object()
|
||||
self.replies = []
|
||||
|
||||
def reply_text(self, text, _message):
|
||||
self.replies.append(text)
|
||||
|
||||
|
||||
class _WebSocket:
|
||||
def __init__(self, messages=()):
|
||||
self.messages = list(messages)
|
||||
self.closed = asyncio.Event()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
await self.close()
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self.messages:
|
||||
return self.messages.pop(0)
|
||||
await self.closed.wait()
|
||||
raise StopAsyncIteration
|
||||
|
||||
async def close(self):
|
||||
self.closed.set()
|
||||
|
||||
|
||||
class _StreamClient:
|
||||
TAG_DISCONNECT = "disconnect"
|
||||
|
||||
def __init__(self, route_result=""):
|
||||
self.route_result = route_result
|
||||
self.websocket = None
|
||||
|
||||
def pre_start(self):
|
||||
return None
|
||||
|
||||
def open_connection(self):
|
||||
return {"endpoint": "wss://example.test/connect", "ticket": "ticket"}
|
||||
|
||||
async def keepalive(self, _websocket):
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def route_message(self, _message):
|
||||
return self.route_result
|
||||
|
||||
|
||||
def _message(text="hello", sender="user-1", conversation="cid-1", conversation_type="1"):
|
||||
return SimpleNamespace(
|
||||
text=SimpleNamespace(content=text),
|
||||
sender_staff_id=sender,
|
||||
conversation_id=conversation,
|
||||
conversation_type=conversation_type,
|
||||
)
|
||||
|
||||
|
||||
def test_session_key_uses_conversation_type_id_and_sender():
|
||||
assert _session_key(_message()) == "1:cid-1:user-1"
|
||||
assert _session_key(_message(sender="user-2")) == "1:cid-1:user-2"
|
||||
assert _session_key(_message(conversation="cid-2", conversation_type="2")) == "2:cid-2:user-1"
|
||||
|
||||
|
||||
def test_renderer_keeps_blocks_in_stream_order_and_finishes_with_elapsed_time(monkeypatch):
|
||||
now = 10.0
|
||||
monkeypatch.setattr("reme.steps.cookbook.dingtalk.wait.time.monotonic", lambda: now)
|
||||
renderer = _CardRenderer()
|
||||
deltas = [
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="中间内容", block_id="content-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="", block_id="content-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="分析问题", block_id="think-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="最终答案", block_id="answer-2")),
|
||||
]
|
||||
|
||||
markdown = "".join(deltas)
|
||||
assert "### 💬 Content\n\n中间内容" in markdown
|
||||
assert "### 🧠 Think\n\n分析问题" in markdown
|
||||
assert "### 💬 Answer" not in markdown
|
||||
assert markdown.count("### 💬 Content") == 2
|
||||
assert markdown.index("中间内容") < markdown.index("### 🧠 Think") < markdown.index("最终答案")
|
||||
|
||||
now = 15.5
|
||||
markdown += renderer.finish()
|
||||
assert "### 💬 Answer" not in markdown
|
||||
assert "### 💬 Content\n\n最终答案" in markdown
|
||||
assert markdown.endswith("最终答案\n\n5.5s")
|
||||
assert all(value not in markdown for value in ("ReMe Agent", "question", "正在生成", "执行轨迹"))
|
||||
assert all(value not in markdown for value in ("✅", "### 当前"))
|
||||
|
||||
|
||||
def test_renderer_limits_non_content_blocks_to_100_characters():
|
||||
renderer = _CardRenderer()
|
||||
deltas = [
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="a" * 101, block_id="think-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_RESULT, chunk="b" * 101, block_id="tool-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_RESULT, chunk="", block_id="tool-1")),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="c" * 101, block_id="answer-1")),
|
||||
renderer.finish(),
|
||||
]
|
||||
markdown = "".join(deltas)
|
||||
for character in "ab":
|
||||
assert f"{character * 100}..." in markdown
|
||||
assert character * 101 not in markdown
|
||||
assert "c" * 101 in markdown
|
||||
assert f"{'c' * 100}..." not in markdown
|
||||
|
||||
|
||||
def test_renderer_keeps_tool_call_and_result_visible():
|
||||
renderer = _CardRenderer()
|
||||
tool_call_deltas = [
|
||||
renderer.feed(
|
||||
StreamChunk(
|
||||
chunk_type=ChunkEnum.TOOL_CALL,
|
||||
chunk='{"name":"Read","id":"tool-1"}',
|
||||
block_id="metadata-1",
|
||||
tool_call_name="Read",
|
||||
),
|
||||
),
|
||||
renderer.feed(
|
||||
StreamChunk(
|
||||
chunk_type=ChunkEnum.TOOL_CALL,
|
||||
chunk='{"path":',
|
||||
block_id="tool-1",
|
||||
tool_call_name="Read",
|
||||
),
|
||||
),
|
||||
renderer.feed(
|
||||
StreamChunk(chunk_type=ChunkEnum.TOOL_CALL, chunk='"memory.md"}', block_id="tool-1"),
|
||||
),
|
||||
renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_CALL, chunk="", block_id="tool-1")),
|
||||
]
|
||||
assert tool_call_deltas[:3] == ["", "", ""]
|
||||
|
||||
deltas = [
|
||||
*tool_call_deltas,
|
||||
renderer.feed(
|
||||
StreamChunk(
|
||||
chunk_type=ChunkEnum.TOOL_RESULT,
|
||||
chunk={"content": '{"message":"memory content","count":1}'},
|
||||
block_id="tool-1",
|
||||
),
|
||||
),
|
||||
renderer.finish(),
|
||||
]
|
||||
markdown = "".join(deltas)
|
||||
assert markdown.count("### 🔧 Tool Call") == 1
|
||||
assert "### 🔧 Tool Call · `Read`" in markdown
|
||||
assert '> {\n> "path": "memory.md"\n> }' in markdown
|
||||
assert "```" not in markdown
|
||||
assert '"name": "Read"' not in markdown
|
||||
assert "### 📦 Tool Result" in markdown
|
||||
assert '> "content": {\n> "message": "memory content",\n> "count": 1\n> }' in markdown
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_resume_session_and_clear_only_the_combined_key(tmp_path):
|
||||
app_context = ApplicationContext(workspace_dir=str(tmp_path))
|
||||
wrapper = _AgentWrapper(app_context=app_context)
|
||||
step = DingTalkWaitStep(app_context=app_context, agent_wrapper=wrapper, card_update_interval=0.05)
|
||||
step.logger = MagicMock()
|
||||
handler = _Handler()
|
||||
sdk = SimpleNamespace(
|
||||
AIMarkdownCardInstance=_Card,
|
||||
AICardStatus=SimpleNamespace(PROCESSING="1"),
|
||||
)
|
||||
sessions = {}
|
||||
message = _message()
|
||||
key = _session_key(message)
|
||||
|
||||
await step._handle_message(message, key, sessions, handler, sdk)
|
||||
await step._handle_message(message, key, sessions, handler, sdk)
|
||||
|
||||
assert sessions == {key: "session-1"}
|
||||
assert wrapper.calls == [("hello", {}), ("hello", {"resume": "session-1"})]
|
||||
assert all(card.finished for card in _Card.instances[-2:])
|
||||
assert all(card.stream_flags for card in _Card.instances[-2:])
|
||||
assert all(all(append for append, _finished, _failed in card.stream_flags) for card in _Card.instances[-2:])
|
||||
assert all("### 💬 Content\n\n回答" in card.content for card in _Card.instances[-2:])
|
||||
assert all(sum(update.count("检查上下文") for update in card.updates) == 1 for card in _Card.instances[-2:])
|
||||
assert all(sum(update.count("回答") for update in card.updates) == 1 for card in _Card.instances[-2:])
|
||||
assert all(card.at_sender for card in _Card.instances[-2:])
|
||||
assert all(card.title is None for card in _Card.instances[-2:])
|
||||
|
||||
other_key = _session_key(_message(sender="user-2"))
|
||||
sessions[other_key] = "session-2"
|
||||
await step._handle_message(_message(text="/clear"), key, sessions, handler, sdk)
|
||||
assert sessions == {other_key: "session-2"}
|
||||
assert handler.replies[-1] == "✅ Conversation cleared. The next message will start a new session."
|
||||
logs = "\n".join(call.args[0] for call in step.logger.info.call_args_list)
|
||||
assert "received DingTalk text" in logs
|
||||
assert "completed DingTalk reply" in logs
|
||||
assert "cleared DingTalk session" in logs
|
||||
assert "conversation_type='1' conversation_id='cid-1' sender_staff_id='user-1'" in logs
|
||||
assert all(value not in logs for value in ("hello", "session-1"))
|
||||
|
||||
|
||||
def test_daily_cookbook_registers_one_step_background_wait_job(monkeypatch):
|
||||
for name in ("DINGTALK_APP_KEY", "DINGTALK_APP_SECRET", "DINGTALK_ROBOT_CODE"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
config = _load_config("daily_cookbook")
|
||||
job = config["jobs"]["dingtalk_wait"]
|
||||
assert job["backend"] == "background"
|
||||
assert job["steps"] == [
|
||||
{
|
||||
"backend": "dingtalk_wait_step",
|
||||
"agent_wrapper": "claude_code",
|
||||
"app_key": "",
|
||||
"app_secret": "",
|
||||
"robot_code": "",
|
||||
"card_update_interval": 1.0,
|
||||
"worker_count": 4,
|
||||
},
|
||||
]
|
||||
assert R.get(ComponentEnum.STEP, "dingtalk_wait_step") is DingTalkWaitStep
|
||||
|
||||
|
||||
def test_daily_cookbook_passes_dingtalk_environment_to_step(monkeypatch):
|
||||
monkeypatch.setenv("DINGTALK_APP_KEY", "app-key")
|
||||
monkeypatch.setenv("DINGTALK_APP_SECRET", "app-secret")
|
||||
monkeypatch.setenv("DINGTALK_ROBOT_CODE", "robot-code")
|
||||
step = _load_config("daily_cookbook")["jobs"]["dingtalk_wait"]["steps"][0]
|
||||
assert (step["app_key"], step["app_secret"], step["robot_code"]) == ("app-key", "app-secret", "robot-code")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_client_closes_when_background_stop_is_set(monkeypatch):
|
||||
websocket = _WebSocket()
|
||||
monkeypatch.setattr("websockets.connect", lambda _uri: websocket)
|
||||
stop_event = asyncio.Event()
|
||||
task = asyncio.create_task(DingTalkWaitStep._run_client(_StreamClient(), stop_event))
|
||||
stop_event.set()
|
||||
await asyncio.wait_for(task, timeout=1)
|
||||
assert websocket.closed.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_client_restarts_after_server_disconnect(monkeypatch):
|
||||
websocket = _WebSocket(["{}"])
|
||||
monkeypatch.setattr("websockets.connect", lambda _uri: websocket)
|
||||
with pytest.raises(ConnectionError, match="WebSocket closed"):
|
||||
await DingTalkWaitStep._run_client(_StreamClient("disconnect"), asyncio.Event())
|
||||
Loading…
Add table
Reference in a new issue