From 1687179f842f3c5d8ff54d45a2f65ff78b723072 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:09:39 +0800 Subject: [PATCH] feat: add Auto Fin cookbook and managed outbound proxy support (#392) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add ssh proxy * feat: add ssh proxy * feat: add ssh proxy * feat: add ssh proxy * feat: add prompt * feat: add agent wrapper * feat: add agent wrapper * feat: add agent wrapper * feat: add tushare skill * feat: add tushare skill * feat: add tushare skill * feat: add none stream * chore(deps): update dependency versions in pyproject.toml - Bump claude-agent-sdk from 0.2.123 to 0.2.126 - Upgrade pre-commit to version 4.6.1 or higher - Upgrade pytest to version 9.1.1 or higher * feat(agent_wrapper): add session compaction support and unify session commands - Introduce compact_session method to BaseAgentWrapper and implement it in AsAgentWrapper, CcAgentWrapper, and CodexAgentWrapper - Add session_command module with SessionCommandResult dataclass and handle_session_command function for /clear and /compact commands - Update __init__.py exports to include session_command handlers - Modify DingTalkWaitStep to handle session commands via handle_session_command function - Remove streaming mode from DingTalkWaitStep and simplify reply handling to final Markdown replies only - Add unit tests for session compaction methods and session command handling across wrappers and DingTalk integration - Clean up and remove obsolete streaming and card rendering code from DingTalk wait step - Adjust daily_cookbook.yaml to remove stream and card_update_interval config entries for DingTalk wait step * feat(auto_fin): add Auto Fin simulated portfolio cookbook workflow - Add comprehensive Auto Fin schema exports for multiple models and enums - Implement base class and helpers for Auto Fin analysis steps - Create file, state, and formatting utilities for Auto Fin with atomic file writes and locking - Define Auto Fin pipeline with four analysis agents: backtest, event, portfolio, and US correlation - Register Auto Fin package in cookbook workflows and schema initialization - Add detailed documentation in markdown describing the system design, workflow, and data contracts * feat(outbound_proxy): add application-scoped outbound HTTP proxy components - Introduce BaseOutboundProxy and OutboundProxyEndpoint as core contracts - Implement FixedHttpOutboundProxy for external HTTP proxy integration - Add SshHttpOutboundProxy providing SSH-backed local HTTP proxy tunnels - Register outbound proxy components in component registry and enumeration - Update components package to include outbound_proxy module - Add dependency on pproxy for SSH HTTP proxy bridging - Include comprehensive unit tests covering proxy lifecycle, validation, environment merging, error handling, readiness, and monitoring mechanisms * refactor(network): replace SSH proxy with explicit HTTP outbound proxy - Remove SSH proxy helper implementation and references in codebase - Add support for explicit HTTP proxy URL in arXiv and HuggingFace clients - Modify clients to use async context manager for consistent resource handling - Update daily paper steps to forward outbound proxy configuration explicitly - Change tests to cover new proxy usage model and remove SSH proxy mocks - Add outbound proxy component configuration in daily_cookbook.yaml - Ensure proxy URL usage disables environment trust in HTTP clients - Fix app context component enum access to be defensive against missing keys * feat(agent_wrapper): add managed proxy support for command environments - Introduce BaseOutboundProxy binding in BaseAgentWrapper for outbound proxy management - Add bash_environment and command_proxy_environment properties to apply proxy settings - Update WorkspaceBackend instantiation in AsAgentWrapper to use bash_environment - Inject managed proxy export commands into Claude Code Bash commands via hooks - Enhance CodexAgentWrapper to include managed proxy in shell environment policy - Modify daily_cookbook.yaml steps to specify outbound_proxy as default where needed - Add comprehensive unit tests verifying managed proxy injection and environment isolation - Ensure subprocess_environment remains unchanged while proxy is applied selectively to commands * refactor(memory): replace search job_tools with memory in daily cookbook config - Change workspace_dir default from .reme to reme_workspace - Replace search job_tools with memory across multiple components and jobs - Update descriptions to reflect long-term memory retrieval instead of search - Modify system prompts to instruct using memory for retrieving notes - Adjust unit tests to verify memory job_tools and job presence instead of search - Ensure consistency in configuration and tests for memory backend usage * refactor(config): rename memory to memory_search in daily cookbook config - Change all occurrences of "memory" to "memory_search" in job_tools and job definitions - Update related system prompts to reflect the new memory_search terminology - Modify unit tests to assert the presence of memory_search instead of memory - Ensure consistency across skills, job tools, and backend configurations in multiple components * feat(auto_fin): add deterministic quantitative research and ranking fusion - Introduce new schema models: EtfScore, RankingMetrics, ExtremeAnalysis, DimensionRanking, and FusionRanking to represent deterministic research outputs - Add ranking data to event, backtest, us_correlation, and portfolio analysis outputs - Implement ranking_section renderer to format Top20 scores and diagnostics in Markdown - Develop AutoFinQuantStep for deterministic ETF ranking using TuShare data, Polars, and a custom extremely randomized tree ensemble - Integrate quantitative rankings into backtest and portfolio analysis steps and reports - Extend auto_fin pipeline with new quant_enabled and quant_required config options - Enforce ranking constraints like unique codes, contiguous ranks, and normalized fusion weights - Update analysis YAMLs with rules limiting data freshness, universe, and ranking usage - Incorporate ranking outputs into all major markdown report bodies in Auto Fin pipeline - Add concurrency-limited asynchronous TuShare client to fetch required market data - Introduce cross-sectional rank correlation and NDCG metrics for ranking quality evaluation * feat(auto_fin): implement stage-wise notification and reporting for analysis pipeline - Refactor notification config in daily_cookbook.yaml to support dispatch steps - Update AutoFinNotificationStep to deduplicate notifications per run stage - Add _notify_stage method in pipeline to send notifications for each analysis stage - Implement persistence and notification for event, backtest, US correlation, and portfolio stages - Modify pipeline flow to persist reports and notify after each stage completion - Adjust metadata to track notifications and errors per stage - Update tests to verify stage-wise notification sending and deduplication - Remove older combined report persistence in favor of modular stage handling * feat(auto_fin): add outbound proxy support for Tushare API usage - Introduce BaseOutboundProxy reference in AutoFinPipelineStep and AutoFinQuantStep - Update TushareResearchClient and trade calendar fetch to accept and use proxy URL - Create _ProxiedTushareApi adapter to route Tushare requests via explicit HTTP proxy - Modify create_tushare_api utility to optionally return proxied API client - Add unit tests covering proxy forwarding and client behavior with managed proxies - Ensure proxy usage respects explicit proxy URL over environment fallback - Integrate outbound proxy into data fetching and quantitative research steps * feat(auto_fin): enforce checkpoint time validation and add state models - Introduce AnalysisState base class and specific states for event, backtest, and US correlation analyses - Replace analysis output types with corresponding state classes in run schemas - Add require_checkpoint_reached method to validate decision_at/data_cutoff against current time - Enforce checkpoint time checks before analysis steps in event, backtest, portfolio, and quant analyses - Refactor quant data loading to include adjustment factors and apply price adjustments without fallback - Update analysis YAML docs to require real-time checkpoint validation and forbid using future data - Improve portfolio run serialization by excluding redundant legacy fields and nested proposed actions - Add helper to extract readable sections from persisted checkpoint documents - Fix event analysis output validation to reject events and sources with future timestamps * feat(auto_fin): auto-select latest reached checkpoint if none specified - Extend checkpoint config to accept empty string for auto selection - Add static method to compute latest checkpoint reached by current time - Modify pipeline step to auto-select checkpoint based on trade calendar and time - Adjust force flag default depending on whether checkpoint is explicit or auto - Log details when checkpoint is auto-selected to improve observability - Add comprehensive tests for auto checkpoint selection logic and edge cases - Remove deprecated default and required constraints from force parameter in config * refactor(auto_fin): unify datetime comparison with compare_datetimes utility - Replace direct datetime comparisons with compare_datetimes function calls - Use cmp_to_key with compare_datetimes for sorting datetime tuples and lists - Update validation logic in backtest, event, analysis, and ledger modules for consistent datetime handling - Add unit tests to verify handling of naive and aware datetime comparisons in event and backtest validations - Ensure marked_at and interval_end timestamps are set and compared consistently using compare_datetimes - Improve correctness of ordering and conditional checks related to timestamps throughout auto_fin steps and ledger code * feat(auto_fin): add datetime comparison helper for mixed timezone data - Implement compare_datetimes function to handle naive and aware datetimes - Ensure naive datetime is interpreted in the known timezone of the counterpart - Facilitate comparisons between legacy and timezone-aware Auto Fin data - Add module docstring explaining purpose of the helpers * docs(auto_fin): enforce unique ETF representative per sub-theme in analysis rules - Update backtest.yaml to recommend or highlight only one ETF per sub-theme for ETF analyses - Modify event.yaml to map only one representative ETF per sub-theme, avoiding duplicate recommendations - Revise portfolio.yaml to restrict holdings/buys to a single ETF per sub-theme, preventing repeated buys of highly overlapping ETFs - Adjust us_correlation.yaml to retain only one representative A-share ETF per sub-theme for mapping or recommendation - Add test to verify presence of new sub-theme uniqueness guidance in step prompts * feat(auto_fin): separate draft model and include deterministic fusion ranking - Introduce _PortfolioProposalDraft pydantic model for agent-authored fields before ranking - Discard any "fusion_ranking" data from draft to prevent conflicts with canonical ranking - Modify AutoFinPortfolioStep to receive draft, enrich with fusion_ranking, and produce final output - Update tests to use _PortfolioProposalDraft and validate deterministic fusion ranking propagation - Add async test verifying fusion ranking is correctly set in portfolio output with no errors * refactor(auto_fin): rewrite and simplify Auto Fin schema and steps - Remove legacy Auto Fin analysis step modules and helpers - Replace complex ranking and portfolio models with simplified current-news models - Update schema to focus on news-case workflow with new domain models - Remove A-share decision checkpoints and backtest details from schema - Simplify recommendation and decision output structures - Clean up deprecated state and utility functions - Update Auto Fin steps initialization to new pipeline steps only - Improve uniqueness validation for themes and ETFs in research plan * feat(auto_fin): implement full local cache and analysis workflow for Auto Fin - Add AutoFinDataStep to prepare and cache daily TuShare data with lookback - Add AutoFinAnalysisStep to analyze cached data and generate Markdown report - Implement detailed time window, ETF filtering, and historical case validation - Introduce YAML prompts for planning and decision-making steps - Update .gitignore to include reme_workspace/ - Clean up config and import structure for auto_fin steps - Remove old pipeline.py and consolidate functionality into new modules - Use polars for efficient CSV reading and data processing - Ensure atomic writes and strict JSON serialization for cache files - Enforce rules on news timing, ETF universe, and historical case usage * fix(auto_fin): restrict news data source to '财联社' in analysis and cache - Update analysis templates to specify current news as from '财联社' only - Modify news fetching functions to filter by source '财联社' - Add validation method to check cached news source correctness - Update news caching logic to exclude non-'财联社' news - Enhance unit tests with multiple sources to ensure filtering works - Confirm news API calls include source filter parameter as '财联社' * refactor(auto_fin): convert I/O methods to asynchronous implementations - Change _news, _dataset, and _theme_data methods to async for improved concurrency - Move JSONL and CSV reading operations to asynchronous wrappers using asyncio.to_thread - Remove synchronous _read_jsonl and _read_csv functions, integrate them as static async class methods - Update cache validation methods to async, awaiting I/O operations accordingly - Adjust usage of dataset and news retrieval in analysis step to await asynchronous methods - Add async unit test to validate JSONL reading with unicode line separators - Preserve existing functionality while enabling non-blocking file and data access * fix(nx_file_graph): defer networkx import and improve dependency handling - Move networkx import inside NxFileGraph constructor for lazy loading - Raise ImportError with original exception context if networkx is missing - Remove module-level fallback assignment of nx to None - Expand test to block loading of multiple optional core dependencies eagerly - Change exception type in test from ModuleNotFoundError to AssertionError - Update test comments to reflect broader optional dependency checks * feat(embedding_store): add quota retry delay mechanism for embedding requests - Introduce quota_retry_delay parameter to configure wait time before retry on quota exhaustion - Implement detection of insufficient quota errors in LocalEmbeddingStore without external SDK - Add retry logic with custom delay when quota is insufficient during embedding requests - Update configuration to set max_retries and quota_retry_delay defaults for embedding store - Add unit tests covering quota exhaustion retry behavior with delay and opt-in control - Ensure existing retry behavior remains unchanged if quota_retry_delay is not set * feat(auto_fin): add detailed logging to analysis and data fetching steps - Add _preview static method for bounded diagnostic output in analysis.py - Log prompt start, completion, errors, and validation details in _reply method - Add info logs for major processing steps in execute method of analysis.py - Add debug and info logs for cache validation, data fetching, and pagination in data.py - Log conditions for skipping reports and cache plans in data.py execute method - Log download summaries and cache writes for news and ETF data - Improve error logging with exception details in cache validation functions - Ensure all logs include context such as record counts, paths, and parameters * refactor(auto_fin): overhaul Auto Fin workflow and schema contracts - Replace old Auto Fin schema models with comprehensive new data classes - Remove legacy Auto Fin analysis step in favor of modular agent-based steps - Introduce AutoFinAgentStep for validating structured agent replies - Simplify data cleaning and JSONL writing utilities for news cache - Remove synchronous and asynchronous dataset methods from analysis step - Redefine Auto Fin analysis configuration for 360-day news retention and multi-step pipeline - Remove embedded analysis prompt templates and replace with agent-driven logic - Update __init__.py exports to match new step implementations and remove deprecated classes - Improve error handling and validation in agent step reply processing - Clean up redundant imports and unused code in analysis and data preparation modules * feat(auto_fin): add detailed logging for analysis and data processing steps - Add timing logs to measure agent prompt processing duration in analysis.py - Log news cache hits and news write paths with record counts in data.py - Include detailed info logs for news download start and completion in data.py - Add start, progress, and completion logs with topic and event counts in history.py - Log start and completion of merge step including path and ETF count in merge.py - Add start and done logs with window and news counts in topic.py * feat(auto_fin): enhance schema and steps with detailed ETF and event modeling - Replace and add multiple AutoFin schema classes to support detailed ETF selection, historical research, market analysis, forecast models, and report output with validation - Implement Shanghai timezone normalization and strict validation in schema models - Remove deprecated AutoFin analysis agent step and consolidate reply handling in base step - Introduce AutoFinStep base class with shared helpers for prompt handling, data fetching, logging, and JSONL file operations - Add AutoFinDataStep to manage daily news data complete with schedule validation, caching, and source validation logic - Update cookbook configuration to customize auto_fin step parameters and simplify outbound proxy settings - Refactor imports and clean unused code for better maintainability * feat(auto_fin): introduce detailed historical event resolution and market similarity analysis - Add AutoFinHistoricalEventReference and AutoFinHistoricalSimilarity models for refined event referencing and similarity judgment - Implement validation to ensure non-empty critical fields and uniqueness of historical news IDs - Develop method to resolve Agent-selected historical event references from workspace files with strict path and existence checks - Enrich historical events with market entry and future returns data after resolution - Redesign market step to calculate similarity-weighted ETF forecasts based on matched historical event similarities - Enforce validation on matched historical events for uniqueness and proper weight summation - Simplify merge step output to final Markdown report without YAML frontmatter and redundant fields - Update user instructions for history search, market, and merge steps to reflect new data structures and responsibilities - Adjust test suite to cover new schema and step behavior changes, including enhanced validation and JSON output formats * feat(auto_fin): add new cron jobs and output analysis jsonl - Add new cron jobs auto_fin_1145_cron and auto_fin_1800_cron with auto_fin_steps - Change auto_fin_0930_cron schedule to run Monday to Sunday - Extend merge step to write analysis data to auto_fin_analysis.jsonl - Update unit tests to verify new cron jobs and their steps configuration * fix(auto_fin): improve atomic file write and refresh daily index - Change temporary file naming to include UUID for uniqueness and hidden prefix - Replace atomic write method from using Path.replace to os.replace with safe unlink - Add import and use os.replace for safer file replace operation - Refresh daily index after writing auto finance markdown and JSONL files - Import and call refresh_day_index in merge step to update file index asynchronously * docs(cookbook): add optional SSH proxy configuration in README files - Introduce optional SSH proxy setup in auto-fin and daily_paper cookbooks - Provide instructions to enable outbound proxy via `daily_cookbook.yaml` and environment variables - Add `REME_PROXY_IP` and `REME_PROXY_ACCOUNT` environment variables descriptions in multiple README files - Update English and Chinese README and README_ZH documents with proxy details - Maintain consistent formatting of environment variable tables across documents * fix(file_io): include schema_version in hidden metadata keys - Added "schema_version" to _INDEX_HIDDEN_METADATA_KEYS in _daily_index.py - Updated _render_notes_block to always include additional keys regardless of schema_version fix(deps): move pproxy dependency to later in pyproject.toml - Removed pproxy from early dependencies list - Added pproxy back near the end of dependency list for better ordering fix(outbound_proxy): require pproxy package for ssh_http proxy - Added importlib.util check for pproxy package presence - Raise RuntimeError if pproxy is not installed when using SSH HTTP outbound proxy - Improved error message suggests installing reme-ai with 'core' extra * docs(readme): update News section with new Cookbook workflows - Clarify introduction of optional Cookbooks with Daily Paper and Auto Fin workflows - Update English README to reflect both paper discovery and file-native ETF event research - Revise Chinese README to include financial news and historical market data research capability - Maintain announcement of paper acceptance at Findings of ACL 2026 * feat(auto_fin): add calculation results to final Markdown output - Implement _calculation_results to summarize forecast for each ETF analyzed - Include program-calculated results in the JSON input for the Markdown report - Update YAML template to incorporate calculation results and adjust recommendation rules - Refine recommendation logic to rely on event impact judgments combined with calculation outputs - Modify tests to verify presence of calculation results and updated report content and format * up prompt * fix(keyword_index): ignore non-indexable chunks during keyword sync - Add is_indexable method to base and BM25 keyword index classes to check text tokenizability - Update local file store to exclude non-indexable chunks from expected document IDs to prevent rebuild - Fix JSONL chunker to correctly handle Unicode line separator U+2028 inside JSON strings without splitting - Add test to ensure non-empty but non-indexable chunk does not trigger keyword index rebuild - Add test to verify U+2028 character does not cause incorrect JSONL record splitting --- .gitignore | 1 + README.md | 25 +- README_ZH.md | 59 +- cookbook/auto-fin/README.md | 324 +++++++ cookbook/auto-fin/README_ZH.md | 304 +++++++ cookbook/daily_paper/README.md | 185 ++-- cookbook/daily_paper/README_ZH.md | 225 +++-- example.env | 4 + pyproject.toml | 11 +- reme/components/__init__.py | 2 + reme/components/agent_wrapper/__init__.py | 11 +- .../agent_wrapper/as_agent_wrapper.py | 10 +- .../agent_wrapper/base_agent_wrapper.py | 26 +- .../agent_wrapper/cc_agent_wrapper.py | 37 + .../agent_wrapper/codex_agent_wrapper.py | 14 + .../agent_wrapper/session_command.py | 29 + .../embedding_store/base_embedding_store.py | 2 + .../embedding_store/local_embedding_store.py | 23 +- .../file_chunker/jsonl_file_chunker.py | 19 +- reme/components/file_graph/nx_file_graph.py | 13 +- .../components/file_store/local_file_store.py | 8 + .../keyword_index/base_keyword_index.py | 4 + reme/components/keyword_index/bm25_index.py | 4 + reme/components/outbound_proxy/__init__.py | 12 + reme/components/outbound_proxy/base.py | 54 ++ reme/components/outbound_proxy/fixed_http.py | 40 + reme/components/outbound_proxy/ssh_http.py | 352 ++++++++ reme/config/daily_cookbook.yaml | 497 ++++++++++- reme/enumeration/component_enum.py | 2 + reme/schema/__init__.py | 40 + reme/schema/auto_fin.py | 358 ++++++++ reme/steps/base_step.py | 2 +- reme/steps/cookbook/__init__.py | 4 +- reme/steps/cookbook/auto_fin/__init__.py | 17 + reme/steps/cookbook/auto_fin/_base.py | 239 +++++ reme/steps/cookbook/auto_fin/data.py | 154 ++++ reme/steps/cookbook/auto_fin/history.py | 93 ++ .../steps/cookbook/auto_fin/history_search.py | 287 ++++++ .../cookbook/auto_fin/history_search.yaml | 39 + reme/steps/cookbook/auto_fin/market.py | 134 +++ reme/steps/cookbook/auto_fin/market.yaml | 35 + reme/steps/cookbook/auto_fin/merge.py | 79 ++ reme/steps/cookbook/auto_fin/merge.yaml | 39 + reme/steps/cookbook/auto_fin/topic.py | 210 +++++ reme/steps/cookbook/auto_fin/topic.yaml | 44 + reme/steps/cookbook/daily_paper/_common.py | 12 +- reme/steps/cookbook/daily_paper/analyze.py | 25 +- reme/steps/cookbook/daily_paper/collect.py | 2 + reme/steps/cookbook/dingtalk/wait.py | 304 +------ reme/steps/evolve/dream/extract.py | 1 + reme/steps/file_io/__init__.py | 3 +- reme/steps/file_io/_daily_index.py | 2 +- reme/utils/arxiv.py | 87 +- reme/utils/huggingface_papers.py | 61 +- reme/utils/logger_utils.py | 1 + reme/utils/tushare.py | 65 ++ skills/tushare-data/SKILL.md | 822 ++++++++++++++++++ skills/tushare-data/agents/openai.yaml | 4 + skills/tushare-data/references/数据接口.md | 237 +++++ skills/tushare-data/scripts/fund_data_demo.py | 87 ++ .../tushare-data/scripts/stock_data_demo.py | 88 ++ tests/unit/test_auto_dream.py | 23 + tests/unit/test_auto_fin.py | 734 ++++++++++++++++ tests/unit/test_base_agent_wrapper.py | 64 +- tests/unit/test_cc_agent_wrapper.py | 99 +++ tests/unit/test_codex_agent_wrapper.py | 71 +- tests/unit/test_config_parser.py | 8 + tests/unit/test_daily_paper.py | 301 ++++++- tests/unit/test_dingtalk_wait.py | 247 ++---- tests/unit/test_file_store_consistency.py | 25 + tests/unit/test_jsonl_file_chunker.py | 33 + tests/unit/test_local_embedding_store.py | 71 ++ tests/unit/test_logging_config.py | 20 + tests/unit/test_outbound_proxy.py | 341 ++++++++ tests/unit/test_reme_cli.py | 33 +- 75 files changed, 7205 insertions(+), 737 deletions(-) create mode 100644 cookbook/auto-fin/README.md create mode 100644 cookbook/auto-fin/README_ZH.md create mode 100644 reme/components/agent_wrapper/session_command.py create mode 100644 reme/components/outbound_proxy/__init__.py create mode 100644 reme/components/outbound_proxy/base.py create mode 100644 reme/components/outbound_proxy/fixed_http.py create mode 100644 reme/components/outbound_proxy/ssh_http.py create mode 100644 reme/schema/auto_fin.py create mode 100644 reme/steps/cookbook/auto_fin/__init__.py create mode 100644 reme/steps/cookbook/auto_fin/_base.py create mode 100644 reme/steps/cookbook/auto_fin/data.py create mode 100644 reme/steps/cookbook/auto_fin/history.py create mode 100644 reme/steps/cookbook/auto_fin/history_search.py create mode 100644 reme/steps/cookbook/auto_fin/history_search.yaml create mode 100644 reme/steps/cookbook/auto_fin/market.py create mode 100644 reme/steps/cookbook/auto_fin/market.yaml create mode 100644 reme/steps/cookbook/auto_fin/merge.py create mode 100644 reme/steps/cookbook/auto_fin/merge.yaml create mode 100644 reme/steps/cookbook/auto_fin/topic.py create mode 100644 reme/steps/cookbook/auto_fin/topic.yaml create mode 100644 reme/utils/tushare.py create mode 100644 skills/tushare-data/SKILL.md create mode 100644 skills/tushare-data/agents/openai.yaml create mode 100644 skills/tushare-data/references/数据接口.md create mode 100644 skills/tushare-data/scripts/fund_data_demo.py create mode 100644 skills/tushare-data/scripts/stock_data_demo.py create mode 100644 tests/unit/test_auto_fin.py create mode 100644 tests/unit/test_outbound_proxy.py diff --git a/.gitignore b/.gitignore index 9ece9bef..3ae8ddff 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ temp*/ # ReMe runtime data .reme/ +reme_workspace/ vault/ *.db *.sqlite diff --git a/README.md b/README.md index 73a168ee..0c7cb76f 100644 --- a/README.md +++ b/README.md @@ -49,18 +49,18 @@ memory, then continuously indexes, links, and consolidates that memory for futur - **Personal assistants**: Give personal assistants such as [QwenPaw](https://github.com/agentscope-ai/QwenPaw), [OpenClaw](https://github.com/openclaw/openclaw), and [Hermes](https://github.com/nousresearch/hermes-agent) a user-editable long-term memory layer. -- **Coding agents**: Preserve coding style, project background, repository decisions, and workflow - experience across sessions when integrating with coding agents such as [Claude Code](plugins/reme). -- **LLM Wiki**: Turn conversations, notes, and resources into a searchable, traceable, and linked Markdown - knowledge base that both users and agents can maintain. +- **Coding agents**: Preserve coding style, project background, repository decisions, and workflow experience across + sessions when integrating with coding agents such as [Claude Code](plugins/reme). +- **LLM Wiki**: Turn conversations, notes, and resources into a searchable, traceable, and linked Markdown knowledge + base that both users and agents can maintain. - **Self-evolving agents**: Support agents that learn from experience by saving successful paths, failed attempts, reusable procedures, and periodic reflections as memory. ## 📰 News -- [2026.07] - Introduced optional Cookbook workflows, starting with - [Daily Paper](cookbook/daily_paper/README.md) for scheduled paper discovery, agent-assisted PDF analysis, reusable - Markdown notes, and five-minute briefs. +- [2026.07] - Introduced optional Cookbooks: [Daily Paper](cookbook/daily_paper/README.md) for paper discovery and + analysis, and [Auto Fin](cookbook/auto-fin/README.md) for file-native ETF event research based on CLS news and + historical market reactions. - [2026.07] - Our paper [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/) has been accepted to Findings of ACL 2026. @@ -170,9 +170,10 @@ Cookbooks are optional, end-to-end workflows assembled from ReMe jobs and steps. configuration; select the cookbook's standalone configuration when starting ReMe. Each new cookbook will be added as another row in this table. -| Cookbook | Capability | Introduction | -|-------------|---------------------------------------------------------------------------------------------------------------|------------------------------------------| -| Daily Paper | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. | [README](cookbook/daily_paper/README.md) | +| Cookbook | Capability | +|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| [Daily Paper](cookbook/daily_paper/README.md) | Discover and rank papers, analyze PDFs with an agent, and generate file-native notes and a five-minute brief. | +| [Auto Fin](cookbook/auto-fin/README.md) | Match CLS events to liquid ETFs, study historical reactions, and generate file-native research reports. | ## 📁 Memory System @@ -314,8 +315,8 @@ are mainly for maintenance, debugging, or advanced integration. Run `reme help` - **Issues and requests**: Check [Open Issues](https://github.com/agentscope-ai/ReMe/issues) first. If there is no related discussion, open a new issue with background, expected behavior, and impact scope. - **Code contributions**: Before making changes, read - the [contribution guide](https://docs.agentscope.io/reme/stable/en/contributing). Source, - schemas, and tests are the authoritative architecture and extension guide. + the [contribution guide](https://docs.agentscope.io/reme/stable/en/contributing). Source, schemas, and tests are the + authoritative architecture and extension guide. - **Documentation contributions**: Submit user-facing documentation changes to the [unified documentation repository](https://github.com/agentscope-ai/docs) under `reme//{en,zh}/`. - **Commit convention**: Conventional Commits are recommended, for example `feat(search): add link expansion option` or diff --git a/README_ZH.md b/README_ZH.md index 6ce4ea81..3c6f2378 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -51,8 +51,8 @@ Agent 能够可靠召回。 ## 📰 新闻 -- [2026.07] - 新增可选的 Cookbook 工作流，首个能力为 [每日论文](cookbook/daily_paper/README_ZH.md)，支持定时发现论文、 - Agent 辅助解析 PDF、沉淀可复用的 Markdown 笔记并生成五分钟简报。 +- [2026.07] - 新增可选 Cookbook 工作流：[每日论文](cookbook/daily_paper/README_ZH.md)用于论文发现与解析， + [Auto Fin](cookbook/auto-fin/README_ZH.md)用于结合财联社新闻和历史行情开展文件化 ETF 事件研究。 - [2026.07] - 我们的论文 [Remember Me, Refine Me: A Dynamic Procedural Memory Framework for Experience-Driven Agent Evolution](https://aclanthology.org/2026.findings-acl.829/) 已被 Findings of ACL 2026 接收。 @@ -79,8 +79,8 @@ pip install -e ".[core]" ### 环境变量 -如果需要 LLM 驱动的记忆演化或 embedding 检索，可以配置环境变量。embedding 默认关闭，因此默认配置不会启动 -embedding 模型，也不需要 embedding API key。 +如果需要 LLM 驱动的记忆演化或 embedding 检索，可以配置环境变量。embedding 默认关闭，因此默认配置不会启动 embedding 模型，也不需要 +embedding API key。 ```bash cat > .env <<'EOF' @@ -158,18 +158,19 @@ ReMe 会把 Agent 记忆保存为可读的 Markdown。 ## 🧑‍🍳 Cookbooks -Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们；启动 ReMe 时选择对应的 -独立配置即可启用。后续新增的 cookbook 会继续在表格中按行追加。 +Cookbook 是由 ReMe jobs 和 steps 组装而成的可选端到端工作流。默认配置不会开启它们；启动 ReMe 时选择对应的 独立配置即可启用。后续新增的 +cookbook 会继续在表格中按行追加。 -| Cookbook | 能力 | 介绍 | -|----------|------------------------------------------|-------------------------------------------| -| 每日论文 | 发现并排序论文，使用 Agent 解读 PDF，生成文件化论文笔记和五分钟简报。 | [使用说明](cookbook/daily_paper/README_ZH.md) | +| Cookbook | 能力 | +|-----------------------------------------------|-----------------------------------------------------------------------| +| [每日论文](cookbook/daily_paper/README_ZH.md) | 发现并排序论文，使用 Agent 解读 PDF，生成文件化论文笔记和五分钟简报。 | +| [Auto Fin](cookbook/auto-fin/README_ZH.md) | 将财联社事件匹配到高流动性 ETF，研究历史反应并生成文件化研究报告。 | ## 📁 记忆系统 > Memory as File, File as Memory. -ReMe 将**记忆视为文件**，让原始对话和外部资料从 `session/`、`resource/` 渐进加工到 `daily/`，再沉淀为 `digest/` +ReMe 将 **记忆视为文件**，让原始对话和外部资料从 `session/`、`resource/` 渐进加工到 `daily/`，再沉淀为 `digest/` 中可长期复用的知识节点。 ### 目录结构 @@ -213,13 +214,13 @@ ReMe 将**记忆视为文件**，让原始对话和外部资料从 `session/`、 ReMe 遵循 capture → index → consolidate → recall 的循环。对话和资料先变成 daily 记忆卡片；后台任务保持文件可检索； `auto_dream` 将稳定知识沉淀到 `digest/`；Agent 再通过搜索、wikilink 或 proactive topics 召回记忆。 -| 能力 | 入口 | 作用 | 输出 | -|---------------------------------------------|----------------------------------|----------------------------------------------------|------------------------------------------------------| -| [`auto_memory`](docs/zh/auto_memory.md) | Agent hook 或 `reme auto_memory` | 提炼有长期价值的对话事实，同时保留原始 session。 | `session/dialog/*.jsonl`、`daily//.md` | -| [`auto_resource`](docs/zh/auto_resource.md) | 资源监听或 `reme auto_resource` | 将 `resource//` 下的文件转为带来源链接的 daily 卡片。 | `daily//.md` | -| [`auto_index`](docs/zh/memory_search.md) | 后台监听或 `reme reindex` | 维护 chunks、BM25 索引、wikilink 图谱及可选的 embedding 索引。 | 可检索的 `daily/`、`digest/`、`resource/` 内容 | -| [`auto_dream`](docs/zh/auto_dream.md) | `dream_cron` 或 `reme auto_dream` | 将变化的 daily 卡片整理为长期 personal、procedure 和 wiki 记忆。 | `digest/**`、`daily//interests.yaml` | -| [`proactive`](docs/zh/proactive.md) | Agent 决定主动行动前调用 `reme proactive` | 读取 `auto_dream` 生成的 topics；是否以及如何提醒用户由宿主 Agent 决定。 | 来自 `daily//interests.yaml` 的结构化 topics | +| 能力 | 入口 | 作用 | 输出 | +|---------------------------------------------|-------------------------------------------|--------------------------------------------------------------------------|-------------------------------------------------------| +| [`auto_memory`](docs/zh/auto_memory.md) | Agent hook 或 `reme auto_memory` | 提炼有长期价值的对话事实，同时保留原始 session。 | `session/dialog/*.jsonl`、`daily//.md` | +| [`auto_resource`](docs/zh/auto_resource.md) | 资源监听或 `reme auto_resource` | 将 `resource//` 下的文件转为带来源链接的 daily 卡片。 | `daily//.md` | +| [`auto_index`](docs/zh/memory_search.md) | 后台监听或 `reme reindex` | 维护 chunks、BM25 索引、wikilink 图谱及可选的 embedding 索引。 | 可检索的 `daily/`、`digest/`、`resource/` 内容 | +| [`auto_dream`](docs/zh/auto_dream.md) | `dream_cron` 或 `reme auto_dream` | 将变化的 daily 卡片整理为长期 personal、procedure 和 wiki 记忆。 | `digest/**`、`daily//interests.yaml` | +| [`proactive`](docs/zh/proactive.md) | Agent 决定主动行动前调用 `reme proactive` | 读取 `auto_dream` 生成的 topics；是否以及如何提醒用户由宿主 Agent 决定。 | 来自 `daily//interests.yaml` 的结构化 topics | @@ -245,10 +246,10 @@ ReMe 遵循 capture → index → consolidate → recall 的循环。对话和 ReMe 作为本地记忆服务运行，并提供 CLI、HTTP API、MCP server 和 SDK 等多种接入方式。不同 Agent 可以选择适合自身 runtime 的路径，同时共享同一个本地 memory workspace。 -| Agent | 推荐接入方式 | 开箱可用能力 | -|------------------------------------------------------|-------------------------------------------------------------------|-----------------------------------------------------------------| -| **QwenPaw** | 通过 Python SDK 嵌入 ReMe。 | 复用应用自身生命周期和模型配置，同时保持 memory 本地、文件化。 | -| **Claude Code** | 以 MCP service 启动 ReMe，并安装 [plugins/reme](plugins/reme)。 | MCP recall tools、`reme-memory` skill，以及自动记录会话的 Stop hook。 | +| Agent | 推荐接入方式 | 开箱可用能力 | +|------------------------------------------------------|-------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| **QwenPaw** | 通过 Python SDK 嵌入 ReMe。 | 复用应用自身生命周期和模型配置，同时保持 memory 本地、文件化。 | +| **Claude Code** | 以 MCP service 启动 ReMe，并安装 [plugins/reme](plugins/reme)。 | MCP recall tools、`reme-memory` skill，以及自动记录会话的 Stop hook。 | | **Other CLI-capable agents (OpenClaw/Hermes/Codex)** | 复制或安装 [skills/reme_memory/SKILL.md](skills/reme_memory/SKILL.md)。 | 通过 CLI 搜索/读取/写入记忆，并调用 `auto_memory`、`auto_dream` 和 `proactive`。 |

集成演示

@@ -284,17 +285,17 @@ ReMe 作为本地记忆服务运行，并提供 CLI、HTTP API、MCP server 和 ReMe 通过 CLI 暴露的统一 job interface 操作 workspace。Agent 通常只需要使用检索、读取、写入、编辑和自动记忆相关命令；更底层的索引、 frontmatter 和文件操作接口主要用于维护、调试或高级集成。完整 job 列表可以运行 `reme help` 查看。 -| 命令 | 作用 | -|-------------------------------------------|---------------------------------------------| -| `reme start` | 启动本地 ReMe 服务。 | -| `reme version` / `reme health_check` | 检查包版本和组件状态。 | +| 命令 | 作用 | +|-------------------------------------------|---------------------------------------------------------------| +| `reme start` | 启动本地 ReMe 服务。 | +| `reme version` / `reme health_check` | 检查包版本和组件状态。 | | `reme status` | 查看有状态数据组件的内存估算及进程 RSS。 | | [`reme search`](docs/zh/memory_search.md) | 默认使用 BM25 和 wikilink 检索，启用后增加向量检索。 | -| `reme read` / `reme write` / `reme edit` | 检查和维护 Markdown 记忆文件。 | -| `reme auto_memory` | 将对话 messages 转为 daily 记忆卡片；需要 LLM 凭证。 | +| `reme read` / `reme write` / `reme edit` | 检查和维护 Markdown 记忆文件。 | +| `reme auto_memory` | 将对话 messages 转为 daily 记忆卡片；需要 LLM 凭证。 | | `reme auto_resource` | 将 `resource/` 下的文件解读为 daily 资料卡片；需要 LLM 凭证。 | -| `reme auto_dream` / `reme proactive` | 将 daily 记忆整理为长期 digest，并暴露值得关注的主题。 | -| `reme reindex` | 基于已有文件重建检索和 wikilink 索引。 | +| `reme auto_dream` / `reme proactive` | 将 daily 记忆整理为长期 digest，并暴露值得关注的主题。 | +| `reme reindex` | 基于已有文件重建检索和 wikilink 索引。 | ## 🤝 社区与支持 diff --git a/cookbook/auto-fin/README.md b/cookbook/auto-fin/README.md new file mode 100644 index 00000000..1baa1595 --- /dev/null +++ b/cookbook/auto-fin/README.md @@ -0,0 +1,324 @@ +# Auto Fin Cookbook + +[中文](README_ZH.md) + +Auto Fin is a local-first, file-native ETF event-research workflow. It identifies market events in CLS news, selects +related liquid ETFs, studies similar historical events and subsequent returns, and produces a Chinese research report. + +> Auto Fin provides event research and holding-period references only. It is not investment advice, does not +> connect to a broker, and does not place or simulate trades. + +## Capabilities + +- Download CLS news through Tushare and maintain up to 360 days of traceable local news records. +- Rank ETF candidates by previous-trading-day turnover, then select representative ETFs related to current events. +- Search ReMe memory and local news files for similar historical events, with strict source-path and news-ID checks. +- Calculate adjusted D1–D10 historical returns in deterministic code instead of asking an Agent to invent numbers. +- Let an Agent judge event similarity, then calculate weights, expected returns, and a reference holding period in code. +- Save readable Markdown and structured JSON/JSONL artifacts, refresh the daily index, and optionally deliver the report + to DingTalk. + +The workflow is assembled by +[`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml). Its public schemas are in +[`reme/schema/auto_fin.py`](../../reme/schema/auto_fin.py), and its steps are in +[`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/). + +## Quick start + +Auto Fin requires Python 3.11 or newer, the `core` dependencies, a Tushare token, and credentials for the configured +Claude Code-compatible endpoint. + +From the repository root: + +```bash +python -m pip install -e ".[core]" +export TUSHARE_TOKEN="your-tushare-token" +export CLAUDE_CODE_API_KEY="your-api-key" +reme start config=daily_cookbook job=auto_fin +``` + +The built-in configuration uses `qwen3.7-max` through DashScope's Anthropic-compatible endpoint. Override these +variables to use another compatible model or provider: + +```bash +export CLAUDE_CODE_MODEL_NAME="your-model" +export CLAUDE_CODE_BASE_URL="https://your-anthropic-compatible-endpoint" +``` + +The default workspace is `reme_workspace/`. This standalone cookbook shares its workspace setting with the daily-paper +workflow: + +```bash +export DAILY_PAPER_WORKSPACE_DIR="/absolute/path/to/reme-workspace" +``` + +To deliver the final Markdown report to DingTalk, set: + +```bash +export DINGTALK_APP_KEY="your-app-key" +export DINGTALK_APP_SECRET="your-app-secret" +export DINGTALK_ROBOT_CODE="your-robot-code" +export DINGTALK_CONVERSATION_IDS="conversation-id-1,conversation-id-2" +``` + +DingTalk delivery is skipped when the required values are empty. + +Dates and times use `Asia/Shanghai`. The optional `date` must be the current date: + +```bash +reme start config=daily_cookbook job=auto_fin date=2026-07-25 +``` + +To refresh every configured news day instead of reusing valid historical files: + +```bash +reme start config=daily_cookbook job=auto_fin force=true +``` + +This may issue many Tushare requests. A normal run reuses valid historical news files and always refreshes today's file. + +### Optional SSH proxy + +The outbound proxy is disabled by default. To enable it, uncomment `components.outbound_proxy.default` in +`daily_cookbook.yaml`, configure non-interactive SSH authentication, and set: + +```bash +export REME_PROXY_IP="your-ssh-proxy-host" +export REME_PROXY_ACCOUNT="your-ssh-account" +``` + +## How it works + +```mermaid +flowchart LR + A[Resolve run date and cutoff] --> B[Maintain CLS news files] + B --> C[Resolve previous A-share trading day] + C --> D[Build current event window] + D --> E[Filter liquid ETF candidates] + E --> F[Agent selects related ETFs] + F --> G{For each ETF} + G --> H[Agent searches historical events] + H --> I[Code resolves original news] + I --> J[Code calculates adjusted D1-D10 returns] + J --> K[Agent judges similarity] + K --> L[Code calculates weighted forecast] + L --> G + G --> M[Agent writes the combined report] + M --> N[Write artifacts and refresh daily index] + N --> O[Optional DingTalk delivery] +``` + +The top-level job contains four Auto Fin steps: + +| Step | Responsibility | Agent | +|-------------------------|-------------------------------------------------------------|-------| +| `auto_fin_data_step` | Maintain news files and resolve the previous trading day | No | +| `auto_fin_topic_step` | Build inputs and select related ETFs and current events | Yes | +| `auto_fin_history_step` | Orchestrate historical research and market analysis per ETF | Yes | +| `auto_fin_merge_step` | Validate results and produce the final Markdown report | Yes | + +For each selected ETF, `auto_fin_history_step` dispatches: + +- `auto_fin_history_search_step`, which asks the Agent for historical news references and then resolves the original + records and calculates their returns in code. +- `auto_fin_market_step`, which asks the Agent only for similarity judgments and then calculates weights and forecasts + in code. + +Agents handle semantic judgments; deterministic code handles source validation and financial calculations. + +## Data and time boundaries + +### News history + +`auto_fin_data_step` reads CLS news from Tushare's `major_news` endpoint: + +- The default lookback is 360 calendar days, including the run date. +- A valid historical file is reused unless `force=true`. +- Today's file is refreshed through the current `decision_at` on every run. +- Large responses are fetched through recursively split time windows. +- Records are ordered and deduplicated before being written with a stable `news_id`. + +The current event window is: + +```text +(previous A-share trading day at 15:00, decision_at] +``` + +Each run rebuilds this complete window; midday and evening runs do not use only the increment since the previous run. + +### ETF candidates + +The candidate universe combines: + +- `etf_basic` for currently listed ETFs and their tracked indexes. +- `fund_daily` for turnover on the previous A-share trading day. + +Code sorts candidates by turnover, removes duplicates by ETF name and index identity, and provides at most 150 +candidates to the Topic Agent. The Agent may return at most 20 ETFs and must copy every ETF code, name, and news ID from +the generated candidate files. + +Turnover is used only to narrow the research universe; it is not a trading signal. + +## Historical research and forecasting + +### Source resolution + +The History Agent searches by event type, entities, transmission mechanism, and expected direction. It first uses +`memory_search` and may then scan: + +```text +daily/YYYY-MM-DD/auto_fin_news_data.jsonl +``` + +Its output contains only a reason, `news_id`, and workspace-relative `source_path`. Code rejects: + +- Current-window news presented as historical evidence. +- Absolute paths, `..` traversal, or paths outside the workspace. +- Sources not named `auto_fin_news_data.jsonl`. +- Missing files or IDs that do not resolve exactly once. +- Records without a usable publication time, title, or body. + +Historical Markdown may guide retrieval, but the original news JSONL is the source of truth. + +### Adjusted returns + +For every resolved historical event, code reads `fund_daily` and `fund_adj` and calculates up to ten future closes: + +- Before 09:30 on a trading day: enter at that day's open. +- From 09:30 until before 15:00: enter at that day's close. +- At or after 15:00, or on a non-trading day: enter at the next trading day's open. +- A daily close later than the current `decision_at` is excluded. + +```text +adjusted_entry = raw_entry × entry_adjustment_factor +adjusted_close = raw_close × close_adjustment_factor +cumulative_return = adjusted_close / adjusted_entry - 1 +``` + +Missing prices, factors, trading days, or horizons become explicit limitations. They are never filled with Agent-made +values. + +### Similarity and forecast + +The Market Agent returns semantic similarity in `[-1, 1]`: + +- Positive values mean a similar mechanism and direction. +- Negative values mean a comparable mechanism but opposite direction. +- Zero means no useful relationship. + +Code clamps out-of-range values, ignores zero-similarity events, normalizes weights from absolute similarity, and +reverses the historical return direction for negative matches. Each D1–D10 horizon is calculated from the samples +available at that horizon. The suggested holding period is the positive-return horizon with the highest expected return, +or empty when none is positive. + +The result also records limited samples, missing horizons, conflicting return directions, and other data limitations. It +is a comparison with a small historical sample, not evidence of statistical significance. + +## Output layout + +```text +reme_workspace/ +├── daily/ +│ ├── YYYY-MM-DD.md +│ └── YYYY-MM-DD/ +│ ├── auto_fin_news_data.jsonl +│ ├── auto_fin_analysis.jsonl +│ └── auto_fin.md +└── resource/ + └── YYYY-MM-DD/ + ├── filtered_news.jsonl + ├── filtered_etf.jsonl + ├── auto_fin_topic_output.jsonl + ├── auto_fin_history___output.json + ├── auto_fin_market___output.json + ├── auto_fin_history_output.jsonl + └── auto_fin_merge_output.json +``` + +Important artifacts: + +- `auto_fin_news_data.jsonl` is the user-owned source used to resolve historical news. +- `filtered_news.jsonl` and `filtered_etf.jsonl` are bounded inputs for the Topic Agent. +- Per-ETF history files contain resolved source news and code-calculated return paths. +- Per-ETF market files contain code-calculated matches, weights, and D1–D10 forecasts. +- `auto_fin_analysis.jsonl` contains the final structured analysis for every selected ETF. +- `auto_fin.md` is the readable report and DingTalk payload. +- `daily/YYYY-MM-DD.md` is refreshed after report generation so the report is discoverable from the daily index. + +News and reports remain ordinary user-owned files. Resource artifacts and search indexes can be rebuilt. + +## Configuration + +### Job parameters + +| Parameter | Default | Meaning | +|-----------|-------------:|---------------------------------------------------------| +| `date` | Current date | Strict `YYYY-MM-DD`; only the current date is supported | +| `force` | `false` | Refresh all configured news days | + +### Environment variables + +| Variable | Required | Meaning | +|-----------------------------|----------|---------------------------------------------------------| +| `TUSHARE_TOKEN` | Yes | News, calendar, ETF daily data, and adjustment factors | +| `CLAUDE_CODE_API_KEY` | Yes | Auto Fin Agent credentials | +| `CLAUDE_CODE_MODEL_NAME` | No | Defaults to `qwen3.7-max` | +| `CLAUDE_CODE_BASE_URL` | No | Anthropic-compatible endpoint | +| `AUTO_FIN_AGENT_BACKEND` | No | Defaults to `claude_code` | +| `AUTO_FIN_PROJECT_PATH` | No | Agent project path; defaults to `..` | +| `REME_PROXY_IP` | No | SSH proxy host; used only when `ssh_http` is enabled | +| `REME_PROXY_ACCOUNT` | No | SSH proxy account; used only when `ssh_http` is enabled | +| `DAILY_PAPER_WORKSPACE_DIR` | No | Standalone cookbook workspace | +| `DINGTALK_*` | No | DingTalk application, robot, and conversation settings | + +Unit tests can inject `tushare_provider` through the runtime context and do not require real credentials. + +### Scheduled jobs + +`daily_cookbook.yaml` defines: + +| Job | Cron | Asia/Shanghai | +|----------------------|---------------|----------------| +| `auto_fin_0930_cron` | `30 9 * * *` | Daily at 09:30 | +| `auto_fin_1145_cron` | `45 11 * * *` | Daily at 11:45 | +| `auto_fin_1800_cron` | `0 18 * * *` | Daily at 18:00 | + +These cron expressions do not exclude weekends or market holidays. The workflow resolves the previous A-share trading +day but does not currently skip a run merely because the run date is not a trading day. + +## Agent and security boundaries + +The Auto Fin wrapper loads the `tushare-data` skill, exposes the `memory_search` job tool, and defaults to +`bypassPermissions`. Prompts constrain each Agent's role, while code revalidates schemas, ETF identities, source paths, +news references, and calculated values. + +The standalone cookbook does not configure an embedding store by default, so `memory_search` normally uses BM25 recall. +Vector and BM25 fusion becomes available only when an embedding store is configured. + +`bypassPermissions` is not an operating-system sandbox. Review the configured project path, workspace, credentials, and +network boundary before deployment. + +## Reruns and limitations + +- Valid historical news files are reused; today's news is always refreshed. +- Outputs use stable per-day paths, so a later same-day run replaces the previous report and resource outputs. +- Auto Fin intentionally has no “report exists, skip” shortcut because its scheduled runs analyze updated news. +- Every successful run attempts DingTalk delivery when configured; notification deduplication is not implemented. +- Missing historical market horizons degrade one sample and are recorded as limitations. +- Invalid dates, missing required services, invalid Agent schemas, unknown ETFs or news IDs, unsafe paths, and + inconsistent cross-step ETF identities fail the job. + +The current implementation does not include stocks, US-market correlation, portfolio accounting, BUY/SELL/HOLD actions, +T+1 execution rules, fees, slippage, broker integration, or real/simulated order execution. + +## Development + +Install development dependencies and run the focused suite: + +```bash +python -m pip install -e ".[dev,core]" +PYTHONPATH=. pytest tests/unit/test_auto_fin.py -v +``` + +The unit suite mocks model and market-data boundaries. Tests requiring real Tushare, model, or DingTalk credentials +should be run separately and only with explicit authorization. diff --git a/cookbook/auto-fin/README_ZH.md b/cookbook/auto-fin/README_ZH.md new file mode 100644 index 00000000..35c897c3 --- /dev/null +++ b/cookbook/auto-fin/README_ZH.md @@ -0,0 +1,304 @@ +# Auto Fin Cookbook + +[English](README.md) + +Auto Fin 是一个本地优先、文件原生的 ETF 事件研究工作流。它从财联社新闻中识别市场事件，选择相关且流动性较好的 +ETF，研究相似历史事件及其后续收益，并生成中文研究报告。 + +> Auto Fin 只提供事件研究和持有时间参考，不构成投资建议。当前实现不连接券商、不提交委托，也不执行模拟交易。 + +## 能力 + +- 通过 Tushare 获取财联社新闻，并维护最多 360 天可追溯的本地新闻记录。 +- 按上一交易日成交额筛选 ETF 候选，再选择与当前事件直接相关的代表性 ETF。 +- 通过 ReMe 记忆检索和本地新闻文件查找相似历史事件，并严格校验来源路径和新闻 ID。 +- 由确定性代码计算复权后的 D1–D10 历史收益，不让 Agent 编造行情数值。 +- 由 Agent 判断事件相似度，再由代码计算权重、预期收益和参考持有时间。 +- 保存可读 Markdown 和结构化 JSON/JSONL，刷新每日索引，并支持可选钉钉投递。 + +工作流由 [`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配，公共 schema 位于 +[`reme/schema/auto_fin.py`](../../reme/schema/auto_fin.py)，各步骤位于 +[`reme/steps/cookbook/auto_fin/`](../../reme/steps/cookbook/auto_fin/)。 + +## 快速开始 + +Auto Fin 要求 Python 3.11 或更高版本、`core` 依赖、Tushare token，以及所配置 Claude Code 兼容 endpoint 的凭据。 + +在仓库根目录运行： + +```bash +python -m pip install -e ".[core]" +export TUSHARE_TOKEN="your-tushare-token" +export CLAUDE_CODE_API_KEY="your-api-key" +reme start config=daily_cookbook job=auto_fin +``` + +内置配置默认通过 DashScope 的 Anthropic 兼容 endpoint 使用 `qwen3.7-max`。如需更换兼容模型或服务商： + +```bash +export CLAUDE_CODE_MODEL_NAME="your-model" +export CLAUDE_CODE_BASE_URL="https://your-anthropic-compatible-endpoint" +``` + +默认 workspace 是 `reme_workspace/`。该 standalone cookbook 与每日论文工作流共用 workspace 配置： + +```bash +export DAILY_PAPER_WORKSPACE_DIR="/absolute/path/to/reme-workspace" +``` + +如需把最终 Markdown 报告发送到钉钉： + +```bash +export DINGTALK_APP_KEY="your-app-key" +export DINGTALK_APP_SECRET="your-app-secret" +export DINGTALK_ROBOT_CODE="your-robot-code" +export DINGTALK_CONVERSATION_IDS="conversation-id-1,conversation-id-2" +``` + +相关配置为空时会跳过钉钉投递。 + +日期和时间均使用 `Asia/Shanghai`。可选的 `date` 必须是当天： + +```bash +reme start config=daily_cookbook job=auto_fin date=2026-07-25 +``` + +如需刷新全部新闻日期，而不是复用有效历史文件： + +```bash +reme start config=daily_cookbook job=auto_fin force=true +``` + +这可能产生大量 Tushare 请求。普通运行会复用有效历史新闻，并始终刷新当天文件。 + +### 可选 SSH 代理 + +出站代理默认关闭。如需启用，请取消 `daily_cookbook.yaml` 中 +`components.outbound_proxy.default` 的注释，配置免交互 SSH 认证，并设置： + +```bash +export REME_PROXY_IP="your-ssh-proxy-host" +export REME_PROXY_ACCOUNT="your-ssh-account" +``` + +## 工作原理 + +```mermaid +flowchart LR + A[确定运行日和截止时间] --> B[维护财联社新闻文件] + B --> C[确定上一 A 股交易日] + C --> D[构建当前事件窗口] + D --> E[筛选高流动性 ETF 候选] + E --> F[Agent 选择相关 ETF] + F --> G{逐只 ETF} + G --> H[Agent 搜索历史事件] + H --> I[代码回查原始新闻] + I --> J[代码计算复权 D1-D10 收益] + J --> K[Agent 判断相似度] + K --> L[代码计算加权预测] + L --> G + G --> M[Agent 编写合并报告] + M --> N[写入产物并刷新每日索引] + N --> O[可选钉钉投递] +``` + +顶层 Job 包含四个 Auto Fin Step： + +| Step | 职责 | Agent | +|-------------------------|-----------------------------------|-------| +| `auto_fin_data_step` | 维护新闻文件并确定上一交易日 | 否 | +| `auto_fin_topic_step` | 构建输入并选择相关 ETF 和当前事件 | 是 | +| `auto_fin_history_step` | 逐只 ETF 编排历史研究和行情分析 | 是 | +| `auto_fin_merge_step` | 校验结果并生成最终 Markdown 报告 | 是 | + +对于每只已选 ETF，`auto_fin_history_step` 会派发： + +- `auto_fin_history_search_step`：Agent 返回历史新闻引用，然后由代码解析原始记录并计算收益。 +- `auto_fin_market_step`：Agent 只判断相似度，然后由代码计算权重和预测。 + +Agent 负责语义判断，确定性代码负责来源校验和金融数值计算。 + +## 数据和时间边界 + +### 新闻历史 + +`auto_fin_data_step` 使用 Tushare `major_news` 接口读取财联社新闻： + +- 默认回看包含运行日在内的 360 个自然日。 +- 有效历史文件会复用，除非设置 `force=true`。 +- 每次运行都会把当天文件刷新到当前 `decision_at`。 +- 数据量接近接口上限时会递归拆分时间窗口。 +- 写入前会排序和去重，并生成稳定 `news_id`。 + +当前事件窗口为： + +```text +(上一 A 股交易日 15:00, decision_at] +``` + +每次运行都会重建完整窗口；午间和晚间运行不是只读取上次运行后的增量。 + +### ETF 候选 + +候选池组合使用： + +- `etf_basic`：当前上市 ETF 及其跟踪指数。 +- `fund_daily`：上一 A 股交易日成交额。 + +代码按成交额排序，并按 ETF 名称和指数标识去重，最多向 Topic Agent 提供 150 个候选。Agent 最多返回 20 只 ETF，且 ETF 代码、名称和新闻 +ID 都必须逐字来自候选文件。 + +成交额仅用于缩小研究范围，不是交易信号。 + +## 历史研究和预测 + +### 来源回查 + +History Agent 按事件类型、关键实体、传导机制和预期方向搜索。它优先使用 `memory_search`，必要时扫描： + +```text +daily/YYYY-MM-DD/auto_fin_news_data.jsonl +``` + +Agent 只返回理由、`news_id` 和 workspace 相对 `source_path`。代码会拒绝： + +- 把当前事件窗口内的新闻当作历史证据； +- 绝对路径、`..` 路径穿越或 workspace 外路径； +- 文件名不是 `auto_fin_news_data.jsonl` 的来源； +- 不存在的文件或不能唯一解析的 ID； +- 缺少有效发布时间、标题或正文的记录。 + +历史 Markdown 只能作为检索线索，原始新闻 JSONL 才是事实来源。 + +### 复权收益 + +对每条已回查的历史事件，代码读取 `fund_daily` 和 `fund_adj`，计算最多十个未来收盘点： + +- 交易日 09:30 前发生：以当日开盘价为 entry。 +- 09:30 至 15:00 前发生：以当日收盘价为 entry。 +- 15:00 或之后、以及非交易日发生：以下一交易日开盘价为 entry。 +- 晚于当前 `decision_at` 的日线收盘数据不会参与计算。 + +```text +adjusted_entry = raw_entry × entry_adjustment_factor +adjusted_close = raw_close × close_adjustment_factor +cumulative_return = adjusted_close / adjusted_entry - 1 +``` + +缺少价格、复权因子、交易日或 horizon 时会记录明确限制，不会用 Agent 生成的数值补齐。 + +### 相似度和预测 + +Market Agent 返回 `[-1, 1]` 范围内的语义相似度： + +- 正值表示机制和方向相似。 +- 负值表示机制可比但方向相反。 +- `0` 表示没有有效关系。 + +代码会截断越界值、忽略零相似度事件，并按相似度绝对值归一化权重。负相似度样本会反转历史收益方向。 每个 D1–D10 horizon 只使用该 +horizon 有数据的样本。参考持有时间取正预期收益中最高的 horizon； 没有正值时留空。 + +结果还会记录样本不足、horizon 缺失、收益方向冲突等限制。它只是有限历史样本比较，不代表统计显著性。 + +## 输出布局 + +```text +reme_workspace/ +├── daily/ +│ ├── YYYY-MM-DD.md +│ └── YYYY-MM-DD/ +│ ├── auto_fin_news_data.jsonl +│ ├── auto_fin_analysis.jsonl +│ └── auto_fin.md +└── resource/ + └── YYYY-MM-DD/ + ├── filtered_news.jsonl + ├── filtered_etf.jsonl + ├── auto_fin_topic_output.jsonl + ├── auto_fin_history_<序号>__output.json + ├── auto_fin_market_<序号>__output.json + ├── auto_fin_history_output.jsonl + └── auto_fin_merge_output.json +``` + +主要产物： + +- `auto_fin_news_data.jsonl`：用户拥有的历史新闻回查来源。 +- `filtered_news.jsonl` 和 `filtered_etf.jsonl`：边界明确的 Topic Agent 输入。 +- 各 ETF history 文件：已回查的原始新闻和代码计算的收益路径。 +- 各 ETF market 文件：代码计算的匹配、权重和 D1–D10 预测。 +- `auto_fin_analysis.jsonl`：全部已选 ETF 的最终结构化分析。 +- `auto_fin.md`：可读报告和钉钉投递内容。 +- `daily/YYYY-MM-DD.md`：报告生成后会刷新，确保每日索引能够发现 Auto Fin 报告。 + +新闻和报告都是用户拥有的普通文件；resource 中间产物和搜索索引均可重建。 + +## 配置 + +### Job 参数 + +| 参数 | 默认值 | 说明 | +|---------|--------:|-----------------------------------| +| `date` | 当天 | 严格 `YYYY-MM-DD`；当前只支持当天 | +| `force` | `false` | 是否刷新全部已配置新闻日期 | + +### 环境变量 + +| 变量 | 必需 | 说明 | +|-----------------------------|------|-------------------------------------| +| `TUSHARE_TOKEN` | 是 | 新闻、交易日历、ETF 日线和复权因子 | +| `CLAUDE_CODE_API_KEY` | 是 | Auto Fin Agent 凭据 | +| `CLAUDE_CODE_MODEL_NAME` | 否 | 默认 `qwen3.7-max` | +| `CLAUDE_CODE_BASE_URL` | 否 | Anthropic 兼容 endpoint | +| `AUTO_FIN_AGENT_BACKEND` | 否 | 默认 `claude_code` | +| `AUTO_FIN_PROJECT_PATH` | 否 | Agent project path，默认 `..` | +| `REME_PROXY_IP` | 否 | 仅启用 `ssh_http` 时使用的 SSH 主机 | +| `REME_PROXY_ACCOUNT` | 否 | 仅启用 `ssh_http` 时使用的 SSH 账户 | +| `DAILY_PAPER_WORKSPACE_DIR` | 否 | standalone cookbook workspace | +| `DINGTALK_*` | 否 | 钉钉应用、机器人和会话设置 | + +单元测试可通过 RuntimeContext 注入 `tushare_provider`，不需要真实凭据。 + +### 定时任务 + +`daily_cookbook.yaml` 定义： + +| Job | Cron | Asia/Shanghai | +|----------------------|---------------|---------------| +| `auto_fin_0930_cron` | `30 9 * * *` | 每天 09:30 | +| `auto_fin_1145_cron` | `45 11 * * *` | 每天 11:45 | +| `auto_fin_1800_cron` | `0 18 * * *` | 每天 18:00 | + +这些 cron 表达式不会排除周末或休市日。工作流会确定上一 A 股交易日，但当前不会仅因为运行日不是交易日而跳过。 + +## Agent 和安全边界 + +Auto Fin wrapper 会加载 `tushare-data` skill、暴露 `memory_search` Job 工具，并默认使用 +`bypassPermissions`。Prompt 会约束各 Agent 的职责，代码则再次校验 schema、ETF 身份、来源路径、 新闻引用和计算值。 + +standalone cookbook 默认没有配置 embedding store，因此 `memory_search` 通常使用 BM25 召回； 配置 embedding store 后才能使用向量与 +BM25 融合。 + +`bypassPermissions` 不是操作系统沙箱。部署前应检查 project path、workspace、凭据和网络边界。 + +## 重跑和限制 + +- 有效历史新闻会复用，当天新闻始终刷新。 +- 输出使用稳定的每日路径，因此同一天后一次运行会替换前一次报告和 resource 输出。 +- Auto Fin 不使用“报告已存在则跳过”，因为定时运行需要分析更新后的新闻。 +- 配置钉钉后，每次成功运行都会尝试投递；当前没有通知去重。 +- 历史样本缺少部分 horizon 时会降级该样本并记录限制。 +- 非法日期、缺少必要服务、Agent schema 错误、未知 ETF/新闻、危险路径或跨步骤 ETF 身份不一致会使 Job 失败。 + +当前没有实现个股、美股关联、组合账本、BUY/SELL/HOLD、T+1 执行、手续费、滑点、券商连接， 也不会执行真实或模拟委托。 + +## 开发 + +安装开发依赖并运行聚焦测试： + +```bash +python -m pip install -e ".[dev,core]" +PYTHONPATH=. pytest tests/unit/test_auto_fin.py -v +``` + +单元测试会 mock 模型和行情数据边界。需要真实 Tushare、模型或钉钉凭据的测试应单独运行，且需要显式授权。 diff --git a/cookbook/daily_paper/README.md b/cookbook/daily_paper/README.md index f205c5a3..b3130c0e 100644 --- a/cookbook/daily_paper/README.md +++ b/cookbook/daily_paper/README.md @@ -2,9 +2,16 @@ [中文](README_ZH.md) -Daily Paper is a local-first, file-native workflow for turning research feeds into a daily reading package. It collects -papers from the Hugging Face weekly and monthly rankings, removes yesterday's papers and recent recommendations, ranks -the remaining candidates, and uses Claude Code to produce detailed Chinese paper notes and a five-minute Chinese brief. +Daily Paper is a local-first, file-native workflow for turning research rankings into a daily reading package. + +## Capabilities + +- Collect papers from the Hugging Face weekly and monthly rankings while excluding yesterday's papers and recent + recommendations. +- Rank and select candidates, then use Claude Code to produce detailed Chinese notes and a five-minute Chinese brief. +- Keep PDFs, notes, and memories as ordinary user-owned files; indexes and caches remain rebuildable. +- Support daily scheduling, optional DingTalk delivery, conversation memory, auto-dream consolidation, and BM25 recall + for the background DingTalk agent. The workflow is assembled by [`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml). Its schemas live in [`reme/schema/daily_paper.py`](../../reme/schema/daily_paper.py), and its steps live in @@ -13,12 +20,13 @@ The workflow is assembled by [`daily_cookbook.yaml`](../../reme/config/daily_coo ## Quick start Daily Paper requires Python 3.11 or later, the `core` dependencies, network access to Hugging Face and arXiv, and -credentials for the configured Claude Code endpoint. +credentials for the configured Claude Code endpoint. Auto-memory and auto-dream additionally require the AgentScope LLM +credentials. From the repository root: ```bash -python -m pip install -e ".[dev,core]" +python -m pip install -e ".[core]" export CLAUDE_CODE_API_KEY="your-api-key" reme start config=daily_cookbook job=daily_paper ``` @@ -26,14 +34,30 @@ reme start config=daily_cookbook job=daily_paper The built-in configuration uses `qwen3.7-max` through DashScope's Anthropic-compatible endpoint. Override `CLAUDE_CODE_MODEL_NAME` and `CLAUDE_CODE_BASE_URL` when using another compatible model or provider. -By default, outputs are written under `.reme/` in the directory where ReMe starts. +This is enough to generate paper notes and the daily brief. To use auto-memory and auto-dream, also configure: + +```bash +export LLM_API_KEY="your-api-key" +``` + +By default, outputs are written under `reme_workspace/` in the directory where ReMe starts. + +### Optional SSH proxy + +The outbound proxy is disabled by default. To enable it, uncomment `components.outbound_proxy.default` in +`daily_cookbook.yaml`, configure non-interactive SSH authentication, and set: + +```bash +export REME_PROXY_IP="your-ssh-proxy-host" +export REME_PROXY_ACCOUNT="your-ssh-account" +``` ## What it creates A successful run writes ordinary PDFs and Markdown files beneath `workspace_dir`: ```text -.reme/ +reme_workspace/ ├── daily/ │ ├── YYYY-MM-DD.md │ └── YYYY-MM-DD/ @@ -44,7 +68,15 @@ A successful run writes ordinary PDFs and Markdown files beneath `workspace_dir` │ └── papers/ │ ├── .pdf │ └── ... +├── digest/ +│ ├── personal/ +│ ├── project/ +│ ├── resource/ +│ └── wiki/ +├── metadata/ +│ └── ... derived catalogs, indexes, and caches └── mem_session/ + ├── agentscope/ └── claude_config/ ``` @@ -53,6 +85,9 @@ A successful run writes ordinary PDFs and Markdown files beneath `workspace_dir` - `daily-paper-brief.md` is a roughly five-minute Chinese digest with wikilinks to every selected paper note. - `daily/YYYY-MM-DD.md` is a derived day index rebuilt from the Markdown files for that date. - `resource/papers/` holds reusable source PDFs. +- `digest/` contains durable auto-dream output; files there remain ordinary user-owned Markdown. +- `metadata/` and search caches are derived state. `reindex` rebuilds the file store, BM25 index, and graph from source + files. The paper notes are the source of truth for recommendation history: their frontmatter contains the `arxiv_id` values used for future deduplication. The day index is derived and can be rebuilt. The workflow does not currently write a @@ -68,7 +103,7 @@ flowchart LR C --> R[2. Rank] R --> S[3. Select] S --> A[4. Analyze PDFs] - A --> D[5. Build digest] + A --> D[5. Build brief] D --> N[6. Notify DingTalk] A --> P[PDFs + paper notes] D --> B[Brief + day index] @@ -76,9 +111,9 @@ flowchart LR ### 1. Collect and deduplicate -The Collect step fetches the weekly ranking for the run date's ISO week, the monthly ranking for its calendar month, -and the Hugging Face Daily Papers IDs for exactly the previous calendar day. It merges weekly and monthly metadata by -arXiv ID and preserves each list's display rank. +The Collect step fetches the weekly ranking for the run date's ISO week, the monthly ranking for its calendar month, and +the Hugging Face Daily Papers IDs for exactly the previous calendar day. It merges weekly and monthly metadata by arXiv +ID and preserves each list's display rank. It then scans `daily//paper-*.md` over the configured history window and excludes IDs found in note frontmatter. The job fails clearly if no eligible papers remain. @@ -128,6 +163,29 @@ The final step sends the brief body, without YAML frontmatter, to each configure conversation IDs it is a no-op. If one group fails, the step still attempts the remaining groups and reports the combined failure afterward. +## Memory and search + +The standalone configuration separates agent wrappers by responsibility: + +- `daily_paper` selects papers, analyzes them, and builds the brief. It keeps Claude Code's normal local tools and + disables `WebSearch`, but currently has no memory-retrieval job configured. +- `dingtalk_wait` runs the background DingTalk agent and exposes `memory_search` as a callable tool. +- `memory` runs auto-memory and the LLM-backed auto-dream steps through AgentScope. Its built-in shell and file tools + are disabled; memory changes go through the narrower ReMe jobs such as `daily_write`, `read`, `edit`, and `write`. + +The built-in `memory_search` job uses BM25 over Markdown under `daily/` and `digest/`. ReMe's search step can fuse +vector results, but this cookbook does not configure an embedding store by default, so vector retrieval is not run. +`node_search` is a narrower digest recall tool used internally by auto-dream. + +`index_update_loop` indexes existing memory files when the service starts and watches those directories for later +changes. Run `reindex` when recovering the derived file store or forcing a complete index rebuild. Source Markdown and +PDFs are not deleted by `reindex`. + +`auto_memory` writes or updates one daily note from caller-supplied conversation messages and a stable `session_id`. +`auto_dream` scans recent daily notes, integrates durable units under `digest/`, and writes interest topics. Both are +on-demand jobs in this cookbook; no auto-dream cron is configured. The DingTalk agent can recall through +`memory_search`, but it does not automatically call `auto_memory` after a conversation. + ## Dates, reruns, and idempotency - `date` must be an exact `YYYY-MM-DD` value. When omitted, the job uses today in the application timezone, which is @@ -138,29 +196,31 @@ combined failure afterward. and digest generation are skipped. The existing brief remains available to the DingTalk notification step. - `force=true` regenerates the notes and brief. Existing valid PDFs are still reused. -Each PDF, detailed note, and final brief uses a temporary file followed by replacement so callers do not see a -partially written file. The complete multi-file workflow is not transactional, and there is no global lock for two -concurrent runs of the same date. +Each PDF, detailed note, and final brief uses a temporary file followed by replacement so callers do not see a partially +written file. The complete multi-file workflow is not transactional, and there is no global lock for two concurrent runs +of the same date. ## Running the cookbook -The standalone configuration defines three jobs: +The main jobs in the standalone configuration are: -| Job | Behavior | -|--------------------|---------------------------------------------------------| -| `daily_paper` | On-demand generation through the CLI or HTTP service | -| `daily_paper_cron` | The same pipeline every day at 08:00 in `Asia/Shanghai` | -| `dingtalk_wait` | A supervised background DingTalk agent | +| Job | Behavior | +|---------------------|-----------------------------------------------------------------| +| `daily_paper` | On-demand generation through the CLI or HTTP service | +| `daily_paper_cron` | The same pipeline every day at 08:00 in `Asia/Shanghai` | +| `dingtalk_wait` | A supervised background DingTalk agent with `memory_search` | +| `auto_memory` | Write or update a daily note from conversation messages | +| `auto_dream` | Consolidate recent daily notes into digest memory and interests | +| `memory_search` | BM25 recall over daily and digest Markdown | +| `reindex` | Rebuild derived search state from existing memory files | +| `index_update_loop` | Initialize and continuously update search state in service mode | + +Supporting jobs such as `node_search`, `daily_list`, `daily_write`, `read`, `write`, `edit`, and frontmatter updates +provide the constrained tools used by the memory agent. ### One-time runs -Generate today's brief: - -```bash -reme start config=daily_cookbook job=daily_paper -``` - -Generate a specific date with selected overrides: +The quick-start command generates today's brief. To generate a specific date with selected overrides: ```bash reme start \ @@ -200,6 +260,19 @@ curl -s http://127.0.0.1:8001/daily_paper \ -d '{"date":"2026-07-21","top_k":3,"force":false}' ``` +Recall memory, record a conversation, consolidate it, or explicitly rebuild the search index: + +```bash +reme memory_search host=127.0.0.1 port=8001 query="agent memory" limit=5 + +reme auto_memory host=127.0.0.1 port=8001 \ + session_id=example-session \ + messages='[{"name":"user","role":"user","content":"I prefer concise paper summaries."}]' + +reme auto_dream host=127.0.0.1 port=8001 date=2026-07-21 +reme reindex host=127.0.0.1 port=8001 +``` + Service and schedule settings can be overridden at startup: ```bash @@ -234,18 +307,22 @@ take precedence over the job defaults. The standalone application also accepts these environment variables: -| Variable | Purpose | -|-----------------------------------------|-------------------------------------------------| -| `DAILY_PAPER_WORKSPACE_DIR` | Overrides the default `.reme` workspace | -| `DAILY_PAPER_PROJECT_PATH` | Repository/project path visible to Claude Code | -| `DAILY_PAPER_HOST` / `DAILY_PAPER_PORT` | HTTP bind address | -| `CLAUDE_CODE_API_KEY` | API key for the configured Claude Code endpoint | -| `CLAUDE_CODE_MODEL_NAME` | Model override | -| `CLAUDE_CODE_BASE_URL` | Anthropic-compatible endpoint override | +| Variable | Purpose | +|-----------------------------------------|------------------------------------------------| +| `DAILY_PAPER_WORKSPACE_DIR` | Overrides the default `reme_workspace` | +| `DAILY_PAPER_PROJECT_PATH` | Repository/project path visible to Claude Code | +| `REME_PROXY_IP` | Optional SSH proxy host | +| `REME_PROXY_ACCOUNT` | Optional SSH proxy account | +| `DAILY_PAPER_HOST` / `DAILY_PAPER_PORT` | HTTP bind address | +| `CLAUDE_CODE_API_KEY` | API key for the Claude Code endpoint | +| `CLAUDE_CODE_MODEL_NAME` | Claude Code model; default `qwen3.7-max` | +| `CLAUDE_CODE_BASE_URL` | Claude Code Anthropic-compatible endpoint | +| `LLM_API_KEY` | API key for the AgentScope memory model | +| `LLM_MODEL_NAME` | Memory model; default `qwen3.7-max` | +| `LLM_BASE_URL` | Memory model's Anthropic-compatible endpoint | -`DAILY_PAPER_PROJECT_PATH` defaults to `..` relative to the workspace. With the default `.reme` workspace, starting -from the repository root resolves it back to the repository. If the workspace lives elsewhere, set both paths -explicitly. +`DAILY_PAPER_PROJECT_PATH` defaults to `..` relative to the workspace. With the default `reme_workspace`, starting from +the repository root resolves it back to the repository. If the workspace lives elsewhere, set both paths explicitly. ReMe loads an uncommitted `.env` file found from the current directory upward, so the same values may be placed there instead of exported in the shell. @@ -266,31 +343,33 @@ first three credentials but not the conversation list. ## Failure recovery and boundaries -| Situation | Behavior | -|--------------------------------------|----------------------------------------------------------------------| -| Temporary Hugging Face failure | Retries with exponential delay up to `hf_max_retries` attempts | -| No eligible papers | Fails before ranking | -| Invalid `top_k` or selection output | Fails after validation; selection output gets one retry | -| Oversized, invalid, or textless PDF | Stops during analysis | -| PDF exceeds page or character limits | Continues with truncated text and records the truncation | -| One paper analysis fails | Stops the job; earlier PDFs and notes remain on disk | -| Brief misses a source-note link | Appends the missing wikilink before writing | -| Existing final brief | Skips generation unless `force=true`; notification can still send it | -| Concurrent runs for one date | No pipeline-level lock; later writes may replace earlier results | +| Situation | Behavior | +|--------------------------------------|----------------------------------------------------------------| +| Temporary Hugging Face failure | Retries with exponential delay up to `hf_max_retries` attempts | +| No eligible papers | Fails before ranking | +| Invalid `top_k` or selection output | Fails after validation; selection output gets one retry | +| Oversized, invalid, or textless PDF | Stops during analysis | +| PDF exceeds page or character limits | Continues with truncated text and records the truncation | +| One paper analysis fails | Stops the job; earlier PDFs and notes remain on disk | +| Brief misses a source-note link | Appends the missing wikilink before writing | +| Auto-dream partial integration | Successful units remain; failed paths are not checkpointed | To recover, inspect the date's notes and PDFs, fix the network, credential, model, or PDF issue, then rerun the same date with `force=true`. Valid cached PDFs will be reused. -The built-in Claude Code component runs with `permission_mode: bypassPermissions`. ReMe disables Claude Code's -`WebSearch` tool, and the analysis/digest prompts constrain what the agent should read, but these steps do not set a -strict per-call tool allowlist or an operating-system sandbox. Run the cookbook only with a trusted project and -workspace, and tighten the agent configuration before shared or production use. +The built-in Claude Code components run with `permission_mode: bypassPermissions` and disable `WebSearch`. +`dingtalk_wait` can call the local `memory_search` job; `daily_paper` currently has no job tools. The analysis and brief +prompts constrain what the agent should read, but these steps do not set a strict per-call tool allowlist or an +operating-system sandbox. The AgentScope memory wrapper disables its built-in shell and filesystem tools, but runs its +ReMe job tools in bypass permission mode. Run the cookbook only with a trusted project and workspace, and tighten the +agent configuration before shared or production use. ## Tests The focused unit suite mocks Hugging Face, arXiv, Claude Code, and DingTalk boundaries: ```bash +python -m pip install -e ".[dev,core]" pytest tests/unit/test_daily_paper.py -v ``` diff --git a/cookbook/daily_paper/README_ZH.md b/cookbook/daily_paper/README_ZH.md index 64895e32..2d605eb7 100644 --- a/cookbook/daily_paper/README_ZH.md +++ b/cookbook/daily_paper/README_ZH.md @@ -2,8 +2,14 @@ [English](README.md) -每日论文是一个本地优先、文件原生的研究资讯工作流。它从 Hugging Face 周榜和月榜采集论文，排除昨日论文和近期 -已经推荐过的论文，对剩余候选进行排序，再使用 Claude Code 生成中文详细论文笔记和约五分钟可读完的中文简报。 +每日论文是一个本地优先、文件原生的研究资讯工作流，用于把研究榜单转化为每日阅读材料。 + +## 能力 + +- 从 Hugging Face 周榜和月榜采集论文，并排除昨日论文和近期已经推荐过的论文。 +- 对候选论文进行排序和精选，再使用 Claude Code 生成中文详细论文笔记和约五分钟可读完的中文简报。 +- 将 PDF、笔记和记忆保存为由用户拥有的普通文件；索引和缓存均可重建。 +- 支持每日定时运行、可选钉钉投递、对话记忆、auto-dream 整理，以及供后台钉钉 Agent 使用的 BM25 检索。 工作流由 [`daily_cookbook.yaml`](../../reme/config/daily_cookbook.yaml) 装配，公共 schema 位于 [`reme/schema/daily_paper.py`](../../reme/schema/daily_paper.py)，各步骤位于 @@ -11,28 +17,44 @@ ## 快速开始 -每日论文要求 Python 3.11 或更高版本、`core` 依赖、可访问 Hugging Face 和 arXiv 的网络，以及所配置 -Claude Code endpoint 的凭据。 +每日论文要求 Python 3.11 或更高版本、`core` 依赖、可访问 Hugging Face 和 arXiv 的网络，以及所配置 Claude Code endpoint +的凭据。auto-memory 和 auto-dream 还需要 AgentScope LLM 凭据。 在仓库根目录运行： ```bash -python -m pip install -e ".[dev,core]" +python -m pip install -e ".[core]" export CLAUDE_CODE_API_KEY="your-api-key" reme start config=daily_cookbook job=daily_paper ``` -内置配置默认通过 DashScope 的 Anthropic 兼容 endpoint 使用 `qwen3.7-max`。如需使用其他兼容模型或服务商， -请覆盖 `CLAUDE_CODE_MODEL_NAME` 和 `CLAUDE_CODE_BASE_URL`。 +内置配置默认通过 DashScope 的 Anthropic 兼容 endpoint 使用 `qwen3.7-max`。如需使用其他兼容模型或服务商， 请覆盖 +`CLAUDE_CODE_MODEL_NAME` 和 `CLAUDE_CODE_BASE_URL`。 -默认情况下，产物写入 ReMe 启动目录下的 `.reme/`。 +以上配置足以生成论文笔记和每日简报。要使用 auto-memory 和 auto-dream，还需配置： + +```bash +export LLM_API_KEY="your-api-key" +``` + +默认情况下，产物写入 ReMe 启动目录下的 `reme_workspace/`。 + +### 可选 SSH 代理 + +出站代理默认关闭。如需启用，请取消 `daily_cookbook.yaml` 中 +`components.outbound_proxy.default` 的注释，配置免交互 SSH 认证，并设置： + +```bash +export REME_PROXY_IP="your-ssh-proxy-host" +export REME_PROXY_ACCOUNT="your-ssh-account" +``` ## 文件产物 一次成功运行会在 `workspace_dir` 下写入普通 PDF 和 Markdown 文件： ```text -.reme/ +reme_workspace/ ├── daily/ │ ├── YYYY-MM-DD.md │ └── YYYY-MM-DD/ @@ -43,7 +65,15 @@ reme start config=daily_cookbook job=daily_paper │ └── papers/ │ ├── .pdf │ └── ... +├── digest/ +│ ├── personal/ +│ ├── project/ +│ ├── resource/ +│ └── wiki/ +├── metadata/ +│ └── ... 派生 catalog、索引和缓存 └── mem_session/ + ├── agentscope/ └── claude_config/ ``` @@ -51,9 +81,11 @@ reme start config=daily_cookbook job=daily_paper - `daily-paper-brief.md` 是约五分钟可读完的中文简报，并包含每篇入选论文笔记的 wikilink。 - `daily/YYYY-MM-DD.md` 是从当日 Markdown 文件重建的派生日索引。 - `resource/papers/` 保存可复用的原始 PDF。 +- `digest/` 保存持久的 auto-dream 产物，其中仍然是由用户拥有的普通 Markdown 文件。 +- `metadata/` 和搜索缓存属于派生状态；`reindex` 会根据源文件重建 file store、BM25 索引和图。 -论文笔记是推荐历史的事实来源：后续排重会读取其 frontmatter 中的 `arxiv_id`。日索引属于可重建的派生文件。 -当前工作流不会另外写入运行 manifest。 +论文笔记是推荐历史的事实来源：后续排重会读取其 frontmatter 中的 `arxiv_id`。日索引属于可重建的派生文件。 当前工作流不会另外写入运行 +manifest。 ## 工作流程 @@ -65,7 +97,7 @@ flowchart LR C --> R[2. Rank] R --> S[3. Select] S --> A[4. Analyze PDFs] - A --> D[5. Build digest] + A --> D[5. Build brief] D --> N[6. Notify DingTalk] A --> P[PDF + 论文笔记] D --> B[简报 + 日索引] @@ -73,11 +105,11 @@ flowchart LR ### 1. 采集与排重 -Collect 会获取运行日所在 ISO week 的周榜、所在自然月的月榜，以及严格前一个自然日的 Hugging Face Daily Papers -ID。周榜和月榜元数据按 arXiv ID 合并，同时保留两个榜单各自的展示排名。 +Collect 会获取运行日所在 ISO week 的周榜、所在自然月的月榜，以及严格前一个自然日的 Hugging Face Daily Papers ID。周榜和月榜元数据按 +arXiv ID 合并，同时保留两个榜单各自的展示排名。 -随后，它会在配置的历史窗口内扫描 `daily//paper-*.md`，排除笔记 frontmatter 中已有的 ID。如果排重后 -没有任何可选论文，Job 会明确失败。 +随后，它会在配置的历史窗口内扫描 `daily//paper-*.md`，排除笔记 frontmatter 中已有的 ID。如果排重后 没有任何可选论文，Job +会明确失败。 ### 2. 候选排序 @@ -88,14 +120,14 @@ score = 1 / (rrf_k + monthly_rank) + weekly_weight / (rrf_k + weekly_rank) ``` -论文缺少某个榜单排名时，该项贡献为零。候选按融合分、upvotes 和 arXiv ID 排序。有界候选池还会为标题或摘要命中 -Agent memory、memory retrieval、continual learning、context compression、knowledge graph、RAG 等记忆相关关键词的 +论文缺少某个榜单排名时，该项贡献为零。候选按融合分、upvotes 和 arXiv ID 排序。有界候选池还会为标题或摘要命中 Agent +memory、memory retrieval、continual learning、context compression、knowledge graph、RAG 等记忆相关关键词的 论文保留若干位置。这个保留策略只是关键词启发式，不是语义分类器。 ### 3. 精选论文 -Claude Code 接收有界候选池，并返回结构化的 `PaperSelection`。实现要求恰好选择 `top_k` 个候选池内的唯一 ID， -且 rank 必须连续。输出不合法时，校验错误会反馈给 Agent 并重试一次；第二次仍不合法则 Job 失败。 +Claude Code 接收有界候选池，并返回结构化的 `PaperSelection`。实现要求恰好选择 `top_k` 个候选池内的唯一 ID， 且 rank +必须连续。输出不合法时，校验错误会反馈给 Agent 并重试一次；第二次仍不合法则 Job 失败。 ### 4. 下载并解读 PDF @@ -107,19 +139,38 @@ Claude Code 接收有界候选池，并返回结构化的 `PaperSelection`。实 4. 请求 Claude Code 返回结构化的详细解读； 5. 写入规范化 frontmatter 和生成的 Markdown 正文。 -当前提取器依赖可用的 PDF 文本层。扫描版或纯图片 PDF 会失败，因为没有 OCR fallback。提取内容超过配置限制时， -笔记会记录输入已被截断。 +当前提取器依赖可用的 PDF 文本层。扫描版或纯图片 PDF 会失败，因为没有 OCR fallback。提取内容超过配置限制时， 笔记会记录输入已被截断。 ### 5. 生成简报与索引 -Claude Code 会读取全部详细笔记并生成当日简报。代码会检查每篇源笔记的 wikilink；如有遗漏，会在写入前自动补齐。 -随后，工作流根据当日 Markdown frontmatter 重建 `daily/YYYY-MM-DD.md`。 +Claude Code 会读取全部详细笔记并生成当日简报。代码会检查每篇源笔记的 wikilink；如有遗漏，会在写入前自动补齐。 随后，工作流根据当日 +Markdown frontmatter 重建 `daily/YYYY-MM-DD.md`。 ### 6. 可选的钉钉通知 最后一步会去掉 YAML frontmatter，把简报正文按顺序发送到每个已配置的钉钉群。未配置群会话 ID 时，该步骤无副作用 跳过。某个群发送失败不会阻止继续尝试其他群，所有发送完成后再汇总报告失败。 +## 记忆与检索 + +独立配置按职责分离 agent wrapper： + +- `daily_paper` 执行论文精选、解读和简报生成。它保留 Claude Code 的常规本地工具并禁用 `WebSearch`，但当前没有 配置记忆检索 + Job。 +- `dingtalk_wait` 运行后台钉钉 Agent，并把 `memory_search` 作为可调用工具。 +- `memory` 通过 AgentScope 执行 auto-memory 和 auto-dream 中依赖 LLM 的步骤。它禁用内置 shell 和文件工具； 记忆变更只能经过 + `daily_write`、`read`、`edit`、`write` 等更窄的 ReMe job。 + +内置配置的 `memory_search` 使用 BM25 检索 `daily/` 和 `digest/` 下的 Markdown。ReMe 的搜索步骤支持融合向量结果， 但本 +cookbook 默认没有配置 embedding store，因此不会执行向量检索。`node_search` 是 auto-dream 内部使用的 digest 节点检索工具。 + +`index_update_loop` 会在服务启动时索引已有记忆文件，并持续监听这些目录的后续变化。修复派生 file store 或需要强制 +完整重建索引时，可运行 `reindex`。`reindex` 不会删除源 Markdown 或 PDF。 + +`auto_memory` 根据调用方传入的对话消息和稳定 `session_id` 写入或更新一篇 daily note。`auto_dream` 扫描近期 daily +notes，把持久记忆单元整合到 `digest/`，并生成兴趣主题。这两个 job 在本 cookbook 中都是按需执行；当前没有配置 auto-dream +cron。钉钉 Agent 可以通过 `memory_search` 召回记忆，但对话结束后不会自动调用 `auto_memory`。 + ## 日期、重跑与幂等 - `date` 必须严格符合 `YYYY-MM-DD`。省略时使用应用配置时区中的今天；内置配置为 `Asia/Shanghai`。 @@ -129,28 +180,28 @@ Claude Code 会读取全部详细笔记并生成当日简报。代码会检查 跳过；已有简报仍会交给钉钉通知步骤。 - `force=true` 会重新生成笔记和简报，但仍会复用已有且有效的 PDF。 -每个 PDF、详细笔记和最终简报都会先写临时文件再替换，避免读取方看到半写状态。整个多文件工作流不是事务， -同一日期的并发运行也没有全局锁。 +每个 PDF、详细笔记和最终简报都会先写临时文件再替换，避免读取方看到半写状态。整个多文件工作流不是事务， 同一日期的并发运行也没有全局锁。 ## 运行方式 -独立配置定义了三个 Job： +独立配置中的主要 Job 如下： -| Job | 行为 | -|--------------------|----------------------------------------| -| `daily_paper` | 通过 CLI 或 HTTP 服务按需生成 | -| `daily_paper_cron` | 每天 08:00（`Asia/Shanghai`）执行相同 pipeline | -| `dingtalk_wait` | 由 supervisor 管理的后台钉钉 Agent | +| Job | 行为 | +|---------------------|---------------------------------------------------------------| +| `daily_paper` | 通过 CLI 或 HTTP 服务按需生成 | +| `daily_paper_cron` | 每天 08:00（`Asia/Shanghai`）执行相同 pipeline | +| `dingtalk_wait` | 由 supervisor 管理、具有 `memory_search` 能力的后台钉钉 Agent | +| `auto_memory` | 根据对话消息写入或更新 daily note | +| `auto_dream` | 把近期 daily notes 整理为 digest 记忆和兴趣主题 | +| `memory_search` | 对 daily 和 digest Markdown 执行 BM25 检索 | +| `reindex` | 根据已有记忆文件重建派生检索状态 | +| `index_update_loop` | 在服务模式下初始化并持续更新检索状态 | + +`node_search`、`daily_list`、`daily_write`、`read`、`write`、`edit` 和 frontmatter 更新等辅助 job 构成 memory agent 使用的受约束工具。 ### 一次性运行 -生成今天的简报： - -```bash -reme start config=daily_cookbook job=daily_paper -``` - -生成指定日期，并覆盖部分参数： +快速开始中的命令会生成今天的简报。要生成指定日期并覆盖部分参数： ```bash reme start \ @@ -177,8 +228,7 @@ reme start config=daily_cookbook job=daily_paper date=2026-07-21 force=true reme start config=daily_cookbook ``` -服务默认监听 `127.0.0.1:8001`，因此可以和默认 ReMe 服务并行运行。在另一个终端中通过 ReMe client 或 HTTP -调用按需任务： +服务默认监听 `127.0.0.1:8001`，因此可以和默认 ReMe 服务并行运行。在另一个终端中通过 ReMe client 或 HTTP 调用按需任务： ```bash reme daily_paper host=127.0.0.1 port=8001 @@ -190,6 +240,19 @@ curl -s http://127.0.0.1:8001/daily_paper \ -d '{"date":"2026-07-21","top_k":3,"force":false}' ``` +检索记忆、记录对话、整理记忆，或显式重建检索索引： + +```bash +reme memory_search host=127.0.0.1 port=8001 query="agent memory" limit=5 + +reme auto_memory host=127.0.0.1 port=8001 \ + session_id=example-session \ + messages='[{"name":"user","role":"user","content":"I prefer concise paper summaries."}]' + +reme auto_dream host=127.0.0.1 port=8001 date=2026-07-21 +reme reindex host=127.0.0.1 port=8001 +``` + 监听地址和调度时间可以在启动时覆盖： ```bash @@ -204,36 +267,41 @@ reme start \ 最常用的 Job 配置如下： -| 配置项 | 默认值 | 用途 | -|-------------------|-----------:|-------------------------| -| `candidate_limit` | `20` | 送入精选阶段的最大论文数 | -| `memory_reserve` | `5` | 记忆关键词启发式保留的候选位置数 | +| 配置项 | 默认值 | 用途 | +|-------------------|-----------:|------------------------------------| +| `candidate_limit` | `20` | 送入精选阶段的最大论文数 | +| `memory_reserve` | `5` | 记忆关键词启发式保留的候选位置数 | | `top_k` | `3` | 最终精选和解读的论文数 | -| `rrf_k` | `60` | RRF 常数 | +| `rrf_k` | `60` | RRF 常数 | | `weekly_weight` | `0.7` | 周榜在融合排序中的权重 | -| `history_days` | `30` | 按 arXiv ID 排除近期推荐的时间窗口 | -| `hf_timeout` | `30` 秒 | Hugging Face 请求 timeout | -| `hf_max_retries` | `3` | Hugging Face 请求最多尝试次数 | -| `pdf_timeout` | `90` 秒 | arXiv 下载 timeout | -| `max_pdf_bytes` | `52428800` | PDF 大小上限（50 MiB） | -| `max_pdf_pages` | `80` | 最多提取的 PDF 页数 | -| `max_pdf_chars` | `240000` | 单篇论文送入模型的最大提取字符数 | +| `history_days` | `30` | 按 arXiv ID 排除近期推荐的时间窗口 | +| `hf_timeout` | `30` 秒 | Hugging Face 请求 timeout | +| `hf_max_retries` | `3` | Hugging Face 请求最多尝试次数 | +| `pdf_timeout` | `90` 秒 | arXiv 下载 timeout | +| `max_pdf_bytes` | `52428800` | PDF 大小上限（50 MiB） | +| `max_pdf_pages` | `80` | 最多提取的 PDF 页数 | +| `max_pdf_chars` | `240000` | 单篇论文送入模型的最大提取字符数 | 公开 Job 参数为 `date`、`force`、`top_k`、`weekly_weight` 和 `history_days`。调用时显式传入的值优先于 Job 默认值。 独立应用还支持以下环境变量： -| 环境变量 | 用途 | -|-----------------------------------------|------------------------------------| -| `DAILY_PAPER_WORKSPACE_DIR` | 覆盖默认 `.reme` workspace | -| `DAILY_PAPER_PROJECT_PATH` | Claude Code 可见的仓库或项目路径 | +| 环境变量 | 用途 | +|-----------------------------------------|----------------------------------------| +| `DAILY_PAPER_WORKSPACE_DIR` | 覆盖默认 `reme_workspace` | +| `DAILY_PAPER_PROJECT_PATH` | Claude Code 可见的仓库或项目路径 | +| `REME_PROXY_IP` | 可选 SSH 代理主机 | +| `REME_PROXY_ACCOUNT` | 可选 SSH 代理账户 | | `DAILY_PAPER_HOST` / `DAILY_PAPER_PORT` | HTTP 监听地址 | -| `CLAUDE_CODE_API_KEY` | 所配置 Claude Code endpoint 的 API key | -| `CLAUDE_CODE_MODEL_NAME` | 覆盖模型名称 | -| `CLAUDE_CODE_BASE_URL` | 覆盖 Anthropic 兼容 endpoint | +| `CLAUDE_CODE_API_KEY` | Claude Code endpoint 的 API key | +| `CLAUDE_CODE_MODEL_NAME` | Claude Code 模型；默认 `qwen3.7-max` | +| `CLAUDE_CODE_BASE_URL` | Claude Code 的 Anthropic 兼容 endpoint | +| `LLM_API_KEY` | AgentScope memory 模型的 API key | +| `LLM_MODEL_NAME` | memory 模型；默认 `qwen3.7-max` | +| `LLM_BASE_URL` | memory 模型的 Anthropic 兼容 endpoint | -`DAILY_PAPER_PROJECT_PATH` 默认是相对于 workspace 的 `..`。使用默认 `.reme` workspace 并从仓库根目录启动时， -它会解析回仓库根目录。如果 workspace 位于其他位置，请显式设置这两个路径。 +`DAILY_PAPER_PROJECT_PATH` 默认是相对于 workspace 的 `..`。使用默认 `reme_workspace` 并从仓库根目录启动时， 它会解析回仓库根目录。如果 +workspace 位于其他位置，请显式设置这两个路径。 ReMe 会从当前目录向上查找未提交的 `.env`，因此也可以把相同变量放在其中，而不是在 shell 中导出。 @@ -252,30 +320,31 @@ DINGTALK_CONVERSATION_IDS=cid-group-one,cid-group-two ## 故障恢复与边界 -| 场景 | 当前行为 | -|-------------------|---------------------------------| -| Hugging Face 暂时失败 | 按指数间隔重试，最多尝试 `hf_max_retries` 次 | -| 没有 eligible 论文 | 在排序前失败 | -| `top_k` 或精选结果不合法 | 校验后失败；精选结果可重试一次 | -| PDF 太大、无效或没有文本层 | 在解读阶段停止 | -| PDF 超过页数或字符数限制 | 使用截断文本继续，并记录截断状态 | -| 某篇论文解读失败 | Job 停止；此前写入的 PDF 和笔记保留 | -| 简报遗漏源笔记链接 | 写入前自动补齐 wikilink | -| 最终简报已存在 | 默认跳过生成，通知仍可发送；`force=true` 可重跑 | -| 同一日期并发运行 | 没有 pipeline 级锁，后写入结果可能替换先前结果 | +| 场景 | 当前行为 | +|----------------------------|----------------------------------------------| +| Hugging Face 暂时失败 | 按指数间隔重试，最多尝试 `hf_max_retries` 次 | +| 没有 eligible 论文 | 在排序前失败 | +| `top_k` 或精选结果不合法 | 校验后失败；精选结果可重试一次 | +| PDF 太大、无效或没有文本层 | 在解读阶段停止 | +| PDF 超过页数或字符数限制 | 使用截断文本继续，并记录截断状态 | +| 某篇论文解读失败 | Job 停止；此前写入的 PDF 和笔记保留 | +| 简报遗漏源笔记链接 | 写入前自动补齐 wikilink | +| auto-dream 部分整合失败 | 成功单元保留，失败路径不会被 checkpoint | -恢复时，先检查该日期已有的笔记和 PDF，修复网络、凭据、模型或 PDF 问题，再使用相同日期和 `force=true` 重跑。 -有效的缓存 PDF 会被复用。 +恢复时，先检查该日期已有的笔记和 PDF，修复网络、凭据、模型或 PDF 问题，再使用相同日期和 `force=true` 重跑。 有效的缓存 PDF +会被复用。 -内置 Claude Code 组件使用 `permission_mode: bypassPermissions`。ReMe 会禁用 Claude Code 的 `WebSearch` 工具， -Analyze 和 Digest prompt 也会限制 Agent 应读取的内容，但这些步骤没有设置严格的逐次调用工具 allowlist，也不是 -操作系统级沙箱。请只在可信的项目和 workspace 中运行；用于共享或生产环境前，应进一步收紧 Agent 配置。 +内置 Claude Code 组件使用 `permission_mode: bypassPermissions`，并禁用 `WebSearch`。`dingtalk_wait` 可以调用本地 +`memory_search`；`daily_paper` 当前没有配置 Job 工具。Analyze 和 Brief prompt 会限制 Agent 应读取的内容，但这些 +步骤没有设置严格的逐次调用工具 allowlist，也不是操作系统级沙箱。AgentScope memory wrapper 禁用了内置 shell 和 文件系统工具，但其 +ReMe job tools 运行于 bypass permission mode。请只在可信的项目和 workspace 中运行；用于共享 或生产环境前，应进一步收紧配置。 ## 测试 聚焦的单元测试会 mock Hugging Face、arXiv、Claude Code 和钉钉边界： ```bash +python -m pip install -e ".[dev,core]" pytest tests/unit/test_daily_paper.py -v ``` diff --git a/example.env b/example.env index 4c093ec4..5910169e 100644 --- a/example.env +++ b/example.env @@ -2,3 +2,7 @@ EMBEDDING_API_KEY=sk-xxx EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 LLM_API_KEY=sk-xxx LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +# Optional SSH jump host for Hugging Face daily-paper downloads. +# REME_PROXY_IP=proxy.example.com +# REME_PROXY_ACCOUNT=your-ssh-account diff --git a/pyproject.toml b/pyproject.toml index d3bdfcdd..2e1efc6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "croniter>=2.0", "fastapi>=0.135.1", "fastmcp>=3.1.0", - "httpx>=0.28.1", + "httpx[socks]>=0.28.1", "loguru>=0.7.3", "mistletoe>=1.5.1", "numpy>=2.2.6", @@ -43,19 +43,22 @@ dependencies = [ [project.optional-dependencies] core = [ "agentscope==2.0.4.post1", - "claude-agent-sdk>=0.2.123", + "claude-agent-sdk>=0.2.126", "dingtalk-stream>=0.24.3", "openai-codex>=0.144.4", + "pproxy>=2.7.9,<3", "faiss-cpu>=1.13.2", "jieba>=0.42.1", "rjieba>=0.2.1", "neo4j>=6.2.0", "networkx>=3.4.2", "pypdf>=5.0.0", + "polars>=1.43.0", + "tushare>=1.4.29", ] dev = [ - "pre-commit", - "pytest>=8.0", + "pre-commit>=4.6.1", + "pytest>=9.1.1", "pytest-asyncio>=0.23", ] full = [ diff --git a/reme/components/__init__.py b/reme/components/__init__.py index a54e4a86..bd2c7c98 100644 --- a/reme/components/__init__.py +++ b/reme/components/__init__.py @@ -11,6 +11,7 @@ from . import file_chunker from . import file_store from . import job from . import keyword_index +from . import outbound_proxy from . import service from . import tokenizer from .application_context import ApplicationContext @@ -39,6 +40,7 @@ __all__ = [ "file_store", "job", "keyword_index", + "outbound_proxy", "service", "tokenizer", ] diff --git a/reme/components/agent_wrapper/__init__.py b/reme/components/agent_wrapper/__init__.py index 7a0ce24c..cc40b55f 100644 --- a/reme/components/agent_wrapper/__init__.py +++ b/reme/components/agent_wrapper/__init__.py @@ -5,5 +5,14 @@ from .as_agent_wrapper import AsAgentWrapper from .cc_agent_wrapper import CcAgentWrapper from .cc_session_store import CcFileSessionStore from .codex_agent_wrapper import CodexAgentWrapper +from .session_command import SessionCommandResult, handle_session_command -__all__ = ["BaseAgentWrapper", "AsAgentWrapper", "CcAgentWrapper", "CcFileSessionStore", "CodexAgentWrapper"] +__all__ = [ + "BaseAgentWrapper", + "AsAgentWrapper", + "CcAgentWrapper", + "CcFileSessionStore", + "CodexAgentWrapper", + "SessionCommandResult", + "handle_session_command", +] diff --git a/reme/components/agent_wrapper/as_agent_wrapper.py b/reme/components/agent_wrapper/as_agent_wrapper.py index 5784b74f..19927171 100644 --- a/reme/components/agent_wrapper/as_agent_wrapper.py +++ b/reme/components/agent_wrapper/as_agent_wrapper.py @@ -178,7 +178,7 @@ class AsAgentWrapper(BaseAgentWrapper): ) -> list[ToolBase]: """Return selected AgentScope built-in tools rooted at ``self.cwd``.""" cwd = str(self.cwd) - backend = WorkspaceBackend(cwd, self.subprocess_environment) + backend = WorkspaceBackend(cwd, self.bash_environment) factories = { "bash": lambda: BypassAnalysisBash(cwd=cwd, backend=backend), "edit": lambda: Edit(backend=backend), @@ -362,6 +362,14 @@ class AsAgentWrapper(BaseAgentWrapper): return result + async def compact_session(self, session_id: str) -> None: + """Force compression of an AgentScope session.""" + kwargs = self._merged_kwargs({"resume": session_id}) + agent, _ = await self._build_agent(None, **kwargs) + config = {**(kwargs.get("context_config") or {}), "trigger_ratio": 1e-9} + await agent.compress_context(ContextConfig(**config)) + await self._dump_state(agent.state) + # ----- StreamChunk conversion ------------------------------------------- @classmethod diff --git a/reme/components/agent_wrapper/base_agent_wrapper.py b/reme/components/agent_wrapper/base_agent_wrapper.py index 957e0146..b571b15f 100644 --- a/reme/components/agent_wrapper/base_agent_wrapper.py +++ b/reme/components/agent_wrapper/base_agent_wrapper.py @@ -9,6 +9,7 @@ from typing import Any, ClassVar, TYPE_CHECKING from pydantic import BaseModel from ..base_component import BaseComponent +from ..outbound_proxy import BaseOutboundProxy from ...enumeration import ChunkEnum, ComponentEnum from ...schema import StreamChunk @@ -31,12 +32,17 @@ class BaseAgentWrapper(BaseComponent): super().__init__(**kwargs) self._cwd = cwd self._project_path = project_path + self.outbound_proxy = self.bind( + "default", + BaseOutboundProxy, + optional=True, + ) if self.SDK_PACKAGE: try: sdk_version = metadata.version(self.SDK_PACKAGE) except metadata.PackageNotFoundError: sdk_version = "unknown" - self.logger.info(f"Agent SDK package={self.SDK_PACKAGE} version={sdk_version}") + self.logger.info(f"Agent SDK name={self.name} package={self.SDK_PACKAGE} version={sdk_version}") @property def cwd(self) -> Path: @@ -121,6 +127,20 @@ class BaseAgentWrapper(BaseComponent): return {} return self.app_context.app_config.environment + @property + def command_proxy_environment(self) -> dict[str, str]: + """Managed proxy variables for agent command tools only.""" + if not isinstance(self.outbound_proxy, BaseOutboundProxy): + return {} + return self.outbound_proxy.merge_environment() + + @property + def bash_environment(self) -> dict[str, str]: + """Configured command environment with the managed proxy applied last.""" + if not isinstance(self.outbound_proxy, BaseOutboundProxy): + return dict(self.subprocess_environment) + return self.outbound_proxy.merge_environment(self.subprocess_environment) + def set_output_schema(self, schema: dict | type[BaseModel]) -> "BaseAgentWrapper": """Set a JSON schema for structured output. Accepts dict or BaseModel class. Returns self for chaining.""" self.kwargs["output_schema"] = self._normalize_output_schema(schema) @@ -171,5 +191,9 @@ class BaseAgentWrapper(BaseComponent): async def reply(self, inputs: Any, **kwargs) -> dict: """Send inputs to the agent and return a dict with session_id and last_message.""" + async def compact_session(self, session_id: str) -> None: + """Request compaction of one persisted agent session.""" + raise NotImplementedError(f"{type(self).__name__} does not support session compaction") + async def reply_stream(self, inputs: Any, **kwargs) -> AsyncGenerator[StreamChunk, None]: """Stream agent events as unified StreamChunk objects.""" diff --git a/reme/components/agent_wrapper/cc_agent_wrapper.py b/reme/components/agent_wrapper/cc_agent_wrapper.py index 86c7a03e..469e208f 100644 --- a/reme/components/agent_wrapper/cc_agent_wrapper.py +++ b/reme/components/agent_wrapper/cc_agent_wrapper.py @@ -1,6 +1,7 @@ """Claude Code SDK backend for the unified agent wrapper.""" import json +import shlex from collections.abc import AsyncGenerator from contextlib import aclosing from dataclasses import asdict, dataclass, fields @@ -94,6 +95,35 @@ class CcAgentWrapper(BaseAgentWrapper): handler=run_job, ) + def _add_bash_proxy_hook(self, opts: Any) -> None: + """Inject managed proxy exports into Claude Code Bash commands.""" + proxy_environment = self.command_proxy_environment + if not proxy_environment: + return + + from claude_agent_sdk import HookMatcher + + exports = " ".join(f"{name}={shlex.quote(value)}" for name, value in proxy_environment.items()) + + async def inject_proxy(hook_input, _tool_use_id, _context): + tool_input = dict(hook_input["tool_input"]) + command = tool_input.get("command") + if not isinstance(command, str): + return {} + tool_input["command"] = f"export {exports}; {command}" + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "updatedInput": tool_input, + }, + } + + hooks = dict(opts.hooks or {}) + pre_tool_use = list(hooks.get("PreToolUse") or []) + pre_tool_use.append(HookMatcher(matcher="Bash", hooks=[inject_proxy])) + hooks["PreToolUse"] = pre_tool_use + opts.hooks = hooks + def _build_options(self, inputs: Any, stream: bool = False, **kwargs) -> Any: """Build ClaudeAgentOptions from kwargs. @@ -147,6 +177,7 @@ class CcAgentWrapper(BaseAgentWrapper): }, ) opts.env.update(extra_env_dict) + self._add_bash_proxy_hook(opts) self.session_path.mkdir(parents=True, exist_ok=True) opts.cwd = opts.cwd or self.cwd claude_config_dir = self.session_path / "claude_config" @@ -384,6 +415,12 @@ class CcAgentWrapper(BaseAgentWrapper): # ----- reply / reply_stream -------------------------------------------- + async def compact_session(self, session_id: str) -> None: + """Compact a Claude Code session through its native command.""" + result = await self.reply("/compact", resume=session_id) + if result["last_message"].get("is_error"): + raise RuntimeError("Claude Code session compaction failed") + async def reply(self, inputs: Any, **kwargs) -> dict: from claude_agent_sdk import query, ResultMessage diff --git a/reme/components/agent_wrapper/codex_agent_wrapper.py b/reme/components/agent_wrapper/codex_agent_wrapper.py index 7da4e181..edef958e 100644 --- a/reme/components/agent_wrapper/codex_agent_wrapper.py +++ b/reme/components/agent_wrapper/codex_agent_wrapper.py @@ -267,6 +267,12 @@ class CodexAgentWrapper(BaseAgentWrapper): def _thread_config(self, kwargs: dict[str, Any]) -> dict[str, Any] | None: config = dict(kwargs.get("config") or {}) + if proxy_environment := self.command_proxy_environment: + shell_environment_policy = dict(config.get("shell_environment_policy") or {}) + environment = dict(shell_environment_policy.get("set") or {}) + environment.update(proxy_environment) + shell_environment_policy["set"] = environment + config["shell_environment_policy"] = shell_environment_policy if server := self._mcp_server_config(kwargs): servers = dict(config.get("mcp_servers") or {}) server_key = hashlib.sha256(json.dumps(server, sort_keys=True).encode()).hexdigest()[:12] @@ -414,6 +420,14 @@ class CodexAgentWrapper(BaseAgentWrapper): raise ValueError("Codex returned invalid JSON for the requested output_schema") from exc return response + async def compact_session(self, session_id: str) -> None: + """Start native compaction of a Codex thread.""" + await self.start() + async with self._turn_lock: + codex = await self._get_codex() + thread = await codex.thread_resume(session_id) + await thread.compact() + @classmethod # pylint: disable=too-many-return-statements def _event_to_chunks(cls, event: Notification, session_id: str) -> list[StreamChunk]: diff --git a/reme/components/agent_wrapper/session_command.py b/reme/components/agent_wrapper/session_command.py new file mode 100644 index 00000000..497958cb --- /dev/null +++ b/reme/components/agent_wrapper/session_command.py @@ -0,0 +1,29 @@ +"""Backend-neutral agent session commands.""" + +from dataclasses import dataclass + +from .base_agent_wrapper import BaseAgentWrapper + + +@dataclass(frozen=True) +class SessionCommandResult: + """Result of a handled session command.""" + + session_id: str | None + answer: str + + +async def handle_session_command( + wrapper: BaseAgentWrapper, + text: str, + session_id: str | None, +) -> SessionCommandResult | None: + """Handle a supported command, or return ``None`` for ordinary input.""" + if text == "/clear": + return SessionCommandResult(None, "✅ Conversation cleared. The next message will start a new session.") + if text != "/compact": + return None + if not session_id: + return SessionCommandResult(None, "No active conversation to compact.") + await wrapper.compact_session(session_id) + return SessionCommandResult(session_id, "✅ Conversation compaction requested.") diff --git a/reme/components/embedding_store/base_embedding_store.py b/reme/components/embedding_store/base_embedding_store.py index cf557304..cc3cbed5 100644 --- a/reme/components/embedding_store/base_embedding_store.py +++ b/reme/components/embedding_store/base_embedding_store.py @@ -24,12 +24,14 @@ class BaseEmbeddingStore(BaseComponent): max_batch_size: int = 10, max_input_length: int = 8192, max_retries: int = 3, + quota_retry_delay: float | None = None, **kwargs, ): super().__init__(**kwargs) self.max_batch_size = max_batch_size self.max_input_length = max_input_length self.max_retries = max_retries + self.quota_retry_delay = quota_retry_delay self.is_healthy: bool = True def _truncate(self, text: str) -> str: diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index bf58d0a4..052a0851 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -133,11 +133,32 @@ class LocalEmbeddingStore(BaseEmbeddingStore): except (TimeoutError, ConnectionError, OSError): if attempt < self.max_retries - 1: await asyncio.sleep(2**attempt) - except Exception: + except Exception as error: + if ( + self.quota_retry_delay is not None + and self._is_insufficient_quota(error) + and attempt < self.max_retries - 1 + ): + self.logger.warning( + f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s", + ) + await asyncio.sleep(self.quota_retry_delay) + continue self.logger.exception("Embedding request failed") return None return None + @staticmethod + def _is_insufficient_quota(error: Exception) -> bool: + """Recognize OpenAI-compatible quota errors without importing a provider SDK.""" + if getattr(error, "code", None) == "insufficient_quota": + return True + body = getattr(error, "body", None) + if not isinstance(body, dict): + return False + details = body.get("error", body) + return isinstance(details, dict) and details.get("code") == "insufficient_quota" + def _validate_dim(self, emb: np.ndarray) -> bool: """Return whether an embedding exactly matches the configured dimension.""" return len(emb) == self.dimensions diff --git a/reme/components/file_chunker/jsonl_file_chunker.py b/reme/components/file_chunker/jsonl_file_chunker.py index 4720fb27..9417282c 100644 --- a/reme/components/file_chunker/jsonl_file_chunker.py +++ b/reme/components/file_chunker/jsonl_file_chunker.py @@ -18,7 +18,7 @@ class JsonlFileChunker(BaseFileChunker): Algorithm --------- 1. From ``start``, greedily accumulate lines until adding the next line - would exceed ``max_chars`` → emit that chunk. + would exceed ``max_chars`` or ``max_lines_per_chunk`` → emit that chunk. 2. From the *end* of the emitted chunk, walk backwards to find the maximum number of trailing lines whose combined size fits within ``max_overlap_chars`` → those lines become the start of the next chunk. @@ -33,6 +33,7 @@ class JsonlFileChunker(BaseFileChunker): encoding: str = "utf-8", max_chars: int = 2000, max_overlap_chars: int = 0, + max_lines_per_chunk: int | None = None, mode: str = "chars", **kwargs, ): @@ -42,6 +43,7 @@ class JsonlFileChunker(BaseFileChunker): self.encoding = encoding self.max_chars = max(64, max_chars) self.max_overlap_chars = max(0, max_overlap_chars) + self.max_lines_per_chunk = max(1, max_lines_per_chunk) if max_lines_per_chunk is not None else None self.mode = mode # ------------------------------------------------------------------ @@ -77,7 +79,11 @@ class JsonlFileChunker(BaseFileChunker): # sum(sizes[start:end]) <= max_chars -- total = 0 end = start - while end < n and total + sizes[end] <= self.max_chars: + while ( + end < n + and total + sizes[end] <= self.max_chars + and (self.max_lines_per_chunk is None or end - start < self.max_lines_per_chunk) + ): total += sizes[end] end += 1 @@ -126,8 +132,13 @@ class JsonlFileChunker(BaseFileChunker): if not text.strip(): return FileNode(path=rel_path, st_mtime=stat.st_mtime), [] - # readlines keeps trailing \n on each line. - lines = text.splitlines(keepends=True) + # JSON Lines records are delimited by LF (with an optional preceding + # CR). str.splitlines() also treats Unicode separators such as U+2028 + # as line boundaries, even though they are valid inside a JSON string. + parts = text.split("\n") + lines = [part + "\n" for part in parts[:-1]] + if parts[-1]: + lines.append(parts[-1]) if not lines: return FileNode(path=rel_path, st_mtime=stat.st_mtime), [] diff --git a/reme/components/file_graph/nx_file_graph.py b/reme/components/file_graph/nx_file_graph.py index 5157d5d0..27257925 100644 --- a/reme/components/file_graph/nx_file_graph.py +++ b/reme/components/file_graph/nx_file_graph.py @@ -3,11 +3,6 @@ import pickle from pathlib import Path -try: - import networkx as nx -except ImportError: - nx = None - from .base_file_graph import BaseFileGraph from ..component_registry import R from ...enumeration import LinkScopeEnum @@ -23,9 +18,11 @@ class NxFileGraph(BaseFileGraph): def __init__(self, **kwargs): super().__init__(**kwargs) - if nx is None: - raise ImportError("NxFileGraph requires networkx — pip install networkx") - self._graph: nx.MultiDiGraph = nx.MultiDiGraph() + try: + import networkx as nx # pylint: disable=import-outside-toplevel + except ImportError as exc: + raise ImportError("NxFileGraph requires networkx — pip install networkx") from exc + self._graph = nx.MultiDiGraph() self.component_metadata_path.mkdir(parents=True, exist_ok=True) self._graph_file: Path = self.component_metadata_path / f"{self.name}.pkl" diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index 3981be09..ef3ed749 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -383,6 +383,14 @@ class LocalFileStore(BaseFileStore): with suppress(Exception): live_ids = set(self.keyword_index.document_ids) + # A non-empty chunk may still be unrepresentable by a lexical backend + # (for example, punctuation-only text produces no BM25 tokens). Check + # only missing IDs so the normal matching path does not tokenize the + # entire corpus during every startup. + if live_ids is not None: + unindexable_ids = {cid for cid in expected_ids - live_ids if not self.keyword_index.is_indexable(docs[cid])} + expected_ids -= unindexable_ids + if live_ids == expected_ids: return diff --git a/reme/components/keyword_index/base_keyword_index.py b/reme/components/keyword_index/base_keyword_index.py index bb64b1b3..3ae551ff 100644 --- a/reme/components/keyword_index/base_keyword_index.py +++ b/reme/components/keyword_index/base_keyword_index.py @@ -37,6 +37,10 @@ class BaseKeywordIndex(BaseComponent): raise RuntimeError("Tokenizer not initialized. Call start() first.") return self.tokenizer.tokenize([text])[0] + def is_indexable(self, text: str) -> bool: + """Return whether this backend can represent *text* as a document.""" + return bool(text) + @abstractmethod async def add_docs(self, docs_dict: dict[str, str]) -> None: """Add or replace documents keyed by id.""" diff --git a/reme/components/keyword_index/bm25_index.py b/reme/components/keyword_index/bm25_index.py index c049f9d6..0cb3db86 100644 --- a/reme/components/keyword_index/bm25_index.py +++ b/reme/components/keyword_index/bm25_index.py @@ -189,6 +189,10 @@ class BM25Index(BaseKeywordIndex): unique_tids = np.fromiter(counts.keys(), dtype=np.int32, count=len(counts)) return unique_tids, len(token_ids), counts + def is_indexable(self, text: str) -> bool: + """Return whether tokenization produces at least one BM25 term.""" + return bool(self._tokenize(text)) + def _append_doc_arrays( self, new_doc_ids: list[str], diff --git a/reme/components/outbound_proxy/__init__.py b/reme/components/outbound_proxy/__init__.py new file mode 100644 index 00000000..1e5e0b72 --- /dev/null +++ b/reme/components/outbound_proxy/__init__.py @@ -0,0 +1,12 @@ +"""Application-scoped outbound HTTP proxy components.""" + +from .base import BaseOutboundProxy, OutboundProxyEndpoint +from .fixed_http import FixedHttpOutboundProxy +from .ssh_http import SshHttpOutboundProxy + +__all__ = [ + "BaseOutboundProxy", + "FixedHttpOutboundProxy", + "OutboundProxyEndpoint", + "SshHttpOutboundProxy", +] diff --git a/reme/components/outbound_proxy/base.py b/reme/components/outbound_proxy/base.py new file mode 100644 index 00000000..9ef838f6 --- /dev/null +++ b/reme/components/outbound_proxy/base.py @@ -0,0 +1,54 @@ +"""Public contract shared by outbound proxy backends.""" + +from collections.abc import Mapping +from dataclasses import dataclass + +from ..base_component import BaseComponent +from ...enumeration import ComponentEnum + +_LOCAL_BYPASS = "127.0.0.1,localhost,::1" +_PROXY_VARIABLES = ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", +) + + +@dataclass(frozen=True, slots=True) +class OutboundProxyEndpoint: + """Stable public endpoint exposed for the lifetime of a started proxy.""" + + http_url: str + + +class BaseOutboundProxy(BaseComponent): + """Base component exposing one application-scoped HTTP proxy endpoint.""" + + component_type = ComponentEnum.OUTBOUND_PROXY + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self._endpoint: OutboundProxyEndpoint | None = None + + @property + def endpoint(self) -> OutboundProxyEndpoint: + """Return the ready endpoint, failing when the component is not started.""" + if self._endpoint is None: + raise RuntimeError("Outbound proxy endpoint is unavailable; start the component first.") + return self._endpoint + + @property + def http_url(self) -> str: + """Return the public HTTP proxy URL.""" + return self.endpoint.http_url + + def merge_environment(self, base: Mapping[str, str] | None = None) -> dict[str, str]: + """Return a proxy-aware environment without mutating the input or process environment.""" + environment = dict(base or {}) + environment.update(dict.fromkeys(_PROXY_VARIABLES, self.http_url)) + environment["NO_PROXY"] = _LOCAL_BYPASS + environment["no_proxy"] = _LOCAL_BYPASS + return environment diff --git a/reme/components/outbound_proxy/fixed_http.py b/reme/components/outbound_proxy/fixed_http.py new file mode 100644 index 00000000..852b5cde --- /dev/null +++ b/reme/components/outbound_proxy/fixed_http.py @@ -0,0 +1,40 @@ +"""Outbound proxy backend wrapping an existing HTTP proxy.""" + +from urllib.parse import urlsplit + +from .base import BaseOutboundProxy, OutboundProxyEndpoint +from ..component_registry import R + + +@R.register("fixed_http") +class FixedHttpOutboundProxy(BaseOutboundProxy): + """Publish an externally managed HTTP proxy without owning its lifecycle.""" + + def __init__(self, url: str = "", **kwargs) -> None: + super().__init__(**kwargs) + self.url = url + + async def _start(self) -> None: + self._endpoint = OutboundProxyEndpoint(http_url=self._validate_url()) + + async def _close(self) -> None: + self._endpoint = None + + def _validate_url(self) -> str: + try: + parsed = urlsplit(self.url) + port = parsed.port + except ValueError as exc: + raise ValueError("Outbound proxy configuration invalid: malformed fixed HTTP proxy URL.") from exc + + if parsed.scheme != "http": + raise ValueError("Outbound proxy configuration invalid: fixed proxy URL must use http://.") + if not parsed.hostname or port is None: + raise ValueError("Outbound proxy configuration invalid: fixed proxy URL must include host and port.") + if parsed.username is not None or parsed.password is not None: + raise ValueError("Outbound proxy configuration invalid: fixed proxy URL must not contain userinfo.") + if parsed.query or parsed.fragment: + raise ValueError( + "Outbound proxy configuration invalid: fixed proxy URL must not contain query or fragment.", + ) + return self.url diff --git a/reme/components/outbound_proxy/ssh_http.py b/reme/components/outbound_proxy/ssh_http.py new file mode 100644 index 00000000..9fbb146d --- /dev/null +++ b/reme/components/outbound_proxy/ssh_http.py @@ -0,0 +1,352 @@ +"""Long-lived SSH-backed HTTP outbound proxy.""" + +import asyncio +import contextlib +import importlib.util +import math +import shutil +import socket +import sys +import time +from asyncio.subprocess import Process + +from .base import BaseOutboundProxy, OutboundProxyEndpoint +from ..component_registry import R + +_LOOPBACK = "127.0.0.1" +_START_ATTEMPTS = 3 +_PROCESS_CLOSE_TIMEOUT = 5.0 +_PROCESS_OUTPUT_LIMIT = 4096 +_PORT_CONFLICT_MARKERS = ("address already in use", "cannot listen to port") + + +class _PortUnavailableError(RuntimeError): + """A selected listener port was claimed before its subprocess bound it.""" + + +@R.register("ssh_http") +class SshHttpOutboundProxy(BaseOutboundProxy): + """Expose a stable local HTTP proxy backed by an OpenSSH SOCKS tunnel.""" + + def __init__( + self, + host: str = "", + account: str = "", + connect_timeout: float = 10.0, + monitor_interval: float = 1.0, + restart_initial_delay: float = 1.0, + restart_max_delay: float = 30.0, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.host = host + self.account = account + self.connect_timeout = connect_timeout + self.monitor_interval = monitor_interval + self.restart_initial_delay = restart_initial_delay + self.restart_max_delay = restart_max_delay + + self._ssh_executable: str | None = None + self._socks_port: int | None = None + self._http_port: int | None = None + self._ssh_process: Process | None = None + self._bridge_process: Process | None = None + self._monitor_task: asyncio.Task[None] | None = None + self._closing = False + + async def _start(self) -> None: + self._validate_configuration() + self._closing = False + + try: + for attempt in range(_START_ATTEMPTS): + self._socks_port, self._http_port = self._pick_distinct_ports() + try: + await self._start_processes() + break + except _PortUnavailableError: + await self._stop_processes() + if attempt + 1 == _START_ATTEMPTS: + raise + self.logger.warning("Outbound proxy listener port was claimed; selecting new ports.") + else: # pragma: no cover - loop either breaks or raises + raise RuntimeError("Outbound proxy failed to allocate listener ports.") + + assert self._http_port is not None + self._endpoint = OutboundProxyEndpoint(http_url=f"http://{_LOOPBACK}:{self._http_port}") + self._monitor_task = asyncio.create_task( + self._monitor_processes(), + name=f"outbound-proxy-monitor:{self.name}", + ) + self.logger.info( + f"Outbound proxy ready http={_LOOPBACK}:{self._http_port} " f"socks={_LOOPBACK}:{self._socks_port}", + ) + except BaseException: + await self._stop_processes() + self._endpoint = None + self._socks_port = None + self._http_port = None + raise + + async def _close(self) -> None: + self._closing = True + cleanup = asyncio.create_task(self._cleanup()) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + await cleanup + raise + + async def _cleanup(self) -> None: + try: + if self._monitor_task is not None: + self._monitor_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._monitor_task + self._monitor_task = None + await self._stop_processes() + finally: + self._endpoint = None + self._socks_port = None + self._http_port = None + + def _validate_configuration(self) -> None: + if not isinstance(self.host, str) or not self.host.strip(): + raise ValueError("Outbound proxy configuration invalid: host is required.") + if not isinstance(self.account, str) or not self.account.strip(): + raise ValueError("Outbound proxy configuration invalid: account is required.") + + numeric_fields = ( + "connect_timeout", + "monitor_interval", + "restart_initial_delay", + "restart_max_delay", + ) + for field in numeric_fields: + value = getattr(self, field) + if isinstance(value, bool): + raise ValueError(f"Outbound proxy configuration invalid: {field} must be positive.") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Outbound proxy configuration invalid: {field} must be positive.") from exc + if not math.isfinite(normalized) or normalized <= 0: + raise ValueError(f"Outbound proxy configuration invalid: {field} must be positive.") + setattr(self, field, normalized) + + self._ssh_executable = shutil.which("ssh") + if self._ssh_executable is None: + raise RuntimeError("Outbound proxy configuration invalid: ssh executable was not found.") + if importlib.util.find_spec("pproxy") is None: + raise RuntimeError( + "SSH HTTP outbound proxy requires pproxy; install ReMe with the 'core' extra: reme-ai[core].", + ) + + @staticmethod + def _pick_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind((_LOOPBACK, 0)) + return int(listener.getsockname()[1]) + + @classmethod + def _pick_distinct_ports(cls) -> tuple[int, int]: + socks_port = cls._pick_free_port() + http_port = cls._pick_free_port() + while http_port == socks_port: + http_port = cls._pick_free_port() + return socks_port, http_port + + async def _start_processes(self) -> None: + await self._start_ssh() + try: + await self._start_bridge() + except BaseException: + await self._stop_process(self._ssh_process) + self._ssh_process = None + raise + + async def _start_ssh(self) -> None: + assert self._ssh_executable is not None + assert self._socks_port is not None + destination = f"{self.account}@{self.host}" + command = ( + self._ssh_executable, + "-N", + "-D", + f"{_LOOPBACK}:{self._socks_port}", + "-o", + "BatchMode=yes", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"ConnectTimeout={max(1, math.ceil(self.connect_timeout))}", + "-o", + "LogLevel=ERROR", + "--", + destination, + ) + self._ssh_process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + try: + await self._wait_for_listener( + self._ssh_process, + self._socks_port, + self.connect_timeout, + "SSH proxy", + ) + except asyncio.CancelledError: + await self._stop_process(self._ssh_process) + raise + except BaseException as exc: + await self._raise_start_error(self._ssh_process, exc, "SSH proxy exited before readiness") + + async def _start_bridge(self) -> None: + assert self._socks_port is not None + assert self._http_port is not None + command = ( + sys.executable, + "-m", + "pproxy", + "-l", + f"http://{_LOOPBACK}:{self._http_port}", + "-r", + f"socks5://{_LOOPBACK}:{self._socks_port}", + ) + self._bridge_process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + try: + await self._wait_for_listener( + self._bridge_process, + self._http_port, + self.connect_timeout, + "HTTP bridge", + ) + except asyncio.CancelledError: + await self._stop_process(self._bridge_process) + raise + except BaseException as exc: + await self._raise_start_error(self._bridge_process, exc, "HTTP bridge exited before readiness") + + async def _raise_start_error(self, process: Process, cause: BaseException, prefix: str) -> None: + detail = await self._read_process_output(process) + await self._stop_process(process) + if self._is_port_conflict(detail): + raise _PortUnavailableError(f"{prefix}: {detail}") from cause + suffix = f": {detail}" if detail else "" + raise RuntimeError(f"{prefix}{suffix}") from cause + + @staticmethod + def _is_port_conflict(detail: str) -> bool: + lowered = detail.lower() + return any(marker in lowered for marker in _PORT_CONFLICT_MARKERS) + + @staticmethod + async def _wait_for_listener(process: Process, port: int, timeout: float, label: str) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if process.returncode is not None: + raise RuntimeError(f"{label} process exited with status {process.returncode}.") + try: + _reader, writer = await asyncio.open_connection(_LOOPBACK, port) + writer.close() + await writer.wait_closed() + return + except OSError: + await asyncio.sleep(min(0.05, max(0.0, deadline - loop.time()))) + raise TimeoutError(f"{label} listener did not become ready on {_LOOPBACK}:{port} within {timeout:g}s.") + + async def _monitor_processes(self) -> None: + attempt = 0 + stable_since = time.monotonic() + while True: + await asyncio.sleep(self.monitor_interval) + if self._closing: + return + + restart_ssh = self._ssh_process is None or self._ssh_process.returncode is not None + restart_bridge = self._bridge_process is None or self._bridge_process.returncode is not None + if not restart_ssh and not restart_bridge: + if time.monotonic() - stable_since >= self.monitor_interval: + attempt = 0 + continue + + delay = min(self.restart_initial_delay * (2**attempt), self.restart_max_delay) + self.logger.warning( + f"Outbound proxy reconnecting ssh={restart_ssh} bridge={restart_bridge} delay={delay:g}s", + ) + await asyncio.sleep(delay) + if self._closing: + return + + try: + if restart_ssh: + await self._restart_ssh() + if restart_bridge: + await self._restart_bridge() + except Exception as exc: + attempt += 1 + stable_since = time.monotonic() + self.logger.error(f"Outbound proxy reconnect failed error={type(exc).__name__}: {exc}") + else: + stable_since = time.monotonic() + + async def _restart_ssh(self) -> None: + await self._stop_process(self._ssh_process) + self._ssh_process = None + try: + await self._start_ssh() + except BaseException: + await self._stop_process(self._ssh_process) + self._ssh_process = None + raise + + async def _restart_bridge(self) -> None: + await self._stop_process(self._bridge_process) + self._bridge_process = None + try: + await self._start_bridge() + except BaseException: + await self._stop_process(self._bridge_process) + self._bridge_process = None + raise + + async def _stop_processes(self) -> None: + await self._stop_process(self._bridge_process) + self._bridge_process = None + await self._stop_process(self._ssh_process) + self._ssh_process = None + + @staticmethod + async def _stop_process(process: Process | None) -> None: + if process is None: + return + if process.returncode is not None: + await process.wait() + return + with contextlib.suppress(ProcessLookupError): + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=_PROCESS_CLOSE_TIMEOUT) + except asyncio.TimeoutError: + with contextlib.suppress(ProcessLookupError): + process.kill() + await process.wait() + + @staticmethod + async def _read_process_output(process: Process) -> str: + stream = process.stderr if process.stderr is not None else process.stdout + if stream is None or process.returncode is None: + return "" + data = await stream.read(_PROCESS_OUTPUT_LIMIT + 1) + truncated = len(data) > _PROCESS_OUTPUT_LIMIT + text = data[:_PROCESS_OUTPUT_LIMIT].decode(errors="replace").strip() + return f"{text}…" if truncated else text diff --git a/reme/config/daily_cookbook.yaml b/reme/config/daily_cookbook.yaml index fcff44d1..031999b8 100644 --- a/reme/config/daily_cookbook.yaml +++ b/reme/config/daily_cookbook.yaml @@ -1,5 +1,5 @@ app_name: ReMe Daily Cookbook -workspace_dir: ${DAILY_PAPER_WORKSPACE_DIR:-.reme} +workspace_dir: ${DAILY_PAPER_WORKSPACE_DIR:-reme_workspace} timezone: Asia/Shanghai language: zh @@ -11,6 +11,369 @@ service: port: ${DAILY_PAPER_PORT:-8001} jobs: + index_update_loop: + backend: background + watch_dirs: [daily_dir, digest_dir] + watch_suffixes: [md, jsonl] + steps: + - backend: init_changes_step + monitor_type: file_store + monitor_name: default + dispatch_steps: [update_index_step] + - backend: watch_changes_step + dispatch_steps: + - backend: update_index_step + persist: false + + auto_dream: + backend: base + description: "Auto-dream: consolidate recent daily notes into digest memory and interest topics." + parameters: + type: object + properties: + date: + type: string + description: "YYYY-MM-DD to scan; defaults to today in the configured timezone." + default: "" + hint: + type: string + description: "Optional guidance for extraction and integration." + default: "" + scan_days: + type: integer + description: "Number of recent daily directories to scan." + default: 2 + max_units: + type: integer + description: "Maximum number of extracted memory units." + default: 5 + topic_count: + type: integer + description: "Maximum number of interest topics to write." + default: 3 + topic_diversity_days: + type: integer + description: "Previous interest-topic days used for de-duplication." + default: 7 + steps: + - backend: dream_extract_step + as_llm: memory + agent_wrapper: memory + file_catalog: dream + topic_session_id: interests + scan_days: 2 + max_units: 5 + - backend: dream_integrate_step + as_llm: memory + agent_wrapper: memory + - backend: dream_topics_step + as_llm: memory + agent_wrapper: memory + topic_count: 3 + topic_diversity_days: 7 + - backend: dream_finish_step + file_catalog: dream + + auto_memory: + backend: base + description: "Auto-memory: record conversation facts into a daily note." + parameters: + type: object + properties: + messages: + type: array + description: "Conversation messages." + items: + type: object + session_id: + type: string + description: "Source conversation session identifier." + default: "" + memory_hint: + type: string + description: "Optional memory-writing guidance." + date: + type: string + description: "YYYY-MM-DD daily-note date; empty infers it from messages or current time." + default: "" + required: [messages] + steps: + - backend: auto_memory_step + agent_wrapper: memory + + reindex: + backend: base + description: "Wipe the derived search store and rebuild it from memory files." + watch_dirs: [daily_dir, digest_dir] + watch_suffixes: [md, jsonl] + parameters: + type: object + properties: {} + steps: + - backend: clear_store_step + - backend: init_changes_step + monitor_type: file_store + monitor_name: default + dispatch_steps: [update_index_step] + + memory_search: + backend: base + description: "Long-term memory retrieval via hybrid workspace search (vector + BM25, RRF-fused)." + parameters: + type: object + properties: + query: + type: string + description: "Search query." + limit: + type: integer + description: "Maximum number of results." + default: 5 + min_score: + type: number + description: "Minimum fused score." + default: 0.0 + start_date: + type: string + description: "Optional inclusive start date (YYYY-MM-DD)." + end_date: + type: string + description: "Optional inclusive end date (YYYY-MM-DD)." + required: [query] + steps: + - backend: search_step + vector_weight: 0.7 + candidate_multiplier: 5.0 + expand_links: true + max_links_per_direction: 10 + + node_search: + backend: base + description: "Recall digest nodes for auto-dream de-duplication and linking." + parameters: + type: object + properties: + query: + type: string + description: "Candidate memory-node name and description." + limit: + type: integer + description: "Maximum number of digest nodes." + default: 20 + required: [query] + steps: + - backend: node_search_step + vector_weight: 0.7 + candidate_multiplier: 5.0 + + daily_list: + backend: base + description: "List notes under one day." + parameters: + type: object + properties: + date: + type: string + description: "YYYY-MM-DD; empty means today." + default: "" + steps: + - backend: daily_list_step + + frontmatter_read: + backend: base + description: "Read a file's frontmatter." + parameters: + type: object + properties: + path: + type: string + description: "Workspace-relative path." + required: [path] + steps: + - backend: frontmatter_read_step + + frontmatter_update: + backend: base + description: "Merge key-values into a file's frontmatter." + parameters: + type: object + properties: + path: + type: string + description: "Workspace-relative path." + metadata: + type: object + description: "Key-values to merge." + required: [path, metadata] + steps: + - backend: frontmatter_update_step + + move: + backend: base + description: "Move or rename a workspace file and retarget inbound wikilinks." + parameters: + type: object + properties: + src_path: + type: string + description: "Workspace-relative source path." + dst_path: + type: string + description: "Workspace-relative destination path." + overwrite: + type: boolean + default: false + retarget: + type: boolean + default: true + required: [src_path, dst_path] + steps: + - backend: move_step + + read: + backend: base + description: "Read a markdown file under the workspace." + parameters: + type: object + properties: + path: + type: string + description: "Workspace-relative markdown path." + start_line: + type: integer + end_line: + type: integer + required: [path] + steps: + - backend: read_step + with_neighbors: false + max_neighbors_per_direction: 10 + + write: + backend: base + description: "Create or overwrite a markdown file with frontmatter." + parameters: + type: object + properties: + path: + type: string + description: "Workspace-relative markdown path." + name: + type: string + description: "Frontmatter name." + description: + type: string + description: "Frontmatter description." + content: + type: string + description: "Markdown body." + metadata: + type: object + description: "Optional extra frontmatter fields." + required: [path, name, description, content] + steps: + - backend: write_step + + daily_write: + backend: base + description: "Write a daily markdown note linked to its source conversation." + parameters: + type: object + properties: + name: + type: string + description: "Filename stem and frontmatter name." + description: + type: string + description: "Frontmatter description." + session_id: + type: string + description: "Source conversation session identifier." + content: + type: string + description: "Markdown body." + date: + type: string + description: "YYYY-MM-DD; empty means today." + default: "" + metadata: + type: object + description: "Optional extra frontmatter fields." + required: [name, description, session_id, content] + steps: + - backend: daily_write_step + + edit: + backend: base + description: "Find and replace text in a markdown file." + parameters: + type: object + properties: + path: + type: string + description: "Workspace-relative path." + old: + type: string + description: "Text to replace." + new: + type: string + description: "Replacement text." + default: "" + required: [path, old, new] + steps: + - backend: edit_step + + auto_fin: + backend: base + description: "Maintain configured daily news history, analyze current topics and historical ETF reactions, then deliver the report." + parameters: + type: object + properties: + date: + type: string + description: "Current date in YYYY-MM-DD; empty means today in Asia/Shanghai." + default: "" + force: + type: boolean + description: "Refresh all configured news days; today's news is always refreshed." + default: false + steps: &auto_fin_steps + - backend: auto_fin_data_step + outbound_proxy: default + lookback_days: 360 + progress_interval: 30 + - backend: auto_fin_topic_step + agent_wrapper: auto_fin + - backend: auto_fin_history_step + agent_wrapper: auto_fin + dispatch_steps: [auto_fin_history_search_step, auto_fin_market_step] + - backend: auto_fin_merge_step + agent_wrapper: auto_fin + - backend: dingtalk_markdown_send_step + input_mapping: + auto_fin_digest_path: markdown_path + app_key: ${DINGTALK_APP_KEY:-} + app_secret: ${DINGTALK_APP_SECRET:-} + robot_code: ${DINGTALK_ROBOT_CODE:-} + conversation_ids: ${DINGTALK_CONVERSATION_IDS:-} + title: ReMe Auto Fin + timeout: 15 + + auto_fin_0930_cron: + backend: cron + cron: "30 9 * * *" + steps: *auto_fin_steps + + auto_fin_1145_cron: + backend: cron + cron: "45 11 * * *" + steps: *auto_fin_steps + + auto_fin_1800_cron: + backend: cron + cron: "0 18 * * *" + steps: *auto_fin_steps + daily_paper: backend: base description: "Build detailed readings and a five-minute brief from Hugging Face weekly/monthly papers." @@ -51,13 +414,15 @@ jobs: default: 30 steps: &daily_paper_steps - backend: daily_paper_collect_step + outbound_proxy: default - backend: daily_paper_rank_step - backend: daily_paper_select_step - agent_wrapper: claude_code + agent_wrapper: daily_paper - backend: daily_paper_analyze_step - agent_wrapper: claude_code + agent_wrapper: daily_paper + outbound_proxy: default - backend: daily_paper_digest_step - agent_wrapper: claude_code + agent_wrapper: daily_paper - backend: dingtalk_markdown_send_step input_mapping: daily_paper_digest_path: markdown_path @@ -91,19 +456,137 @@ jobs: close_timeout: 10 steps: - backend: dingtalk_wait_step - agent_wrapper: claude_code + agent_wrapper: dingtalk_wait app_key: ${DINGTALK_APP_KEY:-} app_secret: ${DINGTALK_APP_SECRET:-} robot_code: ${DINGTALK_ROBOT_CODE:-} - card_update_interval: 1.0 worker_count: 4 components: + outbound_proxy: {} +# default: +# backend: ssh_http +# host: ${REME_PROXY_IP:-} +# account: ${REME_PROXY_ACCOUNT:-} +# connect_timeout: 10 +# monitor_interval: 1 +# restart_initial_delay: 1 +# restart_max_delay: 30 + + tokenizer: + default: + backend: regex + + as_llm: + memory: + backend: anthropic + model: ${LLM_MODEL_NAME:-qwen3.7-max} + stream: true + context_size: 200000 + max_retries: 3 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} + parameters: + max_tokens: 65536 + thinking_enable: false + agent_wrapper: - claude_code: + auto_fin: + backend: ${AUTO_FIN_AGENT_BACKEND:-claude_code} + project_path: ${AUTO_FIN_PROJECT_PATH:-..} + model: ${CLAUDE_CODE_MODEL_NAME:-qwen3.7-max} + api_key: ${CLAUDE_CODE_API_KEY:-} + base_url: ${CLAUDE_CODE_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} + skills: [tushare-data] + job_tools: [memory_search] + permission_mode: bypassPermissions + daily_paper: backend: claude_code project_path: ${DAILY_PAPER_PROJECT_PATH:-..} model: ${CLAUDE_CODE_MODEL_NAME:-qwen3.7-max} api_key: ${CLAUDE_CODE_API_KEY:-} base_url: ${CLAUDE_CODE_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} permission_mode: bypassPermissions + dingtalk_wait: + backend: claude_code + project_path: ${DAILY_PAPER_PROJECT_PATH:-..} + model: ${CLAUDE_CODE_MODEL_NAME:-qwen3.7-max} + api_key: ${CLAUDE_CODE_API_KEY:-} + base_url: ${CLAUDE_CODE_BASE_URL:-https://dashscope.aliyuncs.com/apps/anthropic} + skills: [tushare-data] + job_tools: [memory_search] + system_prompt: + type: preset + preset: claude_code + append: >- + Daily-paper Markdown is stored under the ReMe workspace. Detailed notes, including historical notes, are at + daily/YYYY-MM-DD/paper-.md; daily briefs are at daily/YYYY-MM-DD/daily-paper-brief.md. Use + memory_search to retrieve relevant long-term notes across dates. + permission_mode: bypassPermissions + memory: + backend: agentscope + as_llm: memory + builtin_tools: false + permission_mode: bypass + react_config: + max_iters: 30 + context_config: + trigger_ratio: 0.8 + reserve_ratio: 0.1 + tool_result_limit: 50000 + model_config: + max_retries: 1 + +# as_embedding: +# default: +# backend: openai +# model: ${EMBEDDING_MODEL_NAME:-text-embedding-v4} +# dimensions: 1024 +# max_retries: 0 +# credential: +# api_key: ${EMBEDDING_API_KEY:-} +# base_url: ${EMBEDDING_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1} +# parameters: {} +# +# embedding_store: +# default: +# backend: local +# as_embedding: default +# max_retries: 3 +# quota_retry_delay: 60.0 + + file_graph: + default: + backend: local + + file_catalog: + dream: + backend: local + + file_chunker: + markdown: + backend: markdown + supported_extensions: [md] + embed_toc: true + max_ast_sections: 100 + include_frontmatter_in_metadata: false + include_frontmatter_keys_in_metadata: [] + jsonl: + backend: jsonl + supported_extensions: [jsonl] + max_lines_per_chunk: 1 + + keyword_index: + default: + backend: bm25 + tokenizer: default + + file_store: + default: + backend: local + store_name: local +# embedding_store: default + embedding_store: "" + keyword_index: default + file_graph: default diff --git a/reme/enumeration/component_enum.py b/reme/enumeration/component_enum.py index 61085465..0f02b1f3 100644 --- a/reme/enumeration/component_enum.py +++ b/reme/enumeration/component_enum.py @@ -35,3 +35,5 @@ class ComponentEnum(str, Enum): TOKENIZER = "tokenizer" AGENT_WRAPPER = "agent_wrapper" + + OUTBOUND_PROXY = "outbound_proxy" diff --git a/reme/schema/__init__.py b/reme/schema/__init__.py index 66118cda..6a25163b 100644 --- a/reme/schema/__init__.py +++ b/reme/schema/__init__.py @@ -1,6 +1,27 @@ """Schema""" from .application_config import ApplicationConfig, ComponentConfig, JobConfig +from .auto_fin import ( + AutoFinEtfEventReference, + AutoFinEtfHistoryDetail, + AutoFinEtfHistoricalEvents, + AutoFinEtfHistoricalResearch, + AutoFinEtfSelection, + AutoFinEtfsOutput, + AutoFinDailyEntry, + AutoFinForecastReturnPoint, + AutoFinFutureReturnPoint, + AutoFinHistoricalEvent, + AutoFinHistoricalEventReference, + AutoFinHistoricalMatch, + AutoFinHistoricalSimilarity, + AutoFinMarketSelection, + AutoFinMarketSample, + AutoFinReportOutput, + AutoFinSelectedEvent, + AutoFinSelectedEtfAnalysis, + AutoFinWeightedForecast, +) from .daily_paper import DailyBriefOutput, PaperInfo, PaperNoteOutput, PaperSelection, SelectedPaper from .dream import ( DreamExtractOutput, @@ -22,6 +43,25 @@ from .stream_chunk import StreamChunk __all__ = [ "ApplicationConfig", + "AutoFinEtfEventReference", + "AutoFinEtfHistoryDetail", + "AutoFinEtfHistoricalEvents", + "AutoFinEtfHistoricalResearch", + "AutoFinEtfSelection", + "AutoFinEtfsOutput", + "AutoFinDailyEntry", + "AutoFinForecastReturnPoint", + "AutoFinFutureReturnPoint", + "AutoFinHistoricalEvent", + "AutoFinHistoricalEventReference", + "AutoFinHistoricalMatch", + "AutoFinHistoricalSimilarity", + "AutoFinMarketSelection", + "AutoFinMarketSample", + "AutoFinReportOutput", + "AutoFinSelectedEvent", + "AutoFinSelectedEtfAnalysis", + "AutoFinWeightedForecast", "ComponentConfig", "DailyBriefOutput", "DreamExtractOutput", diff --git a/reme/schema/auto_fin.py b/reme/schema/auto_fin.py new file mode 100644 index 00000000..3c55eb93 --- /dev/null +++ b/reme/schema/auto_fin.py @@ -0,0 +1,358 @@ +"""Public contracts for the Auto Fin workflow.""" + +from __future__ import annotations + +from datetime import date, datetime +from math import isclose +from typing import Annotated, Literal +from zoneinfo import ZoneInfo + +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, model_validator + +_SHANGHAI = ZoneInfo("Asia/Shanghai") + + +def _shanghai_local_time(value): + """Normalize aware input to naive Shanghai wall-clock time.""" + if not isinstance(value, (str, datetime)): + return value + parsed = value if isinstance(value, datetime) else datetime.fromisoformat(value) + if parsed.tzinfo is not None and parsed.utcoffset() is not None: + parsed = parsed.astimezone(_SHANGHAI).replace(tzinfo=None) + return parsed + + +ShanghaiDateTime = Annotated[datetime, BeforeValidator(_shanghai_local_time)] + + +class AutoFinModel(BaseModel): + """Strict base for Agent output.""" + + model_config = ConfigDict(extra="forbid") + + +class AutoFinEtfEventReference(AutoFinModel): + """One selected news item and why it is relevant to an ETF.""" + + reason: str + news_id: str + + @model_validator(mode="after") + def non_empty_values(self) -> "AutoFinEtfEventReference": + """Reject blank event references.""" + self.reason = self.reason.strip() + self.news_id = self.news_id.strip() + if not self.reason or not self.news_id: + raise ValueError("ETF event reason and news ID must be non-empty") + return self + + +class AutoFinSelectedEvent(AutoFinModel): + """A selected current event with its source news reference.""" + + event_time: ShanghaiDateTime + event_content: str + reason: str + news_id: str + event_title: str = "" + + +class AutoFinEtfSelection(AutoFinModel): + """One liquid ETF selected for current news.""" + + etf_code: str + etf_name: str + events: list[AutoFinEtfEventReference] = Field(min_length=1) + + @model_validator(mode="after") + def valid_news_ids(self) -> "AutoFinEtfSelection": + """Reject blank or duplicate news references.""" + news_ids = [event.news_id for event in self.events] + if len(news_ids) != len(set(news_ids)): + raise ValueError("ETF event news IDs must be unique") + return self + + +class AutoFinEtfsOutput(AutoFinModel): + """Liquid ETFs related to current news, deduplicated by name and theme.""" + + etfs: list[AutoFinEtfSelection] = Field(default_factory=list, max_length=20) + + @model_validator(mode="after") + def unique_etfs(self) -> "AutoFinEtfsOutput": + """Reject duplicate ETF codes or names.""" + codes = [item.etf_code.strip().upper() for item in self.etfs] + names = [item.etf_name.strip().casefold() for item in self.etfs] + if any(not code for code in codes) or any(not name for name in names): + raise ValueError("ETF codes and names must be non-empty") + if len(codes) != len(set(codes)) or len(names) != len(set(names)): + raise ValueError("ETF codes and names must be unique") + return self + + +class AutoFinHistoricalEventReference(AutoFinModel): + """One historical news item selected by the search Agent.""" + + reason: str + news_id: str + source_path: str + + @model_validator(mode="after") + def non_empty_values(self) -> "AutoFinHistoricalEventReference": + """Reject references that cannot be resolved deterministically.""" + for field in ("reason", "news_id", "source_path"): + value = getattr(self, field).strip() + if not value: + raise ValueError(f"historical event {field} must not be empty") + setattr(self, field, value) + return self + + +class AutoFinEtfHistoricalEvents(AutoFinModel): + """Historical news references returned by the search Agent.""" + + etf_code: str + etf_name: str + historical_events: list[AutoFinHistoricalEventReference] = Field(default_factory=list) + + +class AutoFinDailyEntry(AutoFinModel): + """First daily open or close that can be traded after an event.""" + + entry_time: ShanghaiDateTime + trade_date: date + price_type: Literal["open", "close"] + raw_price: float = Field(gt=0) + adj_factor: float = Field(gt=0) + + @model_validator(mode="after") + def valid_entry_timestamp(self) -> "AutoFinDailyEntry": + """Require a Shanghai-local timestamp matching the daily price.""" + if self.entry_time.date() != self.trade_date: + raise ValueError("entry time and trade date must match") + expected_clock = (9, 30) if self.price_type == "open" else (15, 0) + if ( + (self.entry_time.hour, self.entry_time.minute) != expected_clock + or self.entry_time.second + or self.entry_time.microsecond + ): + raise ValueError(f"{self.price_type} entry time must use the official daily price timestamp") + return self + + +class AutoFinFutureReturnPoint(AutoFinModel): + """Cumulative adjusted return at one future valid close.""" + + horizon: int = Field(ge=1, le=10) + trade_date: date + raw_close: float = Field(gt=0) + adj_factor: float = Field(gt=0) + cumulative_return: float + + +class AutoFinMarketSample(AutoFinModel): + """Daily adjusted ETF returns following one historical event.""" + + event_time: ShanghaiDateTime + entry: AutoFinDailyEntry | None = None + future_returns: list[AutoFinFutureReturnPoint] = Field(default_factory=list, max_length=10) + reaction_summary: str + + @model_validator(mode="after") + def valid_daily_return_path(self) -> "AutoFinMarketSample": + """Reject look-ahead entries and inconsistent adjusted returns.""" + if self.entry is None: + if self.future_returns: + raise ValueError("future returns require an entry") + return self + if self.entry.entry_time <= self.event_time: + raise ValueError("entry must be strictly after the event") + + expected_horizons = list(range(1, len(self.future_returns) + 1)) + if [point.horizon for point in self.future_returns] != expected_horizons: + raise ValueError("future return horizons must be contiguous and start at 1") + trade_dates = [point.trade_date for point in self.future_returns] + if trade_dates != sorted(set(trade_dates)): + raise ValueError("future return trade dates must be unique and ascending") + if trade_dates: + first_trade_date = trade_dates[0] + if self.entry.price_type == "open" and first_trade_date < self.entry.trade_date: + raise ValueError("an open entry cannot use an earlier close") + if self.entry.price_type == "close" and first_trade_date <= self.entry.trade_date: + raise ValueError("a close entry requires a later close") + + adjusted_entry = self.entry.raw_price * self.entry.adj_factor + for point in self.future_returns: + expected_return = point.raw_close * point.adj_factor / adjusted_entry - 1 + if not isclose(point.cumulative_return, expected_return, rel_tol=1e-6, abs_tol=1e-6): + raise ValueError(f"incorrect adjusted return at horizon {point.horizon}") + return self + + +class AutoFinHistoricalEvent(AutoFinModel): + """One resolved historical news item and its calculated ETF return path.""" + + reason: str + news_id: str + source_path: str + event_time: ShanghaiDateTime + event_title: str + event_content: str + market_entry: AutoFinDailyEntry | None = None + future_returns: list[AutoFinFutureReturnPoint] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def valid_historical_event(self) -> "AutoFinHistoricalEvent": + """Require source identity and validate the embedded market reaction.""" + for field in ("reason", "news_id", "source_path", "event_title", "event_content"): + value = getattr(self, field).strip() + if not value: + raise ValueError(f"historical event {field} must not be empty") + setattr(self, field, value) + AutoFinMarketSample( + event_time=self.event_time, + entry=self.market_entry, + future_returns=self.future_returns, + reaction_summary="", + ) + return self + + +class AutoFinEtfHistoricalResearch(AutoFinModel): + """Resolved historical events with embedded calculated ETF return paths.""" + + etf_code: str + etf_name: str + historical_events: list[AutoFinHistoricalEvent] = Field(default_factory=list) + limitations: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def unique_historical_news(self) -> "AutoFinEtfHistoricalResearch": + """Reject duplicate resolved source records.""" + news_ids = [event.news_id for event in self.historical_events] + if len(news_ids) != len(set(news_ids)): + raise ValueError("historical event news IDs must be unique") + return self + + +class AutoFinHistoricalSimilarity(AutoFinModel): + """One similarity judgment returned by the Market Agent.""" + + reason: str + news_id: str + similarity: float + + @model_validator(mode="after") + def non_empty_values(self) -> "AutoFinHistoricalSimilarity": + """Reject a similarity judgment without source identity or rationale.""" + self.reason = self.reason.strip() + self.news_id = self.news_id.strip() + if not self.reason or not self.news_id: + raise ValueError("historical similarity reason and news ID must be non-empty") + return self + + +class AutoFinMarketSelection(AutoFinModel): + """Historical similarities returned by the Market Agent.""" + + matched_historical_events: list[AutoFinHistoricalSimilarity] = Field(default_factory=list) + + @model_validator(mode="after") + def unique_historical_news(self) -> "AutoFinMarketSelection": + """Reject duplicate similarity judgments.""" + news_ids = [event.news_id for event in self.matched_historical_events] + if len(news_ids) != len(set(news_ids)): + raise ValueError("matched historical event news IDs must be unique") + return self + + +class AutoFinHistoricalMatch(AutoFinModel): + """One historical event selected for the weighted forecast.""" + + reason: str + news_id: str + event_time: ShanghaiDateTime + similarity: float + weight: float = Field(ge=0.0, le=1.0) + + +class AutoFinForecastReturnPoint(AutoFinModel): + """Weighted expected cumulative return for one holding horizon.""" + + horizon: int = Field(ge=1, le=10) + expected_return: float | None = None + + +class AutoFinWeightedForecast(AutoFinModel): + """Program-calculated forecast derived from similar historical events.""" + + returns: list[AutoFinForecastReturnPoint] = Field(min_length=10, max_length=10) + suggested_holding_days: int | None = Field(default=None, ge=1, le=10) + + @model_validator(mode="after") + def complete_horizons(self) -> "AutoFinWeightedForecast": + """Require one ordered forecast point for every D1-D10 horizon.""" + if [point.horizon for point in self.returns] != list(range(1, 11)): + raise ValueError("forecast horizons must be ordered D1-D10") + return self + + +class AutoFinSelectedEtfAnalysis(AutoFinModel): + """Program-calculated weighted forecast for one selected ETF.""" + + etf_code: str + etf_name: str + matched_historical_events: list[AutoFinHistoricalMatch] = Field(default_factory=list) + forecast: AutoFinWeightedForecast + limitations: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def valid_historical_weights(self) -> "AutoFinSelectedEtfAnalysis": + """Reject duplicate matches and invalid normalized weights.""" + news_ids = [event.news_id for event in self.matched_historical_events] + if len(news_ids) != len(set(news_ids)): + raise ValueError("matched historical events must be unique") + if news_ids and not isclose( + sum(event.weight for event in self.matched_historical_events), + 1.0, + rel_tol=1e-6, + abs_tol=1e-6, + ): + raise ValueError("matched historical event weights must sum to 1") + return self + + +class AutoFinEtfHistoryDetail(AutoFinModel): + """Complete historical research and market result for one selected ETF.""" + + etf: AutoFinEtfSelection + current_events: list[AutoFinSelectedEvent] = Field(min_length=1) + historical_research: AutoFinEtfHistoricalResearch + market_analysis: AutoFinSelectedEtfAnalysis + + @model_validator(mode="after") + def consistent_etf_and_events(self) -> "AutoFinEtfHistoryDetail": + """Reject stale or cross-ETF outputs from dispatched steps.""" + identity = (self.etf.etf_code, self.etf.etf_name) + if (self.historical_research.etf_code, self.historical_research.etf_name) != identity: + raise ValueError("historical research ETF must match the selected ETF") + if (self.market_analysis.etf_code, self.market_analysis.etf_name) != identity: + raise ValueError("market analysis ETF must match the selected ETF") + return self + + +class AutoFinReportOutput(AutoFinModel): + """Final Markdown title and body for all selected ETFs.""" + + title: str + body: str + + @model_validator(mode="after") + def non_empty_report(self) -> "AutoFinReportOutput": + """Require both Markdown report fields.""" + for field in ("title", "body"): + value = getattr(self, field).strip() + if not value: + raise ValueError(f"{field} must not be empty") + setattr(self, field, value) + return self diff --git a/reme/steps/base_step.py b/reme/steps/base_step.py index e2ea5a44..2baa1fc9 100644 --- a/reme/steps/base_step.py +++ b/reme/steps/base_step.py @@ -81,7 +81,7 @@ class Ref: if self.optional: return None raise RuntimeError(f"app_context is not set when resolving '{self.key}'") - comp = obj.app_context.components[self.comp_enum].get(name) + comp = obj.app_context.components.get(self.comp_enum, {}).get(name) if comp is None: if self.optional: return None diff --git a/reme/steps/cookbook/__init__.py b/reme/steps/cookbook/__init__.py index 77dc8bd5..782776c2 100644 --- a/reme/steps/cookbook/__init__.py +++ b/reme/steps/cookbook/__init__.py @@ -1,5 +1,5 @@ """Optional, end-to-end cookbook workflows.""" -from . import daily_paper, dingtalk +from . import auto_fin, daily_paper, dingtalk -__all__ = ["daily_paper", "dingtalk"] +__all__ = ["auto_fin", "daily_paper", "dingtalk"] diff --git a/reme/steps/cookbook/auto_fin/__init__.py b/reme/steps/cookbook/auto_fin/__init__.py new file mode 100644 index 00000000..f78ef316 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/__init__.py @@ -0,0 +1,17 @@ +"""Auto Fin news research workflow.""" + +from .data import AutoFinDataStep +from .history_search import AutoFinHistorySearchStep +from .history import AutoFinHistoryStep +from .market import AutoFinMarketStep +from .merge import AutoFinMergeStep +from .topic import AutoFinTopicStep + +__all__ = [ + "AutoFinDataStep", + "AutoFinHistorySearchStep", + "AutoFinHistoryStep", + "AutoFinMarketStep", + "AutoFinMergeStep", + "AutoFinTopicStep", +] diff --git a/reme/steps/cookbook/auto_fin/_base.py b/reme/steps/cookbook/auto_fin/_base.py new file mode 100644 index 00000000..5f98df17 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/_base.py @@ -0,0 +1,239 @@ +"""Shared helpers for the Auto Fin workflow.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import os +from datetime import date, datetime, timedelta +from pathlib import Path +from time import perf_counter +from typing import Any +from uuid import uuid4 +from zoneinfo import ZoneInfo + +from pydantic import BaseModel + +from ....components.outbound_proxy import BaseOutboundProxy +from ....enumeration import ComponentEnum +from ....utils.tushare import create_tushare_api +from ...base_step import BaseStep, Ref + +AGENT_INPUT_LOG_LIMIT = 2000 +AGENT_OUTPUT_LOG_LIMIT = 4000 +SHANGHAI_TIMEZONE = ZoneInfo("Asia/Shanghai") + + +def _news_hash(row: dict[str, Any]) -> str: + src = str(row.get("src") or "") + content = str(row.get("content") or "") + return hashlib.sha256(f"{src}{content}".encode()).hexdigest()[:4] + + +def _news_id(row: dict[str, Any], published_at: datetime) -> str: + return f"{published_at:%Y%m%d%H%M%S}_{_news_hash(row)}" + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") + try: + temporary.write_text(text, encoding="utf-8") + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: + _write( + path, + "".join(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n" for row in records), + ) + + +def _clean(value: Any) -> Any: + if value is None or isinstance(value, (str, int, bool)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, dict): + return {str(key): _clean(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_clean(item) for item in value] + return _clean(value.item()) if hasattr(value, "item") else str(value) + + +def _records(value: Any) -> list[dict[str, Any]]: + if value is None: + return [] + if hasattr(value, "to_dict"): + try: + value = value.to_dict(orient="records") + except TypeError: + value = value.to_dicts() + return [_clean(item) for item in value] + + +class AutoFinStep(BaseStep): + """Shared Auto Fin helpers.""" + + outbound_proxy: BaseOutboundProxy | None = Ref(BaseOutboundProxy, ComponentEnum.OUTBOUND_PROXY, optional=True) + + def _value(self, key: str, default: Any = None) -> Any: + assert self.context is not None + return self.context.get(key, self.kwargs.get(key, default)) + + def _required(self, key: str) -> Any: + assert self.context is not None + if (value := self.context.get(key)) is None: + raise RuntimeError(f"Auto Fin data is missing: {key}") + return value + + async def _reply( + self, + prompt_name: str, + resource_name: str, + model: type[BaseModel], + *, + output_suffix: str = ".json", + jsonl_field: str | None = None, + tool_context_id: str | None = None, + **values: str, + ) -> tuple[BaseModel, Path]: + """Send a complete prompt directly and persist its structured reply.""" + if self.agent_wrapper is None: + raise RuntimeError("Auto Fin analysis requires an agent_wrapper") + if Path(resource_name).name != resource_name: + raise ValueError(f"Invalid Auto Fin resource name: {resource_name}") + if output_suffix not in {".json", ".jsonl"}: + raise ValueError(f"Invalid Auto Fin output suffix: {output_suffix}") + + prompt = self.prompt_format(prompt_name, **values) + output_path = ( + self.workspace_path + / "resource" + / str(self._required("auto_fin_date")) + / f"{resource_name}_output{output_suffix}" + ) + started_at = perf_counter() + serialized_input = json.dumps(prompt, ensure_ascii=False) + input_preview, input_truncated = self._text_preview(serialized_input, AGENT_INPUT_LOG_LIMIT) + self.logger.info( + f"[{self.name}] agent input prompt={prompt_name} schema={model.__name__} " + f"query_chars={len(prompt)} truncated={str(input_truncated).lower()} query={input_preview}", + ) + agent_kwargs: dict[str, Any] = {"output_schema": model} + if tool_context_id: + agent_kwargs["tool_context_id"] = tool_context_id + result = await self.agent_wrapper.reply(prompt, **agent_kwargs) + if not isinstance(result, dict): + raise TypeError("Auto Fin Agent reply must be a dictionary") + value = result.get("structured_output") + if value is None: + raise ValueError(f"Auto Fin Agent returned no structured output: {self._preview(result)}") + output = value if isinstance(value, model) else model.model_validate(value) + payload = output.model_dump(mode="json") + serialized_output = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + if jsonl_field is None: + _write(output_path, f"{serialized_output}\n") + else: + records = payload.get(jsonl_field) + if not isinstance(records, list) or not all(isinstance(record, dict) for record in records): + raise ValueError(f"Auto Fin JSONL field must contain objects: {jsonl_field}") + _write_jsonl(output_path, records) + output_preview, output_truncated = self._text_preview(serialized_output, AGENT_OUTPUT_LOG_LIMIT) + self.logger.info( + f"[{self.name}] agent output prompt={prompt_name} schema={model.__name__} " + f"elapsed={perf_counter() - started_at:.2f}s chars={len(serialized_output)} " + f"resource={output_path} truncated={str(output_truncated).lower()} output={output_preview}", + ) + return output, output_path + + @staticmethod + def _text_preview(text: str, limit: int) -> tuple[str, bool]: + limit = max(0, limit) + truncated = len(text) > limit + return (f"{text[:limit]}..." if truncated else text), truncated + + @staticmethod + def _preview(value: Any, limit: int = 1000) -> str: + text = json.dumps(value, ensure_ascii=False, default=str) + return text if len(text) <= limit else f"{text[:limit]}..." + + @property + def _proxy_url(self) -> str | None: + return self.outbound_proxy.http_url if self.outbound_proxy is not None else None + + async def _fetch(self, endpoint: str, **kwargs) -> list[dict[str, Any]]: + provider = self._value("tushare_provider") + details = " ".join( + f"{key}={kwargs[key]}" for key in ("exchange", "src", "start_date", "end_date") if key in kwargs + ) + provider_name = "injected" if provider is not None else "sdk" + started_at = perf_counter() + self.logger.debug( + f"[{self.name}] tushare fetch start endpoint={endpoint} provider={provider_name} " + f"proxy={bool(self._proxy_url)} {details}", + ) + try: + if provider is not None: + value = provider(endpoint, **kwargs) + rows = _records(await value if asyncio.iscoroutine(value) else value) + else: + token = os.getenv("TUSHARE_TOKEN", "").strip() + if not token: + raise RuntimeError("TUSHARE_TOKEN is required for Auto Fin") + api = create_tushare_api(token, proxy_url=self._proxy_url) + rows = _records(await asyncio.to_thread(getattr(api, endpoint), **kwargs)) + except Exception: + self.logger.exception( + f"[{self.name}] tushare fetch failed endpoint={endpoint} elapsed={perf_counter() - started_at:.2f}s " + f"{details}", + ) + raise + self.logger.debug( + f"[{self.name}] tushare fetch done endpoint={endpoint} records={len(rows)} " + f"elapsed={perf_counter() - started_at:.2f}s {details}", + ) + return rows + + def _news_path(self, day: date) -> Path: + daily_dir = str(self.config_value("daily_dir")) + return self.workspace_path / daily_dir / day.isoformat() / "auto_fin_news_data.jsonl" + + @staticmethod + def _days(start: date, end: date) -> list[date]: + return [start + timedelta(days=offset) for offset in range((end - start).days + 1)] + + @staticmethod + def _read_jsonl_sync(path: Path) -> list[dict[str, Any]]: + with path.open(encoding="utf-8") as stream: + rows = [json.loads(line) for line in stream if line.strip()] + if not all(isinstance(row, dict) for row in rows): + raise ValueError(f"JSONL records must be objects: {path}") + return rows + + @classmethod + async def _read_jsonl(cls, path: Path) -> list[dict[str, Any]]: + return await asyncio.to_thread(cls._read_jsonl_sync, path) + + @staticmethod + def _published_at(row: dict[str, Any]) -> datetime | None: + value = row.get("pub_time") or row.get("published_at") or row.get("datetime") + if not value: + return None + text = str(value).strip() + try: + parsed = datetime.fromisoformat(text) + except ValueError: + try: + parsed = datetime.strptime(text, "%Y%m%d %H:%M:%S") + except ValueError: + return None + if parsed.tzinfo is not None and parsed.utcoffset() is not None: + parsed = parsed.astimezone(SHANGHAI_TIMEZONE).replace(tzinfo=None) + return parsed diff --git a/reme/steps/cookbook/auto_fin/data.py b/reme/steps/cookbook/auto_fin/data.py new file mode 100644 index 00000000..ed1c9bec --- /dev/null +++ b/reme/steps/cookbook/auto_fin/data.py @@ -0,0 +1,154 @@ +"""Download the news required by Auto Fin.""" + +from __future__ import annotations + +import asyncio +from datetime import date, datetime, time, timedelta +from pathlib import Path +from typing import Any + +from ....components import R +from ._base import SHANGHAI_TIMEZONE, AutoFinStep, _news_id, _write_jsonl + + +@R.register("auto_fin_data_step") +class AutoFinDataStep(AutoFinStep): + """Fill missing daily news files and always refresh today's news.""" + + def _schedule(self) -> tuple[date, datetime]: + now_value = self._value("now") + now = datetime.fromisoformat(str(now_value)) if now_value is not None else datetime.now(SHANGHAI_TIMEZONE) + if now.tzinfo is not None and now.utcoffset() is not None: + now = now.astimezone(SHANGHAI_TIMEZONE).replace(tzinfo=None) + requested = str(self._value("date", "")).strip() + run_date = date.fromisoformat(requested) if requested else now.date() + if run_date != now.date(): + raise ValueError("Auto Fin only supports the current date") + return run_date, now + + async def _previous_trade_date(self, run_date: date) -> date: + supplied = self._value("trade_dates") + if supplied is not None: + dates = [date.fromisoformat(str(value)) for value in supplied] + else: + start = run_date - timedelta(days=30) + rows = await self._fetch( + "trade_cal", + exchange="SSE", + start_date=start.strftime("%Y%m%d"), + end_date=run_date.strftime("%Y%m%d"), + fields="cal_date,is_open", + ) + dates = [ + datetime.strptime(str(row["cal_date"]), "%Y%m%d").date() + for row in rows + if int(row.get("is_open", 0)) == 1 + ] + previous = [day for day in dates if day < run_date] + if not previous: + raise ValueError("Auto Fin requires a previous A-share trade date") + return max(previous) + + async def _valid_news(self, path: Path) -> bool: + if not path.is_file(): + return False + try: + rows = await self._read_jsonl(path) + except (OSError, ValueError) as exc: + self.logger.warning( + f"[{self.name}] invalid news cache path={path} error={type(exc).__name__}: {exc}", + ) + return False + valid = all(str(row.get("src") or "") == "财联社" for row in rows) + if not valid: + self.logger.warning(f"[{self.name}] invalid news cache source path={path}") + return valid + + async def _fetch_news(self, start: datetime, end: datetime) -> list[dict[str, Any]]: + rows = await self._fetch( + "major_news", + src="财联社", + start_date=start.strftime("%Y-%m-%d %H:%M:%S"), + end_date=end.strftime("%Y-%m-%d %H:%M:%S"), + fields="title,pub_time,src,content", + ) + if len(rows) < 400 or end - start <= timedelta(minutes=1): + return rows + midpoint = start + (end - start) / 2 + self.logger.debug( + f"[{self.name}] news fetch split start={start.isoformat()} end={end.isoformat()} " + f"midpoint={midpoint.isoformat()} records={len(rows)}", + ) + left, right = await asyncio.gather(self._fetch_news(start, midpoint), self._fetch_news(midpoint, end)) + self.logger.debug( + f"[{self.name}] news fetch split done start={start.isoformat()} end={end.isoformat()} " + f"records={len(left) + len(right)}", + ) + return left + right + + async def _cache_news(self, day: date, decision_at: datetime, refresh: bool) -> bool: + path = self._news_path(day) + if not refresh and await self._valid_news(path): + self.logger.debug(f"[{self.name}] news cache hit date={day.isoformat()} path={path}") + return False + start = datetime.combine(day, time.min) + end = decision_at if day == decision_at.date() else start + timedelta(days=1) + candidates = [] + for row in await self._fetch_news(start, end): + published_at = self._published_at(row) + in_range = ( + published_at is not None + and start <= published_at + and (published_at <= end if day == decision_at.date() else published_at < end) + ) + if in_range and str(row.get("src") or "") == "财联社": + candidates.append((published_at, _news_id(row, published_at), row)) + news = {} + for _published_at, news_id, row in sorted(candidates, key=lambda item: item[:2]): + news.setdefault(news_id, {**row, "news_id": news_id}) + _write_jsonl(path, list(news.values())) + self.logger.debug(f"[{self.name}] news written date={day.isoformat()} records={len(news)} path={path}") + return True + + async def execute(self): + assert self.context is not None + run_date, decision_at = self._schedule() + news_days = int(self._value("lookback_days")) + progress_interval = int(self._value("progress_interval")) + if news_days < 1: + raise ValueError("lookback_days must be at least 1") + if progress_interval < 1: + raise ValueError("progress_interval must be at least 1") + start = run_date - timedelta(days=news_days - 1) + force = bool(self._value("force", False)) + self.logger.info( + f"[{self.name}] start date={run_date.isoformat()} range={start.isoformat()}..{run_date.isoformat()} " + f"days={news_days} force={force} decision_at={decision_at.isoformat()}", + ) + previous_trade_date = await self._previous_trade_date(run_date) + self.logger.info( + f"[{self.name}] trade date resolved date={run_date.isoformat()} " + f"previous_trade_date={previous_trade_date.isoformat()}", + ) + downloaded = 0 + for processed, day in enumerate(self._days(start, run_date), start=1): + downloaded += int(await self._cache_news(day, decision_at, force or day == run_date)) + if processed % progress_interval == 0 and processed < news_days: + self.logger.info( + f"[{self.name}] progress processed={processed}/{news_days} downloaded={downloaded} " + f"cached={processed - downloaded} last_date={day.isoformat()}", + ) + self.context.update( + { + "auto_fin_date": run_date.isoformat(), + "auto_fin_decision_at": decision_at.isoformat(), + "auto_fin_news_start": start.isoformat(), + "auto_fin_previous_trade_date": previous_trade_date.isoformat(), + }, + ) + self.context.response.metadata.update({"date": run_date.isoformat(), "news_downloaded": downloaded}) + self.logger.info( + f"[{self.name}] done downloaded={downloaded} cached={news_days - downloaded} " + f"previous_trade_date={previous_trade_date.isoformat()}", + ) + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/history.py b/reme/steps/cookbook/auto_fin/history.py new file mode 100644 index 00000000..e9da473e --- /dev/null +++ b/reme/steps/cookbook/auto_fin/history.py @@ -0,0 +1,93 @@ +"""Orchestrate historical research and market analysis for selected ETFs.""" + +from __future__ import annotations + +from ....components import R +from ....schema import ( + AutoFinEtfHistoricalResearch, + AutoFinEtfHistoryDetail, + AutoFinEtfSelection, + AutoFinSelectedEtfAnalysis, + AutoFinSelectedEvent, +) +from ._base import AutoFinStep, _write_jsonl + +DEFAULT_DISPATCH_STEPS = ["auto_fin_history_search_step", "auto_fin_market_step"] + + +@R.register("auto_fin_history_step") +class AutoFinHistoryStep(AutoFinStep): + """Dispatch historical and market steps for each selected ETF.""" + + def __init__(self, *args, dispatch_steps=None, **kwargs): + super().__init__( + *args, + dispatch_steps=DEFAULT_DISPATCH_STEPS if dispatch_steps is None else dispatch_steps, + **kwargs, + ) + + async def execute(self): + assert self.context is not None + etfs = [AutoFinEtfSelection.model_validate(item) for item in self._required("auto_fin_etfs")] + history_details = [] + news_rows = await self._read_jsonl(self.workspace_path / str(self._required("auto_fin_filtered_news"))) + news_by_id = {str(row["news_id"]): row for row in news_rows} + current_keys = ( + "auto_fin_current_history", + "auto_fin_current_history_resource", + "auto_fin_current_analysis", + ) + self.logger.info(f"[{self.name}] start etfs={len(etfs)}") + for index, item in enumerate(etfs, 1): + label = f"{item.etf_code}({item.etf_name})" + self.logger.info( + f"[{self.name}] etf start index={index}/{len(etfs)} etf={label!r} events={len(item.events)}", + ) + events = [ + AutoFinSelectedEvent( + reason=event.reason, + news_id=event.news_id, + event_time=news_by_id[event.news_id]["event_time"], + event_title=str(news_by_id[event.news_id].get("title") or ""), + event_content=str( + news_by_id[event.news_id].get("content") or news_by_id[event.news_id].get("title") or "", + ), + ) + for event in item.events + ] + for key in current_keys: + if key in self.context: + del self.context[key] + await self.dispatch_steps( + self.dispatch_step_specs, + agent_wrapper=self.agent_wrapper, + auto_fin_current_index=index, + auto_fin_current_etf=item.model_dump(mode="json"), + auto_fin_current_events=[event.model_dump(mode="json") for event in events], + ) + history = AutoFinEtfHistoricalResearch.model_validate(self._required("auto_fin_current_history")) + analysis = AutoFinSelectedEtfAnalysis.model_validate(self._required("auto_fin_current_analysis")) + detail = AutoFinEtfHistoryDetail( + etf=item, + current_events=events, + historical_research=history, + market_analysis=analysis, + ) + history_details.append(detail.model_dump(mode="json")) + self.logger.info( + f"[{self.name}] etf done index={index}/{len(etfs)} etf={label!r}", + ) + for key in current_keys: + if key in self.context: + del self.context[key] + history_path = ( + self.workspace_path / "resource" / str(self._required("auto_fin_date")) / "auto_fin_history_output.jsonl" + ) + _write_jsonl(history_path, history_details) + self.context["auto_fin_history_details"] = history_details + self.context["auto_fin_history_resource"] = str(history_path) + self.context.response.metadata["analysis_count"] = len(history_details) + self.logger.info( + f"[{self.name}] done analyses={len(history_details)} history_resource={history_path}", + ) + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/history_search.py b/reme/steps/cookbook/auto_fin/history_search.py new file mode 100644 index 00000000..9e7128ba --- /dev/null +++ b/reme/steps/cookbook/auto_fin/history_search.py @@ -0,0 +1,287 @@ +"""Find historical events relevant to one selected ETF.""" + +from __future__ import annotations + +import asyncio +import json +from datetime import date, datetime, time +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from ....components import R +from ....schema import ( + AutoFinEtfHistoricalEvents, + AutoFinEtfHistoricalResearch, + AutoFinEtfSelection, + AutoFinHistoricalEvent, + AutoFinHistoricalEventReference, + AutoFinMarketSample, + AutoFinSelectedEvent, +) +from ...index._dedup import _ToolContextDedupMixin +from ._base import AutoFinStep, _write +from .topic import _plain_text + + +@R.register("auto_fin_history_search_step") +class AutoFinHistorySearchStep(AutoFinStep): + """Find historical events and calculate their adjusted ETF returns.""" + + @staticmethod + def _trade_date(value: Any) -> date | None: + text = str(value or "").replace("-", "") + try: + return datetime.strptime(text, "%Y%m%d").date() + except ValueError: + return None + + @staticmethod + def _positive_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + async def _resolve_historical_events( + self, + references: list[AutoFinHistoricalEventReference], + current_news_ids: set[str], + window_start: datetime, + ) -> list[AutoFinHistoricalEvent]: + """Resolve Agent-selected identities from user-owned source files.""" + workspace = self.workspace_path.resolve() + rows_by_path: dict[Path, list[dict[str, Any]]] = {} + events_by_news_id: dict[str, AutoFinHistoricalEvent] = {} + for reference in references: + if reference.news_id in current_news_ids: + raise ValueError(f"History Agent returned a current news item: {reference.news_id}") + + relative_path = Path(reference.source_path) + if relative_path.is_absolute() or ".." in relative_path.parts: + raise ValueError(f"Historical source path must be workspace-relative: {reference.source_path}") + source_path = (workspace / relative_path).resolve() + try: + normalized_path = source_path.relative_to(workspace) + except ValueError as exc: + raise ValueError(f"Historical source path is outside the workspace: {reference.source_path}") from exc + if source_path.name != "auto_fin_news_data.jsonl": + raise ValueError(f"Historical source must be an Auto Fin news file: {reference.source_path}") + if not source_path.is_file(): + raise ValueError(f"Historical source file does not exist: {reference.source_path}") + + rows = rows_by_path.get(source_path) + if rows is None: + rows = await self._read_jsonl(source_path) + rows_by_path[source_path] = rows + matches = [row for row in rows if str(row.get("news_id") or "") == reference.news_id] + if len(matches) != 1: + raise ValueError( + f"Historical news_id must resolve exactly once in {reference.source_path}: {reference.news_id}", + ) + + row = matches[0] + event_time = self._published_at(row) + if event_time is None: + raise ValueError(f"Historical news has no valid publication time: {reference.news_id}") + if event_time >= window_start: + raise ValueError(f"History Agent returned an event inside the current news window: {reference.news_id}") + event_title = str(row.get("title") or "").strip() + event_content = _plain_text(str(row.get("content") or event_title)) + if not event_title or not event_content: + raise ValueError(f"Historical news has no usable title or content: {reference.news_id}") + + events_by_news_id.setdefault( + reference.news_id, + AutoFinHistoricalEvent( + reason=reference.reason, + news_id=reference.news_id, + source_path=normalized_path.as_posix(), + event_time=event_time, + event_title=event_title, + event_content=event_content, + ), + ) + return sorted(events_by_news_id.values(), key=lambda event: (event.event_time, event.news_id)) + + async def _calculate_samples( + self, + etf_code: str, + events: list[AutoFinHistoricalEvent], + decision_at: datetime, + ) -> tuple[list[AutoFinMarketSample], list[str]]: + if not events: + return [], [] + start_date = min(event.event_time.date() for event in events).strftime("%Y%m%d") + end_date = decision_at.date().strftime("%Y%m%d") + daily, factors = await asyncio.gather( + self._fetch("fund_daily", ts_code=etf_code, start_date=start_date, end_date=end_date), + self._fetch("fund_adj", ts_code=etf_code, start_date=start_date, end_date=end_date), + ) + factors_by_date = { + trade_date: self._positive_float(row.get("adj_factor")) + for row in factors + if (trade_date := self._trade_date(row.get("trade_date"))) is not None + } + daily_by_date = { + trade_date: { + "trade_date": trade_date, + "open": self._positive_float(row.get("open")), + "close": self._positive_float(row.get("close")), + "adj_factor": factors_by_date.get(trade_date), + } + for row in daily + if (trade_date := self._trade_date(row.get("trade_date"))) is not None + and datetime.combine(trade_date, time(15, 0)) <= decision_at + } + rows = [daily_by_date[trade_date] for trade_date in sorted(daily_by_date)] + row_indexes = {row["trade_date"]: index for index, row in enumerate(rows)} + samples = [] + limitations = [] + for event in events: + event_time = event.event_time + event_date = event_time.date() + row_index = row_indexes.get(event_date) + price_type = None + if row_index is not None and event_time.time() < time(9, 30): + price_type = "open" + elif row_index is not None and event_time.time() < time(15, 0): + price_type = "close" + else: + row_index = next( + (index for index, row in enumerate(rows) if row["trade_date"] > event_date), + None, + ) + price_type = "open" if row_index is not None else None + + if row_index is None or price_type is None: + limitations.append(f"{event_time.isoformat()} 之后没有已完成的 ETF 日线") + samples.append( + AutoFinMarketSample( + event_time=event.event_time, + reaction_summary="事件之后没有可用的已完成日线。", + ), + ) + continue + + entry_row = rows[row_index] + raw_price = entry_row[price_type] + adj_factor = entry_row["adj_factor"] + entry_clock = time(9, 30) if price_type == "open" else time(15, 0) + entry_time = datetime.combine(entry_row["trade_date"], entry_clock) + if raw_price is None or adj_factor is None: + limitations.append(f"{entry_row['trade_date']} 缺少有效的 {price_type} 或 ETF 复权因子") + samples.append( + AutoFinMarketSample( + event_time=event.event_time, + reaction_summary="买入点缺少有效价格或复权因子。", + ), + ) + continue + + future_returns = [] + first_close_index = row_index if price_type == "open" else row_index + 1 + adjusted_entry = raw_price * adj_factor + for future_row in rows[first_close_index : first_close_index + 10]: + raw_close = future_row["close"] + close_factor = future_row["adj_factor"] + if raw_close is None or close_factor is None: + limitations.append(f"{future_row['trade_date']} 缺少有效 close 或 ETF 复权因子") + break + future_returns.append( + { + "horizon": len(future_returns) + 1, + "trade_date": future_row["trade_date"], + "raw_close": raw_close, + "adj_factor": close_factor, + "cumulative_return": raw_close * close_factor / adjusted_entry - 1, + }, + ) + if len(future_returns) < 10: + limitations.append(f"{event_time.isoformat()} 只有 {len(future_returns)} 个已完成的未来收盘点") + samples.append( + AutoFinMarketSample.model_validate( + { + "event_time": event.event_time, + "entry": { + "entry_time": entry_time, + "trade_date": entry_row["trade_date"], + "price_type": price_type, + "raw_price": raw_price, + "adj_factor": adj_factor, + }, + "future_returns": future_returns, + "reaction_summary": f"按复权日线计算了 {len(future_returns)} 个未来收盘点。", + }, + ), + ) + return samples, list(dict.fromkeys(limitations)) + + async def execute(self): + assert self.context is not None + item = AutoFinEtfSelection.model_validate(self._required("auto_fin_current_etf")) + events = [AutoFinSelectedEvent.model_validate(event) for event in self._required("auto_fin_current_events")] + index = int(self._required("auto_fin_current_index")) + window_start = datetime.fromisoformat(str(self._required("auto_fin_window_start"))) + decision_at = datetime.fromisoformat(str(self._required("auto_fin_decision_at"))) + search_events = [event.model_dump(mode="json", exclude={"news_id"}) for event in events] + label = f"{item.etf_code}({item.etf_name})" + tool_context_id = f"auto_fin_history_{index:02d}_{item.etf_code}_{uuid4().hex}" + try: + history, history_path = await self._reply( + "history_search_user", + f"auto_fin_history_{index:02d}_{item.etf_code}", + AutoFinEtfHistoricalEvents, + tool_context_id=tool_context_id, + etf_code=item.etf_code, + etf_name=item.etf_name, + events=str(search_events), + window_start=window_start.isoformat(), + workspace_root=str(self.workspace_path), + ) + finally: + if self.app_context is not None: + contexts = self.app_context.metadata.get(_ToolContextDedupMixin.TOOL_CONTEXTS_KEY) + if isinstance(contexts, dict): + contexts.pop(tool_context_id, None) + if not contexts: + self.app_context.metadata.pop(_ToolContextDedupMixin.TOOL_CONTEXTS_KEY, None) + if (history.etf_code, history.etf_name) != (item.etf_code, item.etf_name): + raise ValueError(f"History Agent changed ETF {label!r}") + resolved_events = await self._resolve_historical_events( + history.historical_events, + {event.news_id for event in events}, + window_start, + ) + samples, market_limitations = await self._calculate_samples( + item.etf_code, + resolved_events, + decision_at, + ) + enriched_events = [ + event.model_copy( + update={ + "market_entry": sample.entry, + "future_returns": sample.future_returns, + }, + ) + for event, sample in zip(resolved_events, samples, strict=True) + ] + enriched_history = AutoFinEtfHistoricalResearch( + etf_code=history.etf_code, + etf_name=history.etf_name, + historical_events=enriched_events, + limitations=market_limitations, + ) + _write( + history_path, + json.dumps(enriched_history.model_dump(mode="json"), ensure_ascii=False, indent=2) + "\n", + ) + self.context["auto_fin_current_history"] = enriched_history.model_dump(mode="json") + self.context["auto_fin_current_history_resource"] = str(history_path) + self.logger.info( + f"[{self.name}] ready etf={label!r} events={len(enriched_events)} " + f"limitations={len(enriched_history.limitations)}", + ) + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/history_search.yaml b/reme/steps/cookbook/auto_fin/history_search.yaml new file mode 100644 index 00000000..4f37ebf2 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/history_search.yaml @@ -0,0 +1,39 @@ +history_search_user: | + 你只负责为一个已选定 ETF 选择历史相似新闻，不查询行情、不复制新闻正文、不更换或增加 ETF、 + 不提供预测。时间、标题、正文和行情都由程序在你返回后补充。 + 指定 ETF：{etf_code}（{etf_name}） + 当前事件时间线（仅作为检索线索，不含 news_id）：{events} + 历史截止时间：{window_start} + ReMe workspace：{workspace_root} + + 必须遵守： + 1. 使用 memory_search 多轮搜索 {window_start} 之前的相似新闻；系统会自动过滤本次研究中已经 + 返回过的结果，因此结果不足时继续尝试，并调整事件类型、关键实体、传导机制或影响方向等 + 查询角度。memory_search 无更多有效结果后，使用 Read 或 Python 扫描 workspace 下过去 + 360 天的 `daily/YYYY-MM-DD/auto_fin_news_data.jsonl`。历史 Auto Fin Markdown 只能作为 + 检索线索，最终选择必须能回溯到上述新闻 JSONL。 + 2. 根据当前事件提取事件类型、关键实体、传导机制和影响方向进行搜索，不能只搜索 ETF 名称。 + 3. 当前事件时间线来自最近一次收盘后至本次分析时点，只能作为检索线索，不属于历史事件。 + 即使搜索命中同一条当前新闻，也不得放入 historical_events。 + 4. 每个 historical_events 项只需从原始 JSONL 逐字复制 news_id 和 workspace 相对 + source_path，并用 reason 简洁说明它与当前事件相似的原因。不要返回时间、标题或正文。 + 5. 只选择原始记录中真实存在且带有 news_id 的新闻。程序会严格使用 source_path + news_id + 回查，找不到、重复命中、路径越界或不是 auto_fin_news_data.jsonl 都会失败；严禁编造。 + 6. etf_code 和 etf_name 必须原样返回 `{etf_code}` 和 `{etf_name}`。不要读写或下载行情文件， + 不要计算收益。没有合适新闻时直接返回空 historical_events，无需说明限制。 + 7. 最终只生成 etf_code、etf_name 和 historical_events 三个维度； + historical_events 的每一项只包含 reason、news_id 和 source_path。JSON 示例： + ```json + {{ + "etf_code": "{etf_code}", + "etf_name": "{etf_name}", + "historical_events": [ + {{ + "reason": "事件类型、关键实体、传导机制和影响方向相似", + "news_id": "20260601100000_a3f8", + "source_path": "daily/2026-06-01/auto_fin_news_data.jsonl" + }} + ] + }} + ``` + 8. 最终只生成上述 JSON 对象，不附加解释或 Markdown 正文。无需排序或计算，程序会完成。 diff --git a/reme/steps/cookbook/auto_fin/market.py b/reme/steps/cookbook/auto_fin/market.py new file mode 100644 index 00000000..6c0118c3 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/market.py @@ -0,0 +1,134 @@ +"""Forecast one selected ETF from calculated historical samples.""" + +from __future__ import annotations + +import json + +from ....components import R +from ....schema import ( + AutoFinEtfHistoricalResearch, + AutoFinEtfSelection, + AutoFinMarketSelection, + AutoFinSelectedEtfAnalysis, + AutoFinSelectedEvent, +) +from ._base import AutoFinStep, _write + + +@R.register("auto_fin_market_step") +class AutoFinMarketStep(AutoFinStep): + """Collect similarity judgments and calculate one ETF forecast.""" + + @staticmethod + def _calculate_analysis( + item: AutoFinEtfSelection, + history: AutoFinEtfHistoricalResearch, + selection: AutoFinMarketSelection, + ) -> AutoFinSelectedEtfAnalysis: + """Build all deterministic market fields from Agent-selected news IDs.""" + history_by_news_id = {event.news_id: event for event in history.historical_events} + unknown_news_ids = { + match.news_id for match in selection.matched_historical_events if match.news_id not in history_by_news_id + } + if unknown_news_ids: + raise ValueError(f"Market Agent referenced unknown historical news: {sorted(unknown_news_ids)}") + + selected = [ + (match, min(1.0, max(-1.0, match.similarity))) + for match in selection.matched_historical_events + if min(1.0, max(-1.0, match.similarity)) != 0 + ] + total_similarity = sum(abs(similarity) for _, similarity in selected) + matches = [ + { + "reason": match.reason, + "news_id": match.news_id, + "event_time": history_by_news_id[match.news_id].event_time, + "similarity": similarity, + "weight": abs(similarity) / total_similarity, + } + for match, similarity in selected + ] + + returns = [] + has_missing_horizon = False + has_direction_conflict = False + for horizon in range(1, 11): + available = [] + for match, similarity in selected: + event = history_by_news_id[match.news_id] + point = next((point for point in event.future_returns if point.horizon == horizon), None) + if point is not None: + direction = 1.0 if similarity > 0 else -1.0 + available.append((abs(similarity), direction * point.cumulative_return)) + if not available: + has_missing_horizon = True + expected_return = None + else: + horizon_similarity = sum(similarity for similarity, _ in available) + expected_return = ( + sum(similarity * cumulative_return for similarity, cumulative_return in available) + / horizon_similarity + ) + values = [cumulative_return for _, cumulative_return in available] + has_direction_conflict |= any(value > 0 for value in values) and any(value < 0 for value in values) + returns.append({"horizon": horizon, "expected_return": expected_return}) + + positive_returns = [point for point in returns if (point["expected_return"] or 0) > 0] + suggested_holding_days = ( + max(positive_returns, key=lambda point: (point["expected_return"], -point["horizon"]))["horizon"] + if positive_returns + else None + ) + limitations = list(history.limitations) + if not selected: + limitations.append("没有匹配的历史事件") + elif len(selected) < 2: + limitations.append("相似历史样本少于 2 个") + if has_missing_horizon: + limitations.append("部分持有期缺少可用历史收益") + if has_direction_conflict: + limitations.append("相似历史样本的收益方向存在分歧") + if selected and suggested_holding_days is None: + limitations.append("加权预期收益没有正值") + + return AutoFinSelectedEtfAnalysis.model_validate( + { + "etf_code": item.etf_code, + "etf_name": item.etf_name, + "matched_historical_events": matches, + "forecast": { + "returns": returns, + "suggested_holding_days": suggested_holding_days, + }, + "limitations": list(dict.fromkeys(limitations)), + }, + ) + + async def execute(self): + assert self.context is not None + item = AutoFinEtfSelection.model_validate(self._required("auto_fin_current_etf")) + events = [AutoFinSelectedEvent.model_validate(event) for event in self._required("auto_fin_current_events")] + history = AutoFinEtfHistoricalResearch.model_validate(self._required("auto_fin_current_history")) + index = int(self._required("auto_fin_current_index")) + event_lines = "\n".join( + f"- [{event.event_time.isoformat()}] {event.event_title or event.reason}: {event.event_content}" + for event in events + ) + selection, selection_path = await self._reply( + "market_user", + f"auto_fin_market_{index:02d}_{item.etf_code}", + AutoFinMarketSelection, + etf_code=item.etf_code, + etf_name=item.etf_name, + events=event_lines, + history_path=str(self._required("auto_fin_current_history_resource")), + decision_at=str(self._required("auto_fin_decision_at")), + ) + analysis = self._calculate_analysis(item, history, selection) + _write( + selection_path, + json.dumps(analysis.model_dump(mode="json"), ensure_ascii=False, indent=2) + "\n", + ) + self.context["auto_fin_current_analysis"] = analysis.model_dump(mode="json") + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/market.yaml b/reme/steps/cookbook/auto_fin/market.yaml new file mode 100644 index 00000000..a4b6308a --- /dev/null +++ b/reme/steps/cookbook/auto_fin/market.yaml @@ -0,0 +1,35 @@ +market_user: | + 你只负责判断历史事件与当前事件的相似度，不计算收益、不生成预测、不总结报告。 + ETF：{etf_code}（{etf_name}） + 分析截止时间：{decision_at} + + 当前事件： + {events} + + 已补全的历史事件文件： + {history_path} + + 必须遵守： + 1. 读取历史文件，只从 historical_events 中选择与当前事件相似的事件。 + 2. 综合事件类型、关键实体、传导机制和影响方向判断 similarity，范围为 [-1, 1]。正值表示 + 机制和影响方向相似，负值表示机制可比但影响方向相反，0 表示没有有效关系。例如当前事件是 + 黄金涨价、历史事件是黄金降价时，可以返回负值。程序会将越界值截断到 [-1, 1]。 + 3. 只根据历史事件的 event_time、event_title、event_content 和 reason 判断相似性；不要依据 + market_entry 或 future_returns 选择事件，避免使用事后行情影响相似度判断。 + 4. reason 简洁说明相似之处，news_id 必须从历史文件逐字复制，严禁编造。 + 5. 不需要返回 ETF、事件时间、权重、预测、持有天数、代码、总结或 limitations；程序会校验 + news_id，并完成所有计算。没有相似事件时返回空列表。 + 6. 最终只生成 matched_historical_events；每项字段顺序为 reason、news_id、similarity。 + JSON 示例： + ```json + {{ + "matched_historical_events": [ + {{ + "reason": "事件类型、关键实体、传导机制和影响方向相似", + "news_id": "20260601100000_a3f8", + "similarity": 0.86 + }} + ] + }} + ``` + 7. 最终只生成上述 JSON 对象，不附加解释或 Markdown 正文。 diff --git a/reme/steps/cookbook/auto_fin/merge.py b/reme/steps/cookbook/auto_fin/merge.py new file mode 100644 index 00000000..b0b2e422 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/merge.py @@ -0,0 +1,79 @@ +"""Merge all selected ETF analyses into the final report.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +from ....components import R +from ....schema import AutoFinEtfHistoryDetail, AutoFinReportOutput +from ...file_io import refresh_day_index +from ._base import AutoFinStep, _write, _write_jsonl + + +@R.register("auto_fin_merge_step") +class AutoFinMergeStep(AutoFinStep): + """Ask a fresh Agent for the final Markdown and persist it directly.""" + + @staticmethod + def _calculation_results(history_details: list[AutoFinEtfHistoryDetail]) -> list[dict]: + """Return the program-calculated forecast for every analyzed ETF.""" + results = [] + for item in history_details: + holding_days = item.market_analysis.forecast.suggested_holding_days + results.append( + { + "etf_code": item.etf.etf_code, + "etf_name": item.etf.etf_name, + "suggested_holding_days": holding_days, + "returns": [point.model_dump(mode="json") for point in item.market_analysis.forecast.returns], + }, + ) + return results + + async def execute(self): + assert self.context is not None + etfs = list(self._required("auto_fin_etfs")) + history_details = [ + AutoFinEtfHistoryDetail.model_validate(item) for item in self._required("auto_fin_history_details") + ] + selected = [item.etf.model_dump(mode="json") for item in history_details] + if selected != etfs: + raise ValueError("Auto Fin merge history details must match the selected ETFs") + analyses = [item.market_analysis.model_dump(mode="json") for item in history_details] + calculation_results = self._calculation_results(history_details) + self.logger.info( + f"[{self.name}] start etfs={len(etfs)}", + ) + output, _ = await self._reply( + "merge_user", + "auto_fin_merge", + AutoFinReportOutput, + decision_at=str(self._required("auto_fin_decision_at")), + window_start=str(self._required("auto_fin_window_start")), + etfs_path=str(self._required("auto_fin_etfs_resource")), + history_path=str(self._required("auto_fin_history_resource")), + calculation_results=json.dumps(calculation_results, ensure_ascii=False), + ) + markdown = f"# {output.title}\n\n{output.body}\n\n" + markdown += "> 仅为事件研究和持有时间参考，不构成投资建议，不会执行交易。\n" + day_dir = self.workspace_path / str(self.config_value("daily_dir")) / str(self._required("auto_fin_date")) + report_path = day_dir / "auto_fin.md" + _write_jsonl(day_dir / "auto_fin_analysis.jsonl", analyses) + _write(report_path, markdown) + relative = report_path.relative_to(self.workspace_path).as_posix() + await refresh_day_index( + SimpleNamespace(workspace_path=self.workspace_path), + str(self._required("auto_fin_date")), + str(self.config_value("daily_dir")), + ) + self.context["markdown_path"] = relative + self.context["auto_fin_digest_path"] = relative + self.context.response.answer = output.body + self.context.response.metadata.update( + {"markdown_path": relative, "digest_path": relative, "etf_count": len(history_details)}, + ) + self.logger.info( + f"[{self.name}] done path={relative} etfs={self.context.response.metadata['etf_count']}", + ) + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/merge.yaml b/reme/steps/cookbook/auto_fin/merge.yaml new file mode 100644 index 00000000..dcd72784 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/merge.yaml @@ -0,0 +1,39 @@ +merge_user: | + 你只负责把已经完成的结构化分析写成一份中文 Markdown 报告，不重新搜索新闻、不下载行情、 + 不修改任何数值。 + + 分析截止时间：{decision_at} + 新闻窗口：({window_start}, {decision_at}] + + 已筛选 ETF 及其当前事件时间线： + {etfs_path} + + 各 ETF 的完整历史事件、实际行情样本和程序计算结果： + {history_path} + + 全部 ETF 的程序计算结果（包含正向、负向、零值和缺失值）： + {calculation_results} + + 必须遵守： + 1. 只读取上述两个文件，严格使用其中已有的 ETF、事件、收益、权重、最佳持有天数和 limitations。 + 2. body 只写最终结论，以推荐 ETF 为主要内容。 + 3. 由你根据文件中的当前事件内容和传导关系，自行判断事件对 ETF 的影响方向。推荐 ETF 必须 + 同时满足： + - 经你判断，当前事件对该 ETF 所代表的资产、行业或主题影响明确为正向； + - 程序给出了最佳持有天数，且该天数对应的 expected_return 大于 0。 + 不得使用程序计算结果反推事件方向。事件影响为负向、中性、方向不明，或计算结果为 null、 + 0、负数时，一律不得推荐。 + 4. 若没有同时满足两项条件的 ETF，只输出观望结论。不得因为计算为正就把事件方向不明的 ETF + 推荐出来。 + 5. 如有推荐，结论需要列 ETF code、name、最佳持有天数、对应预估收益。 + 6. 对事件影响为负向或计算结果没有正值的 ETF，在推荐结论后用一句话合并简述；不要逐只展开。 + 中性、方向不明或结果缺失的 ETF 可以不写。 + 7. title 不包含 Markdown 标记，body 不重复一级标题，不生成 YAML frontmatter。 + 8. 最终只生成 title 和 body。JSON 示例： + ```json + {{ + "title": "Auto Fin ETF 结论", + "body": "## 结论\n\n推荐 518880.SH（黄金ETF），参考持有 3 个交易日，当前加权预估收益 +1.2%；......。\n\n负向提示：相关能源 ETF 事件影响偏负，不推荐。" + }} + ``` + 9. 最终只生成上述 JSON 对象，不附加解释或代码块标记。 diff --git a/reme/steps/cookbook/auto_fin/topic.py b/reme/steps/cookbook/auto_fin/topic.py new file mode 100644 index 00000000..f8876820 --- /dev/null +++ b/reme/steps/cookbook/auto_fin/topic.py @@ -0,0 +1,210 @@ +"""Build the current Auto Fin topic timelines.""" + +from __future__ import annotations + +import math +import re +from datetime import date, datetime, time +from html.parser import HTMLParser +from typing import Any + +from ....components import R +from ....schema import AutoFinEtfsOutput +from ._base import AutoFinStep, _news_id, _write_jsonl + +NEWS_TITLE_MAX_CHARS = 200 +NEWS_CONTENT_MAX_CHARS = 1200 +NEWS_TOTAL_CONTENT_MAX_CHARS = 60_000 +ETF_CANDIDATE_LIMIT = 150 + + +class _NewsTextExtractor(HTMLParser): + """Extract visible text without retaining markup, links, or image URLs.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self.ignored_depth = 0 + + def handle_starttag(self, tag: str, _attrs) -> None: + if tag in {"script", "style"}: + self.ignored_depth += 1 + elif tag in {"br", "div", "h1", "h2", "h3", "h4", "li", "p"}: + self.parts.append(" ") + + def handle_endtag(self, tag: str) -> None: + if tag in {"script", "style"} and self.ignored_depth: + self.ignored_depth -= 1 + elif tag in {"div", "h1", "h2", "h3", "h4", "li", "p"}: + self.parts.append(" ") + + def handle_data(self, data: str) -> None: + if not self.ignored_depth: + self.parts.append(data) + + +def _plain_text(value: str) -> str: + parser = _NewsTextExtractor() + parser.feed(value) + parser.close() + return " ".join("".join(parser.parts).split()) + + +def _normalized_key(value: Any) -> str: + """Normalize display labels used only for deterministic deduplication.""" + return re.sub(r"\s+", "", str(value or "")).casefold() + + +@R.register("auto_fin_topic_step") +class AutoFinTopicStep(AutoFinStep): + """Match current news to a small set of liquid, representative ETFs.""" + + async def _current_news(self, start: datetime, end: datetime) -> list[dict]: + news = {} + title_limit = max(0, int(self._value("news_title_max_chars", NEWS_TITLE_MAX_CHARS))) + content_limit = max(0, int(self._value("news_content_max_chars", NEWS_CONTENT_MAX_CHARS))) + total_content_limit = max( + 0, + int(self._value("news_total_content_max_chars", NEWS_TOTAL_CONTENT_MAX_CHARS)), + ) + first_day = date.fromisoformat(str(self._required("auto_fin_news_start"))) + last_day = date.fromisoformat(str(self._required("auto_fin_date"))) + for day in self._days(first_day, last_day): + for row in await self._read_jsonl(self._news_path(day)): + published_at = self._published_at(row) + if published_at is None or not start < published_at <= end: + continue + news_id = str(row.get("news_id") or _news_id(row, published_at)) + news.setdefault( + news_id, + { + "news_id": news_id, + "event_time": published_at.isoformat(), + "title": str(row.get("title") or "").strip()[:title_limit], + "content": _plain_text(str(row.get("content") or "")), + }, + ) + rows = sorted(news.values(), key=lambda row: (row["event_time"], row["news_id"])) + per_news_limit = min(content_limit, total_content_limit // len(rows)) if rows else 0 + for row in rows: + row["content"] = row["content"][:per_news_limit] + return rows + + async def _filtered_etfs(self, trade_date: date) -> list[dict[str, str]]: + basics = await self._fetch( + "etf_basic", + list_status="L", + fields="ts_code,csname,extname,cname,index_code,index_name,list_status", + ) + daily = await self._fetch( + "fund_daily", + trade_date=trade_date.strftime("%Y%m%d"), + fields="ts_code,trade_date,amount", + ) + + basic_by_code: dict[str, dict[str, Any]] = {} + for row in basics: + code = str(row.get("ts_code") or row.get("code") or "").strip().upper() + name = str( + row.get("csname") or row.get("name") or row.get("extname") or row.get("cname") or "", + ).strip() + if code and name and str(row.get("list_status") or "L").upper() == "L": + basic_by_code.setdefault(code, {**row, "code": code, "name": name}) + + amount_by_code: dict[str, float] = {} + for row in daily: + code = str(row.get("ts_code") or row.get("code") or "").strip().upper() + try: + amount = float(row.get("amount")) + except (TypeError, ValueError): + continue + if code in basic_by_code and math.isfinite(amount) and amount >= 0: + amount_by_code[code] = max(amount, amount_by_code.get(code, -math.inf)) + + ranked = sorted( + ({**basic_by_code[code], "amount": amount} for code, amount in amount_by_code.items()), + key=lambda row: (-row["amount"], row["code"]), + ) + selected: list[dict[str, str]] = [] + seen_names: set[str] = set() + seen_indexes: set[str] = set() + limit = max(0, int(self._value("etf_candidate_limit", ETF_CANDIDATE_LIMIT))) + if not limit: + return selected + for row in ranked: + name_key = _normalized_key(row["name"]) + index_keys = { + key + for key in ( + _normalized_key(row.get("index_code")), + _normalized_key(row.get("index_name")), + ) + if key + } + if name_key in seen_names or index_keys & seen_indexes: + continue + selected.append({"code": row["code"], "name": row["name"]}) + seen_names.add(name_key) + seen_indexes.update(index_keys) + if len(selected) >= limit: + break + return selected + + @staticmethod + def _validate_selection( + output: AutoFinEtfsOutput, + news: list[dict[str, Any]], + etfs: list[dict[str, str]], + ) -> None: + news_order = {str(row["news_id"]): index for index, row in enumerate(news)} + news_ids = set(news_order) + candidates = {row["code"]: row["name"] for row in etfs} + for item in output.etfs: + if candidates.get(item.etf_code) != item.etf_name: + raise ValueError(f"Topic Agent returned an ETF outside filtered_etf.jsonl: {item.etf_code}") + selected_news_ids = [event.news_id for event in item.events] + unknown = set(selected_news_ids) - news_ids + if unknown: + raise ValueError(f"Topic Agent returned unknown news IDs: {sorted(unknown)}") + event_order = [news_order[news_id] for news_id in selected_news_ids] + if event_order != sorted(event_order): + raise ValueError(f"Topic Agent returned unsorted news IDs for ETF: {item.etf_code}") + + async def execute(self): + assert self.context is not None + decision_at = datetime.fromisoformat(str(self._required("auto_fin_decision_at"))) + previous = date.fromisoformat(str(self._required("auto_fin_previous_trade_date"))) + window_start = datetime.combine(previous, time(15)) + news = await self._current_news(window_start, decision_at) + self.logger.info( + f"[{self.name}] start window=({window_start.isoformat()},{decision_at.isoformat()}] news={len(news)} " + f"content_chars={sum(len(row['content']) for row in news)}", + ) + resource_dir = self.workspace_path / "resource" / decision_at.date().isoformat() + news_path = resource_dir / "filtered_news.jsonl" + etf_path = resource_dir / "filtered_etf.jsonl" + etfs = await self._filtered_etfs(previous) + _write_jsonl(news_path, news) + _write_jsonl(etf_path, etfs) + output, output_path = await self._reply( + "topic_user", + "auto_fin_topic", + AutoFinEtfsOutput, + output_suffix=".jsonl", + jsonl_field="etfs", + window_start=window_start.isoformat(), + decision_at=decision_at.isoformat(), + filtered_news_path=str(news_path), + filtered_etf_path=str(etf_path), + ) + self._validate_selection(output, news, etfs) + self.context["auto_fin_window_start"] = window_start.isoformat() + self.context["auto_fin_etfs"] = output.model_dump(mode="json")["etfs"] + self.context["auto_fin_etfs_resource"] = str(output_path) + self.context["auto_fin_filtered_news"] = str(news_path) + self.context.response.metadata.update({"news_count": len(news), "etf_count": len(output.etfs)}) + self.logger.info( + f"[{self.name}] done etfs={len(output.etfs)} " + f"events={sum(len(item.events) for item in output.etfs)} news_path={news_path} etf_path={etf_path}", + ) + return self.context.response diff --git a/reme/steps/cookbook/auto_fin/topic.yaml b/reme/steps/cookbook/auto_fin/topic.yaml new file mode 100644 index 00000000..db9f481a --- /dev/null +++ b/reme/steps/cookbook/auto_fin/topic.yaml @@ -0,0 +1,44 @@ +topic_user: | + 你只负责从候选文件中筛选与当前新闻直接相关的代表性 ETF，并返回对应新闻 ID。 + 不搜索或补充新闻、不下载数据、不计算收益、不提供预测或投资建议。 + + 时间边界： + - 新闻窗口严格为 ({window_start}, {decision_at}]。 + - 分析截止时间为 {decision_at}。 + + 输入文件： + - 当前窗口内的全部新闻：{filtered_news_path} + - 已按最近收盘成交额关联、去重并截取 Top 150 的 ETF：{filtered_etf_path} + + 输出要求： + 1. 必须读取上述两个 JSONL 文件；只能从 filtered_etf.jsonl 选择 ETF，只能引用 + filtered_news.jsonl 中的 news_id。 + 2. 只保留其 name 所代表的指数、行业、主题或资产与新闻存在明确传导关系的代表性 ETF。 + 3. 最多返回 20 只 ETF，按与新闻的相关性排序；不要为了凑数纳入弱相关 ETF。 + 4. events 只填写与该 ETF 相关的事件对象，每项包含 reason 和 news_id；reason 简洁说明该条新闻 + 与 ETF 所代表资产之间的直接传导关系。合并重复报道，并按新闻时间升序排列。 + 5. etf_code、etf_name 必须逐字复制候选文件中的 code、name；无相关 ETF 时返回空 etfs。 + 6. reason 不得加入预测；同一 ETF 的不同新闻应分别说明关联理由。 + 7. 最终生成一个对象，只包含 etfs；每只 ETF 只包含 etf_code、etf_name、events，每个 event + 只包含 reason 和 news_id。JSON 示例： + ```json + {{ + "etfs": [ + {{ + "etf_code": "518880.SH", + "etf_name": "华安易富黄金ETF", + "events": [ + {{ + "reason": "避险需求直接影响黄金价格", + "news_id": "20260724070000_4d19" + }}, + {{ + "reason": "美元变化影响黄金计价", + "news_id": "20260724072145_6fa5" + }} + ] + }} + ] + }} + ``` + 8. 最终只生成上述 JSON 对象，不附加解释或 Markdown 正文，不写入 ReMe workspace。 diff --git a/reme/steps/cookbook/daily_paper/_common.py b/reme/steps/cookbook/daily_paper/_common.py index 96cfa07c..f475cc2c 100644 --- a/reme/steps/cookbook/daily_paper/_common.py +++ b/reme/steps/cookbook/daily_paper/_common.py @@ -10,8 +10,10 @@ import aiofiles import frontmatter from pydantic import BaseModel -from ...base_step import BaseStep -from ...file_io._file_io import get_path_lock +from ....components.outbound_proxy import BaseOutboundProxy +from ....enumeration import ComponentEnum +from ...base_step import BaseStep, Ref +from ...file_io import get_path_lock _STATE_PREFIX = "daily_paper_" _FRONTMATTER_PATTERN = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL) @@ -54,6 +56,12 @@ async def write_markdown(path: Path, body: str, metadata: dict[str, Any]) -> Non class DailyPaperStep(BaseStep): """Shared helpers for steps in one daily-paper RuntimeContext.""" + outbound_proxy: BaseOutboundProxy | None = Ref( + BaseOutboundProxy, + ComponentEnum.OUTBOUND_PROXY, + optional=True, + ) + def _skip(self) -> bool: assert self.context is not None return bool(self.context.get(f"{_STATE_PREFIX}skip", False)) diff --git a/reme/steps/cookbook/daily_paper/analyze.py b/reme/steps/cookbook/daily_paper/analyze.py index 5ba6573e..bb4195ee 100644 --- a/reme/steps/cookbook/daily_paper/analyze.py +++ b/reme/steps/cookbook/daily_paper/analyze.py @@ -41,7 +41,12 @@ class DailyPaperAnalyzeStep(DailyPaperStep): raise ValueError(f"No extractable text found in PDF: {path.name}") return content, len(reader.pages), truncated - async def _analyze_one(self, paper: PaperInfo, selected: SelectedPaper) -> tuple[str, str]: + async def _analyze_one( + self, + downloader: ArxivPdfClient, + paper: PaperInfo, + selected: SelectedPaper, + ) -> tuple[str, str]: if self.agent_wrapper is None: raise RuntimeError("Claude Code agent_wrapper is required for paper analysis") day = self._run_day() @@ -56,10 +61,6 @@ class DailyPaperAnalyzeStep(DailyPaperStep): pdf_path, note_path = self.workspace_path / pdf_rel, self.workspace_path / note_rel self.logger.info(f"[{self.name}] paper start arxiv_id={paper.arxiv_id}") - downloader = ArxivPdfClient( - timeout=float(self._value("pdf_timeout", 90.0)), - max_bytes=int(self._value("max_pdf_bytes", 50 * 1024 * 1024)), - ) await downloader.download(paper.arxiv_id, pdf_path) self.logger.info(f"[{self.name}] pdf ready arxiv_id={paper.arxiv_id} path={pdf_rel}") pdf_text, page_count, truncated = await asyncio.to_thread( @@ -129,10 +130,16 @@ class DailyPaperAnalyzeStep(DailyPaperStep): self.logger.info(f"[{self.name}] start papers={len(papers)}") note_paths, pdf_paths = [], [] - for paper, selected in zip(papers, selection.selected): - note_path, pdf_path = await self._analyze_one(paper, selected) - note_paths.append(note_path) - pdf_paths.append(pdf_path) + proxy_url = self.outbound_proxy.http_url if self.outbound_proxy is not None else None + async with ArxivPdfClient( + proxy_url=proxy_url, + timeout=float(self._value("pdf_timeout", 90.0)), + max_bytes=int(self._value("max_pdf_bytes", 50 * 1024 * 1024)), + ) as downloader: + for paper, selected in zip(papers, selection.selected): + note_path, pdf_path = await self._analyze_one(downloader, paper, selected) + note_paths.append(note_path) + pdf_paths.append(pdf_path) self._set_state("note_paths", note_paths) self._set_state("pdf_paths", pdf_paths) self.context.response.answer = f"Claude Code wrote {len(note_paths)} detailed paper notes" diff --git a/reme/steps/cookbook/daily_paper/collect.py b/reme/steps/cookbook/daily_paper/collect.py index ae866b4f..31ecd107 100644 --- a/reme/steps/cookbook/daily_paper/collect.py +++ b/reme/steps/cookbook/daily_paper/collect.py @@ -150,7 +150,9 @@ class DailyPaperCollectStep(DailyPaperStep): self.logger.info( f"[{self.name}] fetch start week={week} month={month} yesterday={yesterday}", ) + proxy_url = self.outbound_proxy.http_url if self.outbound_proxy is not None else None async with HuggingFacePapersClient( + proxy_url=proxy_url, timeout=float(self._value("hf_timeout", 30.0)), max_retries=int(self._value("hf_max_retries", 3)), ) as client: diff --git a/reme/steps/cookbook/dingtalk/wait.py b/reme/steps/cookbook/dingtalk/wait.py index 03c3a1e8..afd370d3 100644 --- a/reme/steps/cookbook/dingtalk/wait.py +++ b/reme/steps/cookbook/dingtalk/wait.py @@ -1,21 +1,15 @@ """Long-running DingTalk Stream bridge for the cookbook application.""" import asyncio -import contextlib import hashlib import json import time -from dataclasses import dataclass, field from typing import Any from urllib.parse import quote_plus from ...base_step import BaseStep from ....components import R -from ....enumeration import ChunkEnum - -_VISIBLE_CHUNKS = {ChunkEnum.THINK, ChunkEnum.TOOL_CALL, ChunkEnum.TOOL_RESULT, ChunkEnum.CONTENT, ChunkEnum.ERROR} -_CODE_CHUNKS = {ChunkEnum.TOOL_CALL, ChunkEnum.TOOL_RESULT} -_BLOCK_CHAR_LIMIT = 100 +from ....components.agent_wrapper import handle_session_command def _session_key(message: Any) -> str: @@ -30,189 +24,20 @@ def _session_key(message: Any) -> str: return ":".join(parts) -def _payload_text(value: Any) -> str: - text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2) - return text.replace("```", "'''") - - def _session_ref(key: str) -> str: """Return a stable log correlation id without exposing DingTalk identifiers.""" return hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] -@dataclass -class _Block: - chunk_type: ChunkEnum - key: str - name: str = "" - parts: list[Any] = field(default_factory=list) - chars: int = 0 - truncated: bool = False - - -class _CardRenderer: - """Render append-only, size-limited DingTalk blocks.""" - - def __init__(self): - self.started_at = time.monotonic() - self.active: _Block | None = None - self.has_output = False - self.error = False - - def feed(self, chunk) -> str: - """Buffer one block and return completed Markdown blocks.""" - if chunk.chunk_type not in _VISIBLE_CHUNKS: - return "" - if chunk.chunk_type == ChunkEnum.ERROR: - self.error = True - - key = chunk.block_id or chunk.tool_call_id or chunk.chunk_type.value - current = (chunk.chunk_type, key) - if not chunk.chunk: - return self._complete_active() if self._active_key() == current else "" - - delta = "" - if self._active_key() != current: - delta += self._complete_active() - self.active = _Block(chunk.chunk_type, key, chunk.tool_call_name or "") - assert self.active is not None - self.has_output = True - if chunk.chunk_type in _CODE_CHUNKS: - self.active.parts.append(chunk.chunk) - return delta - - if self.active.chars == 0: - delta += self._text_title(chunk.chunk_type) - text = _payload_text(chunk.chunk) - if chunk.chunk_type == ChunkEnum.CONTENT: - self.active.chars += len(text) - return delta + text - remaining = _BLOCK_CHAR_LIMIT - self.active.chars - delta += text[:remaining] - self.active.chars += min(len(text), remaining) - if len(text) > remaining and not self.active.truncated: - delta += "..." - self.active.truncated = True - return delta - - def fail(self, message: str) -> str: - """Return an error block for an unexpected local failure.""" - delta = self._complete_active() - self.error = True - self.has_output = True - text = _payload_text(message) - body = text if len(text) <= _BLOCK_CHAR_LIMIT else f"{text[:_BLOCK_CHAR_LIMIT]}..." - return f"{delta}### ⚠️ Error\n\n{body}\n\n" - - def _active_key(self) -> tuple[ChunkEnum, str] | None: - if self.active is None: - return None - return self.active.chunk_type, self.active.key - - def _complete_active(self) -> str: - if self.active is None: - return "" - block, self.active = self.active, None - if block.chunk_type not in _CODE_CHUNKS: - return "\n\n" - if block.chunk_type == ChunkEnum.TOOL_CALL and len(block.parts) == 1 and self._is_call_metadata(block.parts[0]): - return "" - - body = self._block_body(block) - body = body if len(body) <= _BLOCK_CHAR_LIMIT else f"{body[:_BLOCK_CHAR_LIMIT]}..." - if block.chunk_type == ChunkEnum.TOOL_CALL: - name = block.name or self._metadata_name(block) or "tool" - safe_name = name.replace("`", "'") - title = f"### 🔧 Tool Call · `{safe_name}`" - else: - title = "### 📦 Tool Result" - - body = "\n".join(f"> {self._visible_indent(line)}" for line in body.splitlines()) - return f"{title}\n\n{body}\n\n" - - @staticmethod - def _text_title(chunk_type: ChunkEnum) -> str: - if chunk_type == ChunkEnum.THINK: - return "### 🧠 Think\n\n" - if chunk_type == ChunkEnum.ERROR: - return "### ⚠️ Error\n\n" - return "### 💬 Content\n\n" - - def _block_body(self, block: _Block) -> str: - if block.chunk_type not in _CODE_CHUNKS: - return "".join(_payload_text(part) for part in block.parts) - payload, is_json = self._tool_payload(block) - if is_json: - return json.dumps(payload, ensure_ascii=False, indent=2) - return "".join(_payload_text(part) for part in block.parts) - - @staticmethod - def _visible_indent(line: str) -> str: - """Keep JSON indentation after DingTalk collapses ordinary spaces.""" - spaces = len(line) - len(line.lstrip(" ")) - return "　" * (spaces // 2) + line[spaces:] - - def _tool_payload(self, block: _Block) -> tuple[Any, bool]: - parts = block.parts - if block.chunk_type == ChunkEnum.TOOL_CALL and len(parts) > 1 and self._is_call_metadata(parts[0]): - parts = parts[1:] - if len(parts) == 1 and not isinstance(parts[0], str): - return self._expand_nested_json(parts[0]), True - if parts and all(isinstance(part, str) for part in parts): - try: - value = json.loads("".join(parts)) - except json.JSONDecodeError: - pass - else: - return self._expand_nested_json(value), True - return None, False - - def _metadata_name(self, block: _Block) -> str: - if not block.parts or not self._is_call_metadata(block.parts[0]): - return "" - return json.loads(block.parts[0]).get("name") or "" - - @staticmethod - def _is_call_metadata(value: Any) -> bool: - if not isinstance(value, str): - return False - try: - parsed = json.loads(value) - except json.JSONDecodeError: - return False - return isinstance(parsed, dict) and bool(parsed) and set(parsed) <= {"id", "name"} - - @classmethod - def _expand_nested_json(cls, value: Any) -> Any: - if isinstance(value, dict): - return {key: cls._expand_nested_json(item) for key, item in value.items()} - if isinstance(value, list): - return [cls._expand_nested_json(item) for item in value] - if isinstance(value, str) and value.lstrip().startswith(("{", "[")): - try: - return cls._expand_nested_json(json.loads(value)) - except json.JSONDecodeError: - pass - return value - - def finish(self) -> str: - """Close the active block and append elapsed time.""" - delta = self._complete_active() - if not delta and self.has_output: - delta = "\n\n" - return f"{delta}{time.monotonic() - self.started_at:.1f}s" - - @R.register("dingtalk_wait_step") class DingTalkWaitStep(BaseStep): - """Receive DingTalk messages and stream Claude Code replies into AI cards.""" + """Receive DingTalk messages and send final Agent responses as Markdown.""" def __init__( self, app_key: str = "", app_secret: str = "", robot_code: str = "", - card_update_interval: float = 1.0, worker_count: int = 4, **kwargs, ): @@ -220,7 +45,6 @@ class DingTalkWaitStep(BaseStep): self.app_key = app_key self.app_secret = app_secret self.robot_code = robot_code - self.card_update_interval = max(0.05, card_update_interval) self.worker_count = max(1, worker_count) async def execute(self): @@ -243,13 +67,9 @@ class DingTalkWaitStep(BaseStep): sessions = self.app_context.metadata.setdefault("dingtalk_agent_sessions", {}) locks: dict[str, asyncio.Lock] = {} self.logger.info( - f"[{self.name}] starting DingTalk Stream bridge " - f"workers={self.worker_count} card_update_interval={self.card_update_interval:.2f}s", + f"[{self.name}] starting DingTalk Stream bridge workers={self.worker_count}", ) - workers = [ - asyncio.create_task(self._worker(queue, locks, sessions, handler, dingtalk_stream)) - for _ in range(self.worker_count) - ] + workers = [asyncio.create_task(self._worker(queue, locks, sessions, handler)) for _ in range(self.worker_count)] try: await self._run_client(client, self.context.stop_event) finally: @@ -271,20 +91,20 @@ class DingTalkWaitStep(BaseStep): return QueueHandler() - async def _worker(self, queue, locks, sessions, handler, dingtalk_stream) -> None: + async def _worker(self, queue, locks, sessions, handler) -> None: while True: message = await queue.get() try: key = _session_key(message) async with locks.setdefault(key, asyncio.Lock()): - await self._handle_message(message, key, sessions, handler, dingtalk_stream) + await self._handle_message(message, key, sessions, handler) except Exception as exc: # A bad message must not disconnect the Stream client. self.logger.exception(f"Failed to handle DingTalk message: {exc}") await asyncio.to_thread(handler.reply_text, f"处理失败：{exc}", message) finally: queue.task_done() - async def _handle_message(self, message, key, sessions, handler, dingtalk_stream) -> None: + async def _handle_message(self, message, key, sessions, handler) -> None: session_ref = _session_ref(key) self.logger.info( f"[{self.name}] handling DingTalk callback session={session_ref} " @@ -301,97 +121,51 @@ class DingTalkWaitStep(BaseStep): self.logger.info(f"[{self.name}] ignored non-text DingTalk message session={session_ref}") await asyncio.to_thread(handler.reply_text, "暂时只支持文本消息。", message) return - if text == "/clear": - cleared = sessions.pop(key, None) is not None - self.logger.info(f"[{self.name}] cleared DingTalk session session={session_ref} existed={cleared}") - await asyncio.to_thread( - handler.reply_text, - "✅ Conversation cleared. The next message will start a new session.", - message, - ) + command = await handle_session_command(self.agent_wrapper, text, sessions.get(key)) + if command is not None: + if command.session_id is None: + sessions.pop(key, None) + else: + sessions[key] = command.session_id + self.logger.info(f"[{self.name}] handled session command session={session_ref} command={text}") + await asyncio.to_thread(handler.reply_text, command.answer, message) return resumed = key in sessions self.logger.info( f"[{self.name}] received DingTalk text session={session_ref} chars={len(text)} resume={resumed}", ) - renderer = _CardRenderer() - card = dingtalk_stream.AIMarkdownCardInstance(handler.dingtalk_client, message) - card_id = await card.async_create_and_send_card( - card.card_template_id, - card.get_card_data(flow_status=dingtalk_stream.AICardStatus.PROCESSING), - at_sender=True, - ) - if not card_id: - raise RuntimeError("创建钉钉 AI 卡片失败") - self.logger.debug(f"[{self.name}] started DingTalk AI card session={session_ref}") + kwargs = {"resume": sessions[key]} if key in sessions else {} + await self._handle_reply(message, key, sessions, handler, text, kwargs, session_ref) - last_update = 0.0 - pending = "" - chunk_count = 0 - update_count = 0 - streamed_chars = 0 - finalizing = False + async def _handle_reply(self, message, key, sessions, handler, text, kwargs, session_ref) -> None: + """Wait for the final Agent response and send one DingTalk Markdown reply.""" + started_at = time.monotonic() try: - kwargs = {"resume": sessions[key]} if key in sessions else {} - async for chunk in self.agent_wrapper.reply_stream(text, **kwargs): - chunk_count += 1 - if chunk.session_id: - sessions[key] = chunk.session_id - pending += renderer.feed(chunk) - if pending and time.monotonic() - last_update >= self.card_update_interval: - delta = pending - await card.async_streaming( - card_id, - "msgContent", - delta, - append=True, - finished=False, - failed=False, - ) - pending = "" - streamed_chars += len(delta) - update_count += 1 - last_update = time.monotonic() + result = await self.agent_wrapper.reply(text, **kwargs) + if not isinstance(result, dict): + raise TypeError("Agent reply must be a dictionary") + if session_id := result.get("session_id"): + sessions[key] = session_id - pending += renderer.finish() - finalizing = True - await card.async_streaming( - card_id, - "msgContent", - pending, - append=True, - finished=True, - failed=renderer.error, - ) - streamed_chars += len(pending) - pending = "" - log = self.logger.warning if renderer.error else self.logger.info - log( - f"[{self.name}] completed DingTalk reply session={session_ref} success={not renderer.error} " - f"chunks={chunk_count} card_updates={update_count} card_chars={streamed_chars} " - f"elapsed={time.monotonic() - renderer.started_at:.2f}s", + last_message = result.get("last_message") + if isinstance(last_message, dict) and last_message.get("is_error"): + raise RuntimeError("Agent 执行失败") + + answer = result.get("result") + if not isinstance(answer, str) or not (answer := answer.strip()): + raise ValueError("Agent 返回了空回复") + response = await asyncio.to_thread(handler.reply_markdown, "ReMe Agent", answer, message) + if response is None: + raise RuntimeError("发送钉钉 Markdown 回复失败") + self.logger.info( + f"[{self.name}] completed DingTalk reply session={session_ref} success=True " + f"chars={len(answer)} elapsed={time.monotonic() - started_at:.2f}s", ) except Exception: - if not finalizing: - if not renderer.error: - pending += renderer.fail("Agent 执行失败") - pending += renderer.finish() - failure_delta, pending = pending, "" - with contextlib.suppress(Exception): - await card.async_streaming( - card_id, - "msgContent", - failure_delta, - append=True, - finished=True, - failed=True, - ) - streamed_chars += len(failure_delta) self.logger.warning( f"[{self.name}] DingTalk reply failed session={session_ref} " - f"chunks={chunk_count} card_updates={update_count} card_chars={streamed_chars} " - f"elapsed={time.monotonic() - renderer.started_at:.2f}s", + f"elapsed={time.monotonic() - started_at:.2f}s", ) raise diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index 61737e45..e5e67e24 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -104,6 +104,7 @@ class DreamExtractStep(BaseStep): return self._finish(state, True, f"No changed dream input for {', '.join(dates)}") if not llm_available(self): state.errors.append("no llm configured; dream extract requires an LLM") + state.failed_paths = list(changed) self.logger.warning(f"[{self.name}] skip no llm changed={len(changed)}") return self._finish(state, False, state.errors[-1]) diff --git a/reme/steps/file_io/__init__.py b/reme/steps/file_io/__init__.py index b66964f2..932aa6c6 100644 --- a/reme/steps/file_io/__init__.py +++ b/reme/steps/file_io/__init__.py @@ -1,7 +1,7 @@ """File I/O step helpers.""" from ._daily_index import extract_daily_date, parse_daily_date, refresh_day_index, validate_session_id -from ._file_io import write_file_safe +from ._file_io import get_path_lock, write_file_safe from ._path import validate_filename_component from .daily_list import DailyListStep from .daily_reindex import DailyReindexStep @@ -24,6 +24,7 @@ __all__ = [ "parse_daily_date", "validate_session_id", "validate_filename_component", + "get_path_lock", "write_file_safe", "DailyListStep", "DailyReindexStep", diff --git a/reme/steps/file_io/_daily_index.py b/reme/steps/file_io/_daily_index.py index 2c4fc9fe..f5151eeb 100644 --- a/reme/steps/file_io/_daily_index.py +++ b/reme/steps/file_io/_daily_index.py @@ -36,7 +36,7 @@ def parse_daily_date(value) -> str | None: _NOTES_OPEN = "" _NOTES_CLOSE = "" -_INDEX_HIDDEN_METADATA_KEYS = {"session_id", "source_conversation"} +_INDEX_HIDDEN_METADATA_KEYS = {"schema_version", "session_id", "source_conversation"} def _render_notes_block(notes: list[dict]) -> str: diff --git a/reme/utils/arxiv.py b/reme/utils/arxiv.py index 4efa7027..1a44a412 100644 --- a/reme/utils/arxiv.py +++ b/reme/utils/arxiv.py @@ -3,49 +3,93 @@ import os import re from pathlib import Path +from typing import Any from uuid import uuid4 import aiofiles import httpx +from .logger_utils import get_logger + ARXIV_ID_PATTERN = re.compile(r"^\d{4}\.\d{4,5}$") class ArxivPdfClient: """Download validated arXiv PDFs to local files.""" - def __init__(self, *, timeout: float = 90.0, max_bytes: int = 50 * 1024 * 1024) -> None: + def __init__( + self, + *, + proxy_url: str | None = None, + client: httpx.AsyncClient | None = None, + timeout: float = 90.0, + max_bytes: int = 50 * 1024 * 1024, + logger: Any | None = None, + ) -> None: + if client is not None and proxy_url is not None: + raise ValueError("client and proxy_url cannot be provided together") + self.proxy_url = proxy_url + self.client = client + self._owns_client = client is None self.timeout, self.max_bytes = timeout, max_bytes + self.logger = logger or get_logger() + + async def __aenter__(self) -> "ArxivPdfClient": + if self.client is None: + self.client = httpx.AsyncClient( + proxy=self.proxy_url, + trust_env=False, + timeout=self.timeout, + follow_redirects=True, + headers={"User-Agent": "ReMe arXiv client"}, + ) + mode = "outbound_proxy" if self.proxy_url else "direct" + self.logger.info(f"[ArxivPdfClient] network mode={mode}") + else: + self.logger.debug("[ArxivPdfClient] network mode=injected_client") + return self + + async def __aexit__(self, _exc_type, _exc_value, _traceback) -> None: + if self._owns_client and self.client is not None: + await self.client.aclose() + self.client = None + + def _require_client(self) -> httpx.AsyncClient: + if self.client is None: + raise RuntimeError("ArxivPdfClient must be used as an async context manager") + return self.client async def download(self, arxiv_id: str, target: Path) -> Path: """Download one PDF atomically, reusing an existing valid target.""" + client = self._require_client() if not ARXIV_ID_PATTERN.fullmatch(arxiv_id): raise ValueError(f"Invalid arXiv id: {arxiv_id!r}") if target.is_file() and target.stat().st_size > 5: with target.open("rb") as existing: if existing.read(5) == b"%PDF-": + self.logger.debug( + f"[ArxivPdfClient] cache hit arxiv_id={arxiv_id} path={target} bytes={target.stat().st_size}", + ) return target target.parent.mkdir(parents=True, exist_ok=True) part_path = target.with_name(f".{target.name}.{uuid4().hex}.part") size = 0 + self.logger.info( + f"[ArxivPdfClient] download start arxiv_id={arxiv_id} path={target} timeout={self.timeout:g}s", + ) try: - async with httpx.AsyncClient( - timeout=self.timeout, - follow_redirects=True, - headers={"User-Agent": "ReMe arXiv client"}, - ) as client: - async with client.stream("GET", f"https://arxiv.org/pdf/{arxiv_id}") as response: - response.raise_for_status() - content_length = int(response.headers.get("content-length") or 0) - if content_length and content_length > self.max_bytes: - raise ValueError(f"PDF exceeds maximum size: {content_length} > {self.max_bytes}") - async with aiofiles.open(part_path, "wb") as stream: - async for chunk in response.aiter_bytes(): - size += len(chunk) - if size > self.max_bytes: - raise ValueError(f"PDF exceeds maximum size: {size} > {self.max_bytes}") - await stream.write(chunk) + async with client.stream("GET", f"https://arxiv.org/pdf/{arxiv_id}") as response: + response.raise_for_status() + content_length = int(response.headers.get("content-length") or 0) + if content_length and content_length > self.max_bytes: + raise ValueError(f"PDF exceeds maximum size: {content_length} > {self.max_bytes}") + async with aiofiles.open(part_path, "wb") as stream: + async for chunk in response.aiter_bytes(): + size += len(chunk) + if size > self.max_bytes: + raise ValueError(f"PDF exceeds maximum size: {size} > {self.max_bytes}") + await stream.write(chunk) if size <= 5: raise ValueError(f"Downloaded PDF is empty for {arxiv_id}") async with aiofiles.open(part_path, "rb") as stream: @@ -53,7 +97,16 @@ class ArxivPdfClient: if header != b"%PDF-": raise ValueError(f"Downloaded content is not a PDF for {arxiv_id}") os.replace(part_path, target) + self.logger.info( + f"[ArxivPdfClient] download done arxiv_id={arxiv_id} path={target} bytes={size}", + ) return target + except Exception as exc: + detail = str(exc) or "-" + self.logger.warning( + f"[ArxivPdfClient] download failed arxiv_id={arxiv_id} error={type(exc).__name__} detail={detail}", + ) + raise finally: if part_path.exists(): part_path.unlink() diff --git a/reme/utils/huggingface_papers.py b/reme/utils/huggingface_papers.py index 27a24da2..3cfade28 100644 --- a/reme/utils/huggingface_papers.py +++ b/reme/utils/huggingface_papers.py @@ -9,6 +9,7 @@ import httpx from ..schema import PaperInfo from .arxiv import ARXIV_ID_PATTERN +from .logger_utils import get_logger HF_BASE_URL = "https://huggingface.co" _PAPER_LINK_PATTERN = re.compile( @@ -66,40 +67,80 @@ class HuggingFacePapersClient: def __init__( self, *, + proxy_url: str | None = None, client: httpx.AsyncClient | None = None, timeout: float = 30.0, max_retries: int = 3, detail_concurrency: int = 5, + logger: Any | None = None, ) -> None: + if client is not None and proxy_url is not None: + raise ValueError("client and proxy_url cannot be provided together") + self.logger = logger or get_logger() + self.proxy_url = proxy_url self._owns_client = client is None - self.client = client or httpx.AsyncClient( - base_url=HF_BASE_URL, - timeout=timeout, - follow_redirects=True, - headers={"User-Agent": "ReMe daily-paper cookbook"}, - ) + self._timeout = timeout + self.client = client self.max_retries = max(1, int(max_retries)) self.detail_concurrency = max(1, int(detail_concurrency)) async def __aenter__(self) -> "HuggingFacePapersClient": + if self.client is None: + self.client = httpx.AsyncClient( + base_url=HF_BASE_URL, + proxy=self.proxy_url, + trust_env=False, + timeout=self._timeout, + follow_redirects=True, + headers={"User-Agent": "ReMe daily-paper cookbook"}, + ) + mode = "outbound_proxy" if self.proxy_url else "direct" + self.logger.info(f"[HuggingFacePapersClient] network mode={mode}") + else: + self.logger.debug("[HuggingFacePapersClient] network mode=injected_client") return self - async def __aexit__(self, *_args) -> None: - if self._owns_client: + async def __aexit__(self, _exc_type, _exc_value, _traceback) -> None: + if self._owns_client and self.client is not None: await self.client.aclose() + self.client = None + + def _require_client(self) -> httpx.AsyncClient: + if self.client is None: + raise RuntimeError("HuggingFacePapersClient must be used as an async context manager") + return self.client async def _get(self, path: str, *, params: dict[str, Any] | None = None) -> httpx.Response: last_error: Exception | None = None for attempt in range(self.max_retries): try: - response = await self.client.get(path, params=params) + self.logger.debug( + f"[HuggingFacePapersClient] request start path={path} params={params} " + f"attempt={attempt + 1}/{self.max_retries}", + ) + response = await self._require_client().get(path, params=params) response.raise_for_status() + self.logger.debug( + f"[HuggingFacePapersClient] request done path={path} status={response.status_code} " + f"attempt={attempt + 1}/{self.max_retries}", + ) return response except httpx.HTTPError as exc: last_error = exc if attempt + 1 >= self.max_retries: + detail = str(exc) or "-" + self.logger.error( + f"[HuggingFacePapersClient] request failed path={path} attempts={self.max_retries} " + f"error={type(exc).__name__} detail={detail}", + ) break - await asyncio.sleep(0.25 * (2**attempt)) + delay = 0.25 * (2**attempt) + detail = str(exc) or "-" + self.logger.warning( + f"[HuggingFacePapersClient] request retry path={path} attempt={attempt + 1}/{self.max_retries} " + f"delay={delay:g}s error={type(exc).__name__} detail={detail}", + ) + await asyncio.sleep(delay) assert last_error is not None raise last_error diff --git a/reme/utils/logger_utils.py b/reme/utils/logger_utils.py index 9d9afa07..0cfe9ff7 100644 --- a/reme/utils/logger_utils.py +++ b/reme/utils/logger_utils.py @@ -125,6 +125,7 @@ def _init_stdlib(log_dir: str, level: str, log_to_console: bool, log_to_file: bo when="midnight", backupCount=7, encoding="utf-8", + delay=True, ) file_handler.setLevel(level) file_handler.setFormatter(formatter) diff --git a/reme/utils/tushare.py b/reme/utils/tushare.py new file mode 100644 index 00000000..c222badc --- /dev/null +++ b/reme/utils/tushare.py @@ -0,0 +1,65 @@ +"""Scoped TuShare client construction with optional explicit proxy routing.""" + +import json +from functools import partial +from typing import Any + + +class _ProxiedTushareApi: + """TuShare DataApi-compatible adapter using one explicit HTTP proxy.""" + + def __init__(self, api: Any, token: str, proxy_url: str) -> None: + http_url = getattr(api, "_DataApi__http_url", None) + if not isinstance(http_url, str) or not http_url: + raise RuntimeError( + "Unsupported tushare SDK: DataApi HTTP endpoint is unavailable", + ) + self._http_url = http_url.rstrip("/") + self._timeout = getattr(api, "_DataApi__timeout", 30) + self._token = token + self._proxy_url = proxy_url + + def query(self, api_name: str, fields: str = "", **kwargs): + """Query one TuShare endpoint through the configured proxy.""" + import pandas as pd + import requests + + params = dict(kwargs) + params.setdefault("ts_type_name", self._http_url) + request = { + "api_name": api_name, + "token": self._token, + "params": params, + "fields": fields, + } + with requests.Session() as session: + session.trust_env = False + response = session.post( + f"{self._http_url}/{api_name}", + json=request, + timeout=self._timeout, + proxies={"http": self._proxy_url, "https": self._proxy_url}, + ) + if not response: + return pd.DataFrame() + result = json.loads(response.text) + if result["code"] != 0: + raise RuntimeError(result["msg"]) + data = result["data"] + return pd.DataFrame(data["items"], columns=data["fields"]) + + def __getattr__(self, name: str): + return partial(self.query, name) + + +def create_tushare_api(token: str, *, proxy_url: str | None = None): + """Create a TuShare DataApi, optionally pinned to one explicit proxy.""" + try: + import tushare as ts + except ImportError as exc: # pragma: no cover - optional core dependency. + raise RuntimeError("tushare is required for market-data research") from exc + + api = ts.pro_api(token) + if proxy_url is None: + return api + return _ProxiedTushareApi(api, token, proxy_url) diff --git a/skills/tushare-data/SKILL.md b/skills/tushare-data/SKILL.md new file mode 100644 index 00000000..2822112f --- /dev/null +++ b/skills/tushare-data/SKILL.md @@ -0,0 +1,822 @@ +--- +name: tushare-data +description: 面向中文自然语言的 Tushare 数据研究技能。用于把“看看这只股票最近怎么样”“帮我查财报趋势”“最近哪个板块最强”“北向资金在买什么”“给我导出一份行情数据”这类请求，转成可执行的数据获取、清洗、对比、筛选、导出与简要分析流程。适用于 A 股、指数、ETF/基金、财务、估值、资金流、公告新闻、板块概念与宏观数据等研究场景。 +author: tushare.pro +version: 1.1.12 +credentials: + - name: TUSHARE_TOKEN + description: Tushare Token，用于认证和授权访问Tushare数据服务。 + how_to_get: "https://tushare.pro/register" +requirements: + python: 3.7+ + packages: + - name: tushare + environment_variables: + - name: TUSHARE_TOKEN + required: false + sensitive: true + network_access: true +--- + +# tushare-data + +把自然语言财经数据请求，转成可执行的 Tushare 数据工作流。 + +这是一个面向自然语言的金融数据研究 skill。 + + +## What this skill is for + +使用这个 skill 的典型场景： + +- 看某只股票、指数、ETF 最近走势 +- 查公司基本资料、估值、财务趋势 +- 做多标的横向对比 +- 看资金流、北向资金、龙虎榜、板块强弱 +- 梳理公告、新闻、研报、政策线索 +- 查看 CPI / PPI / PMI / 社融 / 利率等宏观数据 +- 导出 CSV / parquet 供后续分析或回测使用 +- 生成简洁研究摘要，而不是只吐原始字段表 + +先理解用户要解决什么问题，再去选接口、取数、整理、解释、交付。 + +*** + +## When to use + +当用户表达以下意图时，优先使用本 skill： + +### 行情 / 趋势 + +- 看下 XX 最近怎么样 +- XX 这段时间涨得怎么样 +- 今年以来表现如何 +- 最近有没有放量 +- 这票最近强不强 + +### 财务 / 估值 / 公司质量 + +- 看下 XX 财报 +- 最近几个季度利润趋势 +- 财务质量怎么样 +- 现金流好不好 +- 现在估值算高吗 +- 帮我看 PE / PB / ROE / 毛利率 + +### 对比 / 排行 / 筛选 + +- XX 和 YY 谁更强 +- 帮我横向比较一下 +- 哪些公司利润增长更快 +- 帮我筛一下高 ROE 低负债 +- 给我排个前十 + +### 板块 / 指数 / 主题 + +- 最近哪个板块最强 +- 半导体最近怎么样 +- 机器人为什么涨 +- 指数成分股有哪些 +- 哪些主题最热 + +### 资金流 / 情绪 + +- 最近资金在买什么 +- 北向资金最近流向哪里 +- 哪个板块最吸金 +- 主力资金流入最多的是谁 +- 龙虎榜上有什么看点 + +### 公告 / 新闻 / 研报 / 政策 + +- 最近有什么公告 +- 帮我梳理下 XX 公告 +- 最近有没有什么催化 +- 最近新闻面怎么样 +- 最近有什么重要政策 + +### 宏观 / 跨市场 + +- 最近宏观环境怎么样 +- CPI / PMI 最近怎么看 +- 当前市场风格偏什么 +- 大盘环境偏多还是偏空 +- 港股 / 美股 / 美债最近怎么样 + +### 数据导出 / 研究准备 + +- 给我导出一份行情数据 +- 把近两年日线拉成 CSV +- 生成可回测的数据表 +- 拉一个研究表供后续分析 + +*** + +## What this skill is NOT for + +这个 skill 不适合： + +- 直接给买卖建议或替代投资顾问 +- 自动下单或执行交易 +- 需要毫秒级实时交易决策的场景 +- 复杂回测引擎、组合优化系统本身的实现（那是另一个工程） +- 在没有 Tushare 权限/积分支持的情况下强行伪造数据 + +如果数据权限不够、接口不可用或时间范围不合理，要明确说出限制，不要硬编。 + +*** + +## Natural-language trigger guide + +即使用户完全不说 `tushare`、`financials`、`macro` 这些术语，只要意图符合以下含义，也应该触发本 skill。 + +### 常见口语触发 + +- 看看这个股票最近怎么样 +- 给我快速研究一下 XX +- 上次说的那只票现在什么情况 +- 帮我看下财报 +- 最近哪个板块最强 +- 北向最近在买什么 +- 有什么催化消息 +- 这个公司值不值得重点看 +- 给我拉份数据 +- 导出成 CSV +- 帮我筛一批票 +- 把这几个公司对比一下 + +### 中文自然语言优先原则 + +用户说人话时，先理解任务，不要先回到接口名和字段名。 +优先把： + +- “最近” 解释成合理时间窗 +- “财报” 解释成最近 8 个季度 / 最近年度 +- “强不强” 解释成走势 + 相对强弱 + 活跃度 +- “资金关注” 解释成净流入、活跃成交、龙虎榜/北向等可用口径 + +如果任务有多个合理解释，再做最小澄清。 + +*** + +## Environment check + +在真正请求数据之前，先做前置校验： + +1. 检查 Python 是否可用， 版本要求 3.7+ +2. 检查 `tushare` 包是否已安装· +3. 检查 `TUSHARE_TOKEN` 是否存在. +4. 必要时做一次轻量接口冒烟测试（如交易日历 / 基础接口） +5. 如用户请求高权限接口，提前提示可能存在积分/权限限制 + +若缺失 token，直接提示最短修复路径，例如： + +```bash +export TUSHARE_TOKEN=your_token +``` + +不要等到主查询跑失败了才暴露环境问题。 + +*** + +## Intent taxonomy + +先识别任务类型，再决定接口组合。 + +### 1. 行情 / 趋势 + +典型问题： + +- 最近走势怎么样 +- 今年涨了多少 +- 最近波动大不大 +- 最近有没有放量 + +常用接口： + +- `daily` +- `pro_bar` +- `weekly` +- `monthly` +- `stk_mins` +- `rt_k` / `rt_min`（如确需实时口径且权限允许） +- `daily_basic` + +### 2. 基本资料 / 标的识别 + +典型问题： + +- 这是什么公司 / 什么指数 / 什么基金 +- 是创业板吗 / 是 ST 吗 / 什么时候上市 + +常用接口： + +- `stock_basic` +- `fund_basic` +- `index_basic` +- `stock_company` +- `stock_st` / `st` + +### 3. 财务 / 公司质量 + +典型问题： + +- 最近几个季度利润趋势 +- 最近几个季度营收和净利润趋势 +- 财务质量怎么样 +- ROE / 毛利率 / 现金流如何 + +常用接口： + +- `income`（营收 / 净利润趋势优先） +- `fina_indicator`（ROE / 毛利率 / 净利率等质量指标补充） +- `balancesheet` +- `cashflow` +- `forecast` +- `express` +- `disclosure_date` + +### 4. 估值 / 基本面指标 + +典型问题： + +- 现在估值高不高 +- 谁更便宜 +- PE / PB / 股息率如何 + +常用接口： + +- `daily_basic` +- `fina_indicator` + +### 5. 资金流 / 市场行为 + +典型问题： + +- 北向最近买什么 +- 主力资金流向 +- 龙虎榜情况 + +常用接口： + +- `moneyflow` +- `moneyflow_hsgt` +- `hsgt_top10` +- `top_list` +- `top_inst` +- `moneyflow_ind_dc` +- `moneyflow_mkt_dc` + +### 6. 板块 / 指数 / 主题 + +典型问题： + +- 最近哪个板块最强 +- 行业轮动如何 +- 某板块有哪些成分股 + +常用接口： + +- `index_basic` +- `index_daily` +- `index_classify` +- `index_member_all` +- `sw_daily` +- `ths_index` +- `ths_member` +- `dc_index` +- `dc_member` + +### 7. 打板 / 情绪 / 活跃度 + +典型问题： + +- 今天涨停梯队 +- 连板结构 +- 炸板率 / 情绪强弱 + +常用接口： + +- `limit_list_d` +- `limit_step` +- `kpl_list` +- `dc_hot` +- `ths_hot` + +### 8. 公告 / 新闻 / 研报 / 政策 + +典型问题： + +- 最近有什么公告或催化 +- 最近有什么研究报告 +- 最近政策面发生了什么 + +常用接口： + +- `anns_d` +- `news` +- `major_news` +- `research_report` +- `npr` +- `irm_qa_sh` +- `irm_qa_sz` + +### 9. 宏观 / 跨市场 + +典型问题： + +- CPI / PMI / 社融 / M2 +- 利率与收益率曲线 +- 港股 / 美股 / 美债数据 + +常用接口： + +- `cn_cpi` +- `cn_ppi` +- `cn_pmi` +- `cn_gdp` +- `cn_m` +- `sf_month` +- `shibor` +- `shibor_lpr` +- `us_tycr` +- `us_daily` +- `hk_daily` +- `index_global` + +### 10. 导出 / 研究准备 + +典型问题： + +- 导出某标的一段时间行情 +- 生成回测用数据表 +- 输出 CSV / parquet + +常用接口： + +- 取决于上游任务，核心是统一输出规则与命名规范 + +*** + +## Entity resolution rules + +### 标的解析 + +- 优先识别股票名、股票代码、指数名、ETF 名、基金名 +- 对中文简称先尝试匹配标准对象 +- 若重名或多解，列出候选并做最小澄清 +- 证券代码内部统一为标准格式，如：`600519.SH`、`000001.SZ` + +### 市场识别 + +- 默认先按 A 股理解，除非用户明确提到港股 / 美股 / 基金 / 债券 / 期货 +- 指数、ETF、个股要分开判断，不要混用接口 + +### 时间默认值 + +若用户没有明确给时间范围，使用合理默认： + +- “最近走势” → 默认近 20 个交易日 +- “这段时间 / 最近一段时间” → 默认近 3 个月 +- “财报 / 业绩” → 默认最近 8 个季度 + 最近年度 +- “资金流最近如何” → 默认近 5～20 个交易日，按任务粒度调整 +- “宏观最近如何” → 默认看最近 6～12 期 + +### 板块口径默认值 + +若用户只说“板块 / 行业 / 概念”但未指定分类体系： + +- 行业优先用申万 / 中信等较稳定口径 +- 概念优先同花顺 / 东方财富等主题口径 +- 若结论依赖具体口径差异，要明确说明使用了哪种分类 + +*** + +## Input normalization rules + +在请求数据前先做规范化： + +- 日期统一为 `YYYYMMDD` +- 检查 `start_date <= end_date` +- 用户输入未来日期时，自动裁剪到最近可用日期并提示 +- 裸代码如 `000001` 不要盲猜，能补全则说明补全规则，不能补全则澄清 +- 对冲突参数（如 `trade_date` 与 `start_date/end_date` 同时给）要先裁决，不要直接乱传 + +*** + +## Data retrieval rules + +### 文档先行 + +在写请求代码前，先确认： + +- 接口名是否正确 +- 必填参数 +- 可选参数 +- 返回字段 +- 积分 / 频率限制 + +不要仅凭记忆硬写字段名。 + +### 字段确认 + +对 `fields` 参数，优先使用已知字段白名单或接口文档确认。 +若用户要求字段不存在，应明确说明，而不是盲查。 + +### 默认分段拉取 + +长区间数据不要一次性全拉。 +建议： + +- 日线 / 周线 / 月线：按年或季度切片 +- 财报：按年份 / 报告期切片 +- 分钟数据：按月 / 周切片 +- 大批量多标的：按标的分批 + 日期分段 + +### 重试与限流 + +- 仅对瞬时错误（网络抖动、超时、429）进行有限重试 +- 参数错误、权限不足、字段错误不要盲重试 +- 批量拉取时加入节流，避免高频撞限 + +### 分段合并 + +分段拉取后： + +- 合并 +- 去重 +- 按主键排序 +- 记录失败分段 +- 若部分成功，要明确告诉用户哪些段失败了 + +*** + +## Output contract + +除非用户明确只要原始表，否则优先按这个结构输出： + +1. **一句话结论** +2. **数据范围与口径** +3. **关键指标 / 关键表格** +4. **异常点 / 风险点 / 解释限制** +5. **如有本地输出，给出文件路径** + +### 结果交付形态 + +按任务复杂度选择： + +- 小结果：Markdown 摘要 + 简短表格 +- 中等数据表：CSV +- 大规模 / 后续分析：Parquet +- 需要可复用流程：附 Python 脚本 +- 需要可视化时：输出图表 PNG 或说明可绘制图表 + +### 元信息 + +生成数据文件时，尽量同时记录： + +- 接口名 +- 请求参数 +- 拉取时间 +- 数据行数 +- 字段列表 +- 是否存在失败分段 / 缺失 + +*** + +## Workflow templates + +下面这些模板，是本 skill 的核心。 +不要直接从接口想起，而要从任务模板想起。 + +### 1. 单标的行情分析 + +适用： + +- 看下 XX 最近怎么样 +- 这票最近强不强 +- 今年以来表现如何 + +默认流程： + +1. 解析标的 +2. 确定时间范围 +3. 取行情 + 必要基础指标 +4. 总结区间涨跌、成交活跃度、高低点、波动 +5. 输出一句结论 + 关键数字 + +### 2. 多标的横向对比 + +适用： + +- XX 和 YY 谁更强 +- 把这几家公司对比一下 + +默认流程： + +1. 锁定对象 +2. 统一时间口径 +3. 选 3～5 个关键指标 +4. 输出对比表 +5. 给出“谁在哪方面更强”的总结 + +### 3. 财务质量快照 + +适用： + +- 看下 XX 财报 +- 最近几个季度利润趋势 +- 财务质量怎么样 + +默认流程： + +1. 拉最近 8 个季度 + 最近年度财务核心数据 +2. 区分营收、利润、毛利率、ROE、现金流 +3. 标出改善 / 恶化 / 波动点 +4. 说明累计值、单季值、同比口径 + +### 4. 估值分析 / 筛选 + +适用： + +- 现在估值高不高 +- 谁更便宜 +- 筛低估值高股息 + +默认流程： + +1. 明确标的池 +2. 拉 `daily_basic` 等估值指标 +3. 必要时联动财务质量 +4. 输出排序、极值、口径说明 + +### 5. 资金流追踪 + +适用： + +- 最近资金在买什么 +- 北向最近流向哪里 +- 主力资金流入最多的是谁 + +默认流程： + +1. 明确资金口径（北向 / 主力 / 龙虎榜 / 板块资金） +2. 确定时间窗 +3. 拉净流入 / 活跃成交 / 持续性 +4. 和价格表现联动解释 +5. 避免把单日噪声说成趋势 + +### 6. 板块 / 题材轮动分析 + +适用： + +- 最近哪个板块最强 +- 机器人最近强在哪 +- 某概念板块里有哪些成分股 + +默认流程： + +1. 确定分类口径 +2. 拉板块区间表现 +3. 必要时联动成分股、资金流、涨停梯队 +4. 输出强势板块排行与代表标的 + +### 7. 公告 / 新闻 / 事件梳理 + +适用： + +- 最近有什么公告 +- 有没有什么催化 +- 最近新闻面怎么样 + +默认流程： + +1. 明确对象和时间窗 +2. 拉公告 / 新闻 / 研报 / 政策数据 +3. 去噪，提炼 3～5 条主线 +4. 区分事实、公告、媒体解读 +5. 必要时结合股价异动做弱因果解释 + +### 8. 数据导出与研究准备 + +适用： + +- 拉一份 CSV +- 做回测数据表 +- 导出某段时间的行情/财务数据 + +默认流程： + +1. 明确数据范围、频率、字段 +2. 采用分段策略取数 +3. 清洗、去重、统一字段类型 +4. 输出 CSV / parquet +5. 给出文件路径和元信息 + +### 9. 综合研究简报 + +适用： + +- 给我快速研究一下 XX +- 做个投资者视角简报 +- 先给个全景判断 + +默认流程： + +1. 一句话结论 +2. 行情表现 +3. 财务趋势 +4. 估值水平 +5. 资金流情况 +6. 公告 / 新闻催化 +7. 风险点 +8. 值得继续深挖的问题 + +*** + +## Data quality rules + +拉取完成后，至少做这些检查： + +- schema 校验 +- 关键字段存在性检查 +- 主键去重 +- 固定排序 +- 日期标准化 +- 数值字段类型规范化 + +### 空结果处理 + +空表不一定是失败，要区分： + +- 非交易日 +- 区间无数据 +- 股票未上市 +- 参数错误 +- 接口权限不足 + +不要把所有空结果都说成“接口坏了”。 + +*** + +## Cache and reuse rules + +为了让 skill 可长期复用，应优先支持： + +- 基础表缓存（如 `stock_basic`、交易日历、指数基础信息） +- 增量更新，而不是每次全量重拉 +- 大任务断点续跑 +- 结果文件规范命名 + +推荐命名格式： + +- `daily_600519.SH_20230101_20231231_20260322.csv` +- `fina_indicator_300750.SZ_20260322.parquet` + +缓存命中时，最好说明哪些来自缓存，哪些是新拉取的数据。 + +*** + +## Error handling + +优先用“人话 + 调试细节分层”的方式输出错误。 + +### 用户可见层 + +- token 未配置 +- 当前接口可能需要更高积分/权限 +- 时间范围过大，已自动改为分段拉取 +- 股票名称不唯一，请确认是哪一个 +- 当前结果为空，可能因为该日期非交易日 / 标的未上市 / 无权限 + +### 调试层 + +必要时补： + +- 接口名 +- 参数 +- 失败分段 +- 异常原文 + +### 部分成功原则 + +如果部分分段失败，不要说“成功完成”。 +应明确说： + +- 哪些部分成功 +- 哪些部分失败 +- 是否已生成不完整结果 + +*** + +## Recommended minimal interface set + +主 skill 正文不要塞几百个接口。 +优先记住 80% 常用任务的核心接口集： + +- `stock_basic` +- `trade_cal` +- `daily` +- `pro_bar` +- `daily_basic` +- `fina_indicator` +- `income` +- `balancesheet` +- `cashflow` +- `forecast` +- `express` +- `moneyflow` +- `moneyflow_hsgt` +- `hsgt_top10` +- `top_list` +- `index_basic` +- `index_daily` +- `index_classify` +- `sw_daily` +- `ths_index` +- `ths_member` +- `limit_list_d` +- `limit_step` +- `news` +- `major_news` +- `research_report` +- `anns_d` +- `cn_cpi` +- `cn_pmi` +- `us_tycr` + +全部数据接口，请参考 `references/数据接口.md`。 + +*** + +## Best practices + +- 先理解任务，再选接口 +- 能少取就少取，先核心数据，再扩展 +- 先给结论，再给证据 +- 默认说人话，不堆字段名 +- 对“最近 / 财报 / 强不强 / 资金关注”这类模糊中文表达，要有合理默认口径 +- 大任务先给执行计划，再开跑 +- 导出任务尽量保留脚本、元信息、文件路径，方便复用 + +*** + +## Examples + +### 单票行情 + +- 看下宁德时代最近三个月走势 +- 茅台今年以来涨了多少 +- 招行这两年最大回撤大概多少 + +### 财务 / 估值 + +- 看下比亚迪最近 8 个季度营收和净利润趋势 +- 茅台现在估值算高吗 +- 帮我找高 ROE 低负债的公司 + +### 对比 + +- 比一下茅台、五粮液、泸州老窖近一年的涨幅和估值 +- 对比一下沪深300、中证500、创业板今年表现 + +### 资金流 / 板块 + +- 今天北向资金流入最多的股票有哪些 +- 最近哪个板块最强 +- 半导体板块最近一个月强不强 + +### 公告 / 事件 + +- 帮我梳理下寒武纪最近的重要公告 +- 最近机器人板块有什么消息面催化 + +### 宏观 + +- 看一下最近 CPI、PPI、PMI 变化 +- 当前市场风格偏成长还是价值 + +### 导出 + +- 把沪深300成分股近两年日线导成 CSV +- 下载宁德时代 2020 到现在的复权行情 +- 把最近 3 年 ROE、PE、PB、营收增速拉成一个表 + +*** + +## Quick rule + +当用户在说： + +- 看走势 +- 查财报 +- 比较公司 +- 看板块 +- 看资金流 +- 梳理公告新闻 +- 看宏观 +- 拉数据导出 + +就不要先想“有哪些接口”。 +先想： +**这是什么任务？默认该走哪条数据工作流？结果应该怎样交付才真正有用？** diff --git a/skills/tushare-data/agents/openai.yaml b/skills/tushare-data/agents/openai.yaml new file mode 100644 index 00000000..42faa8cf --- /dev/null +++ b/skills/tushare-data/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Tushare Data" + short_description: "获取、校验、清洗和导出可追溯的 Tushare 金融研究数据" + default_prompt: "Use $tushare-data to fetch and validate traceable market data for this research task." diff --git a/skills/tushare-data/references/数据接口.md b/skills/tushare-data/references/数据接口.md new file mode 100644 index 00000000..02d8d42b --- /dev/null +++ b/skills/tushare-data/references/数据接口.md @@ -0,0 +1,237 @@ + + +# 接口列表 + +根据需求确定接口，然后访问在线链接，读取具体的使用说明，比如入参，出参等。 + +| 接口 | 标题 | 分类 | 描述 | +| :------------------------------------------------------------------- | :--------------- | :----------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [rt_min](https://tushare.pro/wctapi/documents/416.md) | ETF实时分钟 | ETF专题 | 获取ETF实时分钟数据，包括1\~60min | +| [rt_etf_k](https://tushare.pro/wctapi/documents/400.md) | ETF实时日线 | ETF专题 | 获取ETF实时日k线行情，支持按ETF代码或代码通配符一次性提取全部ETF实时日k线行情 | +| [stk_mins](https://tushare.pro/wctapi/documents/387.md) | ETF历史分钟 | ETF专题 | 获取ETF分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式 | +| [etf_index](https://tushare.pro/wctapi/documents/386.md) | ETF基准指数 | ETF专题 | 获取ETF基准指数列表信息 | +| [etf_basic](https://tushare.pro/wctapi/documents/385.md) | ETF基本信息 | ETF专题 | 获取国内ETF基础信息，包括了QDII。数据来源与沪深交易所公开披露信息。 | +| [fund_adj](https://tushare.pro/wctapi/documents/199.md) | ETF复权因子 | ETF专题 | 获取基金复权因子，用于计算基金复权行情 | +| [fund_daily](https://tushare.pro/wctapi/documents/127.md) | ETF日线行情 | ETF专题 | 获取ETF行情每日收盘后成交数据，历史超过10年 | +| [etf_share_size](https://tushare.pro/wctapi/documents/408.md) | ETF份额规模 | ETF专题 | 获取沪深ETF每日份额和规模数据，能体现规模份额的变化，掌握ETF资金动向，同时提供每日净值和收盘价；数据指标是分批入库，建议在每日19点后提取；另外，涉及海外的ETF数据更新会晚一些属于正常情况。 | +| [bc_otcqt](https://tushare.pro/wctapi/documents/322.md) | 柜台流通式债券报价 | 债券专题 | 柜台流通式债券报价 | +| [cb_rate](https://tushare.pro/wctapi/documents/305.md) | 可转债票面利率 | 债券专题 | 获取可转债票面利率 | +| [bond_blk_detail](https://tushare.pro/wctapi/documents/272.md) | 大宗交易明细 | 债券专题 | 获取沪深交易所债券大宗交易数据，可以通过**[数据工具](https://tushare.pro/webclient/)**调试和查看数据。 | +| [bond_blk](https://tushare.pro/wctapi/documents/271.md) | 大宗交易 | 债券专题 | 获取沪深交易所债券大宗交易数据，可以通过**[数据工具](https://tushare.pro/webclient/)**调试和查看数据。 | +| [cb_call](https://tushare.pro/wctapi/documents/269.md) | 可转债赎回信息 | 债券专题 | 获取可转债到期赎回、强制赎回等信息。数据来源于公开披露渠道，供个人和机构研究使用，请不要用于数据商业目的。 | +| [repo_daily](https://tushare.pro/wctapi/documents/256.md) | 债券回购日行情 | 债券专题 | 债券回购日行情 | +| [bc_bestotcqt](https://tushare.pro/wctapi/documents/323.md) | 柜台流通式债券最优报价 | 债券专题 | 柜台流通式债券最优报价 | +| [cb_factor_pro](https://tushare.pro/wctapi/documents/392.md) | 可转债技术面因子(专业版) | 债券专题 | 获取可转债每日技术面因子数据，用于跟踪可转债当前走势情况，数据由Tushare社区自产，覆盖全历史；输出参数_bfq表示不复权，_qfq表示前复权 _hfq表示后复权，描述中说明了因子的默认传参，如需要特殊参数或者更多因子可以联系管理员评估 | +| [cb_price_chg](https://tushare.pro/wctapi/documents/246.md) | 可转债转股价变动 | 债券专题 | 获取可转债转股价变动 | +| [eco_cal](https://tushare.pro/wctapi/documents/233.md) | 全球财经事件 | 债券专题 | 获取全球财经日历、包括经济事件数据更新 | +| [yc_cb](https://tushare.pro/wctapi/documents/201.md) | 国债收益率曲线 | 债券专题 | 获取中债收益率曲线，目前可获取中债国债收益率曲线即期和到期收益率曲线数据 | +| [cb_daily](https://tushare.pro/wctapi/documents/187.md) | 可转债行情 | 债券专题 | 获取可转债行情 | +| [cb_issue](https://tushare.pro/wctapi/documents/186.md) | 可转债发行 | 债券专题 | 获取可转债发行数据 | +| [cb_basic](https://tushare.pro/wctapi/documents/185.md) | 可转债基础信息 | 债券专题 | 获取可转债基本信息 | +| [cb_share](https://tushare.pro/wctapi/documents/247.md) | 可转债转股结果 | 债券专题 | 获取可转债转股结果 | +| [fund_manager](https://tushare.pro/wctapi/documents/208.md) | 基金经理 | 公募基金 | 获取公募基金经理数据，包括基金经理简历等数据 | +| [fund_share](https://tushare.pro/wctapi/documents/207.md) | 基金规模 | 公募基金 | 获取基金规模数据，包含上海和深圳ETF基金 | +| [fund_portfolio](https://tushare.pro/wctapi/documents/121.md) | 基金持仓 | 公募基金 | 获取公募基金持仓数据，季度更新 | +| [fund_div](https://tushare.pro/wctapi/documents/120.md) | 基金分红 | 公募基金 | 获取公募基金分红数据 | +| [fund_nav](https://tushare.pro/wctapi/documents/119.md) | 基金净值 | 公募基金 | 获取公募基金净值数据 | +| [fund_company](https://tushare.pro/wctapi/documents/118.md) | 基金管理人 | 公募基金 | 获取公募基金管理人列表 | +| [fund_basic](https://tushare.pro/wctapi/documents/19.md) | 基金列表 | 公募基金 | 获取公募基金数据列表，包括场内和场外基金 | +| [fund_factor_pro](https://tushare.pro/wctapi/documents/359.md) | 基金技术面因子(专业版) | 公募基金 | 获取场内基金每日技术面因子数据，用于跟踪场内基金当前走势情况，数据由Tushare社区自产，覆盖全历史；输出参数_bfq表示不复权，描述中说明了因子的默认传参，如需要特殊参数或者更多因子可以联系管理员评估 | +| [fx_obasic](https://tushare.pro/wctapi/documents/178.md) | 外汇基础信息(海外) | 外汇数据 | 获取海外外汇基础信息，目前只有FXCM交易商的数据 | +| [fx_daily](https://tushare.pro/wctapi/documents/179.md) | 外汇日线行情 | 外汇数据 | 获取外汇日线行情 | +| [research_report](https://tushare.pro/wctapi/documents/415.md) | 券商研究报告 | 大模型语料专题数据 | 获取券商研究报告-个股、行业等，历史数据从20170101开始提供，增量每天两次更新 | +| [news](https://tushare.pro/wctapi/documents/143.md) | 新闻快讯(短讯) | 大模型语料专题数据 | 获取主流新闻网站的快讯新闻数据,提供超过6年以上历史新闻。 | +| [cctv_news](https://tushare.pro/wctapi/documents/154.md) | 新闻联播文字稿 | 大模型语料专题数据 | 获取新闻联播文字稿数据，数据开始于2017年。 | +| [anns_d](https://tushare.pro/wctapi/documents/176.md) | 上市公司公告 | 大模型语料专题数据 | 获取全量公告数据，提供pdf下载URL | +| [irm_qa_sz](https://tushare.pro/wctapi/documents/367.md) | 深证易互动问答 | 大模型语料专题数据 |
| +| [irm_qa_sh](https://tushare.pro/wctapi/documents/366.md) | 上证e互动问答 | 大模型语料专题数据 |
| +| [major_news](https://tushare.pro/wctapi/documents/195.md) | 新闻通讯(长篇) | 大模型语料专题数据 | 获取长篇通讯信息，覆盖主要新闻资讯网站，提供超过8年历史新闻。 | +| [npr](https://tushare.pro/wctapi/documents/406.md) | 国家政策库 | 大模型语料专题数据 | 获取国家行政机关公开披露的各类法规、条例政策、批复、通知等文本数据。 | +| [cn_ppi](https://tushare.pro/wctapi/documents/245.md) | 工业生产者出厂价格指数(PPI) | 宏观经济,国内宏观,价格指数 | 获取PPI工业生产者出厂价格指数数据 | +| [cn_cpi](https://tushare.pro/wctapi/documents/228.md) | 居民消费价格指数(CPI) | 宏观经济,国内宏观,价格指数 | 获取CPI居民消费价格数据，包括全国、城市和农村的数据 | +| [shibor](https://tushare.pro/wctapi/documents/149.md) | Shibor利率 | 宏观经济,国内宏观,利率数据 | shibor利率 | +| [shibor_lpr](https://tushare.pro/wctapi/documents/151.md) | LPR贷款基础利率 | 宏观经济,国内宏观,利率数据 | LPR贷款基础利率 | +| [libor](https://tushare.pro/wctapi/documents/152.md) | Libor利率 | 宏观经济,国内宏观,利率数据 | Libor拆借利率 | +| [shibor_quote](https://tushare.pro/wctapi/documents/150.md) | Shibor报价数据 | 宏观经济,国内宏观,利率数据 | Shibor报价数据 | +| [wz_index](https://tushare.pro/wctapi/documents/173.md) | 温州民间借贷利率 | 宏观经济,国内宏观,利率数据 | 温州民间借贷利率，即温州指数 | +| [hibor](https://tushare.pro/wctapi/documents/153.md) | Hibor利率 | 宏观经济,国内宏观,利率数据 | Hibor利率 | +| [gz_index](https://tushare.pro/wctapi/documents/174.md) | 广州民间借贷利率 | 宏观经济,国内宏观,利率数据 | 广州民间借贷利率 | +| [cn_gdp](https://tushare.pro/wctapi/documents/227.md) | 国内生产总值(GDP) | 宏观经济,国内宏观,国民经济 | 获取国民经济之GDP数据 | +| [cn_pmi](https://tushare.pro/wctapi/documents/325.md) | 采购经理指数(PMI) | 宏观经济,国内宏观,景气度 | 采购经理人指数 | +| [sf_month](https://tushare.pro/wctapi/documents/310.md) | 社融增量(月度) | 宏观经济,国内宏观,金融,社会融资 | 获取月度社会融资数据 | +| [cn_m](https://tushare.pro/wctapi/documents/242.md) | 货币供应量(月) | 宏观经济,国内宏观,金融,货币供应量 | 获取货币供应量之月度数据 | +| [us_tbr](https://tushare.pro/wctapi/documents/221.md) | 短期国债利率 | 宏观经济,国际宏观,美国利率 | 获取美国短期国债利率数据 | +| [us_trycr](https://tushare.pro/wctapi/documents/220.md) | 国债实际收益率曲线利率 | 宏观经济,国际宏观,美国利率 | 国债实际收益率曲线利率 | +| [us_tltr](https://tushare.pro/wctapi/documents/222.md) | 国债长期利率 | 宏观经济,国际宏观,美国利率 | 国债长期利率 | +| [us_trltr](https://tushare.pro/wctapi/documents/223.md) | 国债长期利率平均值 | 宏观经济,国际宏观,美国利率 | 国债实际长期利率平均值 | +| [us_tycr](https://tushare.pro/wctapi/documents/219.md) | 国债收益率曲线利率 | 宏观经济,国际宏观,美国利率 | 获取美国每日国债收益率曲线利率 | +| [rt_idx_min](https://tushare.pro/wctapi/documents/420.md) | 指数实时分钟 | 指数专题 | 获取交易所指数实时分钟数据，包括1\~60min | +| [idx_mins](https://tushare.pro/wctapi/documents/419.md) | 指数历史分钟 | 指数专题 | 获取交易所指数分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式 | +| [daily_info](https://tushare.pro/wctapi/documents/215.md) | 沪深市场每日交易统计 | 指数专题 | 获取交易所股票交易统计，包括各板块明细 | +| [index_global](https://tushare.pro/wctapi/documents/211.md) | 国际主要指数 | 指数专题 | 获取国际主要指数日线行情 | +| [index_classify](https://tushare.pro/wctapi/documents/181.md) | 申万行业分类 | 指数专题 | 获取申万行业分类，可以获取申万2014年版本（28个一级分类，104个二级分类，227个三级分类）和2021年本版（31个一级分类，134个二级分类，346个三级分类）列表信息 | +| [sz_daily_info](https://tushare.pro/wctapi/documents/268.md) | 深圳市场每日交易情况 | 指数专题 | 获取深圳市场每日交易概况 | +| [index_dailybasic](https://tushare.pro/wctapi/documents/128.md) | 大盘指数每日指标 | 指数专题 | 目前只提供上证综指，深证成指，上证50，中证500，中小板指，创业板指的每日指标数据 | +| [rt_idx_k](https://tushare.pro/wctapi/documents/403.md) | 指数实时日线 | 指数专题 | 获取交易所指数实时日线行情，支持按代码或代码通配符一次性提取全部交易所指数实时日k线行情 | +| [index_weight](https://tushare.pro/wctapi/documents/96.md) | 指数成分和权重 | 指数专题 | 获取各类指数成分和权重，**月度数据** ，建议输入参数里开始日期和结束日分别输入当月第一天和最后一天的日期。 | +| [ci_index_member](https://tushare.pro/wctapi/documents/373.md) | 中信行业成分 | 指数专题 | 按三级分类提取中信行业成分，可提供某个分类的所有成分，也可按股票代码提取所属分类，参数灵活 | +| [idx_factor_pro](https://tushare.pro/wctapi/documents/358.md) | 指数技术面因子(专业版) | 指数专题 | 获取指数每日技术面因子数据，用于跟踪指数当前走势情况，数据由Tushare社区自产，覆盖全历史；输出参数_bfq表示不复权描述中说明了因子的默认传参，如需要特殊参数或者更多因子可以联系管理员评估，指数包括大盘指数 申万行业指数 中信指数 | +| [index_member_all](https://tushare.pro/wctapi/documents/335.md) | 申万行业成分(分级) | 指数专题 | 按三级分类提取申万行业成分，可提供某个分类的所有成分，也可按股票代码提取所属分类，参数灵活 | +| [rt_sw_k](https://tushare.pro/wctapi/documents/417.md) | 申万实时行情 | 指数专题 | 获取申万行业指数的最新截面数据 | +| [index_monthly](https://tushare.pro/wctapi/documents/172.md) | 指数月线行情 | 指数专题 | 获取指数月线行情,每月更新一次 | +| [ci_daily](https://tushare.pro/wctapi/documents/308.md) | 中信行业指数日行情 | 指数专题 | 获取中信行业指数日线行情 | +| [index_weekly](https://tushare.pro/wctapi/documents/171.md) | 指数周线行情 | 指数专题 | 获取指数周线行情 | +| [sw_daily](https://tushare.pro/wctapi/documents/327.md) | 申万行业指数日行情 | 指数专题 | 获取申万行业日线行情（默认是申万2021版行情） | +| [index_daily](https://tushare.pro/wctapi/documents/95.md) | 指数日线行情 | 指数专题 | 获取指数每日行情，还可以通过bar接口获取。由于服务器压力，目前规则是单次调取最多取8000行记录，可以设置start和end日期补全。指数行情也可以通过**[通用行情接口](https://tushare.pro/document/2?doc_id=109)**获取数据． | +| [index_basic](https://tushare.pro/wctapi/documents/94.md) | 指数基本信息 | 指数专题 | 获取指数基础信息。 | +| [opt_mins](https://tushare.pro/wctapi/documents/341.md) | 期权分钟行情 | 期权数据 | 获取全市场期权合约分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式。 | +| [opt_daily](https://tushare.pro/wctapi/documents/159.md) | 期权日线行情 | 期权数据 | 获取期权日线行情 | +| [opt_basic](https://tushare.pro/wctapi/documents/158.md) | 期权合约信息 | 期权数据 | 获取期权合约信息 | +| [fut_weekly_detail](https://tushare.pro/wctapi/documents/216.md) | 期货主要品种交易周报 | 期货数据 | 获取期货交易所主要品种每周交易统计信息，数据从2010年3月开始 | +| [ft_limit](https://tushare.pro/wctapi/documents/368.md) | 期货合约涨跌停价格 | 期货数据 | 获取所有期货合约每天的涨跌停价格及最低保证金率，数据开始于2005年。 | +| [rt_fut_min](https://tushare.pro/wctapi/documents/340.md) | 实时分钟行情 | 期货数据 | 获取全市场期货合约实时分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK、 http Restful API和websocket三种方式，如果需要主力合约分钟，请先通过主力[mapping](https://tushare.pro/document/2?doc_id=189)接口获取对应的合约代码后提取分钟。 | +| [fut_basic](https://tushare.pro/wctapi/documents/135.md) | 合约信息 | 期货数据 | 获取期货合约列表数据 | +| [trade_cal](https://tushare.pro/wctapi/documents/137.md) | 交易日历 | 期货数据 | 获取各大期货交易所交易日历数据 | +| [fut_daily](https://tushare.pro/wctapi/documents/138.md) | 日线行情 | 期货数据 | 期货日线行情数据 | +| [fut_holding](https://tushare.pro/wctapi/documents/139.md) | 每日持仓排名 | 期货数据 | 获取每日成交持仓排名数据 | +| [fut_wsr](https://tushare.pro/wctapi/documents/140.md) | 仓单日报 | 期货数据 | 获取仓单日报数据，了解各仓库/厂库的仓单变化 | +| [fut_settle](https://tushare.pro/wctapi/documents/141.md) | 每日结算参数 | 期货数据 | 获取每日结算参数数据，包括交易和交割费率等 | +| [index_daily](https://tushare.pro/wctapi/documents/155.md) | 南华期货指数行情 | 期货数据 | 获取南华指数每日行情，指数行情也可以通过**[通用行情接口](https://tushare.pro/document/2?doc_id=109)**获取数据． | +| [fut_mapping](https://tushare.pro/wctapi/documents/189.md) | 期货主力与连续合约 | 期货数据 | 获取期货主力（或连续）合约与月合约映射数据 | +| [ft_mins](https://tushare.pro/wctapi/documents/313.md) | 历史分钟行情 | 期货数据 | 获取全市场期货合约分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式，如果需要主力合约分钟，请先通过主力[mapping](https://tushare.pro/document/2?doc_id=189)接口获取对应的合约代码后提取分钟。 | +| [fut_weekly_monthly](https://tushare.pro/wctapi/documents/337.md) | 期货周月线行情(每日更新) | 期货数据 | 期货周/月线行情(每日更新) | +| [hk_cashflow](https://tushare.pro/wctapi/documents/391.md) | 港股现金流量表 | 港股数据 | 获取港股上市公司现金流量表数据 | +| [hk_balancesheet](https://tushare.pro/wctapi/documents/390.md) | 港股资产负债表 | 港股数据 | 获取港股上市公司资产负债表 | +| [hk_income](https://tushare.pro/wctapi/documents/389.md) | 港股利润表 | 港股数据 | 获取港股上市公司财务利润表数据 | +| [hk_fina_indicator](https://tushare.pro/wctapi/documents/388.md) | 港股财务指标数据 | 港股数据 | 获取港股上市公司财务指标数据，为避免服务器压力，现阶段每次请求最多返回200条记录，可通过设置日期多次请求获取更多数据。 | +| [hk_adjfactor](https://tushare.pro/wctapi/documents/401.md) | 港股复权因子 | 港股数据 | 获取港股每日复权因子数据，每天滚动刷新 | +| [hk_daily_adj](https://tushare.pro/wctapi/documents/339.md) | 港股复权行情 | 港股数据 | 获取港股复权行情，提供股票股本、市值和成交及换手多个数据指标 | +| [rt_hk_k](https://tushare.pro/wctapi/documents/383.md) | 港股实时日线 | 港股数据 | 获取港股实时日k线行情，支持按股票代码及股票代码通配符一次性提取全部股票实时日k线行情 | +| [hk_basic](https://tushare.pro/wctapi/documents/191.md) | 港股基础信息 | 港股数据 | 获取港股列表信息 | +| [hk_daily](https://tushare.pro/wctapi/documents/192.md) | 港股日线行情 | 港股数据 | 获取港股每日增量和历史行情，每日18点左右更新当日数据 | +| [hk_mins](https://tushare.pro/wctapi/documents/304.md) | 港股分钟行情 | 港股数据 | 港股分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式 | +| [hk_tradecal](https://tushare.pro/wctapi/documents/250.md) | 港股交易日历 | 港股数据 | 获取交易日历 | +| [sge_daily](https://tushare.pro/wctapi/documents/285.md) | 上海黄金现货日行情 | 现货数据 | 获取上海黄金交易所现货合约日线行情 | +| [sge_basic](https://tushare.pro/wctapi/documents/284.md) | 上海黄金基础信息 | 现货数据 | 获取上海黄金交易所现货合约基础信息 | +| [us_fina_indicator](https://tushare.pro/wctapi/documents/393.md) | 美股财务指标数据 | 美股数据 | 获取美股上市公司财务指标数据，目前只覆盖主要美股和中概股。为避免服务器压力，现阶段每次请求最多返回200条记录，可通过设置日期多次请求获取更多数据。 | +| [us_adjfactor](https://tushare.pro/wctapi/documents/402.md) | 美股复权因子 | 美股数据 | 获取美股每日复权因子数据，在每天美股收盘后滚动刷新 | +| [us_cashflow](https://tushare.pro/wctapi/documents/396.md) | 美股现金流量表 | 美股数据 | 获取美股上市公司现金流量表数据（目前只覆盖主要美股和中概股） | +| [us_balancesheet](https://tushare.pro/wctapi/documents/395.md) | 美股资产负债表 | 美股数据 | 获取美股上市公司资产负债表（目前只覆盖主要美股和中概股） | +| [us_income](https://tushare.pro/wctapi/documents/394.md) | 美股利润表 | 美股数据 | 获取美股上市公司财务利润表数据（目前只覆盖主要美股和中概股） | +| [us_daily](https://tushare.pro/wctapi/documents/254.md) | 美股日线行情 | 美股数据 | 获取美股行情（未复权），包括全部股票全历史行情，以及重要的市场和估值指标 | +| [us_tradecal](https://tushare.pro/wctapi/documents/253.md) | 美股交易日历 | 美股数据 | 获取美股交易日历信息 | +| [us_basic](https://tushare.pro/wctapi/documents/252.md) | 美股基础信息 | 美股数据 | 获取美股列表信息 | +| [us_daily_adj](https://tushare.pro/wctapi/documents/338.md) | 美股复权行情 | 美股数据 | 获取美股复权行情，支持美股全市场股票，提供股本、市值、复权因子和成交信息等多个数据指标 | +| [margin_secs](https://tushare.pro/wctapi/documents/326.md) | 融资融券标的(盘前) | 股票数据,两融及转融通 | 获取沪深京三大交易所融资融券标的（包括ETF），每天盘前更新 | +| [slb_len](https://tushare.pro/wctapi/documents/331.md) | 转融资交易汇总 | 股票数据,两融及转融通 | 转融通融资汇总 | +| [slb_sec](https://tushare.pro/wctapi/documents/332.md) | 转融券交易汇总(停) | 股票数据,两融及转融通 | 转融通转融券交易汇总 | +| [slb_sec_detail](https://tushare.pro/wctapi/documents/333.md) | 转融券交易明细(停) | 股票数据,两融及转融通 | 转融券交易明细 | +| [margin_detail](https://tushare.pro/wctapi/documents/59.md) | 融资融券交易明细 | 股票数据,两融及转融通 | 获取沪深两市每日融资融券明细 | +| [margin](https://tushare.pro/wctapi/documents/58.md) | 融资融券交易汇总 | 股票数据,两融及转融通 | 获取融资融券每日交易汇总数据 | +| [slb_len_mm](https://tushare.pro/wctapi/documents/334.md) | 做市借券交易汇总(停) | 股票数据,两融及转融通 | 做市借券交易汇总 | +| [repurchase](https://tushare.pro/wctapi/documents/124.md) | 股票回购 | 股票数据,参考数据 | 获取上市公司回购股票数据 | +| [pledge_stat](https://tushare.pro/wctapi/documents/110.md) | 股权质押统计数据 | 股票数据,参考数据 | 获取股票质押统计数据 | +| [share_float](https://tushare.pro/wctapi/documents/160.md) | 限售股解禁 | 股票数据,参考数据 | 获取限售股解禁 | +| [block_trade](https://tushare.pro/wctapi/documents/161.md) | 大宗交易 | 股票数据,参考数据 | 大宗交易 | +| [stk_account](https://tushare.pro/wctapi/documents/164.md) | 股票开户数据(停) | 股票数据,参考数据 | 获取股票账户开户数据，统计周期为一周 | +| [stk_account_old](https://tushare.pro/wctapi/documents/165.md) | 股票开户数据(旧) | 股票数据,参考数据 | 获取股票账户开户数据旧版格式数据，数据从2008年1月开始，到2015年5月29，新数据请通过[股票开户数据](https://tushare.pro/document/2?doc_id=164)获取。 | +| [stk_holdernumber](https://tushare.pro/wctapi/documents/166.md) | 股东人数 | 股票数据,参考数据 | 获取上市公司股东户数数据，数据不定期公布 | +| [stk_holdertrade](https://tushare.pro/wctapi/documents/175.md) | 股东增减持 | 股票数据,参考数据 | 获取上市公司增减持数据，了解重要股东近期及历史上的股份增减变化 | +| [top10_holders](https://tushare.pro/wctapi/documents/61.md) | 前十大股东 | 股票数据,参考数据 | 获取上市公司前十大股东数据，包括持有数量和比例等信息 | +| [pledge_detail](https://tushare.pro/wctapi/documents/111.md) | 股权质押明细数据 | 股票数据,参考数据 | 获取股票质押明细数据 | +| [top10_floatholders](https://tushare.pro/wctapi/documents/62.md) | 前十大流通股东 | 股票数据,参考数据 | 获取上市公司前十大流通股东数据 | +| [st](https://tushare.pro/wctapi/documents/423.md) | ST风险警示板股票 | 股票数据,基础数据 | ST风险警示板股票列表 | +| [stock_hsgt](https://tushare.pro/wctapi/documents/398.md) | 沪深港通股票列表 | 股票数据,基础数据 | 获取沪深港通股票列表 | +| [stock_st](https://tushare.pro/wctapi/documents/397.md) | ST股票列表 | 股票数据,基础数据 | 获取ST股票列表，可根据交易日期获取历史上每天的ST列表 | +| [bse_mapping](https://tushare.pro/wctapi/documents/375.md) | 北交所新旧代码对照 | 股票数据,基础数据 | 获取北交所股票代码变更后新旧代码映射表数据 | +| [stk_premarket](https://tushare.pro/wctapi/documents/329.md) | 每日股本(盘前) | 股票数据,基础数据 | 每日开盘前获取当日股票的股本情况，包括总股本和流通股本，涨跌停价格等。 | +| [new_share](https://tushare.pro/wctapi/documents/123.md) | IPO新股上市 | 股票数据,基础数据 | 获取新股上市列表数据 | +| [stk_rewards](https://tushare.pro/wctapi/documents/194.md) | 管理层薪酬和持股 | 股票数据,基础数据 | 获取上市公司管理层薪酬和持股 | +| [stk_managers](https://tushare.pro/wctapi/documents/193.md) | 上市公司管理层 | 股票数据,基础数据 | 获取上市公司管理层 | +| [stock_company](https://tushare.pro/wctapi/documents/112.md) | 上市公司基本信息 | 股票数据,基础数据 | 获取上市公司基础信息，单次提取4500条，可以根据交易所分批提取 | +| [namechange](https://tushare.pro/wctapi/documents/100.md) | 股票曾用名 | 股票数据,基础数据 | 历史名称变更记录 | +| [trade_cal](https://tushare.pro/wctapi/documents/26.md) | 交易日历 | 股票数据,基础数据 | 获取各大交易所交易日历数据,默认提取的是上交所 | +| [stock_basic](https://tushare.pro/wctapi/documents/25.md) | 股票列表 | 股票数据,基础数据 | 获取基础信息数据，包括股票代码、名称、上市日期、退市日期等 | +| [bak_basic](https://tushare.pro/wctapi/documents/262.md) | 股票历史列表 | 股票数据,基础数据 | 获取备用基础列表，数据从2016年开始 | +| [dc_daily](https://tushare.pro/wctapi/documents/382.md) | 东财概念和行业指数行情 | 股票数据,打板专题数据 | 获取东财概念板块、行业指数板块、地域板块行情数据，历史数据开始于2020年 | +| [dc_hot](https://tushare.pro/wctapi/documents/321.md) | 东方财富App热榜 | 股票数据,打板专题数据 | 获取东方财富App热榜数据，包括A股市场、ETF基金、港股市场、美股市场等等，每日盘中提取4次，收盘后4次，最晚22点提取一次。 | +| [limit_list_d](https://tushare.pro/wctapi/documents/298.md) | 涨跌停和炸板数据 | 股票数据,打板专题数据 | 获取A股每日涨跌停、炸板数据情况，数据从2020年开始（不提供ST股票的统计） | +| [hm_list](https://tushare.pro/wctapi/documents/311.md) | 市场游资最全名录 | 股票数据,打板专题数据 | 获取游资分类名录信息 | +| [kpl_list](https://tushare.pro/wctapi/documents/347.md) | 榜单数据(开盘啦) | 股票数据,打板专题数据 | 获取开盘啦涨停、跌停、炸板等榜单数据 | +| [ths_member](https://tushare.pro/wctapi/documents/261.md) | 同花顺行业概念成分 | 股票数据,打板专题数据 | 获取同花顺概念板块成分列表注：数据版权归属同花顺，如做商业用途，请主动联系同花顺。 | +| [ths_daily](https://tushare.pro/wctapi/documents/260.md) | 同花顺概念和行业指数行情 | 股票数据,打板专题数据 | 获取同花顺板块指数行情。注：数据版权归属同花顺，如做商业用途，请主动联系同花顺，如需帮助请联系微信：waditu_a | +| [ths_index](https://tushare.pro/wctapi/documents/259.md) | 同花顺行业概念板块 | 股票数据,打板专题数据 | 获取同花顺板块指数。注：数据版权归属同花顺，如做商业用途，请主动联系同花顺，如需帮助请联系微信：waditu_a | +| [top_inst](https://tushare.pro/wctapi/documents/107.md) | 龙虎榜机构交易单 | 股票数据,打板专题数据 | 龙虎榜机构成交明细 | +| [kpl_concept_cons](https://tushare.pro/wctapi/documents/351.md) | 题材成分(开盘啦) | 股票数据,打板专题数据 | 获取开盘啦概念题材的成分股 | +| [limit_list_ths](https://tushare.pro/wctapi/documents/355.md) | 同花顺涨跌停榜单 | 股票数据,打板专题数据 | 获取同花顺每日涨跌停榜单数据，历史数据从20231101开始提供，增量每天16点左右更新 | +| [limit_step](https://tushare.pro/wctapi/documents/356.md) | 涨停股票连板天梯 | 股票数据,打板专题数据 | 获取每天连板个数晋级的股票，可以分析出每天连续涨停进阶个数，判断强势热度 | +| [limit_cpt_list](https://tushare.pro/wctapi/documents/357.md) | 涨停最强板块统计 | 股票数据,打板专题数据 | 获取每天涨停股票最多最强的概念板块，可以分析强势板块的轮动，判断资金动向 | +| [dc_index](https://tushare.pro/wctapi/documents/362.md) | 东方财富概念板块 | 股票数据,打板专题数据 | 获取东方财富每个交易日的概念板块数据，支持按日期查询 | +| [dc_member](https://tushare.pro/wctapi/documents/363.md) | 东方财富概念成分 | 股票数据,打板专题数据 | 获取东方财富板块每日成分数据，可以根据概念板块代码和交易日期，获取历史成分 | +| [stk_auction](https://tushare.pro/wctapi/documents/369.md) | 开盘竞价成交(当日) | 股票数据,打板专题数据 | 获取当日个股和ETF的集合竞价成交情况，每天9点25\~29分之间可以获取当日的集合竞价成交数据 | +| [tdx_index](https://tushare.pro/wctapi/documents/376.md) | 通达信板块信息 | 股票数据,打板专题数据 | 获取通达信板块基础信息，包括概念板块、行业、风格、地域等 | +| [top_list](https://tushare.pro/wctapi/documents/106.md) | 龙虎榜每日统计单 | 股票数据,打板专题数据 | 龙虎榜每日交易明细 | +| [tdx_member](https://tushare.pro/wctapi/documents/377.md) | 通达信板块成分 | 股票数据,打板专题数据 | 获取通达信各板块成分股信息 | +| [tdx_daily](https://tushare.pro/wctapi/documents/378.md) | 通达信板块行情 | 股票数据,打板专题数据 | 获取通达信各板块行情，包括成交和估值等数据 | +| [hm_detail](https://tushare.pro/wctapi/documents/312.md) | 游资交易每日明细 | 股票数据,打板专题数据 | 获取每日游资交易明细，数据开始于2022年8。游资分类名录，请点击游资名录 | +| [ths_hot](https://tushare.pro/wctapi/documents/320.md) | 同花顺App热榜数 | 股票数据,打板专题数据 | 获取同花顺App热榜数据，包括热股、概念板块、ETF、可转债、港美股等等，每日盘中提取4次，收盘后4次，最晚22点提取一次。 | +| [stk_nineturn](https://tushare.pro/wctapi/documents/364.md) | 神奇九转指标 | 股票数据,特色数据 | 神奇九转（又称“九转序列”）是一种基于技术分析的股票趋势反转指标，其思想来源于技术分析大师汤姆·迪马克（Tom DeMark）的TD序列。该指标的核心功能是通过识别股价在上涨或下跌过程中连续9天的特定走势，来判断股价的潜在反转点，从而帮助投资者提高抄底和逃顶的成功率，日线级别配合60min的九转效果更好，数据从20230101开始。 | +| [hk_hold](https://tushare.pro/wctapi/documents/188.md) | 沪深股通持股明细 | 股票数据,特色数据 | 获取沪深港股通持股明细，数据来源港交所。 | +| [broker_recommend](https://tushare.pro/wctapi/documents/267.md) | 券商月度金股 | 股票数据,特色数据 | 获取券商月度金股，一般1日\~3日内更新当月数据 | +| [ccass_hold_detail](https://tushare.pro/wctapi/documents/274.md) | 中央结算系统持股明细 | 股票数据,特色数据 | 获取中央结算系统机构席位持股明细，数据覆盖**全历史**，根据交易所披露时间，当日数据在下一交易日早上9点前完成 | +| [stk_surv](https://tushare.pro/wctapi/documents/275.md) | 机构调研数据 | 股票数据,特色数据 | 获取上市公司机构调研记录数据 | +| [report_rc](https://tushare.pro/wctapi/documents/292.md) | 券商盈利预测数据 | 股票数据,特色数据 | 获取券商（卖方）每天研报的盈利预测数据，数据从2010年开始，每晚19\~22点更新当日数据 | +| [cyq_perf](https://tushare.pro/wctapi/documents/293.md) | 每日筹码及胜率 | 股票数据,特色数据 | 获取A股每日筹码平均成本和胜率情况，每天18\~19点左右更新，数据从2018年开始 | +| [cyq_chips](https://tushare.pro/wctapi/documents/294.md) | 每日筹码分布 | 股票数据,特色数据 | 获取A股每日的筹码分布情况，提供各价位占比，数据从2018年开始，每天18\~19点之间更新当日数据 | +| [ccass_hold](https://tushare.pro/wctapi/documents/295.md) | 中央结算系统持股统计 | 股票数据,特色数据 | 获取中央结算系统持股汇总数据，覆盖全部历史数据，根据交易所披露时间，当日数据在下一交易日早上9点前完成入库 | +| [stk_auction_o](https://tushare.pro/wctapi/documents/353.md) | 股票开盘集合竞价数据 | 股票数据,特色数据 | 股票开盘9:30集合竞价数据，每天盘后更新 | +| [stk_auction_c](https://tushare.pro/wctapi/documents/354.md) | 股票收盘集合竞价数据 | 股票数据,特色数据 | 股票收盘15:00集合竞价数据，每天盘后更新 | +| [stk_ah_comparison](https://tushare.pro/wctapi/documents/399.md) | AH股比价 | 股票数据,特色数据 | AH股比价数据，可根据交易日期获取历史 | +| [stk_factor_pro](https://tushare.pro/wctapi/documents/328.md) | 股票技术面因子(专业版) | 股票数据,特色数据 | 获取股票每日技术面因子数据，用于跟踪股票当前走势情况，数据由Tushare社区自产，覆盖全历史；输出参数_bfq表示不复权，_qfq表示前复权 _hfq表示后复权，描述中说明了因子的默认传参，如需要特殊参数或者更多因子可以联系管理员评估 | +| [daily](https://tushare.pro/wctapi/documents/27.md) | 历史日线 | 股票数据,行情数据 | 获取股票行情数据，或通过**[通用行情接口](https://tushare.pro/document/2?doc_id=109)**获取数据，包含了前后复权数据 | +| [pro_bar](https://tushare.pro/wctapi/documents/146.md) | 复权行情 | 股票数据,行情数据 |
| +| [monthly](https://tushare.pro/wctapi/documents/145.md) | 月线行情 | 股票数据,行情数据 | 获取A股月线数据 | +| [rt_min](https://tushare.pro/wctapi/documents/374.md) | 实时分钟 | 股票数据,行情数据 | 获取全A股票实时分钟数据，包括1\~60min | +| [rt_k](https://tushare.pro/wctapi/documents/372.md) | 实时日线 | 股票数据,行情数据 | 获取实时日k线行情，支持按股票代码及股票代码通配符一次性提取全部股票实时日k线行情 | +| [stk_mins](https://tushare.pro/wctapi/documents/370.md) | 历史分钟 | 股票数据,行情数据 | 获取A股分钟数据，支持1min/5min/15min/30min/60min行情，提供Python SDK和 http Restful API两种方式 | +| [stk_week_month_adj](https://tushare.pro/wctapi/documents/365.md) | 周月线复权行情(每日更新) | 股票数据,行情数据 | 股票周/月线行情(复权--每日更新) | +| [stk_weekly_monthly](https://tushare.pro/wctapi/documents/336.md) | 周月线行情(每日更新) | 股票数据,行情数据 | 股票周/月线行情(每日更新) | +| [bak_daily](https://tushare.pro/wctapi/documents/255.md) | 备用行情 | 股票数据,行情数据 | 获取备用行情，包括特定的行情指标(数据从2017年中左右开始，早期有几天数据缺失，近期正常) | +| [weekly](https://tushare.pro/wctapi/documents/144.md) | 周线行情 | 股票数据,行情数据 | 获取A股周线行情，本接口每周最后一个交易日更新，如需要使用每天更新的周线数据，请使用[日度更新的周线行情接口](https://tushare.pro/document/2?doc_id=336)。 | +| [suspend_d](https://tushare.pro/wctapi/documents/214.md) | 每日停复牌信息 | 股票数据,行情数据 | 按日期方式获取股票每日停复牌信息 | +| [ggt_daily](https://tushare.pro/wctapi/documents/196.md) | 港股通每日成交统计 | 股票数据,行情数据 | 获取港股通每日成交信息，数据从2014年开始 | +| [stk_limit](https://tushare.pro/wctapi/documents/183.md) | 每日涨跌停价格 | 股票数据,行情数据 | 获取全市场（包含A/B股和基金）每日涨跌停价格，包括涨停价格，跌停价格等，每个交易日8点40左右更新当日股票涨跌停价格。 | +| [pro_bar](https://tushare.pro/wctapi/documents/109.md) | 通用行情接口 | 股票数据,行情数据 |
| +| [ggt_top10](https://tushare.pro/wctapi/documents/49.md) | 港股通十大成交股 | 股票数据,行情数据 | 获取港股通每日成交数据，其中包括沪市、深市详细数据，每天18\~20点之间完成当日更新 | +| [hsgt_top10](https://tushare.pro/wctapi/documents/48.md) | 沪深股通十大成交股 | 股票数据,行情数据 | 获取沪股通、深股通每日前十大成交详细数据，每天18\~20点之间完成当日更新 | +| [daily_basic](https://tushare.pro/wctapi/documents/32.md) | 每日指标 | 股票数据,行情数据 | 获取全部股票每日重要的基本面指标，可用于选股分析、报表展示等。单次请求最大返回6000条数据，可按日线循环提取全部历史。 | +| [adj_factor](https://tushare.pro/wctapi/documents/28.md) | 复权因子 | 股票数据,行情数据 | 本接口由Tushare自行生产，获取股票复权因子，可提取单只股票全部历史复权因子，也可以提取单日全部股票的复权因子。 | +| [ggt_monthly](https://tushare.pro/wctapi/documents/197.md) | 港股通每月成交统计 | 股票数据,行情数据 | 港股通每月成交信息，数据从2014年开始 | +| [income](https://tushare.pro/wctapi/documents/33.md) | 利润表 | 股票数据,财务数据 | 获取上市公司财务利润表数据 | +| [cashflow](https://tushare.pro/wctapi/documents/44.md) | 现金流量表 | 股票数据,财务数据 | 获取上市公司现金流量表 | +| [balancesheet](https://tushare.pro/wctapi/documents/36.md) | 资产负债表 | 股票数据,财务数据 | 获取上市公司资产负债表 | +| [express](https://tushare.pro/wctapi/documents/46.md) | 业绩快报 | 股票数据,财务数据 | 获取上市公司业绩快报 | +| [fina_indicator](https://tushare.pro/wctapi/documents/79.md) | 财务指标数据 | 股票数据,财务数据 | 获取上市公司财务指标数据，为避免服务器压力，现阶段每次请求最多返回100条记录，可通过设置日期多次请求获取更多数据。 | +| [fina_mainbz](https://tushare.pro/wctapi/documents/81.md) | 主营业务构成 | 股票数据,财务数据 | 获得上市公司主营业务构成，分地区和产品两种方式 | +| [dividend](https://tushare.pro/wctapi/documents/103.md) | 分红送股数据 | 股票数据,财务数据 | 分红送股数据 | +| [disclosure_date](https://tushare.pro/wctapi/documents/162.md) | 财报披露日期表 | 股票数据,财务数据 | 获取财报披露计划日期 | +| [forecast](https://tushare.pro/wctapi/documents/45.md) | 业绩预告 | 股票数据,财务数据 | 获取业绩预告数据 | +| [fina_audit](https://tushare.pro/wctapi/documents/80.md) | 财务审计意见 | 股票数据,财务数据 | 获取上市公司定期财务审计意见数据 | +| [moneyflow_hsgt](https://tushare.pro/wctapi/documents/47.md) | 沪深港通资金流向 | 股票数据,资金流向数据 | 获取沪股通、深股通、港股通每日资金流向数据，每次最多返回300条记录，总量不限制。 | +| [moneyflow](https://tushare.pro/wctapi/documents/170.md) | 个股资金流向 | 股票数据,资金流向数据 | 获取沪深A股票资金流向数据，分析大单小单成交情况，用于判别资金动向，数据开始于2010年。 | +| [moneyflow_ind_ths](https://tushare.pro/wctapi/documents/343.md) | 行业资金流向(THS) | 股票数据,资金流向数据 | 获取同花顺行业资金流向，每日盘后更新 | +| [moneyflow_ind_dc](https://tushare.pro/wctapi/documents/344.md) | 板块资金流向(DC) | 股票数据,资金流向数据 | 获取东方财富板块资金流向，每天盘后更新 | +| [moneyflow_mkt_dc](https://tushare.pro/wctapi/documents/345.md) | 大盘资金流向(DC) | 股票数据,资金流向数据 | 获取东方财富大盘资金流向数据，每日盘后更新 | +| [moneyflow_ths](https://tushare.pro/wctapi/documents/348.md) | 个股资金流向(THS) | 股票数据,资金流向数据 | 获取同花顺个股资金流向数据，每日盘后更新 | +| [moneyflow_dc](https://tushare.pro/wctapi/documents/349.md) | 个股资金流向(DC) | 股票数据,资金流向数据 | 获取东方财富个股资金流向数据，每日盘后更新，数据开始于20230911 | +| [moneyflow_cnt_ths](https://tushare.pro/wctapi/documents/371.md) | 板块资金流向(THS) | 股票数据,资金流向数据 | 获取同花顺概念板块每日资金流向 | +| [film_record](https://tushare.pro/wctapi/documents/156.md) | 全国电影剧本备案数据 | 行业经济,TMT行业 | 获取全国电影剧本备案的公示数据 | +| [teleplay_record](https://tushare.pro/wctapi/documents/180.md) | 全国电视剧备案公示数据 | 行业经济,TMT行业 | 获取2009年以来全国拍摄制作电视剧备案公示数据 | +| [tmt_twincomedetail](https://tushare.pro/wctapi/documents/87.md) | 台湾电子产业月营收明细 | 行业经济,TMT行业 | 获取台湾TMT行业上市公司各类产品月度营收情况。 | +| [tmt_twincome](https://tushare.pro/wctapi/documents/88.md) | 台湾电子产业月营收 | 行业经济,TMT行业 | 获取台湾TMT电子产业领域各类产品月度营收数据。 | +| [bo_monthly](https://tushare.pro/wctapi/documents/113.md) | 电影月度票房 | 行业经济,TMT行业 | 获取电影月度票房数据 | +| [bo_daily](https://tushare.pro/wctapi/documents/115.md) | 电影日度票房 | 行业经济,TMT行业 | 获取电影日度票房 | +| [bo_cinema](https://tushare.pro/wctapi/documents/116.md) | 影院日度票房 | 行业经济,TMT行业 | 获取每日各影院的票房数据 | +| [bo_weekly](https://tushare.pro/wctapi/documents/114.md) | 电影周度票房 | 行业经济,TMT行业 | 获取周度票房数据 | +| [fund_sales_ratio](https://tushare.pro/wctapi/documents/265.md) | 各渠道公募基金销售保有规模占比 | 财富管理,基金销售行业数据 | 获取各渠道公募基金销售保有规模占比数据，年度更新 | +| [fund_sales_vol](https://tushare.pro/wctapi/documents/266.md) | 销售机构公募基金销售保有规模 | 财富管理,基金销售行业数据 | 获取销售机构公募基金销售保有规模数据，本数据从2021年Q1开始公布，季度更新 | diff --git a/skills/tushare-data/scripts/fund_data_demo.py b/skills/tushare-data/scripts/fund_data_demo.py new file mode 100644 index 00000000..bc25e82b --- /dev/null +++ b/skills/tushare-data/scripts/fund_data_demo.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +基金数据获取示例脚本 +""" + +import tushare as ts +import pandas as pd +import os + +# 读取环境变量中的token, 或者读取本地记录的token +token = os.getenv('TUSHARE_TOKEN') or ts.get_token() + +# 初始化pro接口 +pro = ts.pro_api(token) + + +def get_fund_list(): + """ + 获取基金列表 + """ + try: + data = pro.fund_basic(market='E', status='L', fields='ts_code,fund_name,fund_type,found_date,issue_date,delist_date') + print("基金列表获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取基金列表失败：{e}") + return None + + +def get_fund_nav(ts_code, start_date, end_date): + """ + 获取基金净值数据 + """ + try: + data = pro.fund_nav(ts_code=ts_code, start_date=start_date, end_date=end_date) + print(f"{ts_code}基金净值数据获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取基金净值数据失败：{e}") + return None + + +def get_fund_manager(): + """ + 获取基金经理数据 + """ + try: + data = pro.fund_manager(limit=10, fields='ts_code,fund_name,manager_name,begin_date,end_date') + print("基金经理数据获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取基金经理数据失败：{e}") + return None + + +def main(): + """ + 主函数 + """ + print("===== tushare 基金数据获取示例 =====") + + # 获取基金列表 + fund_list = get_fund_list() + + if fund_list is not None: + # 获取第一只基金的代码 + ts_code = fund_list['ts_code'].iloc[0] + print(f"\n使用基金代码：{ts_code}") + + # 获取基金净值数据（最近30天） + import datetime + end_date = datetime.datetime.now().strftime('%Y%m%d') + start_date = (datetime.datetime.now() - datetime.timedelta(days=30)).strftime('%Y%m%d') + print(f"\n获取基金净值数据：{start_date} 至 {end_date}") + get_fund_nav(ts_code, start_date, end_date) + + # 获取基金经理数据 + print("\n获取基金经理数据：") + get_fund_manager() + + +if __name__ == "__main__": + main() diff --git a/skills/tushare-data/scripts/stock_data_demo.py b/skills/tushare-data/scripts/stock_data_demo.py new file mode 100644 index 00000000..e537b347 --- /dev/null +++ b/skills/tushare-data/scripts/stock_data_demo.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +股票数据获取示例脚本 +""" + +import tushare as ts +import pandas as pd +import os + +# 读取环境变量中的token, 或者读取本地记录的token +token = os.getenv('TUSHARE_TOKEN') or ts.get_token() + +# 初始化pro接口 +pro = ts.pro_api(token) + + +def get_stock_list(): + """ + 获取股票列表 + """ + try: + data = pro.stock_basic(exchange='', list_status='L', fields='ts_code,symbol,name,area,industry,list_date') + print("股票列表获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取股票列表失败：{e}") + return None + + +def get_daily_data(ts_code, start_date, end_date): + """ + 获取股票日线数据 + """ + try: + data = pro.daily(ts_code=ts_code, start_date=start_date, end_date=end_date) + print(f"{ts_code}日线数据获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取日线数据失败：{e}") + return None + + +def get_financial_data(ts_code, year, quarter): + """ + 获取财务指标数据 + """ + try: + data = pro.fina_indicator(ts_code=ts_code, year=year, quarter=quarter) + print(f"{ts_code}财务指标数据获取成功：") + print(data.head()) + return data + except Exception as e: + print(f"获取财务指标数据失败：{e}") + return None + + +def main(): + """ + 主函数 + """ + print("===== tushare 股票数据获取示例 =====") + + # 获取股票列表 + stock_list = get_stock_list() + + if stock_list is not None: + # 获取第一只股票的代码 + ts_code = stock_list['ts_code'].iloc[0] + print(f"\n使用股票代码：{ts_code}") + + # 获取日线数据（最近30天） + import datetime + end_date = datetime.datetime.now().strftime('%Y%m%d') + start_date = (datetime.datetime.now() - datetime.timedelta(days=30)).strftime('%Y%m%d') + print(f"\n获取日线数据：{start_date} 至 {end_date}") + get_daily_data(ts_code, start_date, end_date) + + # 获取财务数据（最近一年） + current_year = datetime.datetime.now().year + print(f"\n获取财务数据：{current_year-1}年 第4季度") + get_financial_data(ts_code, current_year-1, 4) + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index b3c62bd7..a6a15402 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -135,6 +135,29 @@ def test_extract_clean_output_respects_max_units(): assert [unit["name"] for unit in state.units] == [f"unit-{i}" for i in range(5)] +def test_extract_without_llm_marks_changed_paths_failed(tmp_path): + """A missing LLM must not let finish checkpoint unprocessed source files.""" + + async def run(): + note = _touch(tmp_path / "daily" / "2026-05-28" / "session.md") + step = DreamExtractStep(app_context=ApplicationContext(workspace_dir=str(tmp_path))) + + response = await step( + RuntimeContext( + date="2026-05-28", + file_catalog=_Catalog(), + file_store=_FileStore(tmp_path), + ), + ) + + dream = response.metadata["dream"] + assert response.success is False + assert str(note.relative_to(tmp_path)) in dream["changed_paths"] + assert dream["failed_paths"] == dream["changed_paths"] + + asyncio.run(run()) + + def test_topics_step_writes_only_target_date_interests(): """Topics are written only to ``state.date`` even when scan dates span multiple days.""" diff --git a/tests/unit/test_auto_fin.py b/tests/unit/test_auto_fin.py new file mode 100644 index 00000000..0807c3b2 --- /dev/null +++ b/tests/unit/test_auto_fin.py @@ -0,0 +1,734 @@ +"""Focused tests for the four-step Auto Fin workflow.""" + +# pylint: disable=missing-function-docstring,protected-access + +import hashlib +import json +from datetime import date, datetime +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from reme.components import ApplicationContext +from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper +from reme.components.runtime_context import RuntimeContext +from reme.config.config_parser import _load_config +from reme.schema import ( + AutoFinEtfHistoricalEvents, + AutoFinEtfHistoricalResearch, + AutoFinEtfSelection, + AutoFinEtfsOutput, + AutoFinHistoricalEvent, + AutoFinMarketSelection, + AutoFinMarketSample, + AutoFinReportOutput, +) +from reme.steps.cookbook.auto_fin._base import _write +from reme.steps.cookbook.auto_fin.data import AutoFinDataStep +from reme.steps.cookbook.auto_fin.history import AutoFinHistoryStep +from reme.steps.cookbook.auto_fin.history_search import AutoFinHistorySearchStep +from reme.steps.cookbook.auto_fin.merge import AutoFinMergeStep +from reme.steps.cookbook.auto_fin.market import AutoFinMarketStep +from reme.steps.cookbook.auto_fin.topic import AutoFinTopicStep, _plain_text + + +def test_atomic_write_preserves_existing_file_and_cleans_temporary_file_on_failure( + tmp_path: Path, + monkeypatch, +): + path = tmp_path / "result.json" + path.write_text("existing", encoding="utf-8") + + def fail_replace(_source, _destination): + raise OSError("replace failed") + + monkeypatch.setattr("reme.steps.cookbook.auto_fin._base.os.replace", fail_replace) + + with pytest.raises(OSError, match="replace failed"): + _write(path, "replacement") + + assert path.read_text(encoding="utf-8") == "existing" + assert not list(tmp_path.glob(".*.tmp")) + + +@pytest.mark.asyncio +async def test_read_jsonl_preserves_unicode_line_separator(tmp_path: Path): + path = tmp_path / "news.jsonl" + rows = [{"title": "包含\u2028行分隔符"}, {"title": "下一条"}] + path.write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + encoding="utf-8", + ) + + assert await AutoFinDataStep._read_jsonl(path) == rows + + +def test_plain_news_text_removes_markup_images_and_hidden_content(): + content = '

甲&乙

丙

' + + assert _plain_text(content) == "甲&乙 丙" + + +def test_published_time_is_normalized_once_to_shanghai_local_time(): + parsed = AutoFinDataStep._published_at({"published_at": "2026-07-24T01:00:00+00:00"}) + + assert parsed == datetime(2026, 7, 24, 9) + assert parsed.tzinfo is None + + +@pytest.mark.asyncio +async def test_current_news_keeps_all_items_with_per_item_and_total_content_limits(tmp_path: Path): + day_dir = tmp_path / "daily" / "2026-07-24" + day_dir.mkdir(parents=True) + rows = [ + { + "title": f"新闻标题{index}", + "pub_time": f"2026-07-24 09:0{index}:00", + "src": "财联社", + "content": f"

正文内容{index}ABCDEFGHIJ

", + } + for index in range(3) + ] + (day_dir / "auto_fin_news_data.jsonl").write_text( + "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), + encoding="utf-8", + ) + step = AutoFinTopicStep(app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")) + step.context = RuntimeContext( + auto_fin_news_start="2026-07-24", + auto_fin_date="2026-07-24", + news_title_max_chars=5, + news_content_max_chars=10, + news_total_content_max_chars=12, + ) + + news = await step._current_news( + datetime.fromisoformat("2026-07-24T08:59:00"), + datetime.fromisoformat("2026-07-24T10:00:00"), + ) + + assert len(news) == 3 + assert all(len(row["title"]) <= 5 for row in news) + assert all(len(row["content"]) <= 4 for row in news) + assert sum(len(row["content"]) for row in news) <= 12 + assert all("<" not in row["content"] for row in news) + assert {row["news_id"] for row in news} == { + f"20260724090{index}00_" f"{hashlib.sha256(f'财联社

正文内容{index}ABCDEFGHIJ

'.encode()).hexdigest()[:4]}" + for index in range(3) + } + + +@pytest.mark.asyncio +async def test_data_fills_missing_news_refreshes_today_and_force_refreshes_history( + tmp_path: Path, +): + calls = [] + + def provider(endpoint: str, **kwargs): + calls.append((endpoint, kwargs)) + if endpoint == "major_news": + day = kwargs["start_date"][:10] + return [ + { + "title": day, + "pub_time": f"{day} 08:00:00", + "src": "财联社", + "content": day, + }, + ] + raise AssertionError(endpoint) + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + context = RuntimeContext( + date="2026-07-24", + now="2026-07-24T09:30:00+08:00", + lookback_days=3, + progress_interval=30, + trade_dates=["2026-07-23"], + tushare_provider=provider, + ) + + await AutoFinDataStep(app_context=app_context)(context) + assert [endpoint for endpoint, _ in calls] == ["major_news"] * 3 + for day in ("2026-07-22", "2026-07-23", "2026-07-24"): + [row] = AutoFinDataStep._read_jsonl_sync( + tmp_path / "daily" / day / "auto_fin_news_data.jsonl", + ) + news_hash = hashlib.sha256(f"财联社{day}".encode()).hexdigest()[:4] + assert row["news_id"] == f"{day.replace('-', '')}080000_{news_hash}" + + calls.clear() + await AutoFinDataStep(app_context=app_context)(context) + assert [endpoint for endpoint, _ in calls] == ["major_news"] + assert calls[0][1]["start_date"] == "2026-07-24 00:00:00" + assert calls[0][1]["end_date"] == "2026-07-24 09:30:00" + + calls.clear() + context["force"] = True + await AutoFinDataStep(app_context=app_context)(context) + assert [endpoint for endpoint, _ in calls] == ["major_news"] * 3 + assert context["auto_fin_previous_trade_date"] == "2026-07-23" + + +@pytest.mark.asyncio +async def test_data_deduplicates_by_publish_time_and_short_hash(tmp_path: Path): + def provider(endpoint: str, **_kwargs): + assert endpoint == "major_news" + return [ + { + "title": "重复新闻的较晚记录", + "pub_time": "2026-07-24 09:00:00", + "src": "财联社", + "content": "相同正文", + }, + { + "title": "另一条新闻", + "pub_time": "2026-07-24 08:00:00", + "src": "财联社", + "content": "另一篇正文", + }, + { + "title": "重复新闻的最早记录", + "pub_time": "2026-07-24 07:00:00", + "src": "财联社", + "content": "相同正文", + }, + { + "title": "同一时间和正文的重复记录", + "pub_time": "2026-07-24 07:00:00", + "src": "财联社", + "content": "相同正文", + }, + ] + + context = RuntimeContext( + date="2026-07-24", + now="2026-07-24T09:30:00+08:00", + lookback_days=1, + progress_interval=30, + trade_dates=["2026-07-23"], + tushare_provider=provider, + ) + await AutoFinDataStep( + app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"), + )(context) + + rows = AutoFinDataStep._read_jsonl_sync( + tmp_path / "daily" / "2026-07-24" / "auto_fin_news_data.jsonl", + ) + assert [row["title"] for row in rows] == ["重复新闻的最早记录", "另一条新闻", "重复新闻的较晚记录"] + assert len({row["news_id"] for row in rows}) == len(rows) + + +@pytest.mark.asyncio +async def test_topic_etfs_join_rank_deduplicate_name_and_index(tmp_path: Path): + def provider(endpoint: str, **kwargs): + if endpoint == "etf_basic": + assert kwargs["list_status"] == "L" + return [ + {"ts_code": "510001.SH", "csname": "同名 ETF", "index_code": "I1", "index_name": "指数一"}, + {"ts_code": "510002.SH", "csname": "同名ETF", "index_code": "I2", "index_name": "指数二"}, + {"ts_code": "510003.SH", "csname": "另一名称", "index_code": "I1", "index_name": "指数一"}, + {"ts_code": "510004.SH", "csname": "独立ETF", "index_code": "I4", "index_name": "指数四"}, + ] + if endpoint == "fund_daily": + assert kwargs["trade_date"] == "20260723" + return [ + {"ts_code": "510002.SH", "amount": 90}, + {"ts_code": "510004.SH", "amount": 70}, + {"ts_code": "510001.SH", "amount": 100}, + {"ts_code": "510003.SH", "amount": 80}, + {"ts_code": "510001.SH", "amount": 95}, + ] + raise AssertionError(endpoint) + + step = AutoFinTopicStep(app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai")) + step.context = RuntimeContext(tushare_provider=provider, etf_candidate_limit=150) + + assert await step._filtered_etfs(date(2026, 7, 23)) == [ + {"code": "510001.SH", "name": "同名 ETF"}, + {"code": "510004.SH", "name": "独立ETF"}, + ] + + +class _Agent(BaseAgentWrapper): + """Return deterministic structured replies for every analysis stage.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = [] + + async def reply(self, inputs, **kwargs): + schema = kwargs["output_schema"] + prompt = str(inputs) + self.calls.append((schema, prompt, kwargs)) + assert "resume" not in kwargs + assert "session_id" not in kwargs + task = prompt + assert "schema" not in task.casefold() + assert "```json" in task + if schema is AutoFinEtfsOutput: + assert "filtered_news.jsonl" in task + assert "filtered_etf.jsonl" in task + assert "Top 150" in task + assert "最多返回 20" in task + value = { + "etfs": [ + { + "etf_code": "159018.SZ", + "etf_name": "油气ETF", + "events": [ + { + "reason": "供应中断直接影响油气产业链盈利预期", + "news_id": ( + f"20260723160000_" + f"{hashlib.sha256('财联社主要产油区供应中断'.encode()).hexdigest()[:4]}" + ), + }, + { + "reason": "供应恢复时间影响油气价格预期", + "news_id": ( + f"20260724090000_" + f"{hashlib.sha256('财联社供应恢复时间仍不确定'.encode()).hexdigest()[:4]}" + ), + }, + ], + }, + ], + } + elif schema is AutoFinEtfHistoricalEvents: + assert "memory_search" in task + assert "不查询行情" in task + assert "159018.SZ（油气ETF）" in task + current_news_id = ( + f"20260724090000_" f"{hashlib.sha256('财联社供应恢复时间仍不确定'.encode()).hexdigest()[:4]}" + ) + assert current_news_id not in task + assert "当前事件时间线（仅作为检索线索，不含 news_id）" in task + assert "时间、标题、正文和行情都由程序在你返回后补充" in task + assert "每一项只包含 reason、news_id 和 source_path" in task + tool_context_id = kwargs.get("tool_context_id", "") + assert tool_context_id.startswith("auto_fin_history_01_159018.SZ_") + assert tool_context_id not in task + assert "系统会自动过滤本次研究中已经" in task + self.app_context.metadata.setdefault("tool_contexts", {})[tool_context_id] = { + "search_seen_chunk_ids": {}, + } + value = { + "etf_code": "159018.SZ", + "etf_name": "油气ETF", + "historical_events": [ + { + "reason": "供应中断的事件类型和传导机制相同", + "news_id": ( + f"20260601100000_" f"{hashlib.sha256('财联社历史供应中断'.encode()).hexdigest()[:4]}" + ), + "source_path": "daily/2026-06-01/auto_fin_news_data.jsonl", + }, + ], + } + elif schema is AutoFinMarketSelection: + assert "ETF：159018.SZ（油气ETF）" in task + assert "[2026-07-23T16:00:00] 原油供应中断" in task + assert "只负责判断历史事件与当前事件的相似度" in task + assert "不要依据" in task + assert "程序会校验" in task + assert "$tushare-data" not in task + history_path = Path( + next(line.strip() for line in task.splitlines() if line.strip().endswith("_output.json")), + ) + history = json.loads(history_path.read_text(encoding="utf-8")) + assert "historical_samples" not in history + assert len(history["historical_events"]) == 1 + assert history["historical_events"][0]["event_title"] == "历史供应中断" + assert len(history["historical_events"][0]["future_returns"]) == 10 + value = { + "matched_historical_events": [ + { + "reason": "供应中断的事件类型和传导机制相同", + "news_id": history["historical_events"][0]["news_id"], + "similarity": 1.2, + }, + ], + } + elif schema is AutoFinReportOutput: + assert "不重新搜索新闻" in task + assert "auto_fin_history_output.jsonl" in task + assert "不生成 YAML frontmatter" in task + assert '"etf_code": "159018.SZ"' in task + assert '"suggested_holding_days": 10' in task + assert '"horizon": 1' in task + assert '"horizon": 10' in task + assert "自行判断事件对 ETF 的影响方向" in task + assert "不得使用程序计算结果反推事件方向" in task + assert "以推荐 ETF 为主要内容" in task + assert "用一句话合并简述" in task + value = { + "title": "Auto Fin ETF 结论", + "body": "## 结论\n\n推荐 159018.SZ（油气ETF），参考持有 10 个交易日，" + "当前加权预估收益 +10%；核心风险：供应恢复。", + } + else: # pragma: no cover + raise AssertionError(schema) + return {"structured_output": schema.model_validate(value)} + + +@pytest.mark.asyncio +async def test_four_step_pipeline_writes_plain_markdown_and_cleans_temporary_data( + tmp_path: Path, +): + def provider(endpoint: str, **_kwargs): + if endpoint == "major_news": + return [ + { + "title": "原油供应中断", + "pub_time": "2026-07-23 16:00:00", + "src": "财联社", + "content": "主要产油区供应中断", + }, + { + "title": "供应恢复时间不确定", + "pub_time": "2026-07-24 09:00:00", + "src": "财联社", + "content": "供应恢复时间仍不确定", + }, + ] + if endpoint == "etf_basic": + return [ + { + "ts_code": "159018.SZ", + "csname": "油气ETF", + "index_code": "930987.CSI", + "index_name": "中证油气产业指数", + "list_status": "L", + }, + ] + if endpoint == "fund_daily": + if "trade_date" in _kwargs: + return [{"ts_code": "159018.SZ", "trade_date": "20260723", "amount": 1000}] + trade_dates = [ + "20260601", + "20260602", + "20260603", + "20260604", + "20260605", + "20260608", + "20260609", + "20260610", + "20260611", + "20260612", + "20260615", + ] + return [ + { + "ts_code": "159018.SZ", + "trade_date": trade_date, + "open": 0.995 + index / 100, + "close": 1.0 + index / 100, + } + for index, trade_date in enumerate(trade_dates) + ] + if endpoint == "fund_adj": + return [ + {"ts_code": "159018.SZ", "trade_date": trade_date, "adj_factor": 1.0} + for trade_date in ( + "20260601", + "20260602", + "20260603", + "20260604", + "20260605", + "20260608", + "20260609", + "20260610", + "20260611", + "20260612", + "20260615", + ) + ] + raise AssertionError(endpoint) + + app_context = ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai") + agent = _Agent(app_context=app_context) + context = RuntimeContext( + date="2026-07-24", + now="2026-07-24T09:30:00+08:00", + lookback_days=2, + progress_interval=30, + trade_dates=["2026-07-23"], + tushare_provider=provider, + ) + + await AutoFinDataStep(app_context=app_context)(context) + historical_path = tmp_path / "daily" / "2026-06-01" / "auto_fin_news_data.jsonl" + historical_path.parent.mkdir(parents=True) + historical_content = "历史供应中断" + historical_news_id = f"20260601100000_" f"{hashlib.sha256(f'财联社{historical_content}'.encode()).hexdigest()[:4]}" + historical_path.write_text( + json.dumps( + { + "title": "历史供应中断", + "pub_time": "2026-06-01 10:00:00", + "src": "财联社", + "content": historical_content, + "news_id": historical_news_id, + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + logs = [] + topic_step = AutoFinTopicStep(app_context=app_context, agent_wrapper=agent) + history_step = AutoFinHistoryStep(app_context=app_context, agent_wrapper=agent) + merge_step = AutoFinMergeStep(app_context=app_context, agent_wrapper=agent) + for step in (topic_step, history_step, merge_step): + step.logger = SimpleNamespace(info=logs.append, debug=lambda _message: None) + await topic_step(context) + await history_step(context) + response = await merge_step(context) + + assert [schema for schema, _, _ in agent.calls] == [ + AutoFinEtfsOutput, + AutoFinEtfHistoricalEvents, + AutoFinMarketSelection, + AutoFinReportOutput, + ] + assert "tool_contexts" not in app_context.metadata + report = (tmp_path / "daily" / "2026-07-24" / "auto_fin.md").read_text(encoding="utf-8") + assert report.startswith("# Auto Fin ETF 结论\n\n") + assert not report.startswith("---") + detail = context["auto_fin_history_details"][0] + assert detail["etf"]["etf_code"] == "159018.SZ" + first_event = detail["etf"]["events"][0] + assert first_event["reason"] == "供应中断直接影响油气产业链盈利预期" + assert first_event["news_id"].startswith("20260723160000_") + analysis = detail["market_analysis"] + assert analysis["matched_historical_events"][0]["weight"] == 1.0 + assert analysis["matched_historical_events"][0]["news_id"] == historical_news_id + assert analysis["matched_historical_events"][0]["similarity"] == 1.0 + assert analysis["forecast"]["suggested_holding_days"] == 10 + assert analysis["forecast"]["returns"][-1]["expected_return"] == pytest.approx(0.1) + assert "calculation_code" not in analysis + assert detail["historical_research"]["historical_events"][0]["event_content"] == "历史供应中断" + assert detail["historical_research"]["historical_events"][0]["reason"] == "供应中断的事件类型和传导机制相同" + assert "当前加权预估收益 +10%" in report + assert "历史事件" not in report + assert not (tmp_path / "daily" / "2026-07-24" / "auto_fin_brief.md").exists() + assert response.answer.startswith("## 结论") + assert response.metadata["etf_count"] == 1 + assert context["markdown_path"] == "daily/2026-07-24/auto_fin.md" + assert context["auto_fin_digest_path"] == "daily/2026-07-24/auto_fin.md" + assert response.metadata["digest_path"] == "daily/2026-07-24/auto_fin.md" + daily_index = (tmp_path / "daily" / "2026-07-24.md").read_text(encoding="utf-8") + assert "[[daily/2026-07-24/auto_fin.md]]" in daily_index + assert sum("agent input prompt=" in line for line in logs) == 2 + assert sum("agent output prompt=" in line for line in logs) == 2 + assert all("agent start prompt=" not in line and "agent done prompt=" not in line for line in logs) + assert any('query="你只负责从候选文件中筛选' in line for line in logs) + assert any('output={"etfs":[{"etf_code":"159018.SZ"' in line for line in logs) + resource_dir = tmp_path / "resource" / "2026-07-24" + filtered_news = AutoFinDataStep._read_jsonl_sync(resource_dir / "filtered_news.jsonl") + filtered_etfs = AutoFinDataStep._read_jsonl_sync(resource_dir / "filtered_etf.jsonl") + topic_etfs = AutoFinDataStep._read_jsonl_sync(resource_dir / "auto_fin_topic_output.jsonl") + history_output = json.loads( + (resource_dir / "auto_fin_history_01_159018.SZ_output.json").read_text(encoding="utf-8"), + ) + history_details = AutoFinDataStep._read_jsonl_sync(resource_dir / "auto_fin_history_output.jsonl") + assert not list(resource_dir.glob("*_input.md")) + assert [row["news_id"] for row in filtered_news] == [event["news_id"] for event in detail["etf"]["events"]] + assert filtered_etfs == [{"code": "159018.SZ", "name": "油气ETF"}] + assert topic_etfs == [detail["etf"]] + historical_event = history_output["historical_events"][0] + assert historical_event["news_id"] == historical_news_id + assert historical_event["event_time"] == "2026-06-01T10:00:00" + assert historical_event["event_title"] == "历史供应中断" + assert historical_event["market_entry"]["price_type"] == "close" + assert [point["horizon"] for point in historical_event["future_returns"]] == list( + range(1, 11), + ) + assert historical_event["future_returns"][-1]["cumulative_return"] == pytest.approx(0.1) + assert '\n "etf_code"' in (resource_dir / "auto_fin_history_01_159018.SZ_output.json").read_text(encoding="utf-8") + assert history_details == context["auto_fin_history_details"] + + +def test_historical_market_sample_rejects_look_ahead_and_incorrect_adjusted_returns(): + sample = { + "event_time": "2026-06-01T10:00:00", + "entry": { + "entry_time": "2026-06-01T15:00:00", + "trade_date": "2026-06-01", + "price_type": "close", + "raw_price": 1.0, + "adj_factor": 1.2, + }, + "future_returns": [ + { + "horizon": 1, + "trade_date": "2026-06-02", + "raw_close": 1.1, + "adj_factor": 1.2, + "cumulative_return": 0.1, + }, + ], + "reaction_summary": "第一个有效收盘点上涨。", + } + + assert AutoFinMarketSample.model_validate(sample).future_returns[0].cumulative_return == pytest.approx(0.1) + + sample["event_time"] = "2026-06-01T16:00:00" + with pytest.raises(ValueError, match="entry must be strictly after the event"): + AutoFinMarketSample.model_validate(sample) + + sample["event_time"] = "2026-06-01T10:00:00" + sample["future_returns"][0]["cumulative_return"] = 0.2 + with pytest.raises(ValueError, match="incorrect adjusted return"): + AutoFinMarketSample.model_validate(sample) + + +def test_market_calculation_clamps_and_reverses_negative_similarity(): + item = AutoFinEtfSelection.model_validate( + { + "etf_code": "518880.SH", + "etf_name": "黄金ETF", + "events": [{"reason": "黄金涨价", "news_id": "20260724090000_abcd"}], + }, + ) + history = AutoFinEtfHistoricalResearch.model_validate( + { + "etf_code": "518880.SH", + "etf_name": "黄金ETF", + "historical_events": [ + { + "reason": "黄金价格方向相反", + "news_id": "20260601100000_abcd", + "source_path": "daily/2026-06-01/auto_fin_news_data.jsonl", + "event_time": "2026-06-01T10:00:00", + "event_title": "黄金价格下跌", + "event_content": "黄金价格出现明显下跌。", + "market_entry": { + "entry_time": "2026-06-01T15:00:00", + "trade_date": "2026-06-01", + "price_type": "close", + "raw_price": 1.0, + "adj_factor": 1.0, + }, + "future_returns": [ + { + "horizon": 1, + "trade_date": "2026-06-02", + "raw_close": 1.1, + "adj_factor": 1.0, + "cumulative_return": 0.1, + }, + ], + }, + ], + }, + ) + selection = AutoFinMarketSelection.model_validate( + { + "matched_historical_events": [ + { + "reason": "机制可比但价格方向相反", + "news_id": "20260601100000_abcd", + "similarity": -2.0, + }, + ], + }, + ) + + analysis = AutoFinMarketStep._calculate_analysis(item, history, selection) + + assert analysis.matched_historical_events[0].similarity == -1.0 + assert analysis.matched_historical_events[0].weight == 1.0 + assert analysis.forecast.returns[0].expected_return == pytest.approx(-0.1) + assert analysis.forecast.suggested_holding_days is None + assert "加权预期收益没有正值" in analysis.limitations + + +@pytest.mark.asyncio +async def test_history_search_calculates_adjusted_returns_for_event_time_boundaries(tmp_path: Path): + calls = [] + + def provider(endpoint: str, **kwargs): + calls.append((endpoint, kwargs)) + if endpoint == "fund_daily": + return [ + {"trade_date": "20260609", "open": 7.5, "close": 8.0}, + {"trade_date": "20260608", "open": 6.0, "close": 7.0}, + {"trade_date": "20260605", "open": 10.0, "close": 11.0}, + ] + if endpoint == "fund_adj": + return [ + {"trade_date": "20260609", "adj_factor": 2.0}, + {"trade_date": "20260608", "adj_factor": 2.0}, + {"trade_date": "20260605", "adj_factor": 1.0}, + ] + raise AssertionError(endpoint) + + step = AutoFinHistorySearchStep( + app_context=ApplicationContext(workspace_dir=str(tmp_path), timezone="Asia/Shanghai"), + ) + step.context = RuntimeContext(tushare_provider=provider) + events = [ + AutoFinHistoricalEvent( + reason="历史行情边界测试", + news_id=f"{event_time.replace('-', '').replace(':', '').replace('T', '')}_abcd", + source_path=f"daily/{event_time[:10]}/auto_fin_news_data.jsonl", + event_time=event_time, + event_title=label, + event_content=label, + ) + for event_time, label in ( + ("2026-06-05T08:00:00", "盘前事件"), + ("2026-06-05T10:00:00", "盘中事件"), + ("2026-06-05T16:00:00", "盘后事件"), + ("2026-06-06T10:00:00", "休市日事件"), + ) + ] + + samples, limitations = await step._calculate_samples( + "159018.SZ", + events, + datetime.fromisoformat("2026-06-10T09:00:00"), + ) + + assert [sample.entry.price_type for sample in samples if sample.entry] == ["open", "close", "open", "open"] + assert [sample.entry.trade_date.isoformat() for sample in samples if sample.entry] == [ + "2026-06-05", + "2026-06-05", + "2026-06-08", + "2026-06-08", + ] + assert samples[0].future_returns[0].cumulative_return == pytest.approx(0.1) + assert samples[1].future_returns[0].cumulative_return == pytest.approx(14 / 11 - 1) + assert samples[2].future_returns[0].cumulative_return == pytest.approx(14 / 12 - 1) + assert samples[3].future_returns[0].cumulative_return == pytest.approx(14 / 12 - 1) + assert limitations + assert [endpoint for endpoint, _ in calls] == ["fund_daily", "fund_adj"] + + +def test_daily_cookbook_wires_enabled_auto_fin_steps_and_tushare_skill(): + config = _load_config("daily_cookbook") + steps = config["jobs"]["auto_fin"]["steps"] + + assert [step["backend"] for step in steps] == [ + "auto_fin_data_step", + "auto_fin_topic_step", + "auto_fin_history_step", + "auto_fin_merge_step", + "dingtalk_markdown_send_step", + ] + expected_cron_jobs = { + "auto_fin_0930_cron": "30 9 * * *", + "auto_fin_1145_cron": "45 11 * * *", + "auto_fin_1800_cron": "0 18 * * *", + } + for job_name, cron in expected_cron_jobs.items(): + assert config["jobs"][job_name]["cron"] == cron + assert config["jobs"][job_name]["steps"] == steps + assert steps[0]["lookback_days"] == 360 + assert steps[0]["progress_interval"] == 30 + assert steps[-1]["input_mapping"] == {"auto_fin_digest_path": "markdown_path"} + assert config["components"]["agent_wrapper"]["auto_fin"]["skills"] == ["tushare-data"] diff --git a/tests/unit/test_base_agent_wrapper.py b/tests/unit/test_base_agent_wrapper.py index 0df6a042..17a1b1fa 100644 --- a/tests/unit/test_base_agent_wrapper.py +++ b/tests/unit/test_base_agent_wrapper.py @@ -5,19 +5,34 @@ from unittest.mock import MagicMock import pytest -from reme.components.agent_wrapper import AsAgentWrapper, BaseAgentWrapper, CcAgentWrapper, CodexAgentWrapper +from reme.components.agent_wrapper import ( + AsAgentWrapper, + BaseAgentWrapper, + CcAgentWrapper, + CodexAgentWrapper, + handle_session_command, +) from reme.components.agent_wrapper.as_agent_wrapper import WorkspaceBackend from reme.components.agent_wrapper import base_agent_wrapper from reme.components.application_context import ApplicationContext +from reme.components.outbound_proxy import FixedHttpOutboundProxy from reme.components import base_component +from reme.enumeration import ComponentEnum class _VersionedAgentWrapper(BaseAgentWrapper): SDK_PACKAGE = "example-agent-sdk" + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.compacted_session = "" + async def reply(self, inputs, **kwargs) -> dict: return {"inputs": inputs, "kwargs": kwargs} + async def compact_session(self, session_id: str) -> None: + self.compacted_session = session_id + def test_init_logs_sdk_version(monkeypatch): """An SDK-backed wrapper logs its installed distribution version.""" @@ -28,7 +43,7 @@ def test_init_logs_sdk_version(monkeypatch): _VersionedAgentWrapper(name="versioned") - logger.info.assert_called_once_with("Agent SDK package=example-agent-sdk version=1.2.3") + logger.info.assert_called_once_with("Agent SDK name=versioned package=example-agent-sdk version=1.2.3") def test_init_logs_unknown_when_sdk_distribution_metadata_is_missing(monkeypatch): @@ -44,7 +59,24 @@ def test_init_logs_unknown_when_sdk_distribution_metadata_is_missing(monkeypatch _VersionedAgentWrapper() - logger.info.assert_called_once_with("Agent SDK package=example-agent-sdk version=unknown") + logger.info.assert_called_once_with( + "Agent SDK name=_VersionedAgentWrapper package=example-agent-sdk version=unknown", + ) + + +@pytest.mark.asyncio +async def test_session_commands_are_backend_neutral(): + """Session commands work independently from a chat transport.""" + wrapper = _VersionedAgentWrapper() + + assert await handle_session_command(wrapper, "hello", "session-1") is None + assert (await handle_session_command(wrapper, "/clear", "session-1")).session_id is None + unavailable = await handle_session_command(wrapper, "/compact", None) + assert unavailable.answer == "No active conversation to compact." + + compacted = await handle_session_command(wrapper, "/compact", "session-1") + assert compacted.session_id == "session-1" + assert wrapper.compacted_session == "session-1" @pytest.mark.parametrize( @@ -105,3 +137,29 @@ async def test_agentscope_backend_passes_configured_environment_to_bash(tmp_path assert result.exit_code == 0 assert result.stdout == b"configured\n" + + +@pytest.mark.asyncio +async def test_agentscope_bash_uses_managed_proxy_without_changing_subprocess_environment(tmp_path): + """AgentScope applies the managed proxy only to its command backend.""" + context = ApplicationContext( + workspace_dir=str(tmp_path), + environment={"TOOL_ENV": "preserved"}, + ) + proxy = FixedHttpOutboundProxy(url="http://127.0.0.1:18080") + await proxy.start() + context.components = {ComponentEnum.OUTBOUND_PROXY: {"default": proxy}} + wrapper = AsAgentWrapper(app_context=context, as_llm="") + + await wrapper.start() + bash = wrapper._builtin_tools(["bash"])[0] # pylint: disable=protected-access + backend = bash._backend # pylint: disable=protected-access + + assert wrapper.subprocess_environment == {"TOOL_ENV": "preserved"} + assert "HTTP_PROXY" not in wrapper.subprocess_environment + assert backend._environment["TOOL_ENV"] == "preserved" # pylint: disable=protected-access + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + assert backend._environment[key] == proxy.http_url # pylint: disable=protected-access + + await wrapper.close() + await proxy.close() diff --git a/tests/unit/test_cc_agent_wrapper.py b/tests/unit/test_cc_agent_wrapper.py index 08723756..cf9ac0ee 100644 --- a/tests/unit/test_cc_agent_wrapper.py +++ b/tests/unit/test_cc_agent_wrapper.py @@ -11,6 +11,7 @@ from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper from reme.components.agent_wrapper.cc_agent_wrapper import CcAgentWrapper from reme.components.agent_wrapper.cc_session_store import CcFileSessionStore from reme.components.application_context import ApplicationContext +from reme.components.outbound_proxy import FixedHttpOutboundProxy from reme.enumeration import ChunkEnum, ComponentEnum # pylint: disable=protected-access @@ -201,6 +202,55 @@ def test_api_credentials_use_only_wrapper_config(tmp_path, monkeypatch): assert empty.env["ANTHROPIC_BASE_URL"] == "" +@pytest.mark.asyncio +async def test_managed_proxy_is_injected_only_into_claude_bash_commands(tmp_path): + """The Claude CLI keeps its model environment while Bash receives proxy exports.""" + from claude_agent_sdk import HookMatcher + + wrapper = _wrapper(tmp_path) + proxy = FixedHttpOutboundProxy(url="http://127.0.0.1:18080") + await proxy.start() + wrapper.app_context.components = {ComponentEnum.OUTBOUND_PROXY: {"default": proxy}} + + async def existing_hook(_hook_input, _tool_use_id, _context): + return {} + + existing = HookMatcher(matcher="Read", hooks=[existing_hook]) + await wrapper.start() + opts = wrapper._build_options( + "hello", + api_key="configured-key", + base_url="https://configured.example.test", + hooks={"PreToolUse": [existing]}, + ) + + assert opts.env["ANTHROPIC_AUTH_TOKEN"] == "configured-key" + assert opts.env["ANTHROPIC_BASE_URL"] == "https://configured.example.test" + assert "HTTP_PROXY" not in opts.env + assert opts.hooks["PreToolUse"][0] is existing + proxy_hook = opts.hooks["PreToolUse"][1] + assert proxy_hook.matcher == "Bash" + + result = await proxy_hook.hooks[0]( + { + "hook_event_name": "PreToolUse", + "tool_name": "Bash", + "tool_input": {"command": "python tushare_analysis.py", "timeout": 30}, + "tool_use_id": "tool-1", + }, + "tool-1", + {"signal": None}, + ) + updated_input = result["hookSpecificOutput"]["updatedInput"] + assert updated_input["command"].endswith("; python tushare_analysis.py") + assert updated_input["timeout"] == 30 + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + assert f"{key}={proxy.http_url}" in updated_input["command"] + + await wrapper.close() + await proxy.close() + + def test_build_options_accepts_empty_output_schema(tmp_path): """An empty schema remains a valid structured-output request.""" opts = _wrapper(tmp_path)._build_options("hello", output_schema={}) @@ -293,6 +343,55 @@ async def test_reply_preserves_falsy_structured_output(tmp_path, monkeypatch): assert result["structured_output"] == {} +@pytest.mark.asyncio +async def test_compact_session_uses_claude_command(tmp_path, monkeypatch): + """Claude Code compaction uses its native slash command.""" + wrapper = _wrapper(tmp_path) + calls = [] + + async def reply(inputs, **kwargs): + calls.append((inputs, kwargs)) + return {"last_message": {"is_error": False}} + + monkeypatch.setattr(wrapper, "reply", reply) + + await wrapper.compact_session("session-1") + + assert calls == [("/compact", {"resume": "session-1"})] + + +@pytest.mark.asyncio +async def test_agentscope_compact_session_forces_and_persists_compression(tmp_path, monkeypatch): + """AgentScope compaction forces the threshold and persists new state.""" + wrapper = AsAgentWrapper(as_llm="", app_context=ApplicationContext(workspace_dir=str(tmp_path))) + observed = {} + + class FakeAgent: + """Minimal AgentScope agent double.""" + + state = SimpleNamespace() + + async def compress_context(self, config): + """Capture the forced context configuration.""" + observed["config"] = config + + async def build_agent(inputs, **kwargs): + observed["build"] = (inputs, kwargs) + return FakeAgent(), inputs + + async def dump_state(state): + observed["state"] = state + + monkeypatch.setattr(wrapper, "_build_agent", build_agent) + monkeypatch.setattr(wrapper, "_dump_state", dump_state) + + await wrapper.compact_session("session-1") + + assert observed["build"] == (None, {"resume": "session-1"}) + assert observed["config"].trigger_ratio == 1e-9 + assert observed["state"] is FakeAgent.state + + def test_error_result_with_success_subtype_is_not_suppressed(): """Latest SDK can report API failures with subtype=success and is_error=True.""" from claude_agent_sdk import ResultMessage diff --git a/tests/unit/test_codex_agent_wrapper.py b/tests/unit/test_codex_agent_wrapper.py index 7d153c94..fa7f4d76 100644 --- a/tests/unit/test_codex_agent_wrapper.py +++ b/tests/unit/test_codex_agent_wrapper.py @@ -17,6 +17,7 @@ from pydantic import BaseModel from reme.components.agent_wrapper.codex_agent_wrapper import CodexAgentWrapper from reme.components.agent_wrapper.codex_mcp_server import _prepare_config from reme.components.job import BackgroundJob +from reme.components.outbound_proxy import FixedHttpOutboundProxy from reme.config import resolve_app_config from reme.enumeration import ChunkEnum, ComponentEnum from reme.schema import ApplicationConfig, Response @@ -54,7 +55,7 @@ def _wrapper(tmp_path, **kwargs): "components": {}, }, ) - context = SimpleNamespace(app_config=config, jobs={job.name: job}) + context = SimpleNamespace(app_config=config, components={}, jobs={job.name: job}) return CodexAgentWrapper(app_context=context, **kwargs), job @@ -88,6 +89,43 @@ def test_thread_config_preserves_other_mcp_servers(tmp_path): assert next(name for name in config["mcp_servers"] if name != "docs").startswith("reme_jobs_") +@pytest.mark.asyncio +async def test_thread_config_injects_proxy_only_into_codex_shell_policy(tmp_path): + wrapper, _job = _wrapper(tmp_path) + proxy = FixedHttpOutboundProxy(url="http://127.0.0.1:18080") + await proxy.start() + wrapper.app_context.components = {ComponentEnum.OUTBOUND_PROXY: {"default": proxy}} + await wrapper.start() + + config = wrapper._thread_config( # pylint: disable=protected-access + { + "config": { + "shell_environment_policy": { + "inherit": "core", + "set": { + "CUSTOM": "preserved", + "HTTP_PROXY": "http://user-proxy.example:8080", + }, + }, + }, + }, + ) + environment = config["shell_environment_policy"]["set"] + + assert config["shell_environment_policy"]["inherit"] == "core" + assert environment["CUSTOM"] == "preserved" + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + assert environment[key] == proxy.http_url + + auth = wrapper._resolve_auth_config("oauth") # pylint: disable=protected-access + client_config = wrapper._build_client_config(auth) # pylint: disable=protected-access + assert "HTTP_PROXY" not in client_config.env + assert "HTTPS_PROXY" not in client_config.env + + await wrapper.close() + await proxy.close() + + def test_mcp_config_rejects_background_jobs(tmp_path): wrapper, _job = _wrapper(tmp_path) wrapper.app_context.jobs["watch"] = BackgroundJob(name="watch", app_context=wrapper.app_context) @@ -520,7 +558,7 @@ async def test_effective_snapshot_exposes_parent_only_custom_job(tmp_path): }, ) job = _Job("only_custom") - context = SimpleNamespace(app_config=app_config, jobs={"only_custom": job}) + context = SimpleNamespace(app_config=app_config, components={}, jobs={"only_custom": job}) wrapper = CodexAgentWrapper(app_context=context) server_config = wrapper._mcp_server_config({"job_tools": ["only_custom"]}) # pylint: disable=protected-access transport = StdioTransport( @@ -562,6 +600,35 @@ async def test_open_thread_defaults_to_full_access(tmp_path): assert observed["sandbox"] == Sandbox.full_access +@pytest.mark.asyncio +async def test_compact_session_uses_native_thread_operation(tmp_path, monkeypatch): + wrapper, _job = _wrapper(tmp_path) + observed = {} + + class FakeThread: + async def compact(self): + observed["compacted"] = True + + async def start(): + observed["started"] = True + + async def resume(session_id): + observed["resume"] = session_id + return FakeThread() + + async def get_codex(): + return SimpleNamespace(thread_resume=resume) + + monkeypatch.setattr(wrapper, "start", start) + monkeypatch.setattr(wrapper, "_get_codex", get_codex) + + await wrapper.compact_session("thread-1") + + assert observed["started"] is True + assert observed["resume"] == "thread-1" + assert observed["compacted"] is True + + @pytest.mark.asyncio async def test_open_thread_forwards_latest_sdk_options(tmp_path): from openai_codex.types import Personality, ThreadSource, ThreadStartSource diff --git a/tests/unit/test_config_parser.py b/tests/unit/test_config_parser.py index 0b7c0472..607389d2 100644 --- a/tests/unit/test_config_parser.py +++ b/tests/unit/test_config_parser.py @@ -63,6 +63,14 @@ def test_default_config_keeps_frontmatter_chunk_metadata_opt_in(): ) in (None, []) +def test_daily_cookbook_chunks_jsonl_one_line_at_a_time(): + """Daily cookbook keeps JSONL records as individually addressable chunks.""" + cfg = _load_config("daily_cookbook.yaml") + + jsonl = cfg["components"]["file_chunker"]["jsonl"] + assert jsonl["max_lines_per_chunk"] == 1 + + def test_parse_args_rejects_non_key_value_extra_argument(): """Extra CLI arguments must use key=value syntax.""" with pytest.raises(ValueError, match="expected key=value"): diff --git a/tests/unit/test_daily_paper.py b/tests/unit/test_daily_paper.py index 2e5ba1bf..db437144 100644 --- a/tests/unit/test_daily_paper.py +++ b/tests/unit/test_daily_paper.py @@ -6,7 +6,7 @@ import json from pathlib import Path import subprocess import sys -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import frontmatter import httpx @@ -14,6 +14,7 @@ import pytest from reme.components import ApplicationContext from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper +from reme.components.outbound_proxy import FixedHttpOutboundProxy from reme.components.runtime_context import RuntimeContext from reme.config.config_parser import _load_config from reme.schema import DailyBriefOutput, PaperInfo, PaperNoteOutput, PaperSelection @@ -29,7 +30,9 @@ from reme.steps.cookbook.daily_paper.rank import build_candidate_pool, rrf_score from reme.steps.cookbook.dingtalk import DingTalkMarkdownSendStep from reme.steps.cookbook.dingtalk import send as dingtalk_send from reme.utils import arxiv as arxiv_utils +from reme.utils import huggingface_papers as hf_utils from reme.utils.huggingface_papers import paper_ids_from_html, paper_info_from_payload +from reme.enumeration import ComponentEnum class _QueuedAgentWrapper(BaseAgentWrapper): @@ -82,6 +85,75 @@ def test_hf_payload_and_html_normalization(): ) == ["2607.16051", "2607.10001"] +@pytest.mark.asyncio +async def test_hf_client_uses_explicit_outbound_proxy(monkeypatch): + """The owned HTTP client uses only the explicitly supplied HTTP proxy.""" + events: list[str] = [] + client_kwargs: dict = {} + logger = MagicMock() + + class FakeAsyncClient: + """Capture construction and close ordering without network access.""" + + def __init__(self, **kwargs): + client_kwargs.update(kwargs) + + async def aclose(self): + """Record deterministic client cleanup.""" + events.append("client-close") + + monkeypatch.setattr(hf_utils.httpx, "AsyncClient", FakeAsyncClient) + + client = hf_utils.HuggingFacePapersClient( + proxy_url="http://127.0.0.1:43124", + timeout=12.0, + logger=logger, + ) + assert client.client is None + async with client: + assert client_kwargs["proxy"] == "http://127.0.0.1:43124" + assert client_kwargs["trust_env"] is False + + assert events == ["client-close"] + info_messages = [call.args[0] for call in logger.info.call_args_list] + assert info_messages == ["[HuggingFacePapersClient] network mode=outbound_proxy"] + + +@pytest.mark.asyncio +async def test_hf_client_logs_retry_after_http_error(monkeypatch): + """Transient HTTP failures report the attempt before retrying.""" + attempts = 0 + logger = MagicMock() + + async def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise httpx.ConnectTimeout("timed out", request=request) + return httpx.Response(200, json=[]) + + sleep = AsyncMock() + monkeypatch.setattr(hf_utils.asyncio, "sleep", sleep) + async with httpx.AsyncClient( + base_url="https://huggingface.co", + transport=httpx.MockTransport(handler), + ) as raw_client: + async with hf_utils.HuggingFacePapersClient( + client=raw_client, + max_retries=2, + logger=logger, + ) as client: + paper_ids = await client.fetch_daily_ids("2026-07-22") + assert raw_client.is_closed is False + + assert paper_ids == set() + assert attempts == 2 + sleep.assert_awaited_once_with(0.25) + warning = logger.warning.call_args.args[0] + assert "request retry path=/api/daily_papers attempt=1/2" in warning + assert "error=ConnectTimeout detail=timed out" in warning + + def test_rrf_and_memory_candidate_reserve(): """RRF is exact and the candidate pool preserves a memory-related slot.""" general = _paper("2607.10001", title="General model", upvotes=100) @@ -137,55 +209,198 @@ async def test_arxiv_pdf_downloads_missing_cache_once(tmp_path: Path, monkeypatc lambda **kwargs: async_client(transport=transport, **kwargs), ) target = tmp_path / "resource" / "papers" / "2607.10001.pdf" - client = arxiv_utils.ArxivPdfClient() + logger = MagicMock() - assert await client.download("2607.10001", target) == target - assert await client.download("2607.10001", target) == target + async with arxiv_utils.ArxivPdfClient(logger=logger) as client: + assert await client.download("2607.10001", target) == target + assert await client.download("2607.10001", target) == target assert target.read_bytes() == b"%PDF-downloaded" assert [str(request.url) for request in requests] == ["https://arxiv.org/pdf/2607.10001"] + info_messages = [call.args[0] for call in logger.info.call_args_list] + assert any("download start arxiv_id=2607.10001" in message for message in info_messages) + assert any("download done arxiv_id=2607.10001" in message for message in info_messages) + assert "cache hit arxiv_id=2607.10001" in logger.debug.call_args.args[0] -def test_standalone_config_uses_only_claude_code_and_eight_am_cron(monkeypatch): - """The standalone config schedules 08:00 and routes all agent work to CC.""" - for name in ( - "DINGTALK_APP_KEY", - "DINGTALK_APP_SECRET", - "DINGTALK_ROBOT_CODE", - "DINGTALK_CONVERSATION_IDS", - ): - monkeypatch.delenv(name, raising=False) - config = _load_config("daily_cookbook") +@pytest.mark.asyncio +async def test_arxiv_pdf_uses_explicit_outbound_proxy(tmp_path: Path, monkeypatch): + """The owned arXiv client uses and closes its explicit HTTP proxy client.""" + events: list[str] = [] + client_kwargs: dict = {} + logger = MagicMock() - assert config.get("extends") is None - assert config["jobs"]["daily_paper_cron"]["cron"] == "0 8 * * *" - steps = config["jobs"]["daily_paper"]["steps"] - assert config["jobs"]["daily_paper_cron"]["steps"] == steps - agent_steps = [ - step - for step in steps - if step["backend"] - in { - "daily_paper_select_step", - "daily_paper_analyze_step", - "daily_paper_digest_step", - } - ] - assert {step.get("agent_wrapper") for step in agent_steps} == {"claude_code"} - assert steps[-1] == { - "backend": "dingtalk_markdown_send_step", - "input_mapping": {"daily_paper_digest_path": "markdown_path"}, - "app_key": "", - "app_secret": "", - "robot_code": "", - "conversation_ids": "", - "title": "ReMe Daily Paper", - "timeout": 15, - } - assert set(config["components"]["agent_wrapper"]) == {"claude_code"} - assert "as_llm" not in config["components"] - assert config["components"]["agent_wrapper"]["claude_code"]["project_path"] == ".." - assert "skills" not in config["components"]["agent_wrapper"]["claude_code"] + class FakeResponse: + """Return one small, valid PDF stream.""" + + headers = {"content-length": "12"} + + def raise_for_status(self): + """Match the successful httpx response interface.""" + + async def aiter_bytes(self): + """Yield the fake PDF body.""" + yield b"%PDF-proxied" + + class FakeStream: + """Async response context returned by the HTTP client.""" + + async def __aenter__(self): + return FakeResponse() + + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + class FakeAsyncClient: + """Capture client construction and cleanup without network access.""" + + def __init__(self, **kwargs): + """Capture HTTPX construction arguments.""" + client_kwargs.update(kwargs) + + async def aclose(self): + """Record deterministic client cleanup.""" + events.append("client-close") + + def stream(self, method, url): + """Return the fake streaming response context.""" + assert (method, url) == ("GET", "https://arxiv.org/pdf/2607.10001") + return FakeStream() + + monkeypatch.setattr(arxiv_utils.httpx, "AsyncClient", FakeAsyncClient) + target = tmp_path / "2607.10001.pdf" + + async with arxiv_utils.ArxivPdfClient( + proxy_url="http://127.0.0.1:43124", + timeout=12.0, + logger=logger, + ) as client: + assert await client.download("2607.10001", target) == target + + assert target.read_bytes() == b"%PDF-proxied" + assert client_kwargs["proxy"] == "http://127.0.0.1:43124" + assert client_kwargs["trust_env"] is False + assert events == ["client-close"] + info_messages = [call.args[0] for call in logger.info.call_args_list] + assert info_messages[0] == "[ArxivPdfClient] network mode=outbound_proxy" + + +@pytest.mark.asyncio +async def test_paper_clients_enforce_context_and_unambiguous_ownership(tmp_path: Path): + """Requests require context entry and injected clients cannot also receive proxy_url.""" + with pytest.raises(RuntimeError, match="async context manager"): + await hf_utils.HuggingFacePapersClient().fetch_daily_ids("2026-07-22") + with pytest.raises(RuntimeError, match="async context manager"): + await arxiv_utils.ArxivPdfClient().download("2607.10001", tmp_path / "paper.pdf") + + async with httpx.AsyncClient() as raw_client: + async with arxiv_utils.ArxivPdfClient(client=raw_client): + pass + assert raw_client.is_closed is False + + with pytest.raises(ValueError, match="client and proxy_url"): + hf_utils.HuggingFacePapersClient( + client=raw_client, + proxy_url="http://127.0.0.1:43124", + ) + with pytest.raises(ValueError, match="client and proxy_url"): + arxiv_utils.ArxivPdfClient( + client=raw_client, + proxy_url="http://127.0.0.1:43124", + ) + + +@pytest.mark.asyncio +async def test_daily_paper_steps_forward_one_managed_proxy_endpoint(tmp_path: Path, monkeypatch): + """Collection and one shared PDF client receive the same application proxy URL.""" + paper = _paper("2607.10001", title="Managed proxy paper") + hf_kwargs: list[dict] = [] + arxiv_kwargs: list[dict] = [] + + class FakeHfClient: + """Return one eligible paper while recording construction.""" + + def __init__(self, **kwargs): + hf_kwargs.append(kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def fetch_scope(self, _scope: str, _value: str): + """Return one ranked paper.""" + return [paper] + + async def fetch_daily_ids(self, _day: str): + """Return no yesterday exclusions.""" + return set() + + class FakeArxivClient: + """Write a fake PDF while recording one shared downloader.""" + + def __init__(self, **kwargs): + arxiv_kwargs.append(kwargs) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def download(self, _arxiv_id: str, target: Path): + """Write one minimal PDF fixture.""" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"%PDF-fake") + return target + + monkeypatch.setattr(collect, "HuggingFacePapersClient", FakeHfClient) + monkeypatch.setattr(analyze, "ArxivPdfClient", FakeArxivClient) + monkeypatch.setattr( + analyze.DailyPaperAnalyzeStep, + "_extract_pdf_text_sync", + lambda *_args: ("--- PAGE 1 ---\nPaper content", 1, False), + ) + + proxy = FixedHttpOutboundProxy(url="http://127.0.0.1:43124") + await proxy.start() + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + app_context.components = {ComponentEnum.OUTBOUND_PROXY: {"default": proxy}} + context = RuntimeContext(date="2026-07-21") + agent = _QueuedAgentWrapper( + [ + { + "description": "Detailed note", + "body": "# Detailed reading\n\nEvidence [p. 1].", + }, + ], + ) + + try: + await DailyPaperCollectStep(app_context=app_context)(context) + context["daily_paper_selection"] = PaperSelection.model_validate( + { + "selection_reasoning": "Only candidate.", + "selected": [ + { + "arxiv_id": paper.arxiv_id, + "rank": 1, + "reason": "Relevant", + "memory_relevance": "low", + }, + ], + "alternates": [], + }, + ) + context["daily_paper_selected_papers"] = [paper] + await DailyPaperAnalyzeStep(app_context=app_context, agent_wrapper=agent)(context) + finally: + await proxy.close() + + assert hf_kwargs[0]["proxy_url"] == "http://127.0.0.1:43124" + assert len(arxiv_kwargs) == 1 + assert arxiv_kwargs[0]["proxy_url"] == "http://127.0.0.1:43124" def test_daily_paper_config_passes_dingtalk_environment(monkeypatch): diff --git a/tests/unit/test_dingtalk_wait.py b/tests/unit/test_dingtalk_wait.py index 9817753f..a3836361 100644 --- a/tests/unit/test_dingtalk_wait.py +++ b/tests/unit/test_dingtalk_wait.py @@ -11,77 +11,44 @@ import pytest from reme.components import ApplicationContext, R from reme.components.agent_wrapper.base_agent_wrapper import BaseAgentWrapper from reme.config.config_parser import _load_config -from reme.enumeration import ChunkEnum, ComponentEnum -from reme.schema import StreamChunk -from reme.steps.cookbook.dingtalk.wait import DingTalkWaitStep, _CardRenderer, _session_key +from reme.enumeration import ComponentEnum +from reme.steps.cookbook.dingtalk.wait import DingTalkWaitStep, _session_key class _AgentWrapper(BaseAgentWrapper): def __init__(self, **kwargs): super().__init__(**kwargs) - self.calls = [] + self.reply_calls = [] + self.compact_calls = [] + self.result_text = "回答" + self.is_error = False + + async def compact_session(self, session_id): + self.compact_calls.append(session_id) async def reply(self, inputs, **kwargs): - raise NotImplementedError - - async def reply_stream(self, inputs, **kwargs): - self.calls.append((inputs, kwargs)) + self.reply_calls.append((inputs, kwargs)) session_id = kwargs.get("resume") or "session-1" - yield StreamChunk( - chunk_type=ChunkEnum.THINK, - chunk="检查上下文", - block_id="think-1", - session_id=session_id, - ) - yield StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1", session_id=session_id) - yield StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="回答", session_id=session_id) - - -class _Card: - instances = [] - - def __init__(self, _client, _message): - self.card_template_id = "template" - self.title = None - self.markdown = "" - self.finished = False - self.failed = False - self.updates = [] - self.stream_flags = [] - self.content = "" - self.at_sender = False - self.__class__.instances.append(self) - - def set_title_and_logo(self, title, _logo): - self.title = title - - def get_card_data(self, flow_status=None): - data = {"msgContent": self.markdown} - if flow_status is not None: - data["flowStatus"] = flow_status - return data - - async def async_create_and_send_card(self, _template, _data, at_sender=False): - self.at_sender = at_sender - return "card-1" - - async def async_streaming(self, _card_id, _key, content, append, finished, failed): - self.updates.append(content) - self.stream_flags.append((append, finished, failed)) - self.content = self.content + content if append else content - if finished: - self.finished = not failed - self.failed = failed + return { + "session_id": session_id, + "last_message": {"is_error": self.is_error}, + "result": self.result_text, + } class _Handler: def __init__(self): - self.dingtalk_client = object() self.replies = [] + self.markdown_replies = [] + self.markdown_result = {"errcode": 0} def reply_text(self, text, _message): self.replies.append(text) + def reply_markdown(self, title, text, _message): + self.markdown_replies.append((title, text)) + return self.markdown_result + class _WebSocket: def __init__(self, messages=()): @@ -142,141 +109,67 @@ def test_session_key_uses_conversation_type_id_and_sender(): assert _session_key(_message(conversation="cid-2", conversation_type="2")) == "2:cid-2:user-1" -def test_renderer_keeps_blocks_in_stream_order_and_finishes_with_elapsed_time(monkeypatch): - now = 10.0 - monkeypatch.setattr("reme.steps.cookbook.dingtalk.wait.time.monotonic", lambda: now) - renderer = _CardRenderer() - deltas = [ - renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="中间内容", block_id="content-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="", block_id="content-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="分析问题", block_id="think-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="最终答案", block_id="answer-2")), - ] - - markdown = "".join(deltas) - assert "### 💬 Content\n\n中间内容" in markdown - assert "### 🧠 Think\n\n分析问题" in markdown - assert "### 💬 Answer" not in markdown - assert markdown.count("### 💬 Content") == 2 - assert markdown.index("中间内容") < markdown.index("### 🧠 Think") < markdown.index("最终答案") - - now = 15.5 - markdown += renderer.finish() - assert "### 💬 Answer" not in markdown - assert "### 💬 Content\n\n最终答案" in markdown - assert markdown.endswith("最终答案\n\n5.5s") - assert all(value not in markdown for value in ("ReMe Agent", "question", "正在生成", "执行轨迹")) - assert all(value not in markdown for value in ("✅", "### 当前")) - - -def test_renderer_limits_non_content_blocks_to_100_characters(): - renderer = _CardRenderer() - deltas = [ - renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="a" * 101, block_id="think-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.THINK, chunk="", block_id="think-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_RESULT, chunk="b" * 101, block_id="tool-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_RESULT, chunk="", block_id="tool-1")), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.CONTENT, chunk="c" * 101, block_id="answer-1")), - renderer.finish(), - ] - markdown = "".join(deltas) - for character in "ab": - assert f"{character * 100}..." in markdown - assert character * 101 not in markdown - assert "c" * 101 in markdown - assert f"{'c' * 100}..." not in markdown - - -def test_renderer_keeps_tool_call_and_result_visible(): - renderer = _CardRenderer() - tool_call_deltas = [ - renderer.feed( - StreamChunk( - chunk_type=ChunkEnum.TOOL_CALL, - chunk='{"name":"Read","id":"tool-1"}', - block_id="metadata-1", - tool_call_name="Read", - ), - ), - renderer.feed( - StreamChunk( - chunk_type=ChunkEnum.TOOL_CALL, - chunk='{"path":', - block_id="tool-1", - tool_call_name="Read", - ), - ), - renderer.feed( - StreamChunk(chunk_type=ChunkEnum.TOOL_CALL, chunk='"memory.md"}', block_id="tool-1"), - ), - renderer.feed(StreamChunk(chunk_type=ChunkEnum.TOOL_CALL, chunk="", block_id="tool-1")), - ] - assert tool_call_deltas[:3] == ["", "", ""] - - deltas = [ - *tool_call_deltas, - renderer.feed( - StreamChunk( - chunk_type=ChunkEnum.TOOL_RESULT, - chunk={"content": '{"message":"memory content","count":1}'}, - block_id="tool-1", - ), - ), - renderer.finish(), - ] - markdown = "".join(deltas) - assert markdown.count("### 🔧 Tool Call") == 1 - assert "### 🔧 Tool Call · `Read`" in markdown - assert '> {\n> 　"path": "memory.md"\n> }' in markdown - assert "```" not in markdown - assert '"name": "Read"' not in markdown - assert "### 📦 Tool Result" in markdown - assert '> 　"content": {\n> 　　"message": "memory content",\n> 　　"count": 1\n> 　}' in markdown - - @pytest.mark.asyncio -async def test_messages_resume_session_and_clear_only_the_combined_key(tmp_path): +async def test_final_reply_resumes_session_and_clear_only_removes_combined_key( + tmp_path, +): app_context = ApplicationContext(workspace_dir=str(tmp_path)) wrapper = _AgentWrapper(app_context=app_context) - step = DingTalkWaitStep(app_context=app_context, agent_wrapper=wrapper, card_update_interval=0.05) + step = DingTalkWaitStep(app_context=app_context, agent_wrapper=wrapper) step.logger = MagicMock() handler = _Handler() - sdk = SimpleNamespace( - AIMarkdownCardInstance=_Card, - AICardStatus=SimpleNamespace(PROCESSING="1"), - ) sessions = {} message = _message() key = _session_key(message) - await step._handle_message(message, key, sessions, handler, sdk) - await step._handle_message(message, key, sessions, handler, sdk) + await step._handle_message(message, key, sessions, handler) + await step._handle_message(message, key, sessions, handler) assert sessions == {key: "session-1"} - assert wrapper.calls == [("hello", {}), ("hello", {"resume": "session-1"})] - assert all(card.finished for card in _Card.instances[-2:]) - assert all(card.stream_flags for card in _Card.instances[-2:]) - assert all(all(append for append, _finished, _failed in card.stream_flags) for card in _Card.instances[-2:]) - assert all("### 💬 Content\n\n回答" in card.content for card in _Card.instances[-2:]) - assert all(sum(update.count("检查上下文") for update in card.updates) == 1 for card in _Card.instances[-2:]) - assert all(sum(update.count("回答") for update in card.updates) == 1 for card in _Card.instances[-2:]) - assert all(card.at_sender for card in _Card.instances[-2:]) - assert all(card.title is None for card in _Card.instances[-2:]) + assert wrapper.reply_calls == [("hello", {}), ("hello", {"resume": "session-1"})] + assert handler.markdown_replies == [("ReMe Agent", "回答"), ("ReMe Agent", "回答")] + + await step._handle_message(_message(text="/compact"), key, sessions, handler) + assert wrapper.compact_calls == ["session-1"] + assert sessions[key] == "session-1" + assert handler.replies[-1] == "✅ Conversation compaction requested." other_key = _session_key(_message(sender="user-2")) sessions[other_key] = "session-2" - await step._handle_message(_message(text="/clear"), key, sessions, handler, sdk) + await step._handle_message(_message(text="/clear"), key, sessions, handler) assert sessions == {other_key: "session-2"} assert handler.replies[-1] == "✅ Conversation cleared. The next message will start a new session." + logs = "\n".join(call.args[0] for call in step.logger.info.call_args_list) assert "received DingTalk text" in logs assert "completed DingTalk reply" in logs - assert "cleared DingTalk session" in logs + assert "handled session command" in logs assert "conversation_type='1' conversation_id='cid-1' sender_staff_id='user-1'" in logs assert all(value not in logs for value in ("hello", "session-1")) +@pytest.mark.asyncio +async def test_final_reply_rejects_empty_agent_reply_and_dingtalk_send_failure( + tmp_path, +): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + wrapper = _AgentWrapper(app_context=app_context) + step = DingTalkWaitStep(app_context=app_context, agent_wrapper=wrapper) + step.logger = MagicMock() + handler = _Handler() + message = _message() + key = _session_key(message) + + wrapper.result_text = " " + with pytest.raises(ValueError, match="空回复"): + await step._handle_message(message, key, {}, handler) + + wrapper.result_text = "回答" + handler.markdown_result = None + with pytest.raises(RuntimeError, match="发送钉钉 Markdown 回复失败"): + await step._handle_message(message, key, {}, handler) + + def test_daily_cookbook_registers_one_step_background_wait_job(monkeypatch): for name in ("DINGTALK_APP_KEY", "DINGTALK_APP_SECRET", "DINGTALK_ROBOT_CODE"): monkeypatch.delenv(name, raising=False) @@ -286,14 +179,26 @@ def test_daily_cookbook_registers_one_step_background_wait_job(monkeypatch): assert job["steps"] == [ { "backend": "dingtalk_wait_step", - "agent_wrapper": "claude_code", + "agent_wrapper": "dingtalk_wait", "app_key": "", "app_secret": "", "robot_code": "", - "card_update_interval": 1.0, "worker_count": 4, }, ] + dingtalk_wait = config["components"]["agent_wrapper"]["dingtalk_wait"] + assert dingtalk_wait["skills"] == ["tushare-data"] + assert dingtalk_wait["job_tools"] == ["memory_search"] + assert dingtalk_wait["system_prompt"] == { + "type": "preset", + "preset": "claude_code", + "append": ( + "Daily-paper Markdown is stored under the ReMe workspace. Detailed notes, including historical notes, " + "are at daily/YYYY-MM-DD/paper-.md; daily briefs are at " + "daily/YYYY-MM-DD/daily-paper-brief.md. Use memory_search to retrieve relevant long-term notes " + "across dates." + ), + } assert R.get(ComponentEnum.STEP, "dingtalk_wait_step") is DingTalkWaitStep @@ -302,7 +207,11 @@ def test_daily_cookbook_passes_dingtalk_environment_to_step(monkeypatch): monkeypatch.setenv("DINGTALK_APP_SECRET", "app-secret") monkeypatch.setenv("DINGTALK_ROBOT_CODE", "robot-code") step = _load_config("daily_cookbook")["jobs"]["dingtalk_wait"]["steps"][0] - assert (step["app_key"], step["app_secret"], step["robot_code"]) == ("app-key", "app-secret", "robot-code") + assert (step["app_key"], step["app_secret"], step["robot_code"]) == ( + "app-key", + "app-secret", + "robot-code", + ) @pytest.mark.asyncio diff --git a/tests/unit/test_file_store_consistency.py b/tests/unit/test_file_store_consistency.py index dbb329e7..f3c48b9e 100644 --- a/tests/unit/test_file_store_consistency.py +++ b/tests/unit/test_file_store_consistency.py @@ -316,6 +316,31 @@ def test_keyword_sync_rebuilds_when_backend_only_exposes_matching_count(): run(go()) +def test_keyword_sync_ignores_nonempty_chunk_with_no_indexable_tokens(monkeypatch): + """A tokenless chunk omitted by BM25 must not trigger a perpetual rebuild.""" + + async def go(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = LocalFileStore(name="t_tokenless_keyword", embedding_store="") + await store.start() + store.file_chunks = { + "indexed": chunk("indexed", "data.jsonl", "searchable content"), + "tokenless": chunk("tokenless", "data.jsonl", "\u2028"), + } + await store.keyword_index.clear() + await store.keyword_index.add_docs({cid: item.text for cid, item in store.file_chunks.items()}) + assert set(store.keyword_index.document_ids) == {"indexed"} + + async def unexpected_rebuild(_docs): + raise AssertionError("tokenless BM25 content must not trigger a rebuild") + + monkeypatch.setattr(store, "_rebuild_keyword_index", unexpected_rebuild) + await store._sync_keyword_index_from_chunks() + await store.close() + + run(go()) + + def test_keyword_sync_rebuilds_in_progress_batches(monkeypatch): """Foreground keyword repair uses bounded batches suitable for progress reporting.""" diff --git a/tests/unit/test_jsonl_file_chunker.py b/tests/unit/test_jsonl_file_chunker.py index 0aaa291a..72720433 100644 --- a/tests/unit/test_jsonl_file_chunker.py +++ b/tests/unit/test_jsonl_file_chunker.py @@ -80,6 +80,22 @@ def test_single_line(): os.unlink(path) +def test_unicode_line_separator_inside_json_string_is_not_a_record_boundary(): + """U+2028 is valid JSON string content, not a JSONL physical newline.""" + record = json.dumps({"id": 1, "text": "before\u2028\u2028after"}, ensure_ascii=False) + path = _write_jsonl([record]) + try: + chunker = JsonlFileChunker(max_lines_per_chunk=1) + node, chunks = _run(chunker.chunk(path)) + + assert len(chunks) == 1 + assert chunks[0].text == record + "\n" + assert chunks[0].start_line == chunks[0].end_line == 1 + assert node.chunk_ids == [chunks[0].id] + finally: + os.unlink(path) + + def test_all_lines_fit_in_one_chunk(): """Small file fits entirely in one chunk.""" lines = _make_records(5, width=30) @@ -95,6 +111,22 @@ def test_all_lines_fit_in_one_chunk(): os.unlink(path) +def test_max_lines_per_chunk_one(): + """max_lines_per_chunk=1 emits exactly one complete line per chunk.""" + lines = _make_records(5, width=10) + path = _write_jsonl(lines) + try: + chunker = JsonlFileChunker(max_chars=5000, max_lines_per_chunk=1) + _, chunks = _run(chunker.chunk(path)) + assert len(chunks) == len(lines) + for line_number, (line, chunk) in enumerate(zip(lines, chunks), start=1): + assert chunk.start_line == line_number + assert chunk.end_line == line_number + assert chunk.text == line + "\n" + finally: + os.unlink(path) + + # --------------------------------------------------------------------------- # Multi-chunk splitting (no overlap) # --------------------------------------------------------------------------- @@ -402,6 +434,7 @@ if __name__ == "__main__": test_empty_file() test_blank_lines_only() test_single_line() + test_unicode_line_separator_inside_json_string_is_not_a_record_boundary() test_all_lines_fit_in_one_chunk() test_multiple_chunks_no_overlap() test_line_aligned_no_intra_line_split() diff --git a/tests/unit/test_local_embedding_store.py b/tests/unit/test_local_embedding_store.py index e3934f39..98d6b93a 100644 --- a/tests/unit/test_local_embedding_store.py +++ b/tests/unit/test_local_embedding_store.py @@ -29,6 +29,27 @@ class BadHealthAsEmbedding: return [[1.0]] +class InsufficientQuotaError(Exception): + """OpenAI-compatible quota error used without importing the provider SDK.""" + + body = {"error": {"code": "insufficient_quota"}} + + +class QuotaThenSuccessAsEmbedding: + """Fail once for quota, then return a valid embedding.""" + + dimensions = 2 + + def __init__(self): + self.calls = 0 + + async def __call__(self, texts: list[str], **_kwargs): + self.calls += 1 + if self.calls == 1: + raise InsufficientQuotaError("quota exhausted") + return [[1.0, 0.0] for _ in texts] + + class BadNodeEmbeddingStore(BaseEmbeddingStore): """Embedding store that returns wrong-dimensional vectors.""" @@ -107,3 +128,53 @@ def test_health_check_rejects_wrong_dimension(): assert store.is_healthy is False run(go()) + + +def test_insufficient_quota_waits_sixty_seconds_before_retry(monkeypatch): + """Quota exhaustion uses the dedicated delay before ReMe retries.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + store = LocalEmbeddingStore( + name="t_local_embedding_quota", + max_retries=2, + quota_retry_delay=60.0, + ) + embedding = QuotaThenSuccessAsEmbedding() + store.as_embedding = embedding + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + result = await store._call_with_retry(["text"]) + + assert result == [[1.0, 0.0]] + assert embedding.calls == 2 + assert sleeps == [60.0] + + run(go()) + + +def test_insufficient_quota_does_not_retry_without_opt_in(monkeypatch): + """The default store behavior remains unchanged for embedded consumers.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + store = LocalEmbeddingStore(name="t_local_embedding_default_quota", max_retries=2) + embedding = QuotaThenSuccessAsEmbedding() + store.as_embedding = embedding + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + result = await store._call_with_retry(["text"]) + + assert result is None + assert embedding.calls == 1 + assert not sleeps + + run(go()) diff --git a/tests/unit/test_logging_config.py b/tests/unit/test_logging_config.py index 1293a74b..d77ff5aa 100644 --- a/tests/unit/test_logging_config.py +++ b/tests/unit/test_logging_config.py @@ -65,6 +65,26 @@ def test_stdlib_formatter_matches_qwenpaw_console_format(monkeypatch, tmp_path, logger_utils.get_logger(log_to_console=False, log_to_file=False, force_init=True) +def test_stdlib_file_creation_is_delayed_until_first_record(monkeypatch, tmp_path): + """Stdlib logging should not leave an empty file when no records are emitted.""" + fixed_datetime = Mock() + fixed_datetime.now.return_value = datetime(2026, 7, 23, 17, 50, 25) + monkeypatch.setattr(logger_utils, "datetime", fixed_datetime) + + logger = logger_utils._init_stdlib(str(tmp_path), "INFO", False, True) # pylint: disable=protected-access + log_path = tmp_path / "2026-07-23_17-50-25.log" + try: + assert not log_path.exists() + + logger.info("Application started") + + assert log_path.read_text(encoding="utf-8").endswith("Application started\n") + finally: + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + def test_stdlib_forwards_screen_and_file_logs_to_qwenpaw(monkeypatch, tmp_path): """Embedded stdlib logging should reuse QwenPaw's active sinks.""" monkeypatch.chdir(tmp_path) diff --git a/tests/unit/test_outbound_proxy.py b/tests/unit/test_outbound_proxy.py new file mode 100644 index 00000000..4a2a6337 --- /dev/null +++ b/tests/unit/test_outbound_proxy.py @@ -0,0 +1,341 @@ +"""Tests for application-scoped outbound proxy components.""" + +# pylint: disable=missing-function-docstring,protected-access + +import asyncio +import os +import sys +from collections.abc import Callable + +import pytest + +from reme.application import Application +from reme.components import R +from reme.components.outbound_proxy import FixedHttpOutboundProxy, SshHttpOutboundProxy +from reme.enumeration import ComponentEnum + + +class FakeProcess: + """Minimal asyncio subprocess stand-in with observable shutdown.""" + + def __init__(self, label: str, events: list[str]) -> None: + self.label = label + self.events = events + self.returncode: int | None = None + self.stdout = asyncio.StreamReader() + self.stderr = asyncio.StreamReader() + self._finished = asyncio.Event() + + async def wait(self) -> int: + await self._finished.wait() + assert self.returncode is not None + return self.returncode + + def terminate(self) -> None: + self.events.append(f"terminate:{self.label}") + self.exit(-15) + + def kill(self) -> None: + self.events.append(f"kill:{self.label}") + self.exit(-9) + + def exit(self, returncode: int, output: str = "") -> None: + if self.returncode is not None: + return + self.returncode = returncode + encoded = output.encode() + self.stdout.feed_data(encoded) + self.stdout.feed_eof() + self.stderr.feed_data(encoded) + self.stderr.feed_eof() + self._finished.set() + + +async def _ready_listener(*_args) -> None: + return None + + +async def _wait_until(predicate: Callable[[], bool], timeout: float = 1.0) -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while not predicate(): + if loop.time() >= deadline: + raise AssertionError("condition did not become true") + await asyncio.sleep(0.005) + + +def test_outbound_proxy_backends_are_registered() -> None: + assert R.get(ComponentEnum.OUTBOUND_PROXY, "fixed_http") is FixedHttpOutboundProxy + assert R.get(ComponentEnum.OUTBOUND_PROXY, "ssh_http") is SshHttpOutboundProxy + + +@pytest.mark.asyncio +async def test_application_builds_and_manages_fixed_http_proxy(tmp_path) -> None: + app = Application( + workspace_dir=str(tmp_path), + enable_logo=False, + log_to_console=False, + log_to_file=False, + service={"backend": "cli"}, + components={ + "outbound_proxy": { + "default": { + "backend": "fixed_http", + "url": "http://127.0.0.1:18080", + }, + }, + }, + ) + + component = app.context.components[ComponentEnum.OUTBOUND_PROXY]["default"] + assert isinstance(component, FixedHttpOutboundProxy) + + await app.start() + assert component.http_url == "http://127.0.0.1:18080" + await app.close() + + with pytest.raises(RuntimeError, match="start the component"): + _ = component.endpoint + + +@pytest.mark.asyncio +async def test_fixed_http_publishes_endpoint_and_merges_environment(monkeypatch) -> None: + monkeypatch.setenv("HTTP_PROXY", "http://ambient.example:8080") + component = FixedHttpOutboundProxy(url="http://127.0.0.1:18080") + base = {"CUSTOM": "value", "NO_PROXY": "example.com"} + + with pytest.raises(RuntimeError, match="start the component"): + _ = component.http_url + + await component.start() + merged = component.merge_environment(base) + + assert component.http_url == "http://127.0.0.1:18080" + assert base == {"CUSTOM": "value", "NO_PROXY": "example.com"} + assert os.environ["HTTP_PROXY"] == "http://ambient.example:8080" + assert merged["CUSTOM"] == "value" + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + assert merged[key] == component.http_url + assert merged["NO_PROXY"] == "127.0.0.1,localhost,::1" + assert merged["no_proxy"] == "127.0.0.1,localhost,::1" + + await component.close() + with pytest.raises(RuntimeError, match="start the component"): + _ = component.endpoint + + +@pytest.mark.parametrize( + ("url", "message"), + [ + ("", "must use http"), + ("https://proxy.example:8080", "must use http"), + ("http://proxy.example", "include host and port"), + ("http://user@proxy.example:8080", "must not contain userinfo"), + ("http://proxy.example:8080?mode=x", "must not contain query or fragment"), + ("http://proxy.example:8080#fragment", "must not contain query or fragment"), + ("http://proxy.example:not-a-port", "malformed"), + ], +) +@pytest.mark.asyncio +async def test_fixed_http_rejects_invalid_urls(url: str, message: str) -> None: + component = FixedHttpOutboundProxy(url=url) + + with pytest.raises(ValueError, match=message): + await component.start() + + assert component.is_started is False + with pytest.raises(RuntimeError, match="start the component"): + _ = component.endpoint + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"host": "", "account": "agent"}, "host is required"), + ({"host": "proxy.example", "account": ""}, "account is required"), + ({"host": "proxy.example", "account": "agent", "connect_timeout": 0}, "connect_timeout"), + ({"host": "proxy.example", "account": "agent", "monitor_interval": -1}, "monitor_interval"), + ({"host": "proxy.example", "account": "agent", "restart_initial_delay": "bad"}, "restart_initial_delay"), + ({"host": "proxy.example", "account": "agent", "restart_max_delay": float("inf")}, "restart_max_delay"), + ], +) +@pytest.mark.asyncio +async def test_ssh_http_validates_configuration(kwargs: dict, message: str) -> None: + component = SshHttpOutboundProxy(**kwargs) + + with pytest.raises(ValueError, match=message): + await component.start() + + +@pytest.mark.asyncio +async def test_ssh_http_requires_ssh_executable(monkeypatch) -> None: + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.shutil.which", lambda _name: None) + component = SshHttpOutboundProxy(host="proxy.example", account="agent") + + with pytest.raises(RuntimeError, match="ssh executable was not found"): + await component.start() + + +@pytest.mark.asyncio +async def test_ssh_http_starts_expected_commands_and_closes_bridge_first(monkeypatch) -> None: + events: list[str] = [] + commands: list[tuple[str, ...]] = [] + processes: list[FakeProcess] = [] + + async def fake_spawn(*command, **kwargs): + assert kwargs + commands.append(command) + label = "ssh" if command[0] == "/usr/bin/ssh" else "bridge" + process = FakeProcess(label, events) + processes.append(process) + return process + + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.shutil.which", lambda _name: "/usr/bin/ssh") + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.asyncio.create_subprocess_exec", fake_spawn) + component = SshHttpOutboundProxy(host="proxy.example", account="agent") + monkeypatch.setattr(component, "_pick_distinct_ports", lambda: (43123, 43124)) + monkeypatch.setattr(component, "_wait_for_listener", _ready_listener) + + await component.start() + + assert component.http_url == "http://127.0.0.1:43124" + assert commands[0] == ( + "/usr/bin/ssh", + "-N", + "-D", + "127.0.0.1:43123", + "-o", + "BatchMode=yes", + "-o", + "ExitOnForwardFailure=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "ConnectTimeout=10", + "-o", + "LogLevel=ERROR", + "--", + "agent@proxy.example", + ) + assert commands[1] == ( + sys.executable, + "-m", + "pproxy", + "-l", + "http://127.0.0.1:43124", + "-r", + "socks5://127.0.0.1:43123", + ) + + await component.close() + + assert events == ["terminate:bridge", "terminate:ssh"] + assert all(process.returncode == -15 for process in processes) + + +@pytest.mark.asyncio +async def test_ssh_http_cleans_up_when_initial_readiness_fails(monkeypatch) -> None: + events: list[str] = [] + process = FakeProcess("ssh", events) + + async def fake_spawn(*_command, **_kwargs): + return process + + async def fail_readiness(*_args): + raise TimeoutError("not ready") + + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.shutil.which", lambda _name: "/usr/bin/ssh") + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.asyncio.create_subprocess_exec", fake_spawn) + component = SshHttpOutboundProxy(host="proxy.example", account="agent") + monkeypatch.setattr(component, "_pick_distinct_ports", lambda: (43123, 43124)) + monkeypatch.setattr(component, "_wait_for_listener", fail_readiness) + + with pytest.raises(RuntimeError, match="SSH proxy exited before readiness"): + await component.start() + + assert events == ["terminate:ssh"] + assert component.is_started is False + with pytest.raises(RuntimeError, match="start the component"): + _ = component.endpoint + + +@pytest.mark.asyncio +async def test_ssh_http_reselects_ports_only_before_endpoint_is_published(monkeypatch) -> None: + events: list[str] = [] + commands: list[tuple[str, ...]] = [] + selected_ports = iter(((43123, 43124), (43125, 43126))) + + async def fake_spawn(*command, **_kwargs): + commands.append(command) + label = "ssh" if command[0] == "/usr/bin/ssh" else "bridge" + process = FakeProcess(label, events) + if label == "ssh" and len(commands) == 1: + process.exit(255, "bind [127.0.0.1]:43123: Address already in use") + return process + + async def fake_readiness(process, *_args): + if process.returncode is not None: + raise RuntimeError("process exited") + + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.shutil.which", lambda _name: "/usr/bin/ssh") + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.asyncio.create_subprocess_exec", fake_spawn) + component = SshHttpOutboundProxy(host="proxy.example", account="agent") + monkeypatch.setattr(component, "_pick_distinct_ports", lambda: next(selected_ports)) + monkeypatch.setattr(component, "_wait_for_listener", fake_readiness) + + await component.start() + + assert component.http_url == "http://127.0.0.1:43126" + assert [command[0] for command in commands] == ["/usr/bin/ssh", "/usr/bin/ssh", sys.executable] + await component.close() + + +@pytest.mark.asyncio +async def test_ssh_http_monitor_restarts_on_original_ports(monkeypatch) -> None: + events: list[str] = [] + commands: list[tuple[str, ...]] = [] + processes: list[FakeProcess] = [] + + async def fake_spawn(*command, **_kwargs): + commands.append(command) + label = "ssh" if command[0] == "/usr/bin/ssh" else "bridge" + process = FakeProcess(label, events) + processes.append(process) + return process + + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.shutil.which", lambda _name: "/usr/bin/ssh") + monkeypatch.setattr("reme.components.outbound_proxy.ssh_http.asyncio.create_subprocess_exec", fake_spawn) + component = SshHttpOutboundProxy( + host="proxy.example", + account="agent", + monitor_interval=0.005, + restart_initial_delay=0.005, + ) + monkeypatch.setattr(component, "_pick_distinct_ports", lambda: (43123, 43124)) + monkeypatch.setattr(component, "_wait_for_listener", _ready_listener) + await component.start() + endpoint = component.endpoint + + processes[0].exit(7) + await _wait_until(lambda: len(processes) == 3) + + assert component.endpoint is endpoint + assert commands[2][0] == "/usr/bin/ssh" + assert "127.0.0.1:43123" in commands[2] + assert sum(command[0] == sys.executable for command in commands) == 1 + + processes[1].exit(8) + await _wait_until(lambda: len(processes) == 4) + + assert component.endpoint is endpoint + assert commands[3] == ( + sys.executable, + "-m", + "pproxy", + "-l", + "http://127.0.0.1:43124", + "-r", + "socks5://127.0.0.1:43123", + ) + + await component.close() diff --git a/tests/unit/test_reme_cli.py b/tests/unit/test_reme_cli.py index 95930f93..ecc1e0ce 100644 --- a/tests/unit/test_reme_cli.py +++ b/tests/unit/test_reme_cli.py @@ -13,29 +13,36 @@ from reme.components.service.cli_service import CliService from reme import reme as reme_module -def test_package_import_does_not_require_optional_agent_sdks(): - """The base package remains importable without Claude or Codex SDKs.""" +def test_package_import_does_not_load_optional_core_dependencies(): + """The base package leaves optional core dependencies unloaded.""" script = """ import importlib.abc -import sys -class BlockOptionalAgentSDKs(importlib.abc.MetaPathFinder): +class BlockOptionalCoreDependencies(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): - blocked = ("claude_agent_sdk", "openai_codex") + blocked = ( + "claude_agent_sdk", + "dingtalk_stream", + "openai_codex", + "faiss", + "jieba", + "rjieba", + "neo4j", + "networkx", + "pypdf", + "polars", + "tushare", + ) if any(fullname == name or fullname.startswith(f"{name}.") for name in blocked): - raise ModuleNotFoundError(f"blocked optional SDK: {fullname}", name=fullname) + raise AssertionError(f"eagerly imported optional core dependency: {fullname}") return None -sys.meta_path.insert(0, BlockOptionalAgentSDKs()) -import reme +import sys -assert not any( - name == sdk or name.startswith(f"{sdk}.") - for name in sys.modules - for sdk in ("claude_agent_sdk", "openai_codex") -) +sys.meta_path.insert(0, BlockOptionalCoreDependencies()) +import reme """ result = subprocess.run( [sys.executable, "-c", script],