From 072cb6a55b274e3187e81204e14f5b989eb23801 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:57:24 +0800 Subject: [PATCH] feat(daily-paper): add opt-in Hugging Face mirror support (#437) * feat(daily-paper): add Hugging Face mirror switch * refactor(daily-paper): simplify the HF mirror switch and warn on ignored env The switch was a three-state bool|None where None preserved the legacy environment-driven selection, but no production caller ever passes None -- collect.py always resolves an explicit bool. Collapse it to a plain bool defaulting to False. HF_MIRROR_URL no longer redirects traffic on its own, so warn when it is configured while the mirror stays disabled; a mirror-only setup would otherwise fall back to the official site with no signal. Both READMEs now record the behavior change and stop presenting the two mirror variables as symmetric -- arXiv remains environment-driven while Hugging Face is gated on the job parameter. Co-Authored-By: Claude Opus 5 * fix(daily-paper): address mirror configuration feedback --------- Co-authored-by: Claude Opus 5 --- cookbook/daily_paper/README.md | 19 ++++-- cookbook/daily_paper/README_ZH.md | 14 ++-- pyproject.toml | 2 +- reme/config/daily_cookbook.yaml | 5 ++ reme/steps/cookbook/daily_paper/collect.py | 1 + reme/utils/huggingface_papers.py | 16 ++++- tests/unit/test_daily_paper.py | 76 +++++++++++++++++++++- 7 files changed, 119 insertions(+), 14 deletions(-) diff --git a/cookbook/daily_paper/README.md b/cookbook/daily_paper/README.md index dd355009..11bf8e3d 100644 --- a/cookbook/daily_paper/README.md +++ b/cookbook/daily_paper/README.md @@ -149,6 +149,7 @@ Public job parameters: |---|---:|---| | `date` | `""` | Run date; empty uses today in the app timezone, otherwise requires `YYYY-MM-DD` | | `force` | `false` | Regenerate even when the day's brief exists | +| `use_hf_mirror` | `false` | Use the Hugging Face mirror from `HF_MIRROR_URL`, or `https://hf-mirror.com` when unset | | `topics` | `""` | Topics to prioritize during selection | | `weekly_weight` | `0.7` | Weekly contribution to RRF | | `history_days` | `30` | Prior recommendation exclusion window | @@ -169,10 +170,14 @@ Step-level settings on the `daily_paper` job: ## Mirrors The data clients use httpx's default environment handling, so `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` take effect -when present. Mirror environment variables independently replace each service's base URL: +when present. The two data sources reach a mirror differently: Hugging Face is gated on the `use_hf_mirror` job +parameter, while arXiv is driven by its environment variable alone. ```dotenv -# Defaults to https://huggingface.co when unset +# Enable the mirror for the built-in daily_paper_cron job +DAILY_PAPER_USE_HF_MIRROR=true + +# Read only when the manual or scheduled job enables the mirror; defaults to https://hf-mirror.com when unset HF_MIRROR_URL=https://hf-mirror.com # Defaults to https://arxiv.org when unset @@ -185,7 +190,12 @@ ARXIV_MIRROR_URL=https://export.arxiv.org `HF_MIRROR_URL` must implement the `/papers/...`, `/api/daily_papers`, and `/api/papers/...` routes used by the current client. `ARXIV_MIRROR_URL` must implement `/pdf/`. A path prefix in either base URL is preserved, and a -trailing slash is optional. When a variable is unset, the official service is used directly; there is no fallback chain. +trailing slash is optional. There is no fallback chain: whichever base URL a client selects is the only one it tries. + +> **Behavior change:** `HF_MIRROR_URL` used to redirect Hugging Face traffic on its own. It is now read only when the +> job runs with `use_hf_mirror=true`; otherwise the official service is used and the client logs a warning that the +> variable was ignored. Pass `use_hf_mirror=true` for manual requests, or set +> `DAILY_PAPER_USE_HF_MIRROR=true` for `daily_paper_cron`, to keep an existing mirror-only setup working. ## Running the workflow @@ -213,7 +223,8 @@ reme start config=daily_cookbook ``` The built-in service listens on `127.0.0.1:8001`. `daily_paper_cron` runs every day at 08:00 in the -`Asia/Shanghai` timezone. Override the bind address with `DAILY_PAPER_HOST`, `DAILY_PAPER_PORT`, or startup arguments. +`Asia/Shanghai` timezone. Set `DAILY_PAPER_USE_HF_MIRROR=true` to make that scheduled job use the Hugging Face mirror. +Override the bind address with `DAILY_PAPER_HOST`, `DAILY_PAPER_PORT`, or startup arguments. ```bash curl -s http://127.0.0.1:8001/daily_paper \ diff --git a/cookbook/daily_paper/README_ZH.md b/cookbook/daily_paper/README_ZH.md index cfa3a5ef..8133d1d1 100644 --- a/cookbook/daily_paper/README_ZH.md +++ b/cookbook/daily_paper/README_ZH.md @@ -138,6 +138,7 @@ reme_workspace/ |---|---:|---| | `date` | `""` | 运行日期；空值使用应用时区当天，非空值必须为 `YYYY-MM-DD` | | `force` | `false` | 已有当日简报时仍重新生成 | +| `use_hf_mirror` | `false` | 是否使用 Hugging Face 镜像站；优先读取 `HF_MIRROR_URL`，未配置时使用 `https://hf-mirror.com` | | `topics` | `""` | 精选论文时优先考虑的主题 | | `weekly_weight` | `0.7` | RRF 中周榜权重 | | `history_days` | `30` | 历史推荐排重窗口 | @@ -157,10 +158,13 @@ reme_workspace/ ## 镜像站 -数据客户端使用 httpx 默认的环境处理，因此存在 `HTTP_PROXY`、`HTTPS_PROXY` 或 `NO_PROXY` 时会自动生效。镜像环境变量独立替换对应数据源的 base URL： +数据客户端使用 httpx 默认的环境处理，因此存在 `HTTP_PROXY`、`HTTPS_PROXY` 或 `NO_PROXY` 时会自动生效。两个数据源启用镜像的方式不同：Hugging Face 由 `use_hf_mirror` 任务参数控制，arXiv 仅由环境变量驱动。 ```dotenv -# 未设置时使用 https://huggingface.co +# 为内置 daily_paper_cron 定时任务启用镜像站 +DAILY_PAPER_USE_HF_MIRROR=true + +# 仅在手动任务或定时任务启用镜像时读取；未配置时使用 https://hf-mirror.com HF_MIRROR_URL=https://hf-mirror.com # 未设置时使用 https://arxiv.org @@ -171,7 +175,9 @@ ARXIV_MIRROR_URL=https://export.arxiv.org # ARXIV_MIRROR_URL=http://relay-host:18080/arxiv ``` -`HF_MIRROR_URL` 必须提供当前代码使用的 `/papers/...`、`/api/daily_papers` 和 `/api/papers/...` 路径。`ARXIV_MIRROR_URL` 必须支持 `/pdf/`。两种 base URL 都会保留路径前缀，末尾 `/` 可有可无；不配置就直接访问官方站点，不会执行备用地址回退。 +`HF_MIRROR_URL` 必须提供当前代码使用的 `/papers/...`、`/api/daily_papers` 和 `/api/papers/...` 路径。`ARXIV_MIRROR_URL` 必须支持 `/pdf/`。两种 base URL 都会保留路径前缀，末尾 `/` 可有可无。不存在备用地址回退：客户端选定哪个 base URL，就只访问该地址。 + +> **行为变更：** 以往只要设置 `HF_MIRROR_URL` 就会改变 Hugging Face 的访问地址；现在该变量仅在任务启用镜像时才会读取，否则直接访问官方站点，并输出一条“已忽略该变量”的告警日志。手动调用需传入 `use_hf_mirror=true`，`daily_paper_cron` 定时任务需设置 `DAILY_PAPER_USE_HF_MIRROR=true`，才能继续走镜像。 ## 运行方式 @@ -198,7 +204,7 @@ reme start config=daily_cookbook job=daily_paper date=2026-08-06 force=true reme start config=daily_cookbook ``` -内置服务监听 `127.0.0.1:8001`，`daily_paper_cron` 按 `Asia/Shanghai` 时区每天 08:00 运行。可通过 `DAILY_PAPER_HOST`、`DAILY_PAPER_PORT` 或启动参数覆盖监听地址和端口。 +内置服务监听 `127.0.0.1:8001`，`daily_paper_cron` 按 `Asia/Shanghai` 时区每天 08:00 运行。设置 `DAILY_PAPER_USE_HF_MIRROR=true` 可让该定时任务使用 Hugging Face 镜像站。可通过 `DAILY_PAPER_HOST`、`DAILY_PAPER_PORT` 或启动参数覆盖监听地址和端口。 ```bash curl -s http://127.0.0.1:8001/daily_paper \ diff --git a/pyproject.toml b/pyproject.toml index cba14f77..b863e98b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "uvicorn>=0.41.0", "watchfiles>=1.1.1", "zstandard>=0.23.0", + "pypdf>=5.0.0", ] [project.optional-dependencies] @@ -53,7 +54,6 @@ core = [ "rjieba>=0.2.1", "neo4j>=6.2.0", "networkx>=3.4.2", - "pypdf>=5.0.0", "polars>=1.43.0", "tushare>=1.4.29", ] diff --git a/reme/config/daily_cookbook.yaml b/reme/config/daily_cookbook.yaml index 55831dd9..40a9fd5f 100644 --- a/reme/config/daily_cookbook.yaml +++ b/reme/config/daily_cookbook.yaml @@ -416,6 +416,10 @@ jobs: type: boolean description: "Regenerate even when that day's final brief already exists." default: false + use_hf_mirror: + type: boolean + description: "Use the Hugging Face mirror configured by HF_MIRROR_URL, or hf-mirror.com when unset." + default: false topics: type: string description: "Optional topics to prioritize when selecting papers." @@ -447,6 +451,7 @@ jobs: daily_paper_cron: backend: cron cron: "0 8 * * *" + use_hf_mirror: ${DAILY_PAPER_USE_HF_MIRROR:-false} candidate_limit: *candidate_limit rrf_k: *rrf_k weekly_weight: *weekly_weight diff --git a/reme/steps/cookbook/daily_paper/collect.py b/reme/steps/cookbook/daily_paper/collect.py index 6cd85fab..716807a0 100644 --- a/reme/steps/cookbook/daily_paper/collect.py +++ b/reme/steps/cookbook/daily_paper/collect.py @@ -119,6 +119,7 @@ class DailyPaperCollectStep(DailyPaperStep): async with HuggingFacePapersClient( timeout=float(self._value("hf_timeout", 600.0)), max_retries=int(self._value("hf_max_retries", 3)), + use_mirror=bool(self._value("use_hf_mirror", False)), ) as client: weekly, monthly, yesterday_ids = await asyncio.gather( client.fetch_scope("week", week), diff --git a/reme/utils/huggingface_papers.py b/reme/utils/huggingface_papers.py index b3c4e544..b00ee792 100644 --- a/reme/utils/huggingface_papers.py +++ b/reme/utils/huggingface_papers.py @@ -13,6 +13,7 @@ from .arxiv import ARXIV_ID_PATTERN from .logger_utils import get_logger HF_BASE_URL = "https://huggingface.co" +HF_MIRROR_BASE_URL = "https://hf-mirror.com" _PAPER_LINK_PATTERN = re.compile( r"href=[\"'](?:https://huggingface\.co)?/papers/(\d{4}\.\d{4,5})(?:[^\"']*)?[\"']", re.IGNORECASE, @@ -73,9 +74,15 @@ class HuggingFacePapersClient: max_retries: int = 3, detail_concurrency: int = 5, logger: Any | None = None, + use_mirror: bool = False, ) -> None: self.logger = logger or get_logger() - self.base_url = os.getenv("HF_MIRROR_URL", "").strip().rstrip("/") or HF_BASE_URL + configured_mirror = os.getenv("HF_MIRROR_URL", "").strip().rstrip("/") + self.base_url = (configured_mirror or HF_MIRROR_BASE_URL) if use_mirror else HF_BASE_URL + # HF_MIRROR_URL alone no longer redirects traffic; warn so a stale + # mirror-only setup is visible instead of silently hitting the official site. + self._ignored_mirror_configured = bool(configured_mirror) and not use_mirror + self._source = "mirror" if use_mirror else "official" self._owns_client = client is None self._timeout = timeout self.client = client @@ -84,13 +91,18 @@ class HuggingFacePapersClient: async def __aenter__(self) -> "HuggingFacePapersClient": if self.client is None: + if self._ignored_mirror_configured: + self.logger.warning( + "[HuggingFacePapersClient] ignoring configured HF_MIRROR_URL " + "because the mirror is disabled; pass use_hf_mirror=true to use it", + ) self.client = httpx.AsyncClient( base_url=self.base_url, timeout=self._timeout, follow_redirects=True, headers={"User-Agent": "ReMe daily-paper cookbook"}, ) - self.logger.info(f"[HuggingFacePapersClient] base_url={self.base_url}") + self.logger.info(f"[HuggingFacePapersClient] source={self._source}") else: self.logger.debug("[HuggingFacePapersClient] network mode=injected_client") return self diff --git a/tests/unit/test_daily_paper.py b/tests/unit/test_daily_paper.py index 1bc4fd3d..b168992f 100644 --- a/tests/unit/test_daily_paper.py +++ b/tests/unit/test_daily_paper.py @@ -85,7 +85,7 @@ def test_hf_payload_and_html_normalization(): @pytest.mark.asyncio async def test_hf_client_uses_configured_mirror(monkeypatch): - """The owned HTTP client uses HF_MIRROR_URL as its source.""" + """The owned HTTP client uses HF_MIRROR_URL once the mirror is enabled.""" events: list[str] = [] client_kwargs: dict = {} logger = MagicMock() @@ -106,6 +106,7 @@ async def test_hf_client_uses_configured_mirror(monkeypatch): client = hf_utils.HuggingFacePapersClient( timeout=12.0, logger=logger, + use_mirror=True, ) assert client.client is None async with client: @@ -115,7 +116,56 @@ async def test_hf_client_uses_configured_mirror(monkeypatch): assert events == ["client-close"] info_messages = [call.args[0] for call in logger.info.call_args_list] - assert info_messages == ["[HuggingFacePapersClient] base_url=https://hf-mirror.com"] + assert info_messages == ["[HuggingFacePapersClient] source=mirror"] + + +def test_hf_client_mirror_switch_controls_source(monkeypatch): + """The switch alone decides the source, even with HF_MIRROR_URL configured.""" + monkeypatch.setenv("HF_MIRROR_URL", "https://relay.example/hf") + + official = hf_utils.HuggingFacePapersClient(use_mirror=False) + configured_mirror = hf_utils.HuggingFacePapersClient(use_mirror=True) + + assert official.base_url == "https://huggingface.co" + assert configured_mirror.base_url == "https://relay.example/hf" + + +def test_hf_client_uses_default_mirror_when_enabled(monkeypatch): + """The mirror switch is useful without requiring another setting.""" + monkeypatch.delenv("HF_MIRROR_URL", raising=False) + + client = hf_utils.HuggingFacePapersClient(use_mirror=True) + + assert client.base_url == "https://hf-mirror.com" + + +@pytest.mark.asyncio +async def test_hf_client_warns_when_mirror_url_is_ignored(monkeypatch): + """A mirror-only setup learns why traffic still reaches the official site.""" + logger = MagicMock() + + class FakeAsyncClient: + """Stand in for the owned client without touching the network.""" + + def __init__(self, **kwargs): + pass + + async def aclose(self): + """Match the owned-client cleanup contract.""" + + monkeypatch.setattr(hf_utils.httpx, "AsyncClient", FakeAsyncClient) + secret_mirror_url = "https://user:password@relay.example/hf?token=secret" + monkeypatch.setenv("HF_MIRROR_URL", secret_mirror_url) + + async with hf_utils.HuggingFacePapersClient(logger=logger) as client: + assert client.base_url == "https://huggingface.co" + + warning = logger.warning.call_args.args[0] + assert "ignoring configured HF_MIRROR_URL" in warning + assert "use_hf_mirror=true" in warning + assert secret_mirror_url not in warning + assert "password" not in warning + assert "secret" not in warning @pytest.mark.asyncio @@ -136,7 +186,7 @@ async def test_hf_client_preserves_mirror_path_prefix(monkeypatch): ) monkeypatch.setenv("HF_MIRROR_URL", "http://relay.example:18080/hf/") - async with hf_utils.HuggingFacePapersClient() as client: + async with hf_utils.HuggingFacePapersClient(use_mirror=True) as client: assert await client.fetch_daily_ids("2026-07-22") == set() assert [str(request.url) for request in requests] == [ @@ -453,6 +503,26 @@ def test_daily_paper_topics_parameter_defaults_to_empty(): } +def test_daily_paper_hf_mirror_parameter_defaults_to_disabled(): + """The public job schema exposes an explicit Hugging Face mirror switch.""" + use_hf_mirror = _load_config("daily_cookbook")["jobs"]["daily_paper"]["parameters"]["properties"]["use_hf_mirror"] + + assert use_hf_mirror == { + "type": "boolean", + "description": "Use the Hugging Face mirror configured by HF_MIRROR_URL, or hf-mirror.com when unset.", + "default": False, + } + + +def test_daily_paper_cron_hf_mirror_uses_explicit_environment_switch(monkeypatch): + """The scheduled workflow can opt in to the Hugging Face mirror.""" + monkeypatch.delenv("DAILY_PAPER_USE_HF_MIRROR", raising=False) + assert _load_config("daily_cookbook")["jobs"]["daily_paper_cron"]["use_hf_mirror"] is False + + monkeypatch.setenv("DAILY_PAPER_USE_HF_MIRROR", "true") + assert _load_config("daily_cookbook")["jobs"]["daily_paper_cron"]["use_hf_mirror"] is True + + def test_paper_pick_list_uses_an_object_root_for_tool_output(): """AgentScope function arguments require an object-root JSON schema.""" schema = PaperPickList.model_json_schema()