From 3ddd06c085e80e09e0571ae64fc8db85928f6e59 Mon Sep 17 00:00:00 2001 From: wang-qisen Date: Thu, 27 Aug 2026 17:38:21 +0800 Subject: [PATCH] feat: wire image resources into the resource watch loop --- docs/en/auto_resource.md | 17 +++++++++++-- docs/zh/auto_resource.md | 8 +++++- pyproject.toml | 2 ++ reme/config/default.yaml | 39 +++++++++++++++++++++++++++++- reme/steps/evolve/auto_resource.py | 29 +++++++++++++++++++++- 5 files changed, 90 insertions(+), 5 deletions(-) diff --git a/docs/en/auto_resource.md b/docs/en/auto_resource.md index 9d9495b1..0ebc19ff 100644 --- a/docs/en/auto_resource.md +++ b/docs/en/auto_resource.md @@ -49,8 +49,21 @@ workspace/ meeting-notes.csv ``` -The current Beta version is best suited to text-based resources such as `md`, `txt`, `json`, `jsonl`, `csv`, `yaml`, and -`html`. +Text resources such as `md`, `txt`, `json`, `jsonl`, `csv`, `yaml`, and `html` are the primary fit. Image resources +(`png`, `jpg`, `jpeg`, `webp`, `gif`, `bmp`, `tiff`, `heic`) produce caption cards as described in +[Image Resources](#image-resources). + +## Image Resources + +Image files are interpreted the same way: a vision model writes a caption card that links back to the original image. +The card body starts with an `![[resource/...]]` embed link and the frontmatter carries `kind: image` and `media_type`, +so text search reaches image content through the caption. + +The vision model is the `vision` instance of `as_llm` when configured, and otherwise falls back to the `default` +instance — a multimodal default model needs no extra configuration. Images larger than the request budget or in +provider-unfriendly formats are downscaled or re-encoded in memory for the request only; the original file under +`resource/` is never modified. When an image changes, its card is rewritten in place; when the image is deleted, the +card is removed with it. ## Resource Cards diff --git a/docs/zh/auto_resource.md b/docs/zh/auto_resource.md index 5a669c09..1253dfdd 100644 --- a/docs/zh/auto_resource.md +++ b/docs/zh/auto_resource.md @@ -46,7 +46,13 @@ workspace/ meeting-notes.csv ``` -当前 Beta 版本更适合处理文本类资源，例如 `md`、`txt`、`json`、`jsonl`、`csv`、`yaml`、`html`。 +当前 Beta 版本以文本类资源为主，例如 `md`、`txt`、`json`、`jsonl`、`csv`、`yaml`、`html`；图像资源（`png`、`jpg`、`jpeg`、`webp`、`gif`、`bmp`、`tiff`、`heic`）会生成 caption 卡片，见下文[图像资源](#图像资源)一节。 + +## 图像资源 + +图像文件的解读方式相同：视觉模型写入一张 caption 卡片并链接原图。卡片正文以 `![[resource/...]]` 嵌入链接开头，frontmatter 携带 `kind: image` 与 `media_type`，文本检索因此可以通过 caption 命中图像内容。 + +视觉模型优先使用配置中的 `as_llm` `vision` 实例，未配置时回退到 `default` 实例——默认模型具备视觉能力时无需额外配置。超过请求预算或格式不被模型接受的图像，仅在请求前于内存中降采样或转码；`resource/` 下的原图文件不会被修改。图像变更时卡片原地重写；图像删除时卡片随之删除。 ## 资源卡片 diff --git a/pyproject.toml b/pyproject.toml index 011e4612..a13655c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "mistletoe>=1.5.1", "numpy>=2.2.6", "openai>=2.26.0", + "pillow>=10.0.0", + "pillow-heif>=0.13.0", "psutil>=5.9", "pydantic>=2.12.5", "python-frontmatter>=1.1.0", diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 95a0bad5..25dd60ac 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -20,7 +20,7 @@ jobs: resource_watch_loop: backend: background watch_dirs: [resource_dir] - watch_suffixes: [md, txt, json, jsonl, csv, yaml, html] + watch_suffixes: [md, txt, json, jsonl, csv, yaml, html, png, jpg, jpeg, webp, gif, bmp, tiff, heic] steps: - backend: init_changes_step monitor_type: file_catalog @@ -29,11 +29,13 @@ jobs: - backend: update_catalog_step file_catalog: resource - backend: auto_resource_step + - backend: auto_image_step - backend: watch_changes_step dispatch_steps: - backend: update_catalog_step file_catalog: resource - backend: auto_resource_step + - backend: auto_image_step digest_watch_loop: backend: background @@ -186,6 +188,30 @@ jobs: steps: - backend: auto_resource_step + auto_image: + backend: base + description: "Auto-image: interpret image resource files into daily caption notes" + parameters: + type: object + properties: + changes: + type: array + description: "image resource change batch, each item has path/file_path and change" + items: + type: object + properties: + path: + type: string + file_path: + type: string + change: + type: string + description: "added/modified/deleted" + required: + - changes + steps: + - backend: auto_image_step + proactive: backend: base description: "Proactive: read daily//interests.yaml and expose the latest user-interest topics." @@ -755,6 +781,17 @@ components: max_tokens: 65536 thinking_enable: false + # Optional dedicated vision model for image resources (auto_image_step). + # Falls back to the "default" instance above when absent; uncomment to + # decouple the vision model from the main LLM. + # vision: + # backend: ${VLM_BACKEND:-openai} + # model: ${VLM_MODEL_NAME:-} + # stream: false + # credential: + # api_key: ${VLM_API_KEY:-} + # base_url: ${VLM_BASE_URL:-} + agent_wrapper: default: backend: agentscope diff --git a/reme/steps/evolve/auto_resource.py b/reme/steps/evolve/auto_resource.py index 5264b5c5..597d6b8b 100644 --- a/reme/steps/evolve/auto_resource.py +++ b/reme/steps/evolve/auto_resource.py @@ -11,7 +11,7 @@ import frontmatter from watchfiles import Change from ..base_step import BaseStep -from ..file_io import refresh_day_index, validate_filename_component +from ..file_io import is_image_file, refresh_day_index, validate_filename_component from ...components import R from ._evolve import agent_reply_result_text, now @@ -473,6 +473,10 @@ class AutoResourceStep(BaseStep): ) self.logger.info(f"[{self.name}] done {note_path} modified={modified}") + def _skip_image_change(self, file_path: str) -> bool: + """Return True for image changes, which auto_image_step interprets.""" + return is_image_file(file_path) + async def _handle_change(self, file_path: str, raw_change) -> dict: assert self.context is not None # Handlers write item-scoped fields into the shared response. Start each @@ -509,6 +513,29 @@ class AutoResourceStep(BaseStep): note_stem = _compute_note_stem(filename) self.logger.info(f"[{self.name}] {change.name} file_path={file_path} note_stem={note_stem}") + if self._skip_image_change(file_path): + # Image resources are binary; the text interpretation below would + # read them as mojibake. They are handled by auto_image_step. + answer = f"Skipped image resource file: {file_path}" + self.context.response.success = True + self.context.response.answer = answer + self.context.response.metadata.update( + { + "path": file_path, + "action": "skipped", + "reason": "image_file", + "modified": False, + }, + ) + self.logger.info(f"[{self.name}] skip change file_path={file_path} reason=image_file") + return { + "success": True, + "path": file_path, + "change": change.name, + "answer": answer, + "metadata": dict(self.context.response.metadata), + } + if change == Change.deleted: await self._handle_delete(file_path, date_str, note_stem) else: