From bfc05bf3b79976c52439008f2b6dd5c4c5a53b0e Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:54:31 +0800 Subject: [PATCH 01/19] feat(decision): allow hosts to inject owned JEV model instances (#652) * feat(decision): support host-owned model injection * fix(decision): finish model cleanup before propagating shutdown cancellation --- docs/en/context_compaction.md | 17 ++ docs/zh/context_compaction.md | 15 ++ .../as_decision_model/agentscope.py | 52 +++++- tests/unit/test_decision_model.py | 159 ++++++++++++++++++ 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/docs/en/context_compaction.md b/docs/en/context_compaction.md index 17ca97b3..c6677668 100644 --- a/docs/en/context_compaction.md +++ b/docs/en/context_compaction.md @@ -159,3 +159,20 @@ the entire gain cannot be attributed to Jev ranking itself. Agent continuation s [Complete experiment: design, results, expected answers and reproduction](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README.md) + +## Host-owned decision models + +Embedded hosts can reuse their existing AgentScope JEV instance, just as they inject an LLM: + +```python +# app contains an as_decision_model.compaction component with backend: agentscope. +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +The YAML `model` value remains the provider model name. The runtime `model` property accepts a callable instance +and does not construct a client on access. Inject before the first request; ReMe validates and converts JSON +questions normally, but does not require separate ReMe credentials or close the borrowed client. The host must +keep the instance alive until ReMe has finished its calls, then close it itself. Close the component before +replacing an internally constructed model or any model with active requests. Shutdown waits for active calls and +owned-client cleanup even when cancelled, then propagates cancellation. Component shutdown detaches the instance; +inject again before restarting a borrowed-model component. Standalone credential-based behavior is unchanged. diff --git a/docs/zh/context_compaction.md b/docs/zh/context_compaction.md index 4de6352a..33accb2e 100644 --- a/docs/zh/context_compaction.md +++ b/docs/zh/context_compaction.md @@ -147,3 +147,18 @@ agent 继续执行的成功率尚未验证。 [完整实验：设计、结果、标准答案与复现](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README_ZH.md) + +## 注入宿主管理的决策模型 + +嵌入式宿主可以复用已有的 AgentScope JEV 实例，与注入 LLM 的方式一致： + +```python +# app 已配置 backend: agentscope 的 as_decision_model.compaction 组件。 +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +YAML 中的 `model` 仍表示供应商模型名称；运行时 `model` 属性接收可调用的模型实例，读取属性不会创建客户端。 +在首次请求之前注入。ReMe 继续校验并转换 JSON 问题，但不再要求额外的 ReMe 凭据，也不会关闭借用的客户端。 +宿主应等待 ReMe 调用结束后自行关闭实例。替换 ReMe 自建模型或存在进行中请求的实例之前，须先关闭组件。 +关闭过程即使被取消，也会等待活动请求和自建客户端清理完成，再传播取消信号。组件关闭时会解除实例引用； +重新启动使用外部模型的组件前需要再次注入。独立运行时按凭据创建模型的行为保持不变。 diff --git a/reme/components/as_decision_model/agentscope.py b/reme/components/as_decision_model/agentscope.py index 2e8ff025..640e3a93 100644 --- a/reme/components/as_decision_model/agentscope.py +++ b/reme/components/as_decision_model/agentscope.py @@ -31,8 +31,8 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): """Wrap AgentScope's TypeSafe decision model with lazy provider loading. ``questions`` accepts AgentScope question objects or JSON dictionaries with - a ``type`` discriminator (``binary``, ``choice``, or ``score``). Only this - component's configuration supplies credentials and transport options. + a ``type`` discriminator (``binary``, ``choice``, or ``score``). Hosts may inject a borrowed model + through ``model``; otherwise this component constructs and owns its client. """ def __init__( @@ -53,10 +53,31 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): retry_delay=retry_delay, ) self._model: "ClassifierModelBase | None" = None + self._owns_model = False self._active_calls = 0 self._idle = asyncio.Event() self._idle.set() + @property + def model(self) -> "ClassifierModelBase | None": + """Return the current instance without constructing a provider client.""" + return self._model + + @model.setter + def model(self, value: "ClassifierModelBase | None") -> None: + """Borrow a host-owned instance, including via Application.update_component. + + Close the component before replacing an internally owned client. An + active request must finish before any replacement or detachment. + """ + if value is self._model: + return + if value is not None and not callable(value): + raise TypeError("An injected decision model must be callable.") + if self._active_calls or self._owns_model: + raise RuntimeError("Close the decision component before replacing its active or owned model.") + self._model = value + @staticmethod def _load_decision_model(): """Load the optional AgentScope adapter on the first decision request.""" @@ -102,6 +123,7 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): credential=credential_cls(**credential), **self.options.model_dump(exclude_none=True), ) + self._owns_model = True except ImportError as exc: raise ImportError( "Decision model inference requires a compatible typesafe-sdk. " @@ -120,10 +142,24 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): self._idle.set() async def _close(self) -> None: + """Finish draining and cleanup before propagating shutdown cancellation.""" + cleanup = asyncio.create_task(self._release_model()) + cancellation = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as exc: + # Keep the lifecycle lock until cleanup finishes, even after + # repeated cancellation. No cleanup task outlives close(). + cancellation = exc + cleanup.result() + if cancellation is not None: + raise cancellation + + async def _release_model(self) -> None: """Release the owned TypeSafe client and allow a later fresh start.""" - try: - await self._idle.wait() - finally: - model, self._model = self._model, None - if model is not None: - await model.client.aclose() + await self._idle.wait() + model, owned = self._model, self._owns_model + self._model, self._owns_model = None, False + if model is not None and owned: + await model.client.aclose() diff --git a/tests/unit/test_decision_model.py b/tests/unit/test_decision_model.py index 1ec36779..1a4c3beb 100644 --- a/tests/unit/test_decision_model.py +++ b/tests/unit/test_decision_model.py @@ -238,6 +238,88 @@ def test_failed_or_cancelled_call_does_not_block_close(provider, failure): asyncio.run(run()) +@pytest.mark.parametrize("borrowed", [False, True]) +def test_cancelled_close_drains_requests_and_releases_model(provider, borrowed): + """Shutdown defers cancellation until active calls and model cleanup finish.""" + _, model = provider + + async def run(): + started, release = asyncio.Event(), asyncio.Event() + + async def reply(**_kwargs): + started.set() + await release.wait() + + model.side_effect = reply + component = AgentScopeAsDecisionModel(credential={"api_key": "test-key"}) + if borrowed: + component.model = model + call = asyncio.create_task(component(state="hello", questions={"q": {"type": "binary"}})) + await asyncio.wait_for(started.wait(), 2) + close = asyncio.create_task(component.close()) + await asyncio.sleep(0) + try: + for _ in range(2): + close.cancel() + await asyncio.sleep(0) + assert not close.done() + assert component.is_started + assert component.model is model + model.client.aclose.assert_not_awaited() + release.set() + await asyncio.wait_for(call, 2) + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(close, 2) + assert component.model is None + assert not component.is_started + assert not component._owns_model + await component.close() + assert model.client.aclose.await_count == (0 if borrowed else 1) + finally: + release.set() + await asyncio.gather(call, close, return_exceptions=True) + + asyncio.run(run()) + + +def test_cancelled_close_finishes_owned_client_cleanup(provider): + """Repeated shutdown cancellation cannot interrupt the client's own close.""" + _, model = provider + + async def run(): + closing, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def close_client(): + closing.set() + await release.wait() + finished.set() + + model.client.aclose.side_effect = close_client + component = AgentScopeAsDecisionModel(credential={"api_key": "test-key"}) + await component(state="hello", questions={"q": {"type": "binary"}}) + close = asyncio.create_task(component.close()) + await asyncio.wait_for(closing.wait(), 2) + try: + for _ in range(2): + close.cancel() + await asyncio.sleep(0) + assert not close.done() + assert component.is_started + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(close, 2) + assert finished.is_set() + assert component.model is None + assert not component.is_started + await component.close() + model.client.aclose.assert_awaited_once() + finally: + release.set() + await asyncio.gather(close, return_exceptions=True) + + asyncio.run(run()) + + def test_client_close_error_clears_cached_model(provider): """A failed close cannot leave a stale client selected on the next start.""" factory, model = provider @@ -350,3 +432,80 @@ def test_adapter_uses_sdk_default_or_explicit_model(monkeypatch, model_name): model.client.aclose.assert_awaited_once() asyncio.run(run()) + + +def test_injected_model_is_borrowed_and_accepts_json_questions(provider, tmp_path): + """Application injection shares the host instance without owning its client.""" + from reme.application import Application + + factory, model = provider + + async def run(): + app = Application( + workspace_dir=str(tmp_path), + enable_logo=False, + log_to_console=False, + log_to_file=False, + service={"backend": "http"}, + components={"as_decision_model": {"compaction": {"backend": "agentscope"}}}, + ) + component = await app.update_component("as_decision_model", "compaction", model=model) + assert component.model is model + async with app: + await component(state="hello", questions={"q": {"type": "binary"}}) + factory.assert_not_called() + model.assert_awaited_once() + model.client.aclose.assert_not_awaited() + assert component.model is None + + asyncio.run(run()) + + +def test_model_replacement_guards_owned_clients(provider): + """Replacing an owned client requires an explicit close to prevent leaks.""" + _, model = provider + + async def run(): + component = AgentScopeAsDecisionModel(credential={"api_key": "test-key"}) + await component(state="hello", questions={"q": {"type": "binary"}}) + component.model = model + with pytest.raises(RuntimeError, match="Close"): + component.model = None + await component.close() + component.model = model + await component.close() + # The borrowed instance is never closed a second time. + model.client.aclose.assert_awaited_once() + with pytest.raises(TypeError, match="callable"): + component.model = "model-name" + + asyncio.run(run()) + + +def test_borrowed_model_cannot_be_replaced_while_in_use(provider): + """Host ownership still obeys the component's in-flight request boundary.""" + _, model = provider + + async def run(): + started, release = asyncio.Event(), asyncio.Event() + + async def reply(**_kwargs): + started.set() + await release.wait() + + model.side_effect = reply + component = AgentScopeAsDecisionModel() + component.model = model + task = asyncio.create_task(component(state="hello", questions={"q": {"type": "binary"}})) + await asyncio.wait_for(started.wait(), 2) + with pytest.raises(RuntimeError, match="Close"): + component.model = None + close = asyncio.create_task(component.close()) + await asyncio.sleep(0) + assert not close.done() + release.set() + await task + await close + model.client.aclose.assert_not_awaited() + + asyncio.run(run()) From 831fa82669fcf021d0a4a68b89ac8e6261973d58 Mon Sep 17 00:00:00 2001 From: DiegoCluv7 <161216254+iluv7@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:13:04 +0800 Subject: [PATCH 02/19] fix(auto-dream): validate extraction before checkpointing inputs (#649) * fix(auto-dream): validate extraction before checkpointing inputs * fix(auto-dream): normalize bucket whitespace before validation --- reme/steps/evolve/dream/extract.py | 91 ++++++------ reme/steps/evolve/dream/extract.yaml | 59 ++++++-- reme/steps/evolve/dream/finish.py | 5 +- reme/steps/evolve/dream/integrate.py | 2 +- reme/steps/evolve/dream/utils.py | 14 +- tests/unit/test_auto_dream.py | 213 +++++++++++++++++++++++---- 6 files changed, 298 insertions(+), 86 deletions(-) diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index 59f6b9c9..d3d4c3b7 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -7,7 +7,7 @@ from ...file_io import refresh_day_index from .._evolve import agent_reply_result_text from ....components import R from ....enumeration import DreamBucketEnum -from ....schema import DreamState +from ....schema import DreamExtractOutput, DreamState from .utils import ( clean_paths, daily_dir, @@ -113,32 +113,43 @@ class DreamExtractStep(BaseStep): return self._finish(state, False, state.errors[-1]) self.logger.info(f"[{self.name}] agent start changed={len(changed)} dates={len(dates)}") - raw_result, meta = "", {} + user_message = self.prompt_format( + "extract_user_message", + date=day, + dates_json=json.dumps(dates, ensure_ascii=False, indent=2), + hint=hint or "(none)", + max_units=max_units, + changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), + material_blob=pack_paths(workspace, changed), + ) + retry_feedback = "" + output_schema = DreamExtractOutput.model_json_schema() + output_schema["required"] = ["units"] for attempt in range(2): try: result = await self.agent_wrapper.reply( - self.prompt_format( - "extract_user_message", - date=day, - dates_json=json.dumps(dates, ensure_ascii=False, indent=2), - hint=hint or "(none)", - max_units=max_units, - changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), - material_blob=pack_paths(workspace, changed), - ), + user_message + retry_feedback, system_prompt=self.prompt_format( "extract_system_prompt", workspace_dir=str(workspace), buckets=", ".join(bucket.value for bucket in DreamBucketEnum), max_units=max_units, + output_schema_json=json.dumps(output_schema, ensure_ascii=False, indent=2), ), job_tools=list(_TOOLS), ) self.logger.info(f"[{self.name}] agent done has_result={bool(result.get('result'))}") raw_result = agent_reply_result_text(result) - meta = parse_structured_reply(raw_result) + state.extract_summary = raw_result + meta = parse_structured_reply(raw_result, strict=True) + self.clean_output(state, meta, max_units=max_units) + break except Exception as e: # noqa: BLE001 if attempt == 0: + retry_feedback = "\n\n" + self.prompt_format( + "extract_retry_message", + error=f"{type(e).__name__}: {e}", + ) self.logger.warning( f"[{self.name}] extract attempt 1 returned no usable receipt; retrying once: " f"{type(e).__name__}: {e}", @@ -150,20 +161,7 @@ class DreamExtractStep(BaseStep): self.logger.error(f"[{self.name}] {error}") return self._finish(state, False, error) - units = meta.get("units") if "units" in meta else meta.get("memory_units") - if isinstance(units, list): - break - if attempt == 0: - self.logger.warning(f"[{self.name}] extract attempt 1 returned an unusable receipt; retrying once") - continue - # Keep the warning-only result checkpointable after one retry so a bad source cannot loop forever. - warning = "dream extract skipped unusable agent receipt after retry; expected a units list" - state.warnings.append(warning) - self.logger.warning(f"[{self.name}] {warning}") - self.logger.info(f"[{self.name}] parse done keys={','.join(sorted(meta.keys())) if meta else '(none)'}") - self.clean_output(state, meta, max_units=max_units) - state.extract_summary = raw_result answer = f"Extracted {len(state.units)} unit(s) from {len(changed)} changed file(s) across {len(dates)} day(s)" return self._finish(state, True, answer) @@ -177,25 +175,34 @@ class DreamExtractStep(BaseStep): return out def clean_output(self, state: DreamState, meta: dict, max_units: int | None = None) -> None: - """Clean up output""" + """Validate the whole receipt before accepting any units or applying the cap.""" + units = meta.get("units") if "units" in meta else meta.get("memory_units") + if not isinstance(units, list): + raise ValueError("expected a units list (use units: [] only when nothing is worth remembering)") + normalized = [] + for raw in units: + if isinstance(raw, dict) and isinstance(raw.get("bucket"), str): + raw = {**raw, "bucket": raw["bucket"].strip()} + if ( + isinstance(raw, dict) + and "bucket" in raw + and raw["bucket"] not in {bucket.value for bucket in DreamBucketEnum} + ): + self.logger.warning( + f"[{self.name}] unit {raw.get('name')!r} emitted an unknown bucket; routing to wiki", + ) + raw = {**raw, "bucket": DreamBucketEnum.WIKI.value} + normalized.append(raw) + output = DreamExtractOutput.model_validate({"units": normalized}) allowed = set(state.changed_paths) - for raw in meta.get("units") or meta.get("memory_units") or []: - if max_units is not None and len(state.units) >= max_units: - break - if not isinstance(raw, dict): - continue - name = str(raw.get("name") or "").strip() - summary = str(raw.get("summary") or "").strip() - raw_bucket = str(raw.get("bucket") or "").strip() - paths = clean_paths(raw.get("paths"), allowed) + cleaned = [] + for index, unit in enumerate(output.units): + name, summary = unit.name.strip(), unit.summary.strip() + paths = clean_paths(unit.paths, allowed) if not name or not summary or not paths: - continue - try: - bucket = DreamBucketEnum(raw_bucket).value - except ValueError: - self.logger.warning(f"[{self.name}] unit {name!r} emitted bucket {raw_bucket!r}; routing to wiki") - bucket = DreamBucketEnum.WIKI.value - state.units.append({"name": name, "bucket": bucket, "summary": summary, "paths": paths}) + raise ValueError(f"units[{index}] requires non-empty name, summary, and paths from changed_paths") + cleaned.append({"name": name, "bucket": unit.bucket.value, "summary": summary, "paths": paths}) + state.units.extend(cleaned if max_units is None else cleaned[: max(max_units - len(state.units), 0)]) def _finish(self, state: DreamState, success: bool, answer: str): assert self.context is not None diff --git a/reme/steps/evolve/dream/extract.yaml b/reme/steps/evolve/dream/extract.yaml index b35847fe..2cdcb942 100644 --- a/reme/steps/evolve/dream/extract.yaml +++ b/reme/steps/evolve/dream/extract.yaml @@ -61,12 +61,20 @@ extract_system_prompt: | ## Output Format - Return only one YAML or JSON object with this exact shape: - units: - - name: - bucket: procedure|personal|wiki - summary: - paths: [, ...] + Return only one valid JSON object, without Markdown fences or surrounding prose. + The units field is required and must be a list. Every unit must contain name, + bucket, summary, and paths. name and summary must be non-empty strings; paths + must be a non-empty list of strings from changed_paths. + Double-quote all keys and string values, and escape embedded double quotes, + backslashes, and newlines according to JSON rules. Code-like text containing + `key: value` must stay inside a quoted string; never emit YAML plain scalars. + Return {{"units": []}} only when there is no reusable memory to extract. + + JSON schema: + {output_schema_json} + + Valid example (replace paths with actual changed_paths): + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout needs `submodules: recursive` to fetch nested dependencies.", "paths": ["daily/2026-10-20/ci.md"]}}]}} extract_system_prompt_zh: | 你是 dream 全局抽取 agent。一起阅读所有发生变化的 daily 文件，输出一个由合并记忆 unit 组成的精简跨文件计划。 @@ -118,12 +126,39 @@ extract_system_prompt_zh: | ## 输出格式 - 只返回一个 YAML 或 JSON object，结构必须严格如下： - units: - - name: <短名称> - bucket: procedure|personal|wiki - summary: <基于证据的可复用抽象> - paths: [, ...] + 只返回一个合法的 JSON object，不要使用 Markdown 代码围栏或添加解释文字。 + 必须包含 units 字段，且它必须是 list。每个 unit 必须包含 name、bucket、summary 和 paths。 + name 和 summary 必须是非空字符串；paths 必须是非空字符串列表，且只能使用 changed_paths 中的路径。 + 所有键和字符串值都必须使用双引号；内部双引号、反斜杠和换行必须按 JSON 规则转义。 + 含有 `key: value` 的代码文本必须放在带引号的字符串内，禁止输出 YAML 无引号标量。 + 只有在没有可复用记忆时才返回 {{"units": []}}。 + + JSON schema： + {output_schema_json} + + 合法示例（路径须替换为实际 changed_paths）： + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout 需要 `submodules: recursive`，以获取嵌套依赖。", "paths": ["daily/2026-10-20/ci.md"]}}]}} + +extract_retry_message: | + The previous extraction failed parsing or validation: + {error} + + Retry extraction from the changed material above and return a complete corrected + JSON object following the system prompt's schema. Include the units list and all + required unit fields. Double-quote every key and string value, including any + `key: value` text, and escape quotes, backslashes, and newlines. Do not use YAML, + Markdown fences, or explanatory prose. A format error does not mean there are no + reusable memories; use {{"units": []}} only when the material warrants it. + +extract_retry_message_zh: | + 上次抽取发生解析或校验错误： + {error} + + 请根据上面的变化内容重新抽取，按系统提示的 schema 返回完整、修正后的 JSON object。 + 必须包含 units 列表和每个 unit 的必填字段。所有键和字符串值都要加双引号， + 含 `key: value` 的文本也不例外；双引号、反斜杠和换行须转义。 + 不要使用 YAML、Markdown 代码围栏或解释文字。格式错误不代表没有可复用记忆； + 只有材料确实不值得记忆时才返回 {{"units": []}}。 extract_user_message: | date: {date} diff --git a/reme/steps/evolve/dream/finish.py b/reme/steps/evolve/dream/finish.py index 243a449a..980f93be 100644 --- a/reme/steps/evolve/dream/finish.py +++ b/reme/steps/evolve/dream/finish.py @@ -74,8 +74,11 @@ class DreamFinishStep(BaseStep): def render_summary(state: DreamState) -> str: """Render a concise user-facing summary.""" dates = ", ".join(state.dates or [state.date]) + title = "AutoDream failed" if state.errors else "AutoDream completed" + if state.warnings and not state.errors: + title += " with warnings" lines = [ - ("AutoDream completed with warnings" if state.warnings else "AutoDream completed"), + title, "", f"- Date: {state.date}", f"- Scan window: {dates}", diff --git a/reme/steps/evolve/dream/integrate.py b/reme/steps/evolve/dream/integrate.py index 12569e7e..f43dbea7 100644 --- a/reme/steps/evolve/dream/integrate.py +++ b/reme/steps/evolve/dream/integrate.py @@ -61,7 +61,7 @@ class DreamIntegrateStep(BaseStep): state = state_from_context(self) if not state.units: self.logger.info(f"[{self.name}] skip no units") - return self._finish(state, True, "No dream units to integrate") + return self._finish(state, not state.errors, "No dream units to integrate") if not llm_available(self): err = "no llm configured; dream integrate requires an LLM" state.errors.append(err) diff --git a/reme/steps/evolve/dream/utils.py b/reme/steps/evolve/dream/utils.py index 2f41d27d..bce5c135 100644 --- a/reme/steps/evolve/dream/utils.py +++ b/reme/steps/evolve/dream/utils.py @@ -134,19 +134,25 @@ def previous_dates(day: str, n_days: int) -> list[str]: return [(base - dt.timedelta(days=i)).isoformat() for i in range(1, max(n_days, 0) + 1)] -def parse_structured_reply(text: str) -> dict: - """Parse a JSON/YAML object from an agent reply, including fenced blocks.""" +def parse_structured_reply(text: str, *, strict: bool = False) -> dict: + """Parse a JSON/YAML object; strict mode rejects lossy fallbacks with diagnostics.""" candidates = [text.strip()] candidates.extend(m.group(1).strip() for m in re.finditer(r"```(?:json|ya?ml)?\s*(.*?)```", text, re.S | re.I)) + error = "expected a JSON/YAML object" for raw in candidates: if not raw: continue try: data = yaml.safe_load(raw) - except yaml.YAMLError: + except yaml.YAMLError as exc: + error = f"{type(exc).__name__}: {exc}" + if strict: + continue data = _parse_scalar_mapping(raw) - if isinstance(data, dict) and data: + if isinstance(data, dict) and (data or strict): return data + if strict: + raise ValueError(f"Invalid structured reply: {error}") return {} diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index 8662c1e8..a808b615 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -30,6 +30,12 @@ from reme.steps.evolve.dream.integrate import DreamIntegrateStep, _snapshot_dige from reme.steps.evolve.dream.utils import parse_structured_reply, recent_dates, scan_day_files from reme.steps.file_io.frontmatter_update import FrontmatterUpdateStep +_MALFORMED_EXTRACT_REPLY = ( + "units:\n- name: checkout\n bucket: procedure\n" + " summary: CI must add `with: submodules: recursive`\n" + " paths: [daily/2026-05-28/session.md]\n" +) + def _touch(path: Path, text: str = "x") -> Path: path.parent.mkdir(parents=True, exist_ok=True) @@ -116,9 +122,11 @@ class _SequenceAgent(BaseAgentWrapper): super().__init__() self.outcomes = list(outcomes) self.calls = 0 + self.requests = [] async def reply(self, _message, **_kwargs): self.calls += 1 + self.requests.append((_message, _kwargs)) outcome = self.outcomes.pop(0) if callable(outcome): outcome = outcome() @@ -245,6 +253,25 @@ def test_parse_structured_reply_handles_fenced_yaml_and_scalar_fallback(): assert data["note"].startswith("Extended node") +def test_extract_parser_preserves_yaml_diagnostics_and_accepts_quoted_colons(): + """Strict extraction rejects the reported YAML while legacy parsing stays compatible.""" + assert parse_structured_reply(_MALFORMED_EXTRACT_REPLY) == {} + with pytest.raises(ValueError, match="ScannerError") as exc: + parse_structured_reply(_MALFORMED_EXTRACT_REPLY, strict=True) + assert "line 4" in str(exc.value) + assert "submodules: recursive" in str(exc.value) + quoted = _MALFORMED_EXTRACT_REPLY.replace( + "summary: CI must add `with: submodules: recursive`", + 'summary: "CI must add `with: submodules: recursive`"', + ) + assert len(parse_structured_reply(quoted, strict=True)["units"]) == 1 + assert len(parse_structured_reply(f"```yaml\n{quoted}```", strict=True)["units"]) == 1 + assert parse_structured_reply("{}", strict=True) == {} + for reply in ("", "not an object", "[]"): + with pytest.raises(ValueError, match="expected a JSON/YAML object"): + parse_structured_reply(reply, strict=True) + + def test_integrate_prompts_require_wikilinks_in_contextual_prose(): """Every digest bucket rejects standalone relation fields and bare links.""" prompt_path = Path(__file__).parents[2] / "reme" / "steps" / "evolve" / "dream" / "integrate.yaml" @@ -277,37 +304,171 @@ def test_extract_clean_output_respects_max_units(): assert [unit["name"] for unit in state.units] == [f"unit-{i}" for i in range(5)] -def test_extract_unusable_receipt_is_a_warning_not_a_failure(tmp_path): - """Malformed best-effort extraction output does not fail the nightly job.""" +@pytest.mark.parametrize("field", ["units", "memory_units"]) +@pytest.mark.parametrize( + ("raw_bucket", "expected_bucket"), + [ + (" procedure ", "procedure"), + ("\tpersonal\n", "personal"), + (" wiki ", "wiki"), + (" unknown ", "wiki"), + (" \t\n ", "wiki"), + ], +) +def test_extract_validation_strips_bucket_whitespace_before_fallback(field, raw_bucket, expected_bucket): + """Padded known buckets retain their destination; unknown and blank values still use wiki.""" + state = DreamState(changed_paths=["daily/a.md"]) + unit = {"name": "unit", "bucket": raw_bucket, "summary": "memory", "paths": ["daily/a.md"]} - async def run(): - _touch(tmp_path / "daily" / "2026-05-28" / "session.md") - step = DreamExtractStep(scan_days=1, app_context=ApplicationContext(workspace_dir=str(tmp_path))) - step.agent_wrapper = _ReplyAgent() + DreamExtractStep().clean_output(state, {field: [unit]}) - with ( - patch("reme.steps.evolve.dream.extract.refresh_day_index", return_value={}), - patch("reme.steps.evolve.dream.extract.llm_available", return_value=True), - ): - response = await step( - RuntimeContext( - date="2026-05-28", - agent_wrapper=step.agent_wrapper, - file_catalog=_Catalog(), - file_store=_FileStore(tmp_path), - ), - ) + assert state.units == [{**unit, "bucket": expected_bucket}] + +def test_extract_validation_preserves_legacy_alias_and_unknown_bucket_fallback(): + """Validation retains the documented bucket fallback and source-path cleanup.""" + state = DreamState(changed_paths=["daily/a.md"]) + DreamExtractStep().clean_output( + state, + { + "memory_units": [ + { + "name": " unit ", + "bucket": "unknown", + "summary": " memory ", + "paths": ["daily/a.md", "daily/a.md", "outside.md"], + }, + ], + }, + ) + assert state.units == [{"name": "unit", "bucket": "wiki", "summary": "memory", "paths": ["daily/a.md"]}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("job_name", ["auto_dream", "dream_cron"]) +@pytest.mark.parametrize( + ("reply", "succeeds"), + [ + (_MALFORMED_EXTRACT_REPLY, False), + ("{}", False), + ('{"units": null}', False), + ('{"units": {}}', False), + ('{"units": [42]}', False), + ('{"units": [{"name": "missing-fields"}]}', False), + ( + '{"units": [{"name": "x", "summary": " ", "bucket": "wiki", "paths": ["daily/2026-05-28/session.md"]}]}', + False, + ), + ('{"units": [{"name": "x", "summary": "x", "bucket": "wiki", "paths": ["outside.md"]}]}', False), + ( + '{"units": [{"name": "valid", "summary": "x", "bucket": "wiki", ' + '"paths": ["daily/2026-05-28/session.md"]}, ' + '{"name": "invalid", "summary": "x", "bucket": "wiki", "paths": []}]}', + False, + ), + ('{"units": []}', True), + ("units: []", True), + ("memory_units: []", True), + ], +) +async def test_dream_extraction_failure_preserves_changed_inputs(tmp_path, monkeypatch, job_name, reply, succeeds): + """Invalid receipts fail both configured pipelines without checkpointing notes or day indexes.""" + + class Catalog(_Catalog): + """Expose versions checkpointed by Finish to the next job invocation.""" + + async def get_nodes(self, paths=None): + return self.upserts + + note = _touch(tmp_path / "daily/2026-05-28/session.md", "CI needs submodules: recursive") + _touch(tmp_path / "daily/2026-05-28.md", "day index") + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + catalog, agent = Catalog(), _ReplyAgent({"result": reply}) + config = resolve_app_config(config="default", log_config=False)["jobs"][job_name] + job = BaseJob(name=job_name, steps=config["steps"], app_context=app_context) + monkeypatch.setattr("reme.steps.evolve.dream.extract.refresh_day_index", AsyncMock(return_value={})) + monkeypatch.setattr("reme.steps.evolve.dream.extract.llm_available", lambda _step: True) + await job.start() + try: + inputs = { + "date": "2026-05-28", + "scan_days": 1, + "agent_wrapper": agent, + "file_catalog": catalog, + "file_store": _FileStore(tmp_path), + } + response = await job(**inputs) dream = response.metadata["dream"] - assert response.success is True + assert response.success is succeeds + assert response.answer.startswith("AutoDream completed" if succeeds else "AutoDream failed") assert dream["units"] == [] - assert dream["failed_paths"] == [] - assert dream["warnings"] == [ - "dream extract skipped unusable agent receipt after retry; expected a units list", - ] - assert step.agent_wrapper.calls == 2 + assert dream["files_changed"] == 2 + assert dream["failed_paths"] == ([] if succeeds else dream["changed_paths"]) + assert bool(dream["errors"]) is not succeeds + assert set(dream["checkpoint_paths"]) == (set(dream["changed_paths"]) if succeeds else set()) + assert catalog.dumps == int(succeeds) + assert agent.calls == (1 if succeeds else 2) + assert note.read_text(encoding="utf-8") == "CI needs submodules: recursive" - asyncio.run(run()) + agent.result = {"result": '{"units": []}'} + response = await job(**inputs) + assert response.success is True + assert response.metadata["dream"]["files_changed"] == (0 if succeeds else 2) + assert agent.calls == (1 if succeeds else 3) + assert catalog.dumps == (2 if succeeds else 1) + finally: + await job.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("language", ["en", "zh"]) +@pytest.mark.parametrize( + ("first_reply", "error"), + [(_MALFORMED_EXTRACT_REPLY, "ScannerError"), ('{"units": [{"name": "missing-fields"}]}', "ValidationError")], +) +async def test_extract_retry_includes_parser_error_and_recovers_json(tmp_path, language, first_reply, error): + """Both localized prompts recover code-like strings using JSON and concrete error feedback.""" + source = "daily/2026-05-28/session.md" + _touch(tmp_path / source) + unit = { + "name": "checkout", + "bucket": "procedure", + "summary": 'CI needs `submodules: recursive`, "quoted" text, and \\paths.\nKeep dependencies.', + "paths": [source], + } + agent = _SequenceAgent({"result": first_reply}, {"result": json.dumps({"units": [unit]})}) + step = DreamExtractStep( + scan_days=1, + language=language, + app_context=ApplicationContext(workspace_dir=str(tmp_path)), + agent_wrapper=agent, + ) + with ( + patch("reme.steps.evolve.dream.extract.refresh_day_index", return_value={}), + patch("reme.steps.evolve.dream.extract.llm_available", return_value=True), + ): + response = await step( + RuntimeContext(date="2026-05-28", file_catalog=_Catalog(), file_store=_FileStore(tmp_path)), + ) + + assert response.success is True + assert response.metadata["dream"]["units"] == [unit] + assert not response.metadata["dream"]["errors"] + first_message, first_kwargs = agent.requests[0] + retry_message, retry_kwargs = agent.requests[1] + assert retry_message.startswith(first_message) + assert error in retry_message + if error == "ScannerError": + assert "line 4" in retry_message + else: + assert "Field required" in retry_message + assert "key: value" in retry_message + assert retry_kwargs["system_prompt"] == first_kwargs["system_prompt"] + system_prompt = first_kwargs["system_prompt"] + assert "JSON schema" in system_prompt + assert "YAML or JSON" not in system_prompt + example = json.loads(system_prompt.splitlines()[-1]) + assert "submodules: recursive" in example["units"][0]["summary"] def test_extract_retries_one_unusable_receipt(tmp_path): @@ -661,7 +822,7 @@ async def test_dream_jobs_tag_outputs_and_preserve_checkpoint_results( await job.close() assert response.success is not integrate_fails - assert response.answer.startswith("AutoDream completed") + assert response.answer.startswith("AutoDream failed" if integrate_fails else "AutoDream completed") assert response.metadata["modified"] is True dream, tagging = response.metadata["dream"], response.metadata["auto_tag"] assert (source in dream["checkpoint_paths"]) is not integrate_fails From beda564def3c60871fea8d7ca3c34936978a13c4 Mon Sep 17 00:00:00 2001 From: WQS Date: Wed, 21 Oct 2026 16:31:47 +0800 Subject: [PATCH 03/19] docs: add MemLens-256K results to the multimodal memory blog (#639) * docs: report multimodal memory results on MemLens-256K * docs: simplify MemLens results with LLM-Judge charts * docs: disclose MemLens evaluation settings and evidence * docs: streamline MemLens evaluation report * docs: clarify integrated benchmark reproduction results * docs: pin MemLens reproduction experiment commit * docs: reference public MemLens evaluation commit * test(docs): validate exact evaluation link destinations * test(docs): simplify evaluation link checks * test(docs): verify Judge settings and visible chart scores --------- Co-authored-by: jinli.yl --- docs/en/blog_20261009.md | 12 ++ docs/en/memlens_256k_evaluation.md | 68 ++++++++++ docs/en/reme-blog.md | 2 + docs/figure/memlens-256k-qwen36-judge.svg | 56 ++++++++ docs/figure/memlens-256k-qwen38-judge.svg | 56 ++++++++ docs/zh/blog_20261009.md | 12 ++ docs/zh/memlens_256k_evaluation.md | 68 ++++++++++ docs/zh/reme-blog.md | 2 + .../tests/memlens-evaluation.test.mjs | 122 ++++++++++++++++++ 9 files changed, 398 insertions(+) create mode 100644 docs/en/memlens_256k_evaluation.md create mode 100644 docs/figure/memlens-256k-qwen36-judge.svg create mode 100644 docs/figure/memlens-256k-qwen38-judge.svg create mode 100644 docs/zh/memlens_256k_evaluation.md create mode 100644 github-pages/tests/memlens-evaluation.test.mjs diff --git a/docs/en/blog_20261009.md b/docs/en/blog_20261009.md index aaec0870..86f41bf4 100644 --- a/docs/en/blog_20261009.md +++ b/docs/en/blog_20261009.md @@ -79,6 +79,18 @@ Image resources then participate in the existing retrieval workflow. A later sea Auto Resource supports PNG, JPG/JPEG, WebP, GIF, BMP, TIFF, and HEIC. Larger images may be resized or converted for the model request; the original resource files remain unchanged. HEIC requires the optional `reme-ai[image-heif]` package. +## Performance: Multimodal Long-History QA on MemLens-256K {#memlens-256k-results} + +On MemLens-256K, ReMe Image achieved LLM-Judge scores of **53.44%** with Qwen3.6-35B-A3B and **72.49%** with Qwen3.8-Max, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +![MemLens-256K LLM-Judge with Qwen3.6-35B-A3B: Full Context Image 47.09%, ReMe Text 41.27%, ReMe Image 53.44%, OpenViking 37.04%, Mem0 38.62%.](../figure/memlens-256k-qwen36-judge.svg) + +![MemLens-256K LLM-Judge with Qwen3.8-Max: Full Context Image 65.61%, ReMe Text 52.91%, ReMe Image 72.49%, OpenViking 42.33%, Mem0 51.32%.](../figure/memlens-256k-qwen38-judge.svg) + +The comparison uses the official LLM-Judge protocol to evaluate each complete system. Six cases triggered model-provider content moderation in at least one method, so all methods use the same filtered set of **189/195** cases. + +ReMe can retrieve original conversations, and Image can reopen source images; OpenViking retrieval in this evaluation is restricted to memory. See the [evaluation notes](./memlens_256k_evaluation.md) for experimental settings, exclusions and the reproduction comparison. + ## Getting Started Both paths require a vision model that supports image input and tool calling, along with a compatible AgentScope wrapper and formatter. The selected wrapper's `as_llm` binding determines which model is used. See [Configuration](./configuration.md) for credential setup. diff --git a/docs/en/memlens_256k_evaluation.md b/docs/en/memlens_256k_evaluation.md new file mode 100644 index 00000000..938a75f3 --- /dev/null +++ b/docs/en/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K Evaluation Notes + +This report describes the experimental settings, data selection and results of the MemLens-256K evaluation in the [multimodal memory article](./blog_20261009.md#memlens-256k-results). + +## Results and Evaluation Set {#results} + +The evaluation uses the 195 questions in the official [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) `agent_subset_195.json`, each with its complete 256K history. After excluding the six questions listed below, all methods are evaluated on the same 189 questions. The official category/subtype-specific LLM-Judge protocol assigns each answer a score of 0 or 1; the reported percentage is `100 × correct / 189`. + +| Model | Method | Judge correct / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image scores **53.44%** and **72.49%** on the two models, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +## Method and Judge Settings {#evaluation-settings} + +| Method | Memory input and answer-time data access | +| --- | --- | +| Full Context Image | The answer model receives the complete conversation text and original images directly. | +| ReMe Text | Builds memory from the original conversation text. The answer Agent uses BM25 to search daily notes, digest notes and original sessions, then reads relevant text. | +| ReMe Image | Builds memory from the original conversation text and images. Retrieval covers the same sources as Text, with `read_image` available to reopen saved original images. | +| OpenViking | Uses native image descriptions and memory processing. Answer-time retrieval is restricted to memory with the default top 10. | +| Mem0 | Builds memory through Mem0 2.0.4's native vision interface. Answer-time retrieval uses FAISS to select the top 20 memories. | + +Each method is evaluated with Qwen3.6-35B-A3B and Qwen3.8-Max as the candidate models. ReMe Image and Mem0 both use original images. ReMe's memory and answer context limits are 200,000 and 400,000 tokens, with temperature 0 and thinking enabled. Memory/compression and answer output budgets are 65,536 and 16,384 tokens. Each question uses an independent answer Agent with at most 30 iterations and native context compression. The `read_image` adapter accesses saved attachments with a 50 MiB read limit and a 4,096-image retention limit. + +Full Context, OpenViking and Mem0 also use temperature 0, thinking enabled and a 16,384-token output budget for answering. Mem0's memory stage uses its native settings: a 2,000-token output budget, temperature 0.1 and top_p 0.1. The OpenViking evaluation adapter includes a JSON parsing correction and explicit error reporting for image-processing failures. + +Scoring follows the [official MemLens protocol](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2). All methods use **Qwen3.8-Max** as Judge, with temperature **0**, thinking **disabled** and a **256-token** output budget. The Judge runs independently of candidate memory applications; reference answers and question-type labels are used only for scoring. + +## Common Content-Moderation Filter {#content-moderation-filter} + +The method/question combinations below triggered content moderation when calling the Qwen model service. Taking the union of affected questions across all methods and both models gives six questions. We exclude this same set from every method and model, leaving **189/195** questions for evaluation. + +In the table, A3B denotes Qwen3.6-35B-A3B and Max denotes Qwen3.8-Max. + +| Question ID | Method / model that triggered moderation | Stage | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | Memory construction | +| `q_1d99a36c` | OpenViking / A3B | Memory construction | +| `q_3b01c02a` | ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B | Memory construction | +| `q_564a657b` | ReMe Image / Max | Answering | +| `q_600a79aa` | ReMe Image / Max | Memory construction | +| `q_c9f8e24d` | ReMe Text / Max | Answering | + +## Independent Reproduction Results {#independent-reproduction} + +We organized and integrated the MemLens benchmark code into ReMe, then independently reran ReMe Image using the same data inputs and scoring settings. See the [run instructions](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md), [method configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml), [Judge configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml) and [Judge scoring rules](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py) for the implementation and settings. + +The corresponding public evaluation code is available in [PR #640](https://github.com/agentscope-ai/ReMe/pull/640) at commit [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8). + +After applying the same six-question filter, one A3B question and two Max questions remain incomplete because conversation-history processing failed. The table compares the 188 and 187 questions completed in both experiments, respectively. + +| Model | Paired questions | Original ReMe Image correct | Independent reproduction correct | Original → reproduction LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +On these matched questions, the reproduced LLM-Judge scores for Qwen3.6-35B-A3B and Qwen3.8-Max are 2.13 and 0.53 percentage points lower than the original results, respectively. Overall, performance remains broadly consistent with the original experiment. diff --git a/docs/en/reme-blog.md b/docs/en/reme-blog.md index 1e7f3cf1..6267b7e3 100644 --- a/docs/en/reme-blog.md +++ b/docs/en/reme-blog.md @@ -276,6 +276,8 @@ BEAM covers ten categories of tasks, including contradiction resolution, event o ReMe also uses $\pi$-Bench to evaluate the potential of multi-session reasoning to improve agent proactivity. The PROC score in $\pi$-Bench evaluates capabilities including directly fulfilling hidden intent, guiding targeted clarification, recovering cross-session preferences, reusing cross-session conventions, inferring cross-task dependencies, and advancing underspecified requests. Across five user personas, ReMe Agent achieved an average PROC score of 0.580, outperforming NanoBot by 2.4% under the same test-model configuration. See the [$\pi$-Bench paper](https://arxiv.org/abs/2605.14678) for details about the benchmark. +On **MemLens-256K**, ReMe Image achieved LLM-Judge scores of **53.44% / 72.49%** with Qwen3.6-35B-A3B / Qwen3.8-Max, exceeding Full Context Image by **6.35 / 6.88** percentage points. All methods use the same **189/195** cases after filtering six cases that triggered model-provider content moderation. See the [multimodal memory evaluation](./blog_20261009.md#memlens-256k-results) for the five-method comparison. + ## Who Is ReMe For? ### People Who Use Agents Directly diff --git a/docs/figure/memlens-256k-qwen36-judge.svg b/docs/figure/memlens-256k-qwen36-judge.svg new file mode 100644 index 00000000..75a02882 --- /dev/null +++ b/docs/figure/memlens-256k-qwen36-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.6-35B-A3B + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 47.09%; ReMe Text: 41.27%; ReMe Image: 53.44%; OpenViking: 37.04%; Mem0: 38.62%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.6-35B-A3B + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 47.09% + Full ContextImage + + + + 41.27% + ReMe Text + + + + 53.44% + ReMe Image + + + + 37.04% + OpenViking + + + + 38.62% + Mem0 + + diff --git a/docs/figure/memlens-256k-qwen38-judge.svg b/docs/figure/memlens-256k-qwen38-judge.svg new file mode 100644 index 00000000..3b5a0800 --- /dev/null +++ b/docs/figure/memlens-256k-qwen38-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.8-Max + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 65.61%; ReMe Text: 52.91%; ReMe Image: 72.49%; OpenViking: 42.33%; Mem0: 51.32%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.8-Max + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 65.61% + Full ContextImage + + + + 52.91% + ReMe Text + + + + 72.49% + ReMe Image + + + + 42.33% + OpenViking + + + + 51.32% + Mem0 + + diff --git a/docs/zh/blog_20261009.md b/docs/zh/blog_20261009.md index 61b5a82d..49438192 100644 --- a/docs/zh/blog_20261009.md +++ b/docs/zh/blog_20261009.md @@ -78,6 +78,18 @@ source_resource: "[[resource/2026-10-09/system-overview.png]]" Auto Resource 支持 PNG、JPG/JPEG、WebP、GIF、BMP、TIFF 和 HEIC。较大的图片会在模型请求中按需缩小或转换格式，原始资源文件保持不变。HEIC 需要额外安装 `reme-ai[image-heif]`。 +## Performance：MemLens-256K 多模态长历史问答 {#memlens-256k-results} + +在 MemLens-256K 上，ReMe Image 的 LLM-Judge 得分在 Qwen3.6-35B-A3B 和 Qwen3.8-Max 下分别达到 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +![Qwen3.6-35B-A3B 的 MemLens-256K LLM-Judge 对比：Full Context Image 47.09%，ReMe Text 41.27%，ReMe Image 53.44%，OpenViking 37.04%，Mem0 38.62%。](../figure/memlens-256k-qwen36-judge.svg) + +![Qwen3.8-Max 的 MemLens-256K LLM-Judge 对比：Full Context Image 65.61%，ReMe Text 52.91%，ReMe Image 72.49%，OpenViking 42.33%，Mem0 51.32%。](../figure/memlens-256k-qwen38-judge.svg) + +评测使用官方 LLM-Judge 协议，比较各方法完整系统的表现。6 个 case 在至少一种方法中触发了模型服务商的内容安全审核，因此对所有方法统一过滤，汇报其余 **189/195** 个 case 的结果。 + +ReMe 可以检索原始会话，Image 还可重新读取原图；本次 OpenViking 的检索范围限定为 memory。实验设置、过滤清单和复现对比见[详细评测说明](./memlens_256k_evaluation.md)。 + ## 如何开始使用 两个入口都需要支持图片和工具调用的视觉模型，以及兼容的 AgentScope wrapper 和 formatter。模型由所选 wrapper 的 `as_llm` 绑定决定。凭证配置见[基础配置](./configuration.md)。 diff --git a/docs/zh/memlens_256k_evaluation.md b/docs/zh/memlens_256k_evaluation.md new file mode 100644 index 00000000..234bf50f --- /dev/null +++ b/docs/zh/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K 评测说明 + +本文介绍[多模态记忆文章](./blog_20261009.md#memlens-256k-results)中 MemLens-256K 评测的实验设置、数据筛选和结果。 + +## 结果与统计范围 {#results} + +评测使用 [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) 官方 `agent_subset_195.json` 中的 195 个问题，每题使用对应的完整 256K 历史。按下文说明过滤六个问题后，所有方法在相同的 189 题上评测。LLM-Judge 按官方类别／子类别评分协议为每题给出 0 或 1 分，得分为 `100 × 正确数 / 189`。 + +| 模型 | 方法 | Judge 正确数 / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image 在两个模型上的得分分别为 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +## 方法与 Judge 设置 {#evaluation-settings} + +| 方法 | 记忆输入与回答阶段的数据访问范围 | +| --- | --- | +| Full Context Image | 回答模型直接读取完整历史文字与原图。 | +| ReMe Text | 使用原始会话文字生成记忆；回答 Agent 通过 BM25 检索 daily、digest 和原始 session，并读取相关文本。 | +| ReMe Image | 使用原始会话文字和图片生成记忆；检索范围与 Text 相同，并可通过 `read_image` 重新读取保存的原图。 | +| OpenViking | 使用原生图片描述和记忆流程；回答阶段检索限定为 memory，采用默认 top 10。 | +| Mem0 | 使用 Mem0 2.0.4 原生视觉入口生成记忆；回答阶段通过 FAISS 检索 top 20 记忆。 | + +各方法分别使用 Qwen3.6-35B-A3B 和 Qwen3.8-Max 作为待测模型。ReMe Image 与 Mem0 均直接使用原图。ReMe 的记忆和回答上下文上限分别为 200,000 / 400,000 tokens，temperature 为 0，开启 thinking；记忆／压缩与回答输出预算分别为 65,536 / 16,384 tokens。每题使用独立回答 Agent，最多运行 30 轮，并采用原生上下文压缩。Image 的 `read_image` 支持读取保存的附件，读取上限为 50 MiB，图片保留上限为 4,096 张。 + +Full Context、OpenViking 和 Mem0 的回答阶段同样使用 temperature 0、thinking 开启、输出预算 16,384 tokens。Mem0 记忆阶段采用原生设置：输出预算 2,000 tokens，temperature 和 top_p 均为 0.1。OpenViking 的评测适配包含 JSON 解析修正，以及图片处理失败时的显式报错。 + +评分采用 [MemLens 官方协议](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2)，所有方法统一使用 **Qwen3.8-Max** 作为 Judge，temperature 为 **0**、thinking **关闭**、输出预算为 **256 tokens**。Judge 与待测方法的记忆应用独立运行；参考答案及题型标签仅用于评分。 + +## 统一内容审核过滤 {#content-moderation-filter} + +调用 Qwen 模型服务时，下表中的方法在处理相应问题时触发了内容安全审核（绿网协议）。我们对各方法、两个模型触发审核的问题取并集，共得到六题，并在所有方法和两个模型中统一过滤，最终统计相同的 **189/195** 题。 + +表中 A3B 指 Qwen3.6-35B-A3B，Max 指 Qwen3.8-Max。 + +| 问题 ID | 触发内容审核的方法 / 模型 | 阶段 | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | 记忆构建 | +| `q_1d99a36c` | OpenViking / A3B | 记忆构建 | +| `q_3b01c02a` | ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B | 记忆构建 | +| `q_564a657b` | ReMe Image / Max | 回答 | +| `q_600a79aa` | ReMe Image / Max | 记忆构建 | +| `q_c9f8e24d` | ReMe Text / Max | 回答 | + +## 独立复现结果 {#independent-reproduction} + +我们将 MemLens benchmark 代码整理并集成到 ReMe 项目中，在相同的数据输入和评分设置下，对 ReMe Image 进行了独立复现。运行方式见[评测说明](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md)，具体参数见[方法配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml)、[Judge 配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml)和[Judge 评分规则](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py)。 + +独立复现对应的公开评测代码见 [PR #640](https://github.com/agentscope-ai/ReMe/pull/640)，commit 为 [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8)。 + +应用相同的六题过滤后，复现实验中 A3B 有 1 题、Max 有 2 题因会话历史处理失败未得到完整结果。下表分别在两次实验均完成的 188 题和 187 题上进行配对比较。 + +| 模型 | 配对题数 | 原实验正确数 | 独立复现正确数 | 原实验 → 独立复现 LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +在上述配对题目上，Qwen3.6-35B-A3B 和 Qwen3.8-Max 的独立复现 LLM-Judge 得分分别比原实验低 2.13 和 0.53 个百分点，整体性能与原实验基本保持一致。 diff --git a/docs/zh/reme-blog.md b/docs/zh/reme-blog.md index 0c4c3826..54ff919f 100644 --- a/docs/zh/reme-blog.md +++ b/docs/zh/reme-blog.md @@ -292,6 +292,8 @@ cases / 400 题上取得 66.1%，在 1M 设置下的 35 cases / 700 题上取得 Agent 在 5 种用户角色（User Persona）上平均取得 0.580 的 PROC 分数，超出相同测试模型配置的 NanoBot 2.4%。关于该基准的详细介绍见 [$\pi$-Bench 论文](https://arxiv.org/abs/2605.14678)。 +在 **MemLens-256K** 上，ReMe Image 的 LLM-Judge 在 Qwen3.6-35B-A3B / Qwen3.8-Max 下达到 **53.44% / 72.49%**，比 Full Context Image 高 **6.35 / 6.88** 个百分点。统一过滤 6 个触发模型服务商内容安全审核的 case 后，汇报 **189/195** 个 case；五种方法的对比见[多模态记忆评测](./blog_20261009.md#memlens-256k-results)。 + ## ReMe 能帮谁？ ### 直接使用 Agent 的人 diff --git a/github-pages/tests/memlens-evaluation.test.mjs b/github-pages/tests/memlens-evaluation.test.mjs new file mode 100644 index 00000000..a33406e9 --- /dev/null +++ b/github-pages/tests/memlens-evaluation.test.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const siteDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const generatedDir = path.join(siteDir, ".generated/site"); +const docsDir = path.resolve(siteDir, "../docs"); +const methods = ["Full Context Image", "ReMe Text", "ReMe Image", "OpenViking", "Mem0"]; +// Published binary verdict counts, not raw benchmark outputs or a second chart generator. +const models = [ + { name: "Qwen3.6-35B-A3B", figure: "qwen36", correct: [89, 78, 101, 70, 73] }, + { name: "Qwen3.8-Max", figure: "qwen38", correct: [124, 100, 137, 80, 97] }, +]; +const excluded = ["q_028cb832", "q_1d99a36c", "q_3b01c02a", "q_564a657b", "q_600a79aa", "q_c9f8e24d"]; +const publicRevision = "962d5cd8401c1d2be17f3bb95f8ac9c185672cd8"; +const reproductionLinks = [ + ...[ + "benchmark/memlens/README.md", + "plugins/memlens/src/reme_memlens/benchmark.yaml", + "plugins/memlens-judge/src/judge_memlens/plugin.yaml", + "plugins/memlens-judge/src/judge_memlens/protocol.py", + ].map((file) => `https://github.com/agentscope-ai/ReMe/blob/${publicRevision}/${file}`), + "https://github.com/agentscope-ai/ReMe/pull/640", + `https://github.com/agentscope-ai/ReMe/commit/${publicRevision}`, +]; + +test("MemLens chart labels and geometry match the published scores", async () => { + for (const model of models) { + const svg = await readFile(path.join(docsDir, "figure", `memlens-256k-${model.figure}-judge.svg`), "utf8"); + const bars = [...svg.matchAll(/\s*]*y="([\d.]+)"[^>]*height="([\d.]+)"[^>]*\/>\s*]*>([\d.]+)%<\/text>/g)]; + assert.equal(bars.length, methods.length); + for (const [index, method] of methods.entries()) { + const score = (100 * model.correct[index] / 189).toFixed(2); + assert.equal(bars[index][1], method); + assert.equal(bars[index][2], score); + assert.equal(bars[index][5], score, `${model.name}/${method}: visible score`); + assert.ok(Math.abs(Number(bars[index][4]) - Number(score) * 3) < 0.001); + assert.ok(Math.abs(Number(bars[index][3]) + Number(bars[index][4]) - 416) < 0.001); + } + } +}); + +test("MemLens bilingual notes and blogs match the published scores", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const rows = notes.split("\n").filter((line) => /^\| Qwen.*\| \d+ \/ 189 \|/.test(line)); + assert.equal(rows.length, 10); + for (const model of models) { + for (const [index, method] of methods.entries()) { + const correct = model.correct[index]; + const score = (100 * correct / 189).toFixed(2); + assert.ok(rows.includes(`| ${model.name} | ${method} | ${correct} / 189 | ${score} |`)); + assert.ok(blog.includes(`${method} ${score}%`), `${language}: alt text for ${model.name}/${method}`); + } + const gain = (100 * (model.correct[2] - model.correct[0]) / 189).toFixed(2); + assert.ok(blog.includes(`**${gain}**`)); + assert.ok(notes.includes(`**${gain}**`)); + } + } +}); + +test("MemLens notes disclose moderation filters, paired results and Judge settings", async () => { + const triggers = { + zh: [ + ["OpenViking / Max", "记忆构建"], + ["OpenViking / A3B", "记忆构建"], + ["ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B", "记忆构建"], + ["ReMe Image / Max", "回答"], + ["ReMe Image / Max", "记忆构建"], + ["ReMe Text / Max", "回答"], + ], + en: [ + ["OpenViking / Max", "Memory construction"], + ["OpenViking / A3B", "Memory construction"], + ["ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B", "Memory construction"], + ["ReMe Image / Max", "Answering"], + ["ReMe Image / Max", "Memory construction"], + ["ReMe Text / Max", "Answering"], + ], + }; + for (const language of ["en", "zh"]) { + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const filter = notes.split("{#content-moderation-filter}")[1].split("{#independent-reproduction}")[0]; + assert.deepEqual([...new Set(filter.match(/q_[0-9a-f]{8}/g))].sort(), excluded); + const rows = filter.split("\n").filter((line) => line.startsWith("| `q_")); + assert.deepEqual(rows, excluded.map((id, index) => { + const [method, stage] = triggers[language][index]; + return `| \`${id}\` | ${method} | ${stage} |`; + })); + assert.match(filter, /189\/195/); + assert.ok(notes.includes("| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 |")); + assert.ok(notes.includes("| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 |")); + const judge = notes.split("\n\n").find((paragraph) => /(?:as|作为) Judge/.test(paragraph)); + assert.ok(judge, `${language}: Judge settings must be documented`); + assert.match(judge, /\*\*Qwen3\.8-Max\*\*/); + assert.match(judge, /temperature (?:为 )?\*\*0\*\*/); + assert.match(judge, /thinking \*\*(?:disabled|关闭)\*\*/); + assert.match(judge, /\*\*256(?:-token| tokens)\*\*/); + assert.match(notes, /memory-only|restricted to memory|限定为 memory/); + } +}); + +test("MemLens notes pin the public evaluation revision without private audit details", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + assert.match(blog, /\]\(\.\/memlens_256k_evaluation\.md\)/); + assert.match(notes, /\]\(\.\/blog_20261009\.md#memlens-256k-results\)/); + assert.doesNotMatch(notes, /\/mnt\/|\/Users\/|docs\/_build|file:\/\//); + const reproduction = notes.split("{#independent-reproduction}")[1]; + const targets = [...reproduction.matchAll(/\]\(([^)\r\n]+)\)/g)].map(([, target]) => target); + assert.deepEqual(targets.sort(), [...reproductionLinks].sort()); + const prose = notes.replace(/\]\([^)]+\)/g, "]"); + assert.doesNotMatch(prose, /\b[a-f0-9]{40,64}\b/); + for (const anchor of ["results", "evaluation-settings", "content-moderation-filter", "independent-reproduction"]) { + assert.ok(notes.includes(`{#${anchor}}`)); + } + } +}); From 6fb402dcd123540e4caef4e0fcbde0fb4aea28f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8C=97=E6=B2=90=E5=9F=8E=E6=AD=8C?= <2690322907@qq.com> Date: Wed, 21 Oct 2026 16:32:49 +0800 Subject: [PATCH 04/19] fix(embedding): keep one bad item from dropping its whole batch (#579) * fix(embedding): keep one bad item from dropping its whole batch A provider-side request error rejects an entire batch, so a single over-limit item used to cost the vectors of every item beside it. QwenPaw #8040 shows two poison chunks turning into exactly twenty lost vectors, while the backfill loop still reported full success. Retry a rejected multi-item request one item at a time so only the item that actually fails is left without a vector. Single-item requests keep their previous behaviour, and provider-level failures (429 / quota) keep using the existing retry paths instead of being split. Also make the backfill loop report what it actually filled, and name the failing chunk ids in the final reindex error. `processed` counts attempts, so it claimed success after a partial loss. * fix(embedding): stop quota failures and transient item errors from multiplying Review follow-up on the two P2 findings in #579: * quota exhaustion no longer falls into the per-item split. The store now gives up explicitly when the provider reports insufficient_quota, including after the opt-in quota retries are spent, because no single item is to blame and splitting only multiplied the rejected requests (3 items -> 4 requests). * the per-item fallback goes back through `_call_with_retry`, so a 429, connection error or timeout on an isolated item keeps the existing backoff and retry policy instead of silently dropping that item's vector. A one-item request still cannot split again. Regression tests cover both findings: a quota failure stays a single request with the default policy and with exhausted quota retries, and a transient failure on one item of a split batch still ends with every healthy sibling embedded. python -m pytest tests/unit/test_local_embedding_store.py tests/unit/test_file_store_consistency.py -q -> 83 passed, 41 skipped * style(test): hoist the sleeps collector out of the transient-failure loop CI / Python quality failed on d5dc9b2: the pylint hook reported tests/unit/test_local_embedding_store.py:935:16: W0640: Cell variable sleeps defined in loop (cell-var-from-loop) `fake_sleep` closes over `sleeps` while the loop rebinds it on every iteration. The collector is now created once before the loop and cleared at the top of each iteration, which keeps the per-iteration assertion (`assert sleeps == [1.0]`) exactly as strict and removes the loop-scoped rebinding. Verification (repo hook arguments): pylint -> Your code has been rated at 10.00/10, exit 0 black --check --line-length=120 --target-version=py311 -> clean flake8 --extend-ignore=E203 --max-line-length=120 -> clean * fix(embedding): keep provider-wide failures out of the per-item split Review follow-up on the P2 raised against 4d8a3c5. `_call_with_retry` sent every unrecognized exception to `_embed_individually` whenever the request had more than one item. Only rate limiting and `insufficient_quota` were excluded, so a 401, a 403 or a 5xx still split a ten-item request into ten extra single-item requests, none of which can succeed: credentials, permissions and server outages are properties of the endpoint, not of any individual input. The split path is now gated on a dedicated classifier, `_is_provider_wide_failure()`, which recognizes 401/403 and 5xx by status code (with the OpenAI SDK class names as a fallback for exceptions that do not expose one). Transient server errors keep their retry policy through the existing backoff; the remaining provider-wide failures stop the loop and report unhealthy, like the quota branch. Input-dependent rejections such as the per-item context-length limit are unchanged and still isolate one item at a time. Regression coverage: test_provider_wide_credentials_errors_are_not_split_into_per_item_requests (401/403 -> a single request of size 10) and test_provider_wide_server_errors_are_retried_without_splitting (500 -> batch sizes [10, 10, 10] with sleeps [1.0, 2.0]). The existing context-length, quota, 429 and per-item transient-retry tests are retained. Local verification, using the repo's own hook arguments: pylint -> Your code has been rated at 10.00/10, exit 0 black -> 2 files would be left unchanged flake8 -> clean Classifier discriminator check (old vs new split decision, extracted from the production file and executed): 400 context-length -> SPLIT / SPLIT 401, 403, 500, 504 -> SPLIT / NO-SPLIT 429 and insufficient_quota keep their own earlier branches. * fix(embedding): retry SDK transport failures instead of splitting the batch Third review follow-up on #579. `_call_with_retry` only caught the built-in `TimeoutError`, `ConnectionError` and `OSError`. `openai.APIConnectionError` and `openai.APITimeoutError` do not inherit from any of them (they go APIError -> OpenAIError -> Exception), so they reached the generic handler: `_is_provider_wide_failure()` did not recognize them either, and a multi-item request fanned out into per-item requests that failed for exactly the same reason. The same gap meant a single-item request got no ReMe-level retry at all after a transport failure. Transport failures are now recognized by a dedicated classifier, `_is_transient_transport_failure()`, covering the built-in socket errors as well as the SDK/httpx exception types (matched across the MRO, so subclasses such as `APITimeoutError` are included) without importing the provider SDK. They retry the original request with the existing backoff and never split; the retry budget is unchanged. Regression coverage: - test_sdk_transport_errors_are_retried_without_splitting APIConnectionError / APITimeoutError -> batch sizes [10, 10, 10], sleeps [1.0, 2.0], then failure with no single-item requests. - test_sdk_transport_error_recovers_without_losing_the_batch one APITimeoutError then success -> [[1.0, 0.0]] * 10, batch sizes [10, 10]. - test_isolated_item_retries_an_sdk_transport_error context-length rejection, then APIConnectionError on one isolated item -> that item's vector is retained. The existing context-length, 401/403, 5xx, quota and rate-limit tests are retained. Local verification with the repo's own hook arguments: pylint -> Your code has been rated at 10.00/10, exit 0 black -> 2 files would be left unchanged flake8 -> clean Branch-order discriminator check (classifiers extracted from the production file and executed against each error type): case old new 400 context-length SPLIT SPLIT 401 / 403 SPLIT STOP 500 SPLIT RETRY SDK APIConnectionError SPLIT RETRY SDK APITimeoutError SPLIT RETRY builtin ConnectionError RETRY RETRY builtin TimeoutError RETRY RETRY 429 RETRY RETRY insufficient_quota STOP STOP * fix(embedding): treat 404 as a provider-wide failure too Fourth review follow-up on #579. The classifier added in c9448ae covered 401/403 and 5xx, but not 404. A missing model or a wrong embedding route answers with 404 (SDK `NotFoundError`), which no individual input can explain, so the batch still fell through to `_embed_individually()`: a ten-item request against a bad model name produced eleven requests, all failing for the same reason. 404 is now part of the provider-wide set (`_PROVIDER_WIDE_STATUS_CODES`) and `NotFoundError` joins the class-name fallback. Like 401/403 it is not transient, so the request is reported as failed once instead of being retried or split. The context-length `400` is deliberately left out of that set: an over-limit item is the one status a single input can explain by itself, so it has to stay isolatable. Regression coverage: - test_provider_wide_statuses_exclude_input_dependent_bad_request 401/403/404/500 -> provider-wide; an over-limit 400 -> not provider-wide. - test_provider_wide_not_found_errors_are_not_split 404 -> batch sizes [10], no sleeps, no single-item requests. The existing context-length isolation, 401/403, 5xx, quota, rate-limit and SDK transport tests are retained. Local verification with the repo's own hook arguments: pylint -> Your code has been rated at 10.00/10, exit 0 black -> 2 files would be left unchanged flake8 -> clean Branch-order discriminator check (classifiers extracted from the production file and executed against each error type): case old new 400 context-length SPLIT SPLIT 401 / 403 SPLIT STOP 404 NotFoundError SPLIT STOP 500 SPLIT RETRY SDK APIConnectionError SPLIT RETRY SDK APITimeoutError SPLIT RETRY builtin ConnectionError RETRY RETRY builtin TimeoutError RETRY RETRY 429 RETRY RETRY insufficient_quota STOP STOP * fix(embedding): classify every provider status shape and split only input errors The per-item fallback still multiplied service failures into one request per item. Both gaps come from the fallback this PR adds, so they are fixed here. Status extraction - `_http_status()` reads the status from `status_code`, an integer `code` (Google GenAI exposes the HTTP status there), `response.status_code` (httpx-style wrappers), and the structured `status_code` embedded in the message (AgentScope's DashScope embedding turns every non-200 response into a plain `RuntimeError`). - Only integers are read from `code`. A string `code` names an error (`insufficient_quota`, `rate_limit_exceeded`, `InvalidApiKey`); reading it as a status would misclassify quota and rate-limit errors. - `_is_provider_wide_failure`, `_is_transient_provider_failure`, `_is_rate_limited` and `_is_input_dependent_failure` all classify through it. Splitting - The generic fallback splits only when `_is_input_dependent_failure()` matches: a context-length rejection, the one error a single over-long item explains by itself. Anything unrecognized is provider-wide and is never answered with one request per item. - This narrows the isolation this PR adds. An input-dependent error that carries neither a status nor an input-length signal is no longer split, so it costs its batch instead of multiplying the requests. Erring towards the batch keeps one provider failure from becoming N failures and losing the healthy vectors anyway. - `_embed_individually()` drops the redundant `len(single) == 1` check: the `_call_with_retry([text])` contract only returns a result of the expected length. Verified against the real exception classes: `google.genai.errors.APIError.code` is an `int`, and a real DashScope response yields a `RuntimeError` whose message carries `"status_code"`. Regression coverage uses the real classes when available and mirrors their shape otherwise. * fix(embedding): stop per-item isolation on a provider-wide failure and read serialized codes - `_embed_individually` now stops asking for the remaining items as soon as one isolated item fails provider-wide (credentials, missing model, exhausted quota, exhausted transient retries). No remaining input can explain that failure, so continuing only multiplies the rejected requests. The vectors already recovered are kept and the items that were never answered stay `None`. - `_call_with_retry` reports that outcome through an optional `terminal_failure` out-parameter, so its return type and every existing call site stay unchanged. - `_error_code` reads the business error code from the attribute, the structured body, or the serialized message. AgentScope's DashScope embedding raises a plain `RuntimeError` with the response embedded in the message, so a quota error was classified as a rate limit and never used the configured `quota_retry_delay`. Regression tests cover both halves of the isolation boundary (a provider-wide failure stops the loop; an input-dependent failure does not) and the serialized DashScope quota path with quota retries enabled. * fix(embedding): treat an input-length 413 as a per-item rejection * fix(embedding): keep the health state after a partial success `_compute_batch()` set `is_healthy = True` whenever the batch produced at least one vector. Per-item isolation can recover an early item and then meet a provider-wide failure (exhausted quota, authentication, exhausted transient retries), which `_call_with_retry()` correctly records as unhealthy before it stops. The recovered vector then overwrote that state, so a request that ended in an outage reported the provider as healthy. The terminal-failure flag is now threaded from `_embed_individually()` back to `_compute_batch()`. The vectors recovered before the failure are still returned and cached; only the health state now follows the failure. This matters because `LocalFileStore._recover_after_real_request()` schedules missing-vector backfill off the unhealthy-to-healthy transition: a hidden outage would either trigger backfill while the provider is still unavailable or suppress a later genuine recovery. Adds a regression test through `get_embeddings()` that checks both the retained partial vector and the final unhealthy state. The existing isolation test covers `_call_with_retry()` directly, so it never saw the overwrite. * fix(embedding): leave provider 5xx retries to the layers that own them A store-level retry for a provider 5xx stacked on top of the retries the embedding wrapper and the provider SDK already perform, so a batch the SDK answered after 12 attempts became 36. Retry ownership stays with those layers; the store answers a provider-wide 5xx once and gives up. Transport failures keep their retry because they describe a connection that no lower layer has reported, and the opt-in quota policy is untouched. Tests updated to pin the new boundary: a 503 is answered once for the OpenAI-shaped, Gemini-shaped and DashScope-shaped errors, while a 429 keeps its retry from the rate-limit branch. * fix(embedding): make transport retry ownership explicit The store retried a connection failure or a timeout on top of the embedding wrapper and the provider SDK, which already retry both, so a persistent timeout on a batch reached the provider 36 times instead of 12: the OpenAI model retries a batch max_retries + 1 times and the SDK retries each of those attempts. Transport failures stay provider-wide -- the request is never split per item -- but the retry is now owned explicitly instead of implicitly: * a whole-batch request leaves it to the layers below, which already retry it; * an isolated single-item attempt keeps the store's backoff, because that attempt is the last chance to keep the one vector the isolation step is recovering. Rate limiting is unchanged. The transport cases now pin both budgets, and three tests drive a real OpenAIAsEmbedding wrapper and a real openai.AsyncClient over an httpx.MockTransport, so the provider attempt count is measured rather than assumed. * fix(embedding): check the failing error against the provider retry policy --- .../embedding_store/local_embedding_store.py | 442 +++++- .../components/file_store/local_file_store.py | 12 +- tests/unit/test_file_store_consistency.py | 45 + tests/unit/test_local_embedding_store.py | 1256 ++++++++++++++++- 4 files changed, 1729 insertions(+), 26 deletions(-) diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index 4fad8361..abb08caa 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -2,6 +2,7 @@ import asyncio import hashlib +import re from collections import OrderedDict from pathlib import Path @@ -14,6 +15,81 @@ from ..as_embedding import BaseAsEmbedding Miss = tuple[int, str, str] # (result_index, text, cache_key) _MAX_VECTOR_SPACE_ATTEMPTS = 3 +_PROVIDER_WIDE_STATUS_CODES = frozenset({401, 403, 404}) + +# Transport failures raised by the OpenAI SDK and httpx. They do not inherit from the built-in +# TimeoutError / ConnectionError / OSError, so the built-in check alone lets them slip into the +# generic handler and fan out into per-item requests. Matching on class names (across the MRO, +# so subclasses such as APITimeoutError are covered) keeps this module free of SDK imports. +_TRANSPORT_FAILURE_TYPES = frozenset( + { + "APIConnectionError", + "APITimeoutError", + "ConnectError", + "ConnectTimeout", + "PoolTimeout", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "WriteError", + "WriteTimeout", + }, +) + +# Transport errors an SDK retries on its own. A positive numeric retry budget is not evidence: +# AgentScope's Ollama model keeps the inherited ``max_retries=3`` next to an empty policy, and +# the number never buys a retry. Only a client that both has a budget and retries this class of +# error owns the failure; otherwise the store keeps its backoff. +_SDK_TRANSPORT_RETRY_TYPES = frozenset({"APIConnectionError", "APITimeoutError"}) + +# The status a provider reports can live in more than one place. ``status_code`` is the +# OpenAI-compatible attribute, Google GenAI exposes an integer ``code``, httpx-style wrappers +# keep it on ``response``, and AgentScope's DashScope implementation raises a plain +# ``RuntimeError`` whose message embeds the structured response, so the status has to be read +# back out of the message. A string ``code`` names an error (``insufficient_quota``, +# ``rate_limit_exceeded``, ``InvalidApiKey``) and is never a status. +_STATUS_IN_MESSAGE = re.compile(r'"status_code"\s*:\s*(\d{3})') + +# The business error code travels the same way the status does: a string ``code`` on the +# OpenAI-compatible SDKs, ``error.code`` inside a structured body, and a serialized +# ``"code"`` in the message for AgentScope's DashScope ``RuntimeError``. Reading only the +# attribute loses the DashScope case and misclassifies a quota error as a rate limit. +_CODE_IN_MESSAGE = re.compile(r'"code"\s*:\s*"([^"]+)"') + +# A rejected batch can only be paid back one item at a time when the provider blames an input. +# A context-length rejection is the one status an over-long item explains by itself, so it is +# the only error allowed to fan out into per-item requests. +_INPUT_DEPENDENT_STATUS_CODES = frozenset({400}) + +# A 413 is ambiguous. Providers such as Ollama v0.40.3 normalize an embedding input-limit +# rejection to 413, but a bare 413 can also describe an unrelated request-body limit that no +# single input explains. It therefore only qualifies as input-dependent when the error itself +# names the input length, which is why it is checked through the marker list instead of being +# accepted outright. +_INPUT_DEPENDENT_AMBIGUOUS_STATUS_CODES = frozenset({413}) + +_INPUT_DEPENDENT_ERROR_TYPES = frozenset( + { + "ContextLengthError", + "ContextWindowExceededError", + "InputLengthError", + "TokenLimitError", + }, +) + +_INPUT_DEPENDENT_MARKERS = ( + "context length", + "context_length", + "context window", + "input length", + "input is too long", + "maximum context", + "reduce the length", + "token limit", + "too many tokens", + "too long", +) + @R.register("local") class LocalEmbeddingStore(BaseEmbeddingStore): @@ -162,7 +238,17 @@ class LocalEmbeddingStore(BaseEmbeddingStore): async def _compute_batch(self, batch: list[Miss], **kwargs) -> list[tuple[int, str, np.ndarray]]: texts = [text for _, text, _ in batch] - embeddings = await self._call_with_retry(texts, **kwargs) + # A provider-wide failure can end a request after some of its vectors were already + # recovered. Those vectors are still worth keeping, but they must not report the + # provider as healthy again: the unhealthy-to-healthy transition is what schedules + # missing-vector backfill, so a hidden outage would either trigger it while the + # provider is still unavailable or suppress a later genuine recovery. + terminal_failure: list[bool] = [] + embeddings = await self._call_with_retry( + texts, + terminal_failure=terminal_failure, + **kwargs, + ) if not embeddings or len(embeddings) != len(texts): return [] out: list[tuple[int, str, np.ndarray]] = [] @@ -178,22 +264,75 @@ class LocalEmbeddingStore(BaseEmbeddingStore): if bad_dims: details = ", ".join(f"{count} with dim {dim}" for dim, count in sorted(bad_dims.items())) self.logger.error(f"Embedding dimension mismatch in batch: expected {self.dimensions}; rejected {details}") - if out: + if out and not terminal_failure: self.is_healthy = True else: self.is_healthy = False return out - async def _call_with_retry(self, texts: list[str], **kwargs) -> list[list[float] | None] | None: + async def _call_with_retry( + self, + texts: list[str], + *, + terminal_failure: list[bool] | None = None, + **kwargs, + ) -> list[list[float] | None] | None: + """Answer a batch, retrying transient failures and isolating input ones. + + Transport failures are provider-wide: no individual input can explain them, so the + request is never split per item. Which layer retries them depends on the provider, not + on where the request came from: when the embedding model below already retries the + transport exceptions it raises (AgentScope's OpenAI model does, and the OpenAI SDK + retries each of those attempts), the store answers once and leaves that budget there. + A provider whose model has no retry policy -- Ollama inherits an empty retryable tuple + and its SDK turns ``httpx.ConnectError`` into a plain ``ConnectionError`` -- gets the + store's backoff instead, because otherwise a single dropped connection would cost the + vector. + + ``terminal_failure`` is an optional out-parameter. When the caller passes a list, + it is set to ``[True]`` if this call gave up on a failure that no remaining item + can explain (a provider-wide error, exhausted quota, or exhausted transient + retries). The per-item isolation loop uses it to stop instead of asking for every + remaining item and multiplying the rejected requests, and it forwards the same + flag to its caller so a request that recovered some vectors before the failure is + still not reported as healthy. + """ + + def stop() -> None: + """Record that this failure ends the work for the whole batch.""" + if terminal_failure is not None: + terminal_failure[:] = [True] + for attempt in range(self.max_retries): try: result = await self.as_embedding(texts, **kwargs) if result and len(result) == len(texts): return result - except (TimeoutError, ConnectionError, OSError): - if attempt < self.max_retries - 1: - await asyncio.sleep(2**attempt) except Exception as error: + if self._is_transport_failure(error): + # A connection failure or a timeout describes the endpoint, not any single + # item, so this request is never split per item. Whether the store retries + # it is a property of the provider: when the model below has its own retry + # policy, that layer (and the SDK under it) already answers the failure -- + # a batch they answer after 12 attempts used to reach the provider 36 times + # with this store retrying on top of them, and an isolated item used to + # multiply the same budget a third time. + if self._provider_owns_transport_retry(error): + self.logger.exception( + "Embedding request failed: transport error owned by the provider, not splitting", + ) + self.is_healthy = False + stop() + break + if attempt < self.max_retries - 1: + await asyncio.sleep(2**attempt) + continue + self.logger.exception( + "Embedding request failed: transport error after exhausting retries", + ) + self.is_healthy = False + stop() + break if self._is_rate_limited(error): if attempt < self.max_retries - 1: delay = 2**attempt @@ -202,6 +341,7 @@ class LocalEmbeddingStore(BaseEmbeddingStore): continue self.logger.exception("Embedding request failed after exhausting rate-limit retries") self.is_healthy = False + stop() return None if ( self.quota_retry_delay is not None @@ -213,35 +353,291 @@ class LocalEmbeddingStore(BaseEmbeddingStore): ) await asyncio.sleep(self.quota_retry_delay) continue - self.logger.exception("Embedding request failed") - self.is_healthy = False - return None + if self._is_insufficient_quota(error): + # Quota is a service-level failure: no single item is to blame, so splitting the + # batch cannot isolate anything and would only multiply the rejected requests. + # This also covers the case where the quota retries above are exhausted. + self.logger.exception("Embedding request failed: quota exhausted") + self.is_healthy = False + stop() + return None + if self._is_provider_wide_failure(error): + # Credentials, permissions and server outages describe the endpoint, not any + # single item: isolation cannot fix them, so give up on the whole batch + # instead of multiplying the rejected requests. + # + # A 5xx is deliberately *not* retried here. The embedding wrapper and the + # provider SDK already retry transient server errors, and every extra layer + # that retries multiplies the attempts and the waiting time of the layers + # below it -- a batch the SDK answered after 12 attempts became 36 once this + # store retried on top of it. Retry ownership therefore stays with those + # layers: a caller that wants a longer budget configures it there instead of + # stacking another silent loop here. A transport failure is left to them for + # the same reason; it only skips the per-item split. + self.logger.exception("Embedding request failed: provider-wide error, not splitting") + stop() + break + if self._is_input_dependent_failure(error): + # A rejection a single input explains by itself is the one case the per-item + # split can resolve: isolating it keeps the remaining vectors usable. + self.logger.exception("Embedding request failed: input-dependent error") + self.is_healthy = False + if len(texts) > 1: + return await self._embed_individually( + texts, + terminal_failure=terminal_failure, + **kwargs, + ) + return None + # Anything else counts as provider-wide. Splitting cannot recover a vector unless + # the failing input is identified, so an unrecognized error is never answered + # with one request per item. + self.logger.exception("Embedding request failed: unrecognized error, not splitting") + stop() + break self.is_healthy = False + stop() return None + async def _embed_individually( + self, + texts: list[str], + *, + terminal_failure: list[bool] | None = None, + **kwargs, + ) -> list[list[float] | None]: + """Retry a rejected batch one item at a time so one bad item cannot drop the rest. + + Provider-side request errors, such as a per-item context-length limit, reject the + whole batch and would otherwise cost every vector in it. Isolating each item bounds + the damage to the item that actually failed and keeps the remaining vectors usable. + Each single-item request goes back through ``_call_with_retry`` so that a rate limit and + a transient transport failure still get the retries that layer owns instead of losing + the vector; a provider-wide failure ends the work for the whole batch rather than being + asked for every remaining item, and a one-item request never splits again. + When the loop stops on a provider-wide failure, ``terminal_failure`` is set for the + caller so the vectors recovered before it are not read as a healthy provider. + """ + results: list[list[float] | None] = [] + for index, text in enumerate(texts): + terminal: list[bool] = [] + single = await self._call_with_retry( + [text], + terminal_failure=terminal, + **kwargs, + ) + if single: + results.append(single[0]) + continue + self.logger.error(f"Embedding request failed for a single item (chars={len(text)})") + results.append(None) + if terminal and terminal[0]: + # No remaining item can explain a provider-wide failure, so asking for the + # rest would only multiply the rejected requests. Keep the vectors already + # recovered and leave the items that were never answered as None. The batch + # caller has to learn about it as well: a partial success must not report + # the provider as healthy again. + if terminal_failure is not None: + terminal_failure[:] = [True] + self.logger.error( + f"Stopping per-item isolation after {index + 1}/{len(texts)} items: " + "the failure is not specific to any input", + ) + results.extend([None] * (len(texts) - index - 1)) + break + return results + + @staticmethod + def _is_transport_failure(error: Exception) -> bool: + """Return whether an error describes a transport failure rather than a rejected input. + + Covers the built-in socket errors as well as the provider/httpx exceptions that describe + a connection failure or a timeout. Transport failures are provider-wide: no individual + input can be blamed, so the request is never split per item. Who retries them is decided + per provider by ``_provider_owns_transport_retry`` rather than per call site. + """ + if isinstance(error, (TimeoutError, ConnectionError, OSError)): + return True + return any(cls.__name__ in _TRANSPORT_FAILURE_TYPES for cls in type(error).__mro__) + + def _provider_owns_transport_retry(self, error: Exception) -> bool: + """Return whether a lower layer retries *this* transport failure. + + Retry ownership is a property of the provider and of the error, not of the call site. + A model whose ``_get_retryable_exceptions()`` covers the error owns the retry, and so + does an SDK that retries that class of error. Everything else must keep the store's + backoff, or one dropped connection costs the vector. + + Two shapes are deliberately not treated as ownership. AgentScope's Ollama embedding + model declares an empty policy but keeps the inherited ``max_retries=3``: its loop + re-raises every exception outside the empty tuple, so the number never buys a retry. + AgentScope's DashScope model declares ``(RuntimeError,)``, which does not cover a + connection error raised underneath it. A nonempty policy is therefore checked against + the error that actually occurred rather than trusted as a whole. + + A model that exposes no policy at all -- test doubles and custom providers -- is only + trusted to an SDK that has a positive retry budget *and* retries this class of + transport error. Otherwise the store answers, as the base implementation did. + """ + model = getattr(self.as_embedding, "model", None) + if model is None: + return False + policy = self._declared_retry_policy(model) + if policy: + max_retries = getattr(model, "max_retries", None) + if isinstance(max_retries, int) and max_retries > 0 and self._policy_covers(policy, error): + return True + return self._sdk_owns_transport_retry(model, error) + + @staticmethod + def _declared_retry_policy(model) -> tuple[type[Exception], ...] | None: + """Return the model's declared retryable types, or ``None`` when it declares none.""" + retryable = getattr(model, "_get_retryable_exceptions", None) + if not callable(retryable): + return None + try: + return tuple(retryable()) # pylint: disable=not-callable + except Exception: # pragma: no cover - a broken policy owns nothing + return None + + @staticmethod + def _policy_covers(policy: tuple[type[Exception], ...], error: Exception) -> bool: + """Return whether the declared policy actually retries this exception.""" + try: + return isinstance(error, policy) + except TypeError: # pragma: no cover - a malformed policy covers nothing + return False + + @staticmethod + def _sdk_owns_transport_retry(model, error: Exception) -> bool: + """Return whether the SDK under the model retries this transport error itself. + + The numeric ``max_retries`` only counts once the client also identifies this error as + retryable. The OpenAI client retries ``APIConnectionError`` / ``APITimeoutError``; + Ollama's client has no retry budget at all, and AgentScope's DashScope path calls the + module directly, so neither has a client to consult. + """ + if not LocalEmbeddingStore._is_transport_failure(error): + return False + client = getattr(model, "client", None) + if client is None: + return False + max_retries = getattr(client, "max_retries", None) + if not isinstance(max_retries, int) or max_retries <= 0: + return False + return any(cls.__name__ in _SDK_TRANSPORT_RETRY_TYPES for cls in type(error).__mro__) + + @staticmethod + def _http_status(error: Exception) -> int | None: + """Return the HTTP status an exception carries, across the provider SDK shapes. + + ``status_code`` covers the OpenAI-compatible SDKs, an integer ``code`` covers Google + GenAI, ``response.status_code`` covers httpx-style wrappers, and the structured + ``status_code`` embedded in the message covers AgentScope's DashScope ``RuntimeError``. + Only integers are accepted: a string ``code`` is a provider error name such as + ``insufficient_quota``, and reading it as a status would misclassify those errors. + """ + for attribute in ("status_code", "code"): + value = getattr(error, attribute, None) + if isinstance(value, int) and not isinstance(value, bool): + return value + value = getattr(getattr(error, "response", None), "status_code", None) + if isinstance(value, int) and not isinstance(value, bool): + return value + match = _STATUS_IN_MESSAGE.search(str(error)) + return int(match.group(1)) if match else None + + @staticmethod + def _is_provider_wide_failure(error: Exception) -> bool: + """Return whether an error describes the provider rather than a single input. + + Authentication, permission, missing model or route and server errors belong to the + endpoint, the credentials or the deployment: no individual input can be blamed, so + splitting the batch cannot isolate anything and would only multiply the rejected + requests. Rate limits and quota exhaustion have their own branches above and never + reach this classifier. A context-length ``400`` stays input-dependent on purpose, + because that is the one status an over-limit item can explain by itself. + """ + status = LocalEmbeddingStore._http_status(error) + if status is not None and (status in _PROVIDER_WIDE_STATUS_CODES or 500 <= status < 600): + return True + return type(error).__name__ in ( + "AuthenticationError", + "PermissionDeniedError", + "NotFoundError", + "InternalServerError", + ) + + @staticmethod + def _is_input_dependent_failure(error: Exception) -> bool: + """Return whether a single input can be blamed for the rejected request. + + Only a context-length rejection qualifies: it is the one error an over-long item + explains by itself, and therefore the only one a per-item retry can resolve. It can + arrive as a ``400`` or, from a provider that normalizes the rejection (Ollama reports + ``413``), as an ambiguous status that is accepted only when the error names the input + itself. Anything else stays on the provider-wide path, so a client never answers a + service failure with one request per item. + """ + status = LocalEmbeddingStore._http_status(error) + if status is not None and status not in _INPUT_DEPENDENT_STATUS_CODES: + if status not in _INPUT_DEPENDENT_AMBIGUOUS_STATUS_CODES: + return False + # The status alone does not name an input for an ambiguous code, so the error text + # has to carry the signal; otherwise this is a request-level rejection. + text = LocalEmbeddingStore._error_text(error) + return any(marker in text for marker in _INPUT_DEPENDENT_MARKERS) + if type(error).__name__ in _INPUT_DEPENDENT_ERROR_TYPES: + return True + text = LocalEmbeddingStore._error_text(error) + return any(marker in text for marker in _INPUT_DEPENDENT_MARKERS) + + @staticmethod + def _error_text(error: Exception) -> str: + """Flatten an exception and its OpenAI-compatible body into one lowercase string.""" + parts = [str(error)] + body = getattr(error, "body", None) + if isinstance(body, dict): + parts.append(str(body)) + return " ".join(parts).lower() + + @staticmethod + def _error_code(error: Exception) -> str | None: + """Return the provider's business error code, across the supported error shapes. + + ``code`` covers the OpenAI-compatible SDKs when the provider puts the error name + there, ``error.code`` inside ``body`` covers the structured OpenAI payload, and the + serialized message covers AgentScope's DashScope ``RuntimeError`` — the same shape + ``_http_status`` already reads the status back out of. Without the last case a + DashScope quota error is indistinguishable from an ordinary 429. + """ + code = getattr(error, "code", None) + if isinstance(code, str) and code: + return code + body = getattr(error, "body", None) + if isinstance(body, dict): + details = body.get("error", body) + if isinstance(details, dict): + inner = details.get("code") + if isinstance(inner, str) and inner: + return inner + match = _CODE_IN_MESSAGE.search(str(error)) + return match.group(1) if match else None + @staticmethod def _is_rate_limited(error: Exception) -> bool: """Recognize an OpenAI-compatible 429 response without importing a provider SDK.""" - if LocalEmbeddingStore._is_insufficient_quota(error): + code = LocalEmbeddingStore._error_code(error) + if code == "insufficient_quota": return False - if getattr(error, "status_code", None) == 429: + if code == "rate_limit_exceeded": return True - body = getattr(error, "body", None) - if not isinstance(body, dict): - return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "rate_limit_exceeded" + return LocalEmbeddingStore._http_status(error) == 429 @staticmethod def _is_insufficient_quota(error: Exception) -> bool: """Recognize OpenAI-compatible quota errors without importing a provider SDK.""" - if getattr(error, "code", None) == "insufficient_quota": - return True - body = getattr(error, "body", None) - if not isinstance(body, dict): - return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "insufficient_quota" + return LocalEmbeddingStore._error_code(error) == "insufficient_quota" def _validate_dim(self, emb: np.ndarray) -> bool: """Return whether an embedding exactly matches the configured dimension.""" diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index e5448f65..dcbe6177 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -43,6 +43,7 @@ _SEARCH_FILTER_RESERVED_KEYS = frozenset( "strict_date_filter", }, ) +_FAILED_CHUNK_PREVIEW = 5 @R.register("local") @@ -440,7 +441,12 @@ class LocalFileStore(BaseFileStore): if chunk.text and not self._embedding_dim_matches(chunk.embedding) ] if missing: - raise RuntimeError(f"embedding reindex incomplete: {len(missing)} chunks failed") + preview = ", ".join(missing[:_FAILED_CHUNK_PREVIEW]) + if len(missing) > _FAILED_CHUNK_PREVIEW: + preview += f" (+{len(missing) - _FAILED_CHUNK_PREVIEW} more)" + raise RuntimeError( + f"embedding reindex incomplete: {len(missing)} chunks failed: {preview}", + ) await self._finalize_embedding_reindex() await self._dump_owned_state() if self.embedding_store is not None: @@ -544,6 +550,7 @@ class LocalFileStore(BaseFileStore): return processed = 0 + filled = 0 batch_count = 0 embedding_started_at = time.monotonic() next_percent = _PROGRESS_LOG_PERCENT_STEP @@ -552,10 +559,11 @@ class LocalFileStore(BaseFileStore): await self.embedding_store.get_node_embeddings(batch) self._drop_stale_embeddings(batch, "backfill") processed += len(batch) + filled += sum(1 for chunk in batch if chunk.embedding is not None) batch_count += 1 next_percent = self._log_progress("embedding backfill", processed, total, next_percent) self.logger.info( - f"{self.name}: embedding batches complete: processed={processed}/{total}, " + f"{self.name}: embedding batches complete: processed={processed}/{total}, filled={filled}, " f"batches={batch_count}, elapsed={time.monotonic() - embedding_started_at:.3f}s", ) except asyncio.CancelledError: diff --git a/tests/unit/test_file_store_consistency.py b/tests/unit/test_file_store_consistency.py index b7750017..9e6af475 100644 --- a/tests/unit/test_file_store_consistency.py +++ b/tests/unit/test_file_store_consistency.py @@ -118,6 +118,16 @@ class RecoveringEmbeddingStore(CountingFakeEmbeddingStore): return await super().get_node_embeddings(nodes, **kwargs) +class PartialEmbeddingStore(FakeEmbeddingStore): + """Fake provider that leaves chunks whose text contains "beta" without a vector.""" + + async def get_node_embeddings(self, nodes: list[FileChunk], **_kwargs) -> list[FileChunk]: + for chunk_node in nodes: + if "beta" not in chunk_node.text: + chunk_node.embedding = self._embed(chunk_node.text) + return nodes + + class HealthCountingEmbeddingStore(FakeEmbeddingStore): """Fake provider that records eager health checks.""" @@ -838,6 +848,41 @@ def test_verified_resume_without_chunks_supersedes_inflight_health_check(): run(go()) +@pytest.mark.parametrize("store_factory", [_new_local_store, _new_faiss_store, _new_zvec_store]) +def test_embedding_reindex_error_names_the_chunks_without_vectors(store_factory): + """The final reindex error must name the chunks that still have no vector.""" + + async def go(): + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = store_factory("t_reindex_failure_ids") + await store.start() + await set_chunks_with_graph( + store, + { + "good": chunk("good", "a.md", "alpha text"), + "bad": chunk("bad", "b.md", "beta text"), + }, + ) + store.embedding_store = PartialEmbeddingStore() + _ensure_zvec_collection(store) + if isinstance(store, FaissLocalFileStore): + store._rebuild_index() + elif isinstance(store, ZvecLocalFileStore): + store._rebuild_collection() + await store.require_embedding_rebuild() + + with pytest.raises(RuntimeError) as failure: + await store.reindex("embedding") + + message = str(failure.value) + assert "1 chunks failed" in message + assert "bad" in message + assert "good" not in message + await store.close() + + run(go()) + + @pytest.mark.parametrize("store_factory", [_new_local_store, _new_faiss_store, _new_zvec_store]) def test_embedding_reindex_is_explicit_and_keeps_vectors_disabled_until_success( store_factory, diff --git a/tests/unit/test_local_embedding_store.py b/tests/unit/test_local_embedding_store.py index 51b7bf63..912c1789 100644 --- a/tests/unit/test_local_embedding_store.py +++ b/tests/unit/test_local_embedding_store.py @@ -1,11 +1,14 @@ """Regression tests for LocalEmbeddingStore dimension and vector space handling.""" -# pylint: disable=protected-access +# pylint: disable=protected-access,too-many-lines import asyncio +import json from types import SimpleNamespace +import httpx import numpy as np +import openai from reme.components.as_embedding import DashScopeAsEmbedding, OllamaAsEmbedding, OpenAIAsEmbedding from reme.components.embedding_store.base_embedding_store import BaseEmbeddingStore @@ -126,6 +129,296 @@ class QuotaThenSuccessAsEmbedding: return [[1.0, 0.0] for _ in texts] +class ContextLengthError(Exception): + """OpenAI-compatible 400 raised when one item exceeds the provider's per-item window.""" + + status_code = 400 + body = {"error": {"message": "the input length exceeds the context length"}} + + +class PerItemLimitAsEmbedding: + """Reject a whole request when any single text is over the provider's per-item limit. + + Mirrors an OpenAI-compatible gateway: one over-limit item makes the provider answer + HTTP 400 for the entire batch, so a client that gives up on the batch loses every + vector in it. + """ + + dimensions = 2 + vector_space_id = "fakespace000" + max_chars = 32 + + def __init__(self): + self.batch_sizes: list[int] = [] + + def initialize_model(self): + """Mirror the real component's idempotent initialization hook.""" + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + if any(len(text) > self.max_chars for text in texts): + raise ContextLengthError("the input length exceeds the context length") + return [[1.0, 0.0] for _ in texts] + + +class AlwaysInsufficientQuotaAsEmbedding: + """Always fail with quota exhaustion, recording every requested batch size.""" + + dimensions = 2 + + def __init__(self): + self.batch_sizes: list[int] = [] + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + raise InsufficientQuotaError("quota exhausted") + + +class PerItemLimitThenTransientErrorAsEmbedding: + """Reject an over-limit batch, then fail the first single-item request once. + + Mirrors the review case: the batch is rejected because one item exceeds the per-item + window, and a sibling item then hits a transient provider failure while retried alone. + """ + + dimensions = 2 + vector_space_id = "fakespace000" + max_chars = 32 + + def __init__(self, transient: Exception): + self.transient = transient + self.batch_sizes: list[int] = [] + self.transient_failures = 0 + + def initialize_model(self): + """Mirror the real component's idempotent initialization hook.""" + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + if any(len(text) > self.max_chars for text in texts): + raise ContextLengthError("the input length exceeds the context length") + if len(texts) == 1 and self.transient_failures == 0: + self.transient_failures += 1 + raise self.transient + return [[1.0, 0.0] for _ in texts] + + +class PerItemLimitThenQuotaAsEmbedding: + """Reject an over-limit batch, recover the first isolated item, then exhaust the quota. + + Mirrors the review case: isolation starts, the first single-item request comes back, and + every later call then fails provider-wide. Asking for the remaining items would only + multiply the rejected requests, because no single item explains a quota failure. + """ + + dimensions = 2 + vector_space_id = "fakespace000" + max_chars = 32 + + def __init__(self): + self.batch_sizes: list[int] = [] + self.singles = 0 + + def initialize_model(self): + """Mirror the real component's idempotent initialization hook.""" + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + if any(len(text) > self.max_chars for text in texts): + raise ContextLengthError("the input length exceeds the context length") + self.singles += 1 + if self.singles > 1: + raise InsufficientQuotaError("quota exhausted") + return [[1.0, 0.0] for _ in texts] + + +class AuthenticationError(Exception): + """OpenAI-compatible 401 error used without importing the provider SDK.""" + + status_code = 401 + + +class PermissionDeniedError(Exception): + """OpenAI-compatible 403 error used without importing the provider SDK.""" + + status_code = 403 + + +class InternalServerError(Exception): + """OpenAI-compatible 500 error used without importing the provider SDK.""" + + status_code = 500 + + +class NotFoundError(Exception): + """OpenAI-compatible 404 error, e.g. a missing model or endpoint.""" + + status_code = 404 + + +class APIError(Exception): + """Base of the OpenAI SDK error hierarchy, reproduced without importing the SDK.""" + + +class APIConnectionError(APIError): + """OpenAI SDK connection failure: it does not inherit from the built-in ConnectionError.""" + + +class APITimeoutError(APIConnectionError): + """OpenAI SDK timeout, a subclass of APIConnectionError.""" + + +class _RetryingPolicyModel: + """Stand-in for AgentScope's OpenAI embedding model: it retries transport errors.""" + + max_retries = 3 + + @classmethod + def _get_retryable_exceptions(cls): + """Mirror the OpenAI policy the real wrapper consults.""" + return (APIConnectionError,) + + +class AlwaysProviderWideFailureAsEmbedding: + """Always fail with the same provider-wide error, recording every requested batch size.""" + + dimensions = 2 + + def __init__(self, error: Exception): + self.error = error + self.batch_sizes: list[int] = [] + # This double stands for an error raised by the SDK beneath an OpenAI-style model, so it + # advertises the same lower-layer retry policy: the store must not add a third loop. + self.model = _RetryingPolicyModel() + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + raise self.error + + +class TransientThenSuccessAsEmbedding: + """Fail the first `failures` calls with one error, then return valid embeddings.""" + + dimensions = 2 + vector_space_id = "fakespace000" + + def __init__(self, error: Exception, failures: int = 1): + self.error = error + self.failures = failures + self.batch_sizes: list[int] = [] + + def initialize_model(self): + """Mirror the real component's idempotent initialization hook.""" + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + if len(self.batch_sizes) <= self.failures: + raise self.error + return [[1.0, 0.0] for _ in texts] + + +_REAL_OPENAI_ASYNC_CLIENT = openai.AsyncClient +_OFFLINE_OPENAI_BASE_URL = "http://127.0.0.1:9/v1" + + +class CountingOpenAIWrapper(OpenAIAsEmbedding): + """Real OpenAI wrapper that records how often the store asked it for embeddings.""" + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.calls = 0 + + async def __call__(self, inputs, **kwargs): + self.calls += 1 + return await super().__call__(inputs, **kwargs) + + +class OfflineOpenAIEndpoint: + """Real openai.AsyncClient wired to an offline httpx transport that counts provider attempts. + + Gives the wrapper-and-SDK layers their real code path, so a regression test can assert how + many attempts the provider would see instead of trusting the store's own loop. + """ + + def __init__(self, handler, sdk_max_retries: int = 2) -> None: + self.requests = 0 + self.handler = handler + self.client = _REAL_OPENAI_ASYNC_CLIENT( + api_key="test-key", + base_url=_OFFLINE_OPENAI_BASE_URL, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(self._handle)), + max_retries=sdk_max_retries, + ) + + def _handle(self, request): + self.requests += 1 + return self.handler(request) + + def install(self, monkeypatch) -> "OfflineOpenAIEndpoint": + """Route every openai.AsyncClient built during this test to the offline client.""" + monkeypatch.setattr(openai, "AsyncClient", lambda **kwargs: self.client) + return self + + +def offline_openai_endpoint(handler) -> OfflineOpenAIEndpoint: + """Build an offline endpoint whose transport follows ``handler``.""" + return OfflineOpenAIEndpoint(handler) + + +def openai_wrapper(name: str) -> CountingOpenAIWrapper: + """Build a real OpenAI embedding wrapper pointed at the offline endpoint.""" + return CountingOpenAIWrapper( + name=name, + model="text-embedding-v4", + backend="openai", + dimensions=2, + credential={"base_url": _OFFLINE_OPENAI_BASE_URL, "api_key": "test-key"}, + retry_delay=0, + pass_dimensions=False, + ) + + +def embedding_payload(request) -> httpx.Response: + """Answer an embeddings request the way the provider would.""" + body = json.loads(request.content.decode("utf-8")) + inputs = body["input"] + if isinstance(inputs, str): + inputs = [inputs] + return httpx.Response( + 200, + json={ + "object": "list", + "data": [ + {"object": "embedding", "index": position, "embedding": [0.0, 1.0]} for position in range(len(inputs)) + ], + "model": body.get("model", "text-embedding-v4"), + "usage": {"prompt_tokens": len(inputs), "total_tokens": len(inputs)}, + }, + ) + + +def context_length_rejection() -> httpx.Response: + """Answer a batch the way an OpenAI-compatible provider reports an over-long input.""" + return httpx.Response( + 400, + json={ + "error": { + "message": ( + "This model's maximum context length is 8191 tokens. " "Please reduce the length of the input." + ), + "type": "invalid_request_error", + "param": None, + "code": "context_length_exceeded", + }, + }, + ) + + +def read_timeout(request) -> httpx.Response: + """Fail one attempt the way a read timeout reaches the SDK.""" + raise httpx.ReadTimeout("simulated read timeout", request=request) + + class BadNodeEmbeddingStore(BaseEmbeddingStore): """Embedding store that returns wrong-dimensional vectors.""" @@ -730,3 +1023,964 @@ def test_start_ignores_cache_file_without_vector_space_tag(monkeypatch, tmp_path assert not store._cache run(go()) + + +def test_batch_failure_falls_back_to_per_item_embedding(): + """One over-limit item must cost one vector, not its whole batch.""" + + async def go(): + provider = PerItemLimitAsEmbedding() + store = LocalEmbeddingStore(name="t_local_embedding_poison_batch", enable_cache=False) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [[1.0, 0.0], None, [1.0, 0.0]] + assert provider.batch_sizes == [3, 1, 1, 1] + + run(go()) + + +def test_poison_chunk_keeps_its_batch_mates(): + """A rejected chunk must not clear the vectors of the chunks beside it.""" + + async def go(): + store = LocalEmbeddingStore(name="t_local_embedding_poison_nodes", enable_cache=False) + store.as_embedding = PerItemLimitAsEmbedding() + nodes = [EmbNode(text="first"), EmbNode(text="y" * 64), EmbNode(text="third")] + + await store.get_node_embeddings(nodes) + + assert nodes[0].embedding is not None + assert nodes[1].embedding is None + assert nodes[2].embedding is not None + + run(go()) + + +def test_single_item_failure_still_reports_a_failed_batch(): + """A one-item request has nothing to isolate, so the existing contract stands.""" + + async def go(): + store = LocalEmbeddingStore(name="t_local_embedding_single_poison", enable_cache=False) + store.as_embedding = PerItemLimitAsEmbedding() + + assert await store._call_with_retry(["x" * 64]) is None + + run(go()) + + +def test_rate_limited_batch_is_not_split_into_per_item_requests(monkeypatch): + """A 429 is a provider-level failure; splitting it would multiply the load.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + store = LocalEmbeddingStore(name="t_local_embedding_rate_limit_batch", max_retries=2) + embedding = AlwaysRateLimitedAsEmbedding() + store.as_embedding = embedding + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + assert await store._call_with_retry(["a", "b"]) is None + assert embedding.calls == 2 + assert sleeps == [1.0] + + run(go()) + + +def test_insufficient_quota_batch_is_not_split_into_per_item_requests(monkeypatch): + """Quota exhaustion is a service-level failure; splitting it would multiply the load.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + store = LocalEmbeddingStore(name="t_local_embedding_quota_batch", max_retries=2) + embedding = AlwaysInsufficientQuotaAsEmbedding() + store.as_embedding = embedding + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + assert await store._call_with_retry(["a", "b", "c"]) is None + assert embedding.batch_sizes == [3] + assert store.is_healthy is False + assert not sleeps + + run(go()) + + +def test_exhausted_quota_retries_still_do_not_split_the_batch(monkeypatch): + """Once the opt-in quota retries run out, the store gives up instead of splitting.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + store = LocalEmbeddingStore( + name="t_local_embedding_quota_batch_exhausted", + max_retries=3, + quota_retry_delay=5.0, + ) + embedding = AlwaysInsufficientQuotaAsEmbedding() + store.as_embedding = embedding + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + assert await store._call_with_retry(["a", "b"]) is None + assert embedding.batch_sizes == [2, 2, 2] + assert sleeps == [5.0, 5.0] + + run(go()) + + +def test_per_item_fallback_keeps_a_rate_limited_vector(monkeypatch): + """A 429 on one isolated item still gets the store's backoff and keeps its vector. + + Rate limiting is the failure class the store still owns: no lower layer reports it as a + provider-wide error worth its own budget, so the retry lives here. Connection errors and + timeouts are left to the wrapper and the SDK instead; see the transport tests above. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + provider = PerItemLimitThenTransientErrorAsEmbedding( + RateLimitError("Requests are too frequent"), + ) + store = LocalEmbeddingStore( + name="t_local_embedding_fallback_rate_limit", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [[1.0, 0.0], None, [1.0, 0.0]] + assert provider.batch_sizes == [3, 1, 1, 1, 1] + assert provider.transient_failures == 1 + assert sleeps == [1.0] + + run(go()) + + +def test_provider_wide_credentials_errors_are_not_split_into_per_item_requests(): + """401/403 belong to the credentials: splitting ten items only multiplies the failures.""" + + async def go(): + for error in ( + AuthenticationError("invalid api key"), + PermissionDeniedError("no access to model"), + ): + provider = AlwaysProviderWideFailureAsEmbedding(error) + store = LocalEmbeddingStore(name="t_local_embedding_provider_wide", max_retries=3) + store.as_embedding = provider + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert provider.batch_sizes == [10] + assert store.is_healthy is False + + run(go()) + + +def test_provider_wide_server_errors_are_not_retried_without_splitting(monkeypatch): + """A 500 is left to the layers that own retries, so this store asks exactly once.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + provider = AlwaysProviderWideFailureAsEmbedding(InternalServerError("upstream unavailable")) + store = LocalEmbeddingStore(name="t_local_embedding_provider_server", max_retries=3) + store.as_embedding = provider + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + # One request only: the wrapper and the SDK already retry a 5xx, and retrying here + # as well multiplied their attempts (12 -> 36 in the reported reproduction). + assert provider.batch_sizes == [10] + assert not sleeps + assert store.is_healthy is False + + run(go()) + + +def test_sdk_transport_errors_are_provider_wide_and_not_retried_here(monkeypatch): + """SDK connection/timeout errors are provider-wide: never split, and not retried here. + + The wrapper and the SDK already retry a transport failure, so the store answers once and + leaves the retry budget to those layers instead of multiplying their attempts. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + for error in (APIConnectionError("connection reset"), APITimeoutError("timed out")): + sleeps.clear() + provider = AlwaysProviderWideFailureAsEmbedding(error) + store = LocalEmbeddingStore(name="t_local_embedding_sdk_transport", max_retries=3) + store.as_embedding = provider + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + # One request only: the wrapper and the SDK already retry a transport failure + # (4 wrapper attempts, each retried by the SDK), so another loop here only + # multiplies them -- a persistent timeout used to cost 36 attempts instead of 12. + assert provider.batch_sizes == [10] + assert not sleeps + assert store.is_healthy is False + + run(go()) + + +def test_a_provider_without_a_retry_policy_keeps_the_store_backoff(monkeypatch): + """A provider with no lower-layer retry policy still gets the store's backoff. + + This fake exposes no ``model`` with a retry policy, which is the same position Ollama is + in: its embedding model inherits an empty retryable tuple and its SDK turns a connection + error into a plain ``ConnectionError`` without retrying. The store's retry is therefore the + only layer that can keep that vector. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + for transient in ( + APIConnectionError("connection reset"), + ConnectionError("connection reset"), + TimeoutError("request timed out"), + ): + sleeps.clear() + + provider = PerItemLimitThenTransientErrorAsEmbedding(transient) + store = LocalEmbeddingStore( + name="t_local_embedding_sdk_isolated", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [[1.0, 0.0], None, [1.0, 0.0]] + assert provider.batch_sizes == [3, 1, 1, 1, 1] + assert sleeps == [1.0] + + run(go()) + + +def test_a_persistent_transport_error_is_not_multiplied_by_the_store(monkeypatch): + """A timeout is asked once: the wrapper and the SDK own the retry budget. + + The provider sees the 4 wrapper attempts x 3 SDK attempts those layers make, while the + store's max_retries=3 no longer multiplies them -- a persistent timeout used to cost 36 + provider attempts instead of 12. + """ + + async def go(): + endpoint = offline_openai_endpoint(read_timeout).install(monkeypatch) + provider = openai_wrapper("t_openai_transport_budget") + store = LocalEmbeddingStore( + name="t_local_embedding_sdk_transport_budget", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + assert await store._call_with_retry([f"item{i}" for i in range(3)]) is None + assert provider.calls == 1 + assert endpoint.requests == 12 + assert store.is_healthy is False + + run(go()) + + +def test_a_transient_transport_error_still_recovers_through_the_sdk(monkeypatch): + """One timeout is answered by the SDK's own retry: the batch survives and never splits.""" + + async def go(): + state = {"timeouts": 1} + + def handler(request): + if state["timeouts"] > 0: + state["timeouts"] -= 1 + return read_timeout(request) + return embedding_payload(request) + + endpoint = offline_openai_endpoint(handler).install(monkeypatch) + provider = openai_wrapper("t_openai_transport_recover") + store = LocalEmbeddingStore( + name="t_local_embedding_sdk_transport_recover", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry([f"item{i}" for i in range(3)]) + + assert results == [[0.0, 1.0]] * 3 + assert provider.calls == 1 + assert endpoint.requests == 2 + + run(go()) + + +def test_an_isolated_item_still_recovers_through_the_sdk(monkeypatch): + """After a context-length rejection, a timeout on one remaining item is still recovered. + + The store asks the wrapper once per item; the retry that saves that item belongs to the SDK. + """ + + async def go(): + state = {"timeout_pending": True} + + def handler(request): + body = json.loads(request.content.decode("utf-8")) + inputs = body["input"] + if isinstance(inputs, str): + inputs = [inputs] + if any(len(item) > 32 for item in inputs): + return context_length_rejection() + if len(inputs) == 1 and state["timeout_pending"]: + state["timeout_pending"] = False + return read_timeout(request) + return embedding_payload(request) + + endpoint = offline_openai_endpoint(handler).install(monkeypatch) + provider = openai_wrapper("t_openai_transport_isolated") + store = LocalEmbeddingStore( + name="t_local_embedding_sdk_transport_isolated", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [[0.0, 1.0], None, [0.0, 1.0]] + assert provider.calls == 4 + assert endpoint.requests == 5 + + run(go()) + + +def test_provider_wide_statuses_exclude_input_dependent_bad_request(): + """404 is provider-wide, but an over-limit 400 must stay input-dependent.""" + + async def go(): + store = LocalEmbeddingStore(name="t_local_embedding_classifier") + for error in ( + AuthenticationError("bad key"), + PermissionDeniedError("no access"), + NotFoundError("model_not_found"), + InternalServerError("boom"), + ): + assert store._is_provider_wide_failure(error) is True + over_limit = ContextLengthError("the input length exceeds the context length") + assert store._is_provider_wide_failure(over_limit) is False + + run(go()) + + +def test_provider_wide_not_found_errors_are_not_split(monkeypatch): + """A missing model or endpoint is provider-wide: one request, then failure, never a split.""" + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + provider = AlwaysProviderWideFailureAsEmbedding(NotFoundError("model_not_found")) + store = LocalEmbeddingStore(name="t_local_embedding_not_found", max_retries=3) + store.as_embedding = provider + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert provider.batch_sizes == [10] + assert not sleeps + + run(go()) + + +class GenAiClientError(Exception): + """Google GenAI client error: the HTTP status is an integer ``code``, not ``status_code``.""" + + def __init__(self, code: int, message: str = "client error"): + super().__init__(message) + self.code = code + + +class GenAiServerError(Exception): + """Google GenAI server error: the HTTP status is an integer ``code``, not ``status_code``.""" + + def __init__(self, code: int, message: str = "server error"): + super().__init__(message) + self.code = code + + +class StringCodeError(Exception): + """OpenAI-compatible error whose ``code`` names the error instead of carrying a status.""" + + code = "invalid_api_key" + + +class BadRequestError(Exception): + """OpenAI-compatible 400 that names no input, such as a malformed payload.""" + + status_code = 400 + + +class ResponseLike: + """httpx-style wrapper that keeps the status on a ``response`` attribute.""" + + def __init__(self, status_code: int): + self.status_code = status_code + + +class WrappedTransportError(Exception): + """Error that only exposes its status through ``response.status_code``.""" + + def __init__(self, status_code: int): + super().__init__("wrapped") + self.response = ResponseLike(status_code) + + +class DashScopeRuntimeError(RuntimeError): + """AgentScope turns every non-200 DashScope response into a plain ``RuntimeError``. + + The structured response is embedded in the message instead of being kept as an attribute, + so a client that only reads exception attributes cannot classify it. + """ + + @staticmethod + def for_status(status_code: int, code: str = "InvalidApiKey") -> "DashScopeRuntimeError": + """Build the exception AgentScope raises for a non-200 DashScope response.""" + response = { + "status_code": status_code, + "request_id": "req-0", + "code": code, + "message": "request rejected", + "output": None, + "usage": None, + } + return DashScopeRuntimeError(f"Failed to get embedding from DashScope API: {json.dumps(response)}") + + +def test_http_status_reads_every_supported_exception_shape(): + """Read the status from status_code, an integer code, response.status_code or the message.""" + + store = LocalEmbeddingStore(name="t_local_embedding_status_shapes") + + assert store._http_status(AuthenticationError("bad key")) == 401 + assert store._http_status(NotFoundError("model_not_found")) == 404 + assert store._http_status(GenAiClientError(401)) == 401 + assert store._http_status(GenAiServerError(503)) == 503 + assert store._http_status(WrappedTransportError(404)) == 404 + assert store._http_status(DashScopeRuntimeError.for_status(401)) == 401 + assert store._http_status(ValueError("no status here")) is None + + +def test_http_status_ignores_a_string_code(): + """A string ``code`` names the error; reading it as a status would misclassify quota errors.""" + + store = LocalEmbeddingStore(name="t_local_embedding_status_string_code") + + assert store._http_status(StringCodeError("invalid key")) is None + assert store._is_insufficient_quota(StringCodeError("invalid key")) is False + assert store._http_status(InsufficientQuotaError("quota exhausted")) == 429 + assert store._is_insufficient_quota(InsufficientQuotaError("quota exhausted")) is True + + +def test_google_genai_service_failures_are_not_split_into_per_item_requests(monkeypatch): + """A Gemini 401 / 429 / 503 describes the service, so the batch is never split per item. + + A 503 is answered once: the wrapper and the SDK own the retry for a 5xx, and retrying + here as well multiplied their attempts. A 429 keeps its retry because the rate-limit + branch above this classifier is the one place that reacts to it. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + for error, expected_batches in ( + (GenAiClientError(401), [10]), + (GenAiServerError(503), [10]), + (GenAiClientError(429), [10, 10, 10]), + ): + sleeps.clear() + provider = AlwaysProviderWideFailureAsEmbedding(error) + store = LocalEmbeddingStore( + name="t_local_embedding_genai", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert provider.batch_sizes == expected_batches + + run(go()) + + +def test_dashscope_runtime_errors_are_not_split_into_per_item_requests(monkeypatch): + """A DashScope failure arrives as a plain RuntimeError; the status lives in its message. + + As with the Gemini shapes, a 503 is answered once (the lower layers own that retry) and + a 429 keeps the retry from the rate-limit branch. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + for status, expected_batches in ((401, [10]), (503, [10]), (429, [10, 10, 10])): + sleeps.clear() + provider = AlwaysProviderWideFailureAsEmbedding(DashScopeRuntimeError.for_status(status)) + store = LocalEmbeddingStore( + name="t_local_embedding_dashscope", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert provider.batch_sizes == expected_batches + + run(go()) + + +def test_unrecognized_errors_are_not_split_into_per_item_requests(): + """Only an input-dependent error may split; anything unrecognized stays a batch failure.""" + + async def go(): + provider = AlwaysProviderWideFailureAsEmbedding(RuntimeError("provider said no")) + store = LocalEmbeddingStore(name="t_local_embedding_unrecognized", max_retries=3) + store.as_embedding = provider + + assert await store._call_with_retry(["a", "b", "c"]) is None + assert provider.batch_sizes == [3] + + run(go()) + + +def test_input_dependent_classifier_pins_both_halves_of_the_boundary(): + """The isolation must keep working for the one error a single item can explain.""" + + store = LocalEmbeddingStore(name="t_local_embedding_input_dependent") + + assert ( + store._is_input_dependent_failure( + ContextLengthError("the input length exceeds the context length"), + ) + is True + ) + assert store._is_input_dependent_failure(ContextLengthError("too long")) is True + assert store._is_input_dependent_failure(AuthenticationError("bad key")) is False + assert store._is_input_dependent_failure(InternalServerError("boom")) is False + assert store._is_input_dependent_failure(GenAiClientError(400)) is False + assert store._is_input_dependent_failure(RuntimeError("provider said no")) is False + + +def test_bad_request_without_an_input_length_signal_is_not_split(): + """A bare 400 does not name an input, so the batch is not divided over it.""" + + async def go(): + provider = AlwaysProviderWideFailureAsEmbedding(BadRequestError("malformed payload")) + store = LocalEmbeddingStore(name="t_local_embedding_bare_400", max_retries=2) + store.as_embedding = provider + + assert await store._call_with_retry(["a", "b", "c"]) is None + assert provider.batch_sizes == [3] + + run(go()) + + +def test_isolation_stops_when_a_provider_wide_failure_hits(): + """Once isolation meets a provider-wide failure, the remaining items are not requested. + + The vectors already recovered stay in the result: only the items that were never answered + are dropped, and no further request is sent for them. + """ + + async def go(): + provider = PerItemLimitThenQuotaAsEmbedding() + store = LocalEmbeddingStore( + name="t_local_embedding_isolation_stops", + max_retries=2, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) + + assert results == [[1.0, 0.0], None, None, None] + # One rejected batch, the recovered first item, the over-limit item, then the quota + # failure. The fourth item is never requested once the quota failure is seen. + assert provider.batch_sizes == [4, 1, 1, 1] + + run(go()) + + +def test_partial_success_keeps_a_failed_provider_unhealthy(): + """Recovered vectors must not overwrite the health state of a provider-wide failure. + + Isolation can recover an early item and then meet a provider-wide failure such as + exhausted quota. ``_call_with_retry`` marks the store unhealthy and stops, but the + vectors recovered before it are still returned. Treating that partial success as a + healthy provider would hide the outage, and it would break the unhealthy-to-healthy + transition the file store uses to schedule missing-vector backfill. The existing + isolation test drives ``_call_with_retry`` directly, so it never sees the overwrite. + """ + + async def go(): + provider = PerItemLimitThenQuotaAsEmbedding() + store = LocalEmbeddingStore( + name="t_local_embedding_partial_health", + max_retries=2, + enable_cache=False, + ) + store.as_embedding = provider + + vectors = await store.get_embeddings(["ok", "x" * 64, "also ok", "still ok"]) + + # The vector recovered before the outage is kept ... + assert vectors[0] is not None + assert vectors[1] is None and vectors[2] is None and vectors[3] is None + # ... and the provider-wide failure still decides the health state. + assert store.is_healthy is False + # One rejected batch, the recovered first item, the over-limit item, then the quota + # failure: the fourth item is never requested. + assert provider.batch_sizes == [4, 1, 1, 1] + + run(go()) + + +def test_isolation_keeps_going_while_only_inputs_fail(): + """An input-dependent failure is local to its item, so isolation must continue. + + This pins the other half of the boundary: stopping on every isolated failure would lose + the vectors of the items that are perfectly embeddable on their own. + """ + + async def go(): + provider = PerItemLimitAsEmbedding() + store = LocalEmbeddingStore( + name="t_local_embedding_isolation_keeps_going", + max_retries=2, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) + + assert results == [[1.0, 0.0], None, [1.0, 0.0], [1.0, 0.0]] + assert provider.batch_sizes == [4, 1, 1, 1, 1] + + run(go()) + + +def test_dashscope_serialized_quota_errors_keep_their_quota_policy(monkeypatch): + """A serialized DashScope quota error must classify as quota, not as a rate limit. + + AgentScope embeds the structured response in the message, so the business error code has to + be read back out of it. Without that the opt-in quota retry policy is never applied and the + error is retried with the generic backoff instead. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + error = DashScopeRuntimeError.for_status(429, code="insufficient_quota") + assert LocalEmbeddingStore._is_insufficient_quota(error) is True + assert LocalEmbeddingStore._is_rate_limited(error) is False + + provider = AlwaysProviderWideFailureAsEmbedding(error) + store = LocalEmbeddingStore( + name="t_local_embedding_dashscope_quota", + max_retries=3, + quota_retry_delay=7.0, + enable_cache=False, + ) + store.as_embedding = provider + + assert await store._call_with_retry(["a", "b"]) is None + assert provider.batch_sizes == [2, 2, 2] + assert sleeps == [7.0, 7.0] + + run(go()) + + +class ContextLength413Error(Exception): + """An over-limit input reported as HTTP 413 instead of 400. + + Ollama v0.40.3 normalizes an embedding input-limit rejection to 413 and propagates that + status through the embedding endpoint, so the same over-long item that arrives as a 400 + from an OpenAI-compatible gateway is a 413 for that provider. + """ + + status_code = 413 + body = {"error": {"message": "the input length exceeds the context length"}} + + +class RequestBodyTooLargeError(Exception): + """A bare 413 that never names an input: the request body itself is too large.""" + + status_code = 413 + + +class PerItemLimitAs413Embedding: + """Reject a whole request with HTTP 413 when any single text is over the per-item limit.""" + + dimensions = 2 + vector_space_id = "fakespace000" + max_chars = 32 + + def __init__(self): + self.batch_sizes: list[int] = [] + + def initialize_model(self): + """Mirror the real component's idempotent initialization hook.""" + + async def __call__(self, texts: list[str], **_kwargs): + self.batch_sizes.append(len(texts)) + if any(len(text) > self.max_chars for text in texts): + raise ContextLength413Error("the input length exceeds the context length") + return [[1.0, 0.0] for _ in texts] + + +def test_context_length_rejection_reported_as_http_413_is_isolated(): + """An over-limit input carried by a 413 must still reach the per-item isolation path. + + Ollama normalizes an embedding input-limit rejection to 413, so a classifier that only + accepts 400 rejects the batch instead of isolating the item that caused it, and every + healthy vector in the request is lost. + """ + + async def go(): + provider = PerItemLimitAs413Embedding() + store = LocalEmbeddingStore( + name="t_local_embedding_413_isolated", + max_retries=2, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [[1.0, 0.0], None, [1.0, 0.0]] + assert provider.batch_sizes == [3, 1, 1, 1] + + run(go()) + + +def test_bare_http_413_without_an_input_signal_is_not_split(): + """A 413 that names no input stays provider-wide, so the batch is not divided over it. + + The status alone is ambiguous: it can describe an unrelated request-body limit, which no + single input explains, so only a 413 that blames the input itself may fan out per item. + """ + + async def go(): + provider = AlwaysProviderWideFailureAsEmbedding( + RequestBodyTooLargeError("request entity too large"), + ) + store = LocalEmbeddingStore(name="t_local_embedding_bare_413", max_retries=2) + store.as_embedding = provider + + assert await store._call_with_retry(["a", "b", "c"]) is None + assert provider.batch_sizes == [3] + + run(go()) + + +def test_input_dependent_classifier_accepts_a_413_only_with_an_input_signal(): + """Pin both halves of the 413 boundary, and that the 400 behaviour is unchanged.""" + + store = LocalEmbeddingStore(name="t_local_embedding_413_classifier") + + assert ( + store._is_input_dependent_failure( + ContextLength413Error("the input length exceeds the context length"), + ) + is True + ) + assert store._is_input_dependent_failure(ContextLength413Error("too long")) is True + assert store._is_input_dependent_failure(RequestBodyTooLargeError("request entity too large")) is False + assert store._is_input_dependent_failure(ContextLengthError("too long")) is True + assert store._is_input_dependent_failure(GenAiClientError(400)) is False + + +class _NoRetryPolicyModel: + """Stand-in for AgentScope's Ollama embedding model: an empty policy plus a numeric budget. + + The real model keeps the inherited ``max_retries=3`` next to the empty tuple, so the double + has to carry both fields: omitting the number is what let the previous helper mistake the + budget for a policy. + """ + + max_retries = 3 + + @classmethod + def _get_retryable_exceptions(cls): + """Mirror the inherited empty policy.""" + return () + + +class _PartialRetryPolicyModel: + """Stand-in for AgentScope's DashScope embedding model: ``(RuntimeError,)`` and nothing else.""" + + max_retries = 3 + + @classmethod + def _get_retryable_exceptions(cls): + """Mirror the DashScope policy, which does not cover a connection error.""" + return (RuntimeError,) + + +class OllamaLikeTransientProvider(TransientThenSuccessAsEmbedding): + """An Ollama-shaped provider: the bound model declares no retry policy.""" + + def __init__(self, error: Exception, failures: int = 1): + super().__init__(error, failures) + self.model = _NoRetryPolicyModel() + + +def test_a_provider_with_an_empty_retry_policy_recovers_a_connection_error(monkeypatch): + """The store is the retry owner when the model below declares no policy. + + Ollama's embedding model inherits an empty retryable tuple and its SDK converts + ``httpx.ConnectError`` into a plain ``ConnectionError`` without retrying, so leaving the + retry to the lower layers would turn one dropped connection into a lost vector. The base + implementation answered that case with two requests; so must this one. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + provider = OllamaLikeTransientProvider(ConnectionError("connection reset")) + store = LocalEmbeddingStore( + name="t_local_embedding_empty_retry_policy", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["a", "b", "c"]) + + assert results == [[1.0, 0.0]] * 3 + assert provider.batch_sizes == [3, 3] + assert sleeps == [1.0] + + run(go()) + + +def test_a_policy_that_does_not_cover_the_error_keeps_the_store_backoff(monkeypatch): + """A nonempty policy is not ownership unless it covers the error that actually occurred. + + AgentScope's DashScope model declares ``(RuntimeError,)``, which does not cover a connection + error raised by ``requests`` underneath it. Trusting the whole tuple would leave that retry + to a layer that re-raises the error immediately, so one dropped connection would still cost + the vector. The store has to answer it as the base implementation did. + """ + + async def go(): + sleeps = [] + + async def fake_sleep(delay): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", fake_sleep) + + provider = OllamaLikeTransientProvider(ConnectionError("connection reset")) + provider.model = _PartialRetryPolicyModel() + store = LocalEmbeddingStore( + name="t_local_embedding_partial_policy", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["a", "b", "c"]) + + assert results == [[1.0, 0.0]] * 3 + assert provider.batch_sizes == [3, 3] + assert sleeps == [1.0] + + run(go()) + + +def test_an_isolated_item_does_not_multiply_sdk_transport_retries(monkeypatch): + """Isolating an item must not add a third retry loop over the SDK's budget. + + A rejected batch costs one request; the first isolated item then hits a persistent timeout, + which the wrapper and the SDK already try 12 times. The store must stop there (13 requests + in total) instead of asking that item three more times (37), and the failure is provider + wide, so the remaining items are left unanswered rather than probed one by one. + """ + + async def go(): + def handler(request): + body = json.loads(request.content.decode("utf-8")) + inputs = body["input"] + if isinstance(inputs, str): + inputs = [inputs] + if any(len(item) > 32 for item in inputs): + return context_length_rejection() + return read_timeout(request) + + endpoint = offline_openai_endpoint(handler).install(monkeypatch) + provider = openai_wrapper("t_openai_transport_isolated_budget") + store = LocalEmbeddingStore( + name="t_local_embedding_sdk_isolated_budget", + max_retries=3, + enable_cache=False, + ) + store.as_embedding = provider + + results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + + assert results == [None, None, None] + assert provider.calls == 2 + assert endpoint.requests == 13 + assert store.is_healthy is False + + run(go()) From a4f606c6a8ad0d90743c658bfe19151450416236 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:58:29 +0800 Subject: [PATCH 05/19] fix(dsh): recover memory backlogs and protect endpoint credentials (#653) * fix(dsh): recover memory backlogs and protect endpoint credentials * fix(dsh): address outbox and idempotency review findings --- docs/en/auto_memory.md | 5 + docs/zh/auto_memory.md | 4 + integrations/dsh/README.md | 46 +- integrations/dsh/README_ZH.md | 44 +- integrations/dsh/package-lock.json | 390 +++++----- integrations/dsh/package.json | 24 +- integrations/dsh/scripts/test-package.mjs | 1 + integrations/dsh/src/client/index.tsx | 56 +- integrations/dsh/src/config.ts | 47 +- integrations/dsh/src/endpoint.ts | 22 + integrations/dsh/src/index.ts | 9 +- integrations/dsh/src/outbox.ts | 231 ++++++ integrations/dsh/src/reme/client.ts | 16 +- integrations/dsh/src/reme/types.ts | 2 + integrations/dsh/src/runtime.ts | 491 +++++++------ integrations/dsh/src/types.ts | 11 +- integrations/dsh/tests/bundle.test.mjs | 1 + .../dsh/tests/client-settings.test.mjs | 29 +- integrations/dsh/tests/client.test.mjs | 69 ++ integrations/dsh/tests/config.test.mjs | 10 + integrations/dsh/tests/dsh-index.test.mjs | 34 +- integrations/dsh/tests/outbox.test.mjs | 197 +++++ integrations/dsh/tests/runtime.test.mjs | 676 +++++++++++++----- reme/config/default.yaml | 3 + reme/steps/evolve/auto_memory.py | 65 +- tests/unit/test_auto_memory_idempotency.py | 148 ++++ 26 files changed, 1949 insertions(+), 682 deletions(-) create mode 100644 integrations/dsh/src/endpoint.ts create mode 100644 integrations/dsh/src/outbox.ts create mode 100644 integrations/dsh/tests/outbox.test.mjs create mode 100644 tests/unit/test_auto_memory_idempotency.py diff --git a/docs/en/auto_memory.md b/docs/en/auto_memory.md index 02064837..e9c4597f 100644 --- a/docs/en/auto_memory.md +++ b/docs/en/auto_memory.md @@ -63,6 +63,11 @@ retarget inbound wikilinks. To see what happened on a day, start with `YYYY-MM-D ## Preserving the Original Information +For retrying clients, `auto_memory` accepts an optional `request_id` of 1–128 characters. Keep the ID and input unchanged +when retrying a batch: completed results are replayed without another memory-agent call; reuse with different input is +rejected. Receipts live under the configured `metadata_dir/auto_memory_requests`. A process interruption before receipt +commit may repeat an unfinished operation; arbitrary Agent file edits are not transactional. + The distilled daily note is optimized for readability; a filtered source conversation record is retained for trust and verification. diff --git a/docs/zh/auto_memory.md b/docs/zh/auto_memory.md index 7e946f15..aa16cf01 100644 --- a/docs/zh/auto_memory.md +++ b/docs/zh/auto_memory.md @@ -57,6 +57,10 @@ source_conversation: "[[session/dialog/session-a.jsonl]]" ## 同时保存原始信息 +需要重试的客户端可为 `auto_memory` 传入 1–128 字符的可选 `request_id`。重试同一批次时保持 ID 与输入一致： +已完成请求直接重放结果，不再调用记忆 Agent；同一个 ID 对应不同输入会报错。回执保存在配置的 +`metadata_dir/auto_memory_requests` 下。回执提交前发生进程中断，未完成的操作仍可能再次执行；任意 Agent 文件修改不具备事务性。 + 整理后的 daily note 负责“好读”，过滤后的对话来源记录负责“可信”。 Auto Memory 在生成记忆卡片的同时，也会保存对话来源消息： diff --git a/integrations/dsh/README.md b/integrations/dsh/README.md index d84c4bf6..b3f4aa28 100644 --- a/integrations/dsh/README.md +++ b/integrations/dsh/README.md @@ -4,7 +4,7 @@ This guide explains how to install, configure, and use `@agentscope-ai/reme-dsh-plugin` with DeepSeek Harness (DSH), including memory guidance injection, the `reme_search` tool, automatic memory, daily consolidation, and the ReMe Status page. -The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, and `0.2.1-alpha.4`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. +The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, and `0.2.1-rc.2`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. ## 1. How the plugin works @@ -26,7 +26,7 @@ The DSH adapter injects **usage guidance**, not every historical memory. Relevan ## 2. Requirements - ReMe is installed and its configuration exposes the `search`, `auto_memory`, and `auto_dream` jobs. -- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, or `0.2.1-alpha.4`. +- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, or `0.2.1-rc.2`. - Node.js `^22.19.0` or `>=24.0.0`, matching the current DSH engine range. - The browser running DSH can reach the configured ReMe HTTP endpoint. Cross-machine deployments must also allow the DSH browser origin. @@ -110,21 +110,23 @@ If an earlier upgrade already renamed `settings.yaml` to `settings.yaml.imported ![ReMe Memory plugin configuration](./figures/reme-memory-settings.png) -| UI meaning | Configuration key | Default | Description | -| ---------------------- | --------------------- | ----------------------- | --------------------------------------------------------------- | -| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute ReMe HTTP URL using `http` or `https`. | -| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | -| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | -| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | -| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | -| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | -| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | -| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | -| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | -| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | -| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | -| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | -| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| UI meaning | Configuration key | Default | Description | +| ---------------------- | ---------------------- | ----------------------- | --------------------------------------------------------------- | +| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute HTTP(S) URL without credentials, query, or fragment. | +| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | +| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | +| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | +| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | +| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | +| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | +| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | +| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | +| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | +| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | +| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | +| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| Retry delay | `autoMemoryRetryMs` | `1000` | Initial retry delay; doubles after each failure. | +| Maximum retry delay | `autoMemoryRetryMaxMs` | `60000` | Upper bound for retry delay. | Deployment configuration also supports `REME_URL`, or `REME_HOST` together with `REME_PORT`. The timer-only test option `dreamIntervalMs` is intentionally excluded from user settings. @@ -169,9 +171,17 @@ An empty successful response becomes `No relevant memory found.`. Service failur With `autoMemoryEnabled=true`, the plugin listens to DSH session events and collects completed user and assistant messages per session. When `autoMemoryInterval` is reached, the batch enters a background queue and calls ReMe `auto_memory`. Plugin-generated context and tool results are excluded from capture so they cannot be laundered back into long-term memory. +Each completed turn is saved in a private JSON outbox before submission. Failures use capped exponential backoff independently of new turns; recovery drains every ready batch, including ended sessions. Payload/job rejections, such as HTTP 413, pause the affected session while other sessions continue; network, authentication, rate-limit, and service failures pause the endpoint. Each endpoint drains independently and reads one batch at a time, so a stalled old destination cannot block a newly configured service. Disabling automatic memory pauses capture and further sends; queued files remain available when re-enabled. + +The outbox defaults to `$DSH_HOME/reme-memory/outbox` (`~/.dsh/reme-memory/outbox` without `DSH_HOME`). Set the deployment-only `autoMemoryQueueDir` field and restart to choose another directory; simultaneous Hosts must use separate directories. Batches retain their original service endpoint across settings changes. An exclusive owner lock prevents concurrent writes, and a restart recovers an exited owner's queue. Queue files contain conversation text: keep them private. Disk/permission failures are surfaced as errors; preserve DSH session logs for recovery. + +Owner-lock recovery is serialized. If startup reports a busy or interrupted `.lock-recovery`, retry first; if the error persists, stop all Hosts using that queue, inspect it, and remove only the empty `.lock-recovery` directory before restarting. Preserve `.lock` and all queued JSON files. + +Sealed batches keep a stable `request_id` across timeouts and restarts. The updated ReMe service stores completed request receipts under its configured metadata directory, rejects reuse with different input, and replays completed results without invoking the memory agent again. An interruption before a receipt is committed can still repeat an unfinished operation; receipts do not make arbitrary agent file edits transactional. + ![Automatic-memory activity](./figures/reme-status-auto-memory.png) -Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until running and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. +Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until queued turns, running tasks, and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. ## 8. ReMe Status tabs diff --git a/integrations/dsh/README_ZH.md b/integrations/dsh/README_ZH.md index 08262b95..34222a7b 100644 --- a/integrations/dsh/README_ZH.md +++ b/integrations/dsh/README_ZH.md @@ -26,7 +26,7 @@ DSH 启动新会话时，插件向根 Agent 注入一段“如何使用长期记 ## 2. 环境要求 - ReMe Python 服务已安装，且配置中提供 `search`、`auto_memory` 和 `auto_dream` Job。 -- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2` 或 `0.2.1-alpha.4`。 +- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4` 或 `0.2.1-rc.2`。 - Node.js `^22.19.0` 或 `>=24.0.0`，与当前 DSH 的 engine 范围一致。 - DSH 页面能够访问 ReMe HTTP 地址；跨机器部署时还要允许 DSH 页面所在的浏览器 Origin。 @@ -115,21 +115,23 @@ dsh web --no-open --port 3090 截图中的测试配置使用 `http://127.0.0.1:3457`、English 指引、默认搜索数量 5、搜索超时 10 秒，并启用了自动记忆和“Exclude subagents”。完整字段如下： -| 界面含义 | 配置键 | 默认值 | 说明 | -| -------------------- | --------------------- | ----------------------- | -------------------------------------------------------------------------- | -| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的 ReMe HTTP 服务绝对地址，只支持 `http`/`https`。 | -| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | -| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | -| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | -| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | -| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | -| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | -| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | -| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | -| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | -| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | -| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | -| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 界面含义 | 配置键 | 默认值 | 说明 | +| -------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------- | +| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的绝对 HTTP(S) 地址，不支持 URL 内嵌凭据、查询参数或片段。 | +| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | +| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | +| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | +| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | +| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | +| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | +| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | +| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | +| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | +| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | +| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | +| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 失败重试间隔（毫秒） | `autoMemoryRetryMs` | `1000` | 首次失败后的等待时间，连续失败时翻倍。 | +| 最大重试间隔（毫秒） | `autoMemoryRetryMaxMs` | `60000` | 重试等待时间的上限。 | 部署层还支持 `REME_URL`，或组合使用 `REME_HOST` 与 `REME_PORT`。仅供定时器测试的 `dreamIntervalMs` 不出现在用户设置中。 @@ -180,11 +182,19 @@ only on retrieved memory and cite the returned paths. 启用 `autoMemoryEnabled` 后，插件监听 DSH `session/event`，按会话收集完成的用户和助手消息。达到 `autoMemoryInterval` 后进入提交队列，后台调用 ReMe `auto_memory`。插件生成的上下文以及工具结果不会再次进入自动记忆，避免把指引或检索回显循环写回长期记忆。 +每个完成回合会先写入私有 JSON 提交队列。失败后按有上限的指数退避独立重试，不依赖下一轮对话；恢复后持续排空可发送批次，也包括已结束的会话。请求内容或 Job 被拒绝（例如 HTTP 413）时，仅暂停受影响的会话，其他会话继续处理；网络、认证、限流或服务故障会暂停对应服务地址。每个服务地址独立排空队列，每次读取一个批次，旧地址挂起不会阻塞新配置的服务。关闭自动记忆会暂停捕获及后续发送，重新启用后可以继续处理保留的队列。 + +队列默认位于 `$DSH_HOME/reme-memory/outbox`，未配置 `DSH_HOME` 时使用 `~/.dsh/reme-memory/outbox`。可在部署配置中设置 `autoMemoryQueueDir`，重启后生效；同时运行的 Host 必须分别使用不同目录。积压批次保留捕获时的服务地址，修改地址不会把旧内容发送到另一个服务。独占锁阻止并发写入，重启时可以恢复已退出进程的积压。队列包含对话正文，请保留私有权限。磁盘或权限错误会明确报告，原始 DSH 会话日志应保留用于恢复。 + +进程锁的回收会串行执行。若启动报告 `.lock-recovery` 正忙或恢复中断，先重试；错误持续时，停止所有使用该队列的 Host，检查队列后仅移除空的 `.lock-recovery` 目录，再重新启动。保留 `.lock` 和所有排队的 JSON 文件。 + +已封存的批次在超时、重启后沿用相同的 `request_id`。更新后的 ReMe 服务在配置的 metadata 目录保存已完成请求的回执，拒绝把同一个 ID 用于不同输入，并直接重放已完成结果，避免重复调用记忆 Agent。回执提交前发生中断，未完成的操作仍可能重试；回执不会把任意 Agent 文件修改变成事务。 + 截图测试把间隔临时设为 1；运行记录中可以看到英文测试会话形成的已完成提交： ![自动记忆运行记录](./figures/reme-status-auto-memory.png) -自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 +自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“待处理回合”“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 ## 8. ReMe 状态页六个标签 diff --git a/integrations/dsh/package-lock.json b/integrations/dsh/package-lock.json index 94cd94bc..f6796838 100644 --- a/integrations/dsh/package-lock.json +++ b/integrations/dsh/package-lock.json @@ -9,18 +9,18 @@ "version": "0.1.3", "license": "Apache-2.0", "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -328,21 +328,21 @@ } }, "node_modules/@deepseek-ai/cordis": { - "version": "4.0.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5-alpha.2.tgz", - "integrity": "sha512-rd5Qp6PNrMk+0saWVMLUaVv08s/WBwf2TXM84ST6tppFqlHS0Yl9l/NwPrA4oVOuS/1BKCYOGlZ6zJRj+JG6IQ==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5.tgz", + "integrity": "sha512-Sktq5axy0owk9PdnivBRNOJjHEVEibCt0+TDDZcv0+UH2KS1M910fX4kn5y45YFsrpZIi1mJf/1s5VzywMUnyg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" }, "bin": { "cordis": "bin.js" }, "peerDependencies": { - "@deepseek-ai/cordis-plugin-include": "~1.0.10-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "~1.0.6-alpha.2" + "@deepseek-ai/cordis-plugin-include": "~1.0.10", + "@deepseek-ai/cordis-plugin-loader": "~1.0.6" }, "peerDependenciesMeta": { "@deepseek-ai/cordis-plugin-include": { @@ -354,17 +354,17 @@ } }, "node_modules/@deepseek-ai/cordis-plugin-loader": { - "version": "1.0.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6-alpha.2.tgz", - "integrity": "sha512-ea4WjgWsrUxVJ0UYujTbGgaSIR/mV46myY/s6isYRTQRCmlkyVPGagkTxrRR2c/MqGwwbC4KXxusT4uM7H9BXg==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6.tgz", + "integrity": "sha512-fr011A782oEjkJXCY4pePt2qaPpkGvem7Rmst6Rhwgg6icgwI4eN9QBRb6S/q9XOYGVQUdvL0wXUWyNxO8WipA==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/cosmokit": "~1.8.6", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", + "@deepseek-ai/cordis": "~4.0.5", "node-addon-require-builtin": "^0.1.9" }, "peerDependenciesMeta": { @@ -374,28 +374,28 @@ } }, "node_modules/@deepseek-ai/cosmokit": { - "version": "1.8.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6-alpha.2.tgz", - "integrity": "sha512-FcxdMUal80CeFURh3lPYsw9TPDMVSGpovWnA3JtTGrBG15XFli4k38z1Egso1lgvifKpIW4Hx42X6UTelLEOAA==", + "version": "1.8.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6.tgz", + "integrity": "sha512-ZSD/fN52L498LBIOYwjIOlaLv5veY6ZEwI2yzvZUKQtctn0QAPqu8tKdJ+OF5+Tv6NBq1U+wJWH+QI6OGy5EvQ==", "dev": true, "license": "MIT" }, "node_modules/@deepseek-ai/dsh-agent": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-alpha.4.tgz", - "integrity": "sha512-QIL78OhM5csPWkrdGfhUJQr3zdyUd/DZyM4euwUnkMS0BA5Hp21hCWg7N69HY73bqvV20oyU10g5huPtqJHSyQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-rc.2.tgz", + "integrity": "sha512-e96Tnq8g/RNCDDUW7kAypY/4cho5QxjLHMhk86BGGIEu9b31XEkSFqwqQIwtYXQez3cDR5z7LZpAM8Uqx9Y8KA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/dsh-workspace": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/dsh-workspace": "0.2.1-rc.2" }, "peerDependenciesMeta": { "@deepseek-ai/dsh-workspace": { @@ -404,148 +404,148 @@ } }, "node_modules/@deepseek-ai/dsh-brand": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-alpha.4.tgz", - "integrity": "sha512-SlrYMisnHPNREhc9jsohRylcZmpQPuzcul7DxUCJNjK08fDUUuzcyrQI7uDjvbw1n34IwYHNNV6o4mrhIIForQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-rc.2.tgz", + "integrity": "sha512-T2kcygikRS8rv8cKjgho2x381EgGprv5yQmvHl4/iYbuHE19OW+SLGtHWbV27PGdSHz5U8CJxasYzFosKW4e2A==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-primitives": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-alpha.4.tgz", - "integrity": "sha512-nHzG2k7l8UuNG2rcIkzny1ko/fn/mQjiWrSh6wIe6X9d/qQi55Tdo87N3usmwce1akcZrvsXQ27v451tC1gzwA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-rc.2.tgz", + "integrity": "sha512-3K6W2ID4sbDUDHhZHs8cNO15XDXBWAYFnEzJ/iD8ZASqidazcJActxlEf8GO6NHmbbeX0lEq0P20Ftwhovt+Mw==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-settings": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-alpha.4.tgz", - "integrity": "sha512-SG6wzQyBmkNUuVt+5Da4tSnGeVVgNE8cJ6Qa6MG4zOJqWnEkJTCIRuOmsN1+7IGcWSNooFG76RGJpO+5uKkDdA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-rc.2.tgz", + "integrity": "sha512-oU8f/0+DsdbgF0XhM+Po7W4PDJmTbhifQVwIoAcKfj29JN/4tLxKqIKS0IRcMbIGD39BGFITne78KIMgrGJy3Q==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-fs": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-alpha.4.tgz", - "integrity": "sha512-K5TK7QxC3fRM8s1Uvfy1O3C4kaybSXOLGFMSb9WLwfTe1QGsq7JZN0v8jmiv3VFA0cFUkTPybz/PokyDtRWvRg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-rc.2.tgz", + "integrity": "sha512-Z6d2ZKsAiJc0FaiES8wC66vuUQMGohl9tCc2p3bAsPFYkJ0+L2zTViQ9Y5BArtRe9Br8qhNjNgLlN0jb/uwv1g==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-llm": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-alpha.4.tgz", - "integrity": "sha512-rjAT6x0iNUNLgCC3pCNaQxrtQxlcMAAdCh8x27C4PiGCXMaa7kNge8HnjPC3iaZoopSrr5XRBLcyIFmg46Cc+A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-rc.2.tgz", + "integrity": "sha512-3xfoMDHEpUJGcH2Lu+lNXUJYRBclYfBkdCG9Au02VzIjYAm9BrFmc/MNuaVTg0vwgvPpTsd6Fx1IeGtfmQFvlg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-timeout": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-crypto": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-timeout": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-crypto": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-ptc-runtime": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-alpha.4.tgz", - "integrity": "sha512-r+4GYR1UYbr0zyuu9tOO+iJuQvxYjtCJyTHr6YR9UZNJrfbwkrSeqlRIbR1cw5NaIh7dplZK1+a90FHURkrgWA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-rc.2.tgz", + "integrity": "sha512-e81rbNwp3YNIpzIbkFI4NsWQzTD3uAP8zXU16diL9wLmITUWjMS+kvrWEXtrkWAWp5cnK2Pgn4xCbqVZ6n4I8w==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-alpha.4.tgz", - "integrity": "sha512-PRCb2xDAvX609LagauDjLvYFmzYLBO3QYAYXcc7btfiqOdgfeDb7FgstzKjdbz2o31iVdvKUdDK1iFzXjVg6eg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-rc.2.tgz", + "integrity": "sha512-nnxNIkgi4A4tkPrTV8sbVMse3c32OT0U+mwYhLqDz/+cIm5tUGufQD+Ax0zqVeClyzXsxZo0/pVtcqDJrsp+7w==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox-policy": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-alpha.4.tgz", - "integrity": "sha512-xEfAiquD87cg1aOzdPtFoB82e0P4hzWkVEimwRk1s85ESb8YiV+DRzWLrIGprfi9fKhsxSFFIFnCKgzOKjURuA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-rc.2.tgz", + "integrity": "sha512-fBb8zQ13CLKM60ynl3EHvFmO8F0GARofQYf7CXUkqJEuBcjxXUG14rzDKRS/PhE8oJ023lEvd5HJjE6qUjslhw==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-scope": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-alpha.4.tgz", - "integrity": "sha512-oo1AraYsu5D3oYGdKim43HGg0Hf6NbYLmV5LZO8jiDnaXiE+xfNDqI8ARwq049lODJMxP0ONX2hTo5DTG5zjuQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-rc.2.tgz", + "integrity": "sha512-CVGPI4lzmUy7O3zg8X1LavUSmD3JKFkUdykfHN7VkyWXG5oAXSvig9s5PLnIb+sxRItmSpiqPXGnT/48X1T1qg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-session": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-alpha.4.tgz", - "integrity": "sha512-YMdG+ZdvM+jXFDkfzO+ksCn6YpUFuoN4YuMEIcbSqLnwNo0Q7ba0GJWJEH7oP5v0sCwGtMKQySKSgqBjkUyw+w==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-rc.2.tgz", + "integrity": "sha512-WDA4lIrXtG5nf1dn2wrA4UzEr7D6e8T3AdPpVSZ7eY4N4ICMDZ0RYuD4GkM+iSb7WkLojdO8ljr0etiIMEbqJg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-session-projection": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-alpha.4.tgz", - "integrity": "sha512-+Tm9xJd7ETbcLlolf9wx+Nm2z1O0SFqXermWU3GZ7ffQqo+tRKQ2oEa6NhSS5Mw1wVkY0xuxye5RRnVk3gbb4g==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-rc.2.tgz", + "integrity": "sha512-GBGgIAaC+Cw1rV7tpyGy9aaENUOkQCosP+LJHcmzqPWg64Ufts36mDEU3f0lC/nyGb0z66Ov+RbyaLLsCnZMwA==", "dev": true, "license": "MIT", "peer": true, @@ -553,144 +553,144 @@ "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-system-prompt": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-alpha.4.tgz", - "integrity": "sha512-oHk8b0F2ecwrcN58ZtkMkR8FIRTULNSMTkHB1KbOz1LpTYuD6M01VFhWG0FvzZw01xPeuDlbO9wNmYtYi6/RsA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-rc.2.tgz", + "integrity": "sha512-X1WG2FMyfZapAkcCYyTl0UO5vVqDcCsA9wECJY7eeDTLHJDsqrilQJg2DRgHQzM+2/7T01QyzFZhKd7hfALJxg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-timeout": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-alpha.4.tgz", - "integrity": "sha512-hXCp1YWwDZAaTrNNon3ypb/Z6ZgDNKi/3zdC2JLHb9gU2f7DdSexbvRs37xKfyXO8HhtXDKvp/FUCWvI6rpRlg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-rc.2.tgz", + "integrity": "sha512-R75r+GkYP+ZhtIsIfn8MMuRXPkugJr0OyQ1Y+rauoTae2sdVfnE8v5zvQORRCHdh7MiOcmh10dQN70KjHh7qhQ==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-tools": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-alpha.4.tgz", - "integrity": "sha512-oTurHJ91LmlirKbEMyU0K+ES8LYtfJW4MAGrLjSBSaRKrq8eERTWNJafcxlhEKNpTWeUNU7h3OHaX/rqcMmLyA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-rc.2.tgz", + "integrity": "sha512-K3gET3VPbtxM66pkeRsEGrZ+bQKiBqrTbNXiIreBKu0hP+qPvlSwhiSI5MTnFp2tMFK5rpuYMZosRIdbfrbJ/g==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-ptc-runtime": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox-policy": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-user-approval": "0.2.1-alpha.4", - "@deepseek-ai/dsh-working-directory": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-ptc-runtime": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox-policy": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-user-approval": "0.2.1-rc.2", + "@deepseek-ai/dsh-working-directory": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-typert-protocol": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-alpha.4.tgz", - "integrity": "sha512-LBNUgiwbRqhyRNPvaAca++eLAxJ1jmmFJnd3RJAa8pT+oULJgZvqPgr9jVnTfX/P5pQT0p7AMZeelTc7QpQ/OA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-rc.2.tgz", + "integrity": "sha512-rtsHJHGE9SWX+zOgtDbPUqz3h7r+qFm3h1WASzRJxDnULqo7JaBdIiGshNTFPmetsX+r1T69CKFdTZAJmHwMug==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-user-approval": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-alpha.4.tgz", - "integrity": "sha512-J6bcZV3rFuQiRDuMej5S/DalSeztfVIFngoGA15M31khb0E74kfrgWUIxvS7pRwy2j1MKz89wR8BbbDzufHTRw==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-rc.2.tgz", + "integrity": "sha512-pwhw1U1IZZm5tKyPayj2APZBq5I7/I0C0ad+ohd4vKuNb6twDIltt1GEHdCNt7I19IGHLRiUY+/Q9S/8+q5K0A==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-util-crypto": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-alpha.4.tgz", - "integrity": "sha512-BLMlmD3y+HhTZPGij4viPRFWx8DNdsPNMauUW5XxyE2Iq5KG9ohQ+6xULju/J2Ah9JldVr3XAku7ItDPDk8Mpg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-rc.2.tgz", + "integrity": "sha512-V4SxvXL4ZbY5RYJckVtmo3qz4YsA52pYKvDF13+8hsDCEmQIv9YfmLRMA61pjK/i05pGMe3p1pPhAB6L+NOIEA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-util-values": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-alpha.4.tgz", - "integrity": "sha512-RM3gAmdyUsldAb4AIsU++KYbBY98UvHRCskcHX2bojZ6kefGd1a8P1Yl/Fo+bUWIAHZj/VoA1pnAl7r5e5456A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-rc.2.tgz", + "integrity": "sha512-mgnByPadOgZtoBxbwMI184li494O6sNn66/NvdqN0RIrzAJmFGpFxpjFkQtQN/yetwWhlx7Ic17+6p7eKl8DNg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-working-directory": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-alpha.4.tgz", - "integrity": "sha512-VtgMMlvYJ4iOPtpasI+ey6ioBekJy5MLSw3ky6lvLdLnKPmj0zobVOFhldr7KEvZFRizKYN5ktU8D85qKZG9Cg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-rc.2.tgz", + "integrity": "sha512-zhPtvAaCxa9U0WUN9OEG37zVBq7GOXXXwnDx2Jds4DFksGY5Lmpl3ZK1QMGz+F/Aye4VHt9zoISI7m4wDA8DEg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-fs": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-fs": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/schemastery": { - "version": "3.18.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5-alpha.2.tgz", - "integrity": "sha512-W+2TSjdeKOEKqaii9BNpw7ev2Yf8C+J74i/BE/VBx56O7BX0zxV2zQKwRY4ADLTXD5O69iJ+/Skm91/ES7iMgw==", + "version": "3.18.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5.tgz", + "integrity": "sha512-Mnobvqf9rHsZYv4g5edj5ddxS1oF1kIdSlz0DunUgnY3LWG5F26MqvH3YOOsSaodCgyzJ16OMN9O99/s5Crtdw==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" } }, @@ -1488,9 +1488,9 @@ } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz", - "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==", + "version": "7.0.13", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.13.tgz", + "integrity": "sha512-2E8Q6rh+1RRSq/YdaGsNfXsEZy9FVqv6BSSrkmZJP7BnaoPnRfKL+YaofPvP+5rzdk144qSm6i4+hvPgUcbV4Q==", "dev": true, "license": "MIT", "engines": { @@ -1811,9 +1811,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.11.29", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.29.tgz", - "integrity": "sha512-YeesxhUmR4h5y5V8KF8iBCqJb8YFrBJ++XBTeGcWIaQOYXGN19JM0FoiOjAhwuDoln8owMgXcWBmgHtYSkky2w==", + "version": "2.11.30", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.30.tgz", + "integrity": "sha512-YJs6Pz17RMNv8HoyKREtCY38E3BV1nPkWVmIb+XN1uHC1Vqt3KZCH/uJH9ZKRoiKb/icZ6tvehyee6RZNbFfRA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1835,9 +1835,9 @@ } }, "node_modules/browserslist": { - "version": "4.29.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", - "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "version": "4.29.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.4.tgz", + "integrity": "sha512-2TcbbiwXEq+N0kD36tykpCfyyQigUXmuwNQbnPp4Xk4PX9JOqVIKjJlIDjpH7uY6tmRGoAd+apeT1p60T8wsdA==", "dev": true, "funding": [ { @@ -1855,11 +1855,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.11.26", - "caniuse-lite": "^1.0.30001813", - "electron-to-chromium": "^1.5.439", - "node-releases": "^2.0.57", - "update-browserslist-db": "^1.3.3" + "baseline-browser-mapping": "^2.11.29", + "caniuse-lite": "^1.0.30001816", + "electron-to-chromium": "^1.5.459", + "node-releases": "^2.0.59", + "update-browserslist-db": "^1.3.4" }, "bin": { "browserslist": "cli.js" @@ -1879,9 +1879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001816", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001816.tgz", - "integrity": "sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==", + "version": "1.0.30001817", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001817.tgz", + "integrity": "sha512-uZ1JG80BwhtMalIAGPxPfmNhHyd/B/nTdr3hITWlRamSyAdUStP1Gop01e7SD5q6J90ixJvWDbvmiFVfDgn4bw==", "dev": true, "funding": [ { @@ -1998,9 +1998,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.457", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.457.tgz", - "integrity": "sha512-RNKsgErRmkzRgC8nOV1wzFhSbfGlYJ6k4LT/aE7oFrDYaQgHhTh5oTp0ZgSbRRybKR8HlZIdCWc38Y+CDTk1Jw==", + "version": "1.5.463", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.463.tgz", + "integrity": "sha512-nVWNU6ltFjopvOdD44J4zoBkgO0M+WZiFKKFV7EjebOH9z0KNphk6bCEjplXtJofL21ARIuuwWOCsFkBVklm6w==", "dev": true, "license": "ISC" }, diff --git a/integrations/dsh/package.json b/integrations/dsh/package.json index 86388a31..0ec3eaa9 100644 --- a/integrations/dsh/package.json +++ b/integrations/dsh/package.json @@ -79,14 +79,14 @@ } }, "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -98,10 +98,10 @@ "react": "^18.2.0", "typescript": "^5.9.2", "typescript-eslint": "8.59.3", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", "react-test-renderer": "^18.3.1", "semver": "^7.8.5" }, diff --git a/integrations/dsh/scripts/test-package.mjs b/integrations/dsh/scripts/test-package.mjs index 67884661..3e8f3f46 100644 --- a/integrations/dsh/scripts/test-package.mjs +++ b/integrations/dsh/scripts/test-package.mjs @@ -37,6 +37,7 @@ const releases = [ loader: "1.0.6-alpha.2", schema: "3.18.5-alpha.2", }, + { dsh: "0.2.1-rc.2", cordis: "4.0.5", loader: "1.0.6", schema: "3.18.5" }, ]; async function run(command, args, cwd) { diff --git a/integrations/dsh/src/client/index.tsx b/integrations/dsh/src/client/index.tsx index f8674bd0..d14a0151 100644 --- a/integrations/dsh/src/client/index.tsx +++ b/integrations/dsh/src/client/index.tsx @@ -2,6 +2,7 @@ import { useState, useSyncExternalStore } from "react"; import type { ConfigForm } from "@deepseek-ai/dsh-client-ui-settings/client"; import type { ReMeSettings } from "../types.js"; +import { validEndpoint } from "../endpoint.js"; import { ReMeStatusPage, statusEn, @@ -19,6 +20,8 @@ const FIELDS = [ "requestTimeoutMs", "backgroundTimeoutMs", "shutdownTimeoutMs", + "autoMemoryRetryMs", + "autoMemoryRetryMaxMs", "autoMemoryEnabled", "autoMemoryInterval", "autoDreamEnabled", @@ -65,6 +68,8 @@ interface Draft { requestTimeoutMs: string; backgroundTimeoutMs: string; shutdownTimeoutMs: string; + autoMemoryRetryMs: string; + autoMemoryRetryMaxMs: string; autoMemoryEnabled: boolean; autoMemoryInterval: string; autoDreamEnabled: boolean; @@ -113,6 +118,8 @@ const en = { advanced: "Advanced", backgroundTimeout: "Background request timeout (ms)", shutdownTimeout: "Shutdown flush timeout (ms)", + retryDelay: "Retry delay (ms)", + retryMaxDelay: "Maximum retry delay (ms)", diagnostics: "Service diagnostics", processMemory: "Process memory", componentMemory: "Component memory", @@ -186,6 +193,8 @@ const zh: typeof en = { advanced: "高级设置", backgroundTimeout: "后台请求超时（毫秒）", shutdownTimeout: "关闭时写入等待（毫秒）", + retryDelay: "失败重试间隔（毫秒）", + retryMaxDelay: "最大重试间隔（毫秒）", diagnostics: "服务诊断", processMemory: "进程内存", componentMemory: "组件内存", @@ -568,6 +577,28 @@ function SettingsForm({ } /> + + + update("autoMemoryRetryMs", event.target.value) + } + /> + + + + update("autoMemoryRetryMaxMs", event.target.value) + } + /> + value === undefined) @@ -676,6 +721,8 @@ function parseDraft(draft: Draft): ReMeSettings | undefined { requestTimeoutMs: requestTimeoutMs!, backgroundTimeoutMs: backgroundTimeoutMs!, shutdownTimeoutMs: shutdownTimeoutMs!, + autoMemoryRetryMs: autoMemoryRetryMs!, + autoMemoryRetryMaxMs: autoMemoryRetryMaxMs!, autoMemoryEnabled: draft.autoMemoryEnabled, autoMemoryInterval: autoMemoryInterval!, autoDreamEnabled: draft.autoDreamEnabled, @@ -699,15 +746,6 @@ function boundedInteger( : undefined; } -function validEndpoint(value: string): boolean { - try { - const url = new URL(value.trim()); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - function validTimezone(value: string): boolean { try { new Intl.DateTimeFormat("en", { timeZone: value.trim() }).format(0); diff --git a/integrations/dsh/src/config.ts b/integrations/dsh/src/config.ts index ce4421bc..e517814c 100644 --- a/integrations/dsh/src/config.ts +++ b/integrations/dsh/src/config.ts @@ -1,4 +1,7 @@ import z from "@deepseek-ai/schemastery"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { validEndpoint } from "./endpoint.js"; import { validTimezone, validateDailyCron } from "./scheduling.js"; import type { ReMeConfig, ReMeConfigInput } from "./types.js"; @@ -26,6 +29,17 @@ export const Config = z.object({ shutdownTimeoutMs: z.natural().min(100).max(60000).default(5000).volatile(), autoMemoryEnabled: z.boolean().default(true).volatile(), autoMemoryInterval: z.natural().min(1).max(1000).default(5).volatile(), + autoMemoryQueueDir: z + .string() + .default("") + .description("Private persistent automatic-memory queue directory"), + autoMemoryRetryMs: z.natural().min(1000).max(60000).default(1000).volatile(), + autoMemoryRetryMaxMs: z + .natural() + .min(1000) + .max(3600000) + .default(60000) + .volatile(), autoDreamEnabled: z.boolean().default(true).volatile(), dreamCron: checkedString( DEFAULT_DREAM_CRON, @@ -77,6 +91,9 @@ const DEFAULT_CONFIG: Readonly = Object.freeze({ shutdownTimeoutMs: 5000, autoMemoryEnabled: true, autoMemoryInterval: 5, + autoMemoryQueueDir: "", + autoMemoryRetryMs: 1000, + autoMemoryRetryMaxMs: 60000, autoDreamEnabled: true, dreamCron: "0 23 * * *", dreamHint: "", @@ -121,6 +138,23 @@ export function resolveConfig( config.endpoint = stripTrailingSlashes(String(config.endpoint)); assertEndpoint(config.endpoint); + const home = env.DSH_HOME?.trim() || `${homedir()}/.dsh`; + const queueDir = config.autoMemoryQueueDir || `${home}/reme-memory/outbox`; + config.autoMemoryQueueDir = resolve( + queueDir.replace(/^~(?=\/|$)/, homedir()), + ); + config.autoMemoryRetryMs = integer( + config.autoMemoryRetryMs, + 1000, + 60000, + 1000, + ); + config.autoMemoryRetryMaxMs = integer( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs, + 3600000, + 60000, + ); config.requestTimeoutMs = integer( config.requestTimeoutMs, 1000, @@ -163,15 +197,10 @@ export function resolveConfig( } function assertEndpoint(value: string): void { - let endpoint: URL; - try { - endpoint = new URL(value); - } catch { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } - if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } + if (!validEndpoint(value)) + throw new TypeError( + "ReMe endpoint must be an absolute http(s) URL without credentials, query, or fragment", + ); } function stripTrailingSlashes(value: string): string { diff --git a/integrations/dsh/src/endpoint.ts b/integrations/dsh/src/endpoint.ts new file mode 100644 index 00000000..b7b8f99b --- /dev/null +++ b/integrations/dsh/src/endpoint.ts @@ -0,0 +1,22 @@ +/** HTTP endpoints exclude credentials, query strings, and fragments. */ +export function validEndpoint(value: string): boolean { + try { + const url = new URL(value.trim()); + return ( + (url.protocol === "http:" || url.protocol === "https:") && + !url.username && + !url.password && + !url.search && + !url.hash && + !/[?#]/.test(value) + ); + } catch { + return false; + } +} + +/** Remove URL userinfo before errors cross model, browser, or log boundaries. */ +export function safeError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message.replace(/(https?:\/\/)[^\s/]+@/gi, "$1[redacted]@"); +} diff --git a/integrations/dsh/src/index.ts b/integrations/dsh/src/index.ts index fbdc3640..8a14eebd 100644 --- a/integrations/dsh/src/index.ts +++ b/integrations/dsh/src/index.ts @@ -20,6 +20,10 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { const current = () => config; const client = new ReMeClient(current); const runtime = new ReMeRuntime(client, current, ctx.logger); + ctx.effect(() => { + runtime.start(); + return () => runtime.disposeAll(); + }, "remeMemory.lifecycle()"); ctx.provide("remeMemory", runtime); void ctx.plugin(ReMeStatusGateway); ctx.effect( @@ -27,11 +31,6 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { "remeMemory.tools()", ); - ctx.effect(() => { - runtime.start(); - return () => runtime.disposeAll(); - }, "remeMemory.lifecycle()"); - ctx.on("loader/volatile-update", () => { config = resolveConfig(input); runtime.reconfigure(); diff --git a/integrations/dsh/src/outbox.ts b/integrations/dsh/src/outbox.ts new file mode 100644 index 00000000..3becfacd --- /dev/null +++ b/integrations/dsh/src/outbox.ts @@ -0,0 +1,231 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + opendirSync, + readFileSync, + renameSync, + rmdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import type { ReMeMessage } from "./reme/types.js"; +import { validEndpoint } from "./endpoint.js"; + +export interface MemoryBatch { + version: 1; + id: string; + createdAt: number; + endpoint: string; + sessionId: string; + date: string; + turns: ReMeMessage[][]; +} + +/** Private JSON outbox: seal batches before sending and remove only after acknowledgement. */ +export class MemoryOutbox { + private readonly lock: string; + private readonly owner = `${process.pid}:${randomUUID()}`; + + constructor(readonly directory: string) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + this.lock = join(directory, ".lock"); + // Serialize inspection, stale-owner removal, and replacement as one operation. + // An interrupted recovery fails closed; never reclaim this guard automatically. + const guard = join(directory, ".lock-recovery"); + try { + mkdirSync(guard, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + throw new Error( + "ReMe outbox lock recovery is busy or interrupted; retry, or inspect .lock-recovery after stopping all Hosts", + ); + } + try { + if (existsSync(this.lock)) { + const pid = Number(readFileSync(this.lock, "utf8").split(":")[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) + throw new Error( + "Invalid ReMe outbox lock; inspect the queue directory", + ); + try { + process.kill(pid, 0); + throw new Error( + "ReMe outbox is in use; configure a separate autoMemoryQueueDir for each Host", + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + unlinkSync(this.lock); + } + const fd = openSync(this.lock, "wx", 0o600); + try { + writeFileSync(fd, this.owner); + } finally { + closeSync(fd); + } + } finally { + rmdirSync(guard); + } + } + + append( + endpoint: string, + sessionId: string, + date: string, + messages: ReMeMessage[], + interval: number, + ): void { + const key = createHash("sha256") + .update(JSON.stringify([endpoint, sessionId, date])) + .digest("hex"); + const name = `partial-${key}.json`; + const path = join(this.directory, name); + const batch: MemoryBatch = existsSync(path) + ? this.read(name) + : { + version: 1, + id: randomUUID(), + createdAt: Date.now(), + endpoint, + sessionId, + date, + turns: [], + }; + batch.turns.push(messages); + this.write(name, batch); + if (batch.turns.length >= interval) this.seal(name, batch); + // Finish the previous workspace day without combining dates in one request. + for (const file of this.files()) { + if (!file.startsWith("partial-") || file === name) continue; + const previous = this.read(file); + if ( + previous.endpoint === endpoint && + previous.sessionId === sessionId && + previous.date !== date + ) + this.seal(file, previous); + } + } + + flush(sessionId?: string): void { + for (const name of this.files()) { + if (!name.startsWith("partial-")) continue; + const batch = this.read(name); + if (sessionId === undefined || batch.sessionId === sessionId) + this.seal(name, batch); + } + } + + next( + accepts: (batch: MemoryBatch) => boolean = () => true, + ): { name: string; batch: MemoryBatch } | undefined { + let oldest: string | undefined; + for (const name of this.files()) { + if ( + name.startsWith("ready-") && + (oldest === undefined || name < oldest) && + accepts(this.read(name)) + ) + oldest = name; + } + return oldest === undefined + ? undefined + : { name: oldest, batch: this.read(oldest) }; + } + + acknowledge(name: string): void { + unlinkSync(join(this.directory, name)); + } + + stats(): { turns: number; batches: number } { + let turns = 0; + let batches = 0; + for (const name of this.files()) { + turns += this.read(name).turns.length; + if (name.startsWith("ready-")) batches += 1; + } + return { turns, batches }; + } + + close(): void { + if (existsSync(this.lock) && readFileSync(this.lock, "utf8") === this.owner) + unlinkSync(this.lock); + } + + private *files(): Generator { + const dir = opendirSync(this.directory); + try { + for (let entry = dir.readSync(); entry; entry = dir.readSync()) + if (entry.isFile() && /^(partial|ready)-.*\.json$/.test(entry.name)) + yield entry.name; + } finally { + dir.closeSync(); + } + } + + private read(name: string): MemoryBatch { + try { + const batch = JSON.parse( + readFileSync(join(this.directory, name), "utf8"), + ) as MemoryBatch; + if ( + batch.version !== 1 || + typeof batch.id !== "string" || + !/^[a-f0-9-]{36}$/.test(batch.id) || + !validEndpoint(batch.endpoint) || + typeof batch.sessionId !== "string" || + typeof batch.date !== "string" || + !Number.isFinite(batch.createdAt) || + !Array.isArray(batch.turns) || + !batch.turns.length || + !batch.turns.every( + (turn) => + Array.isArray(turn) && + turn.every( + (message) => + typeof message.id === "string" && + ["user", "assistant"].includes(message.role) && + Array.isArray(message.content) && + message.content.every( + (part) => + part.type === "text" && typeof part.text === "string", + ), + ), + ) + ) + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + return batch; + } catch { + // JSON parser errors can quote message content; leave the record untouched. + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + } + } + + private write(name: string, batch: MemoryBatch): void { + const temporary = join(this.directory, `${randomUUID()}.tmp`); + const fd = openSync(temporary, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify(batch)); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, join(this.directory, name)); + } + + private seal(name: string, batch: MemoryBatch): void { + renameSync( + join(this.directory, name), + join(this.directory, `ready-${batch.createdAt}-${batch.id}.json`), + ); + } +} diff --git a/integrations/dsh/src/reme/client.ts b/integrations/dsh/src/reme/client.ts index ff0bccbc..10d9050c 100644 --- a/integrations/dsh/src/reme/client.ts +++ b/integrations/dsh/src/reme/client.ts @@ -13,6 +13,8 @@ import type { SearchOptions, } from "./types.js"; +import { safeError, validEndpoint } from "../endpoint.js"; + interface ReMeResponseBody { success?: boolean; answer?: unknown; @@ -50,7 +52,10 @@ export class ReMeClient { sessionId: string, options: AutoMemoryOptions = {}, ): Promise { - const config = this.configSource(); + const config = { + ...this.configSource(), + ...(options.endpoint ? { endpoint: options.endpoint } : {}), + }; return this.request( "auto_memory", { @@ -58,6 +63,7 @@ export class ReMeClient { session_id: sessionId, memory_hint: options.memoryHint || "", date: options.date || "", + ...(options.requestId ? { request_id: options.requestId } : {}), }, config, config.backgroundTimeoutMs, @@ -189,6 +195,10 @@ export class ReMeClient { ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal; try { + if (!validEndpoint(config.endpoint)) + throw new TypeError( + "Invalid ReMe endpoint: credentials, query, and fragment are unsupported", + ); const response = await fetch(`${config.endpoint}/${job}`, { method: "POST", headers: { @@ -208,7 +218,7 @@ export class ReMeClient { metadata: body.metadata ?? {}, error: ok ? "" - : String(body.answer || body.detail || `HTTP ${response.status}`), + : safeError(body.answer || body.detail || `HTTP ${response.status}`), }; } catch (error) { return { @@ -216,7 +226,7 @@ export class ReMeClient { status: 0, answer: "", metadata: {}, - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }; } finally { clearTimeout(timer); diff --git a/integrations/dsh/src/reme/types.ts b/integrations/dsh/src/reme/types.ts index a6864637..cf32dcb2 100644 --- a/integrations/dsh/src/reme/types.ts +++ b/integrations/dsh/src/reme/types.ts @@ -95,6 +95,8 @@ export interface SearchOptions { /** Automatic-memory request controls supported by the plugin client. */ export interface AutoMemoryOptions { + endpoint?: string; + requestId?: string; date?: string; memoryHint?: string; signal?: AbortSignal; diff --git a/integrations/dsh/src/runtime.ts b/integrations/dsh/src/runtime.ts index 407ec3f4..986a86f7 100644 --- a/integrations/dsh/src/runtime.ts +++ b/integrations/dsh/src/runtime.ts @@ -1,35 +1,37 @@ import type { LoggerLike, ReMeClientLike, ReMeMessage } from "./reme/types.js"; - import { messagesDay, nextDailyRun } from "./scheduling.js"; import { captureMessage, remeSessionId } from "./messages.js"; +import { MemoryOutbox, type MemoryBatch } from "./outbox.js"; +import { safeError } from "./endpoint.js"; import type { DshSession, ReMeConfig, SessionEvent } from "./types.js"; import type { ReMeRuntimeSnapshot, ReMeRuntimeTask } from "./runtime-status.js"; -interface PendingTurn { - messages: ReMeMessage[]; - day: string; -} - -interface SessionState { - session: DshSession; - sessionId: string; - activeTurn: unknown; - activeMessages: ReMeMessage[]; - pendingTurns: PendingTurn[]; - unconfirmedTurns: number; - writes: Promise; - requestController: AbortController; +interface MemoryWorker { + controller: AbortController; + task: Promise; + active?: ReMeRuntimeTask; } export class ReMeRuntime { - readonly states = new Map(); + private readonly states = new Map(); private readonly configSource: () => ReMeConfig; + private readonly outbox: MemoryOutbox; + private memoryTask: Promise | null = null; + private readonly memoryWorkers = new Map(); + private readonly retryTimers = new Map< + string, + ReturnType + >(); + private readonly failures = new Map(); + private memoryError: string | undefined; + private captureError: string | undefined; private dreamTimer: ReturnType | null = null; private dreamScheduleGeneration = 0; private dreamTask: Promise | null = null; private dreamController: AbortController | null = null; private started = false; private stopping = false; + private closed = false; private taskSequence = 0; private readonly recentTasks: ReMeRuntimeTask[] = []; private nextDreamAt: string | undefined; @@ -44,148 +46,232 @@ export class ReMeRuntime { readonly logger: LoggerLike = console, ) { this.configSource = typeof config === "function" ? config : () => config; - } - - stateFor(session: DshSession): SessionState { - const existing = this.states.get(session.id); - if (existing) { - existing.session = session; - if (existing.requestController.signal.aborted) - existing.requestController = new AbortController(); - return existing; - } - const state: SessionState = { - session, - sessionId: remeSessionId(session.id), - activeTurn: null, - activeMessages: [], - pendingTurns: [], - unconfirmedTurns: 0, - writes: Promise.resolve(), - requestController: new AbortController(), - }; - this.states.set(session.id, state); - return state; + this.outbox = new MemoryOutbox(this.configSource().autoMemoryQueueDir); } capture(session: DshSession, event: SessionEvent): void { const config = this.configSource(); - if (!config.autoMemoryEnabled) return; - const state = this.stateFor(session); - const data = isRecord(event.data) ? event.data : undefined; + if (this.stopping || !config.autoMemoryEnabled) return; if (event.type === "turn/start") { - state.activeTurn = data?.turn ?? null; - state.activeMessages = []; + this.states.set(session.id, []); return; } + const messages = this.states.get(session.id); + if (!messages) return; const message = captureMessage(event, session.id); - if (message) state.activeMessages.push(message); + if (message) messages.push(message); if (event.type !== "turn/end") return; - - const reason = data?.reason; - const reasonKind = isRecord(reason) ? reason.kind : undefined; - const completed = reasonKind === "completed" || reasonKind === "max-tokens"; - const hasUser = state.activeMessages.some((item) => item.role === "user"); - const hasAssistant = state.activeMessages.some( - (item) => item.role === "assistant", - ); - if (completed && hasUser && hasAssistant) { - const day = messagesDay(state.activeMessages, config.timezone); - const previousDay = state.pendingTurns.at(-1)?.day; - if (previousDay && day && previousDay !== day) - this.scheduleAutoMemory(state, true); - state.pendingTurns.push({ messages: state.activeMessages, day }); + const data = isRecord(event.data) ? event.data : undefined; + const reason = isRecord(data?.reason) ? data.reason.kind : undefined; + if ( + (reason === "completed" || reason === "max-tokens") && + messages.some((item) => item.role === "user") && + messages.some((item) => item.role === "assistant") + ) { + try { + this.outbox.append( + config.endpoint, + remeSessionId(session.id), + messagesDay(messages, config.timezone), + messages, + config.autoMemoryInterval, + ); + } catch (error) { + this.captureError = safeError(error); + this.recordMemoryError(error); + } } - state.activeTurn = null; - state.activeMessages = []; - this.scheduleAutoMemory(state); + this.states.set(session.id, []); + void this.drain(); } - private scheduleAutoMemory(state: SessionState, force = false): void { - const interval = this.configSource().autoMemoryInterval; - const firstDay = state.pendingTurns[0]?.day; - const dayCount = state.pendingTurns.findIndex((turn) => - Boolean(firstDay && turn.day && turn.day !== firstDay), - ); - const available = dayCount === -1 ? state.pendingTurns.length : dayCount; - const crossesDayBoundary = dayCount !== -1; - if (!force && !crossesDayBoundary && available < interval) return; - const count = force || crossesDayBoundary ? available : interval; - if (count === 0) return; - const turns = state.pendingTurns.splice(0, count); - const messages = turns.flatMap((turn) => turn.messages); - const date = turns[0]?.day || ""; - const task: ReMeRuntimeTask = { - id: `auto-memory-${++this.taskSequence}`, - phase: "queued", - queuedAt: new Date().toISOString(), - turns: turns.length, - messages: messages.length, - }; - this.recentTasks.unshift(task); - this.recentTasks.length = Math.min(this.recentTasks.length, 20); - state.unconfirmedTurns += turns.length; - state.writes = state.writes.then(async () => { - task.phase = "running"; - task.startedAt = new Date().toISOString(); - try { - const result = await this.client.autoMemory(messages, state.sessionId, { - date, - signal: state.requestController.signal, - }); - if (result.ok) { - task.phase = "completed"; - task.result = resultSummary(result.answer, result.metadata); - this.log("debug", "auto_memory_complete", { - sessionId: state.sessionId, - turns: turns.length, + /** Drain each endpoint independently; failed batches stay on disk for retry. */ + drain(): Promise { + if (this.closed) return Promise.resolve(); + if (this.retryTimers.has("*") || !this.configSource().autoMemoryEnabled) + return this.memoryTask ?? Promise.resolve(); + try { + for (;;) { + const next = this.outbox.next( + (batch) => + !this.memoryWorkers.has(batch.endpoint) && !this.blocked(batch), + ); + if (!next) break; + const { endpoint } = next.batch; + const worker: MemoryWorker = { + controller: new AbortController(), + task: Promise.resolve(), + }; + this.memoryWorkers.set(endpoint, worker); + worker.task = Promise.resolve() + .then(() => this.drainEndpoint(endpoint, worker)) + .catch((error: unknown) => { + this.recordMemoryError(error); + this.scheduleRetry(); + }) + .finally(() => { + this.memoryWorkers.delete(endpoint); + // A capture may have arrived after this worker observed an empty queue. + if (!this.stopping) void this.drain(); }); + } + } catch (error) { + this.recordMemoryError(error); + this.scheduleRetry(); + } + if (this.memoryTask) return this.memoryTask; + if (!this.memoryWorkers.size) return Promise.resolve(); + // Share one completion promise instead of accumulating waiters on stalled requests. + this.memoryTask = Promise.resolve() + .then(async () => { + while (this.memoryWorkers.size) + await Promise.all( + [...this.memoryWorkers.values()].map((worker) => worker.task), + ); + }) + .finally(() => { + this.memoryTask = null; + if (!this.stopping) return this.drain(); + }); + return this.memoryTask; + } + + private blocked(batch: MemoryBatch): boolean { + return ( + this.retryTimers.has(endpointKey(batch.endpoint)) || + this.retryTimers.has(sessionKey(batch)) + ); + } + + private async drainEndpoint( + endpoint: string, + worker: MemoryWorker, + ): Promise { + const { signal } = worker.controller; + while ( + !signal.aborted && + !this.closed && + this.configSource().autoMemoryEnabled && + !this.retryTimers.has("*") + ) { + const next = this.outbox.next( + (batch) => batch.endpoint === endpoint && !this.blocked(batch), + ); + if (!next) return; + const { name, batch } = next; + const messages = batch.turns.flat(); + const task: ReMeRuntimeTask = { + id: `${batch.id}-${++this.taskSequence}`, + phase: "running", + queuedAt: new Date(batch.createdAt).toISOString(), + startedAt: new Date().toISOString(), + turns: batch.turns.length, + messages: messages.length, + }; + worker.active = task; + this.recentTasks.unshift(task); + this.recentTasks.length = Math.min(this.recentTasks.length, 20); + let retryKey = endpointKey(endpoint); + try { + const result = await this.client.autoMemory(messages, batch.sessionId, { + date: batch.date, + endpoint, + requestId: batch.id, + signal, + }); + if (signal.aborted) { + task.phase = "cancelled"; return; } - state.pendingTurns.unshift(...turns); - task.phase = "failed"; - task.error = - result.error || "ReMe rejected the automatic-memory request"; - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: result.error, - }); + if (!result.ok) { + // Payload/job errors affect this session; network, auth and service errors affect the endpoint. + if (requestFailure(result.status)) retryKey = sessionKey(batch); + throw new Error( + `ReMe automatic-memory request failed (HTTP ${result.status ?? 0})`, + ); + } + this.outbox.acknowledge(name); + task.phase = "completed"; + // Reply text and file names can contain personal information. + this.failures.delete(endpointKey(endpoint)); + this.failures.delete(sessionKey(batch)); + if (!this.retryTimers.size) this.memoryError = undefined; } catch (error) { - state.pendingTurns.unshift(...turns); - task.phase = state.requestController.signal.aborted - ? "cancelled" - : "failed"; - task.error = error instanceof Error ? error.message : String(error); - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: error instanceof Error ? error.message : String(error), - }); + task.phase = signal.aborted ? "cancelled" : "failed"; + task.error = safeError(error); + this.recordMemoryError(error); + if (this.stopping || signal.aborted) return; + this.scheduleRetry(retryKey); } finally { task.finishedAt = new Date().toISOString(); - state.unconfirmedTurns -= turns.length; + worker.active = undefined; } - }); + } + } + + private recordMemoryError(error: unknown): void { + this.memoryError = safeError(error); + this.log("warn", "auto_memory_failed", { error: this.memoryError }); + } + + private scheduleRetry(key = "*"): void { + if ( + this.stopping || + this.retryTimers.has(key) || + !this.configSource().autoMemoryEnabled + ) + return; + const config = this.configSource(); + const failures = this.failures.get(key) ?? 0; + this.failures.set(key, failures + 1); + const delay = Math.min( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs * 2 ** Math.min(failures, 20), + ); + const timer = setTimeout(() => { + this.retryTimers.delete(key); + void this.drain(); + }, delay); + timer.unref?.(); + this.retryTimers.set(key, timer); + } + + private clearRetries(): void { + for (const timer of this.retryTimers.values()) clearTimeout(timer); + this.retryTimers.clear(); + this.failures.clear(); } start(): void { + if (this.started || this.stopping) return; this.started = true; - if (!this.configSource().autoDreamEnabled || this.stopping) return; - this.scheduleDream(this.dreamScheduleGeneration); + // A previous Host may have stopped before the batching interval was reached. + try { + this.outbox.flush(); + } catch (error) { + this.outbox.close(); + throw error; + } + void this.drain(); + if (this.configSource().autoDreamEnabled) + this.scheduleDream(this.dreamScheduleGeneration); } - /** Apply a changed settings snapshot to pending batching and dream scheduling. */ + /** Apply settings without discarding captured work or replacing the runtime. */ reconfigure(): void { if (this.stopping) return; - const config = this.configSource(); - if (config.autoMemoryEnabled) { - for (const state of this.states.values()) this.scheduleAutoMemory(state); + this.clearRetries(); + if (!this.configSource().autoMemoryEnabled) this.states.clear(); + if (this.configSource().autoMemoryEnabled) { + this.outbox.flush(); + void this.drain(); } if (!this.started) return; this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); this.dreamTimer = null; this.nextDreamAt = undefined; - if (config.autoDreamEnabled) + if (this.configSource().autoDreamEnabled) this.scheduleDream(this.dreamScheduleGeneration); } @@ -206,7 +292,7 @@ export class ReMeRuntime { Date.now(); } catch (error) { this.log("warn", "auto_dream_schedule_invalid", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); return; } @@ -220,6 +306,7 @@ export class ReMeRuntime { } async runDream(): Promise { + if (this.stopping) return; if (this.dreamTask) return this.dreamTask; const config = this.configSource(); this.dreamController = new AbortController(); @@ -237,21 +324,20 @@ export class ReMeRuntime { result.ok ? "debug" : "warn", result.ok ? "auto_dream_complete" : "auto_dream_failed", { - error: result.ok ? undefined : result.error, + error: result.ok ? undefined : safeError(result.error), }, ); this.dreamLastResult = result.ok ? "completed" : "failed"; this.dreamLastError = result.ok ? undefined - : result.error || "ReMe rejected the Auto Dream request"; + : safeError(result.error || "ReMe rejected the Auto Dream request"); } catch (error) { this.dreamLastResult = this.dreamController?.signal.aborted ? "cancelled" : "failed"; - this.dreamLastError = - error instanceof Error ? error.message : String(error); + this.dreamLastError = safeError(error); this.log("warn", "auto_dream_failed", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); } })().finally(() => { @@ -263,84 +349,66 @@ export class ReMeRuntime { } async dispose(session: DshSession): Promise { - const state = this.states.get(session.id); - if (!state) return; - if (state.requestController.signal.aborted) - state.requestController = new AbortController(); - const flush = (async () => { - await state.writes; - const retryCount = state.pendingTurns.length; - let scheduled = 0; - while (state.pendingTurns.length && scheduled < retryCount) { - const before = state.pendingTurns.length; - this.scheduleAutoMemory(state, true); - scheduled += before - state.pendingTurns.length; - } - await state.writes; - })(); - const completed = await this.withinShutdownBudget(flush, () => - state.requestController.abort(), - ); - const unsentTurns = state.pendingTurns.length + state.unconfirmedTurns; - if (unsentTurns) { - this.log( - "warn", - completed ? "auto_memory_retained" : "auto_memory_shutdown_timeout", - { - sessionId: state.sessionId, - unsentTurns, - }, - ); - } else { - this.states.delete(session.id); - } + this.states.delete(session.id); + this.outbox.flush(remeSessionId(session.id)); + void this.drain(); } async disposeAll(): Promise { + if (this.stopping) return; this.stopping = true; this.started = false; + this.states.clear(); this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); + this.clearRetries(); this.dreamTimer = null; this.nextDreamAt = undefined; this.dreamController?.abort(); - const shutdown = Promise.all([ - ...[...this.states.values()].map((state) => this.dispose(state.session)), - ...(this.dreamTask ? [this.dreamTask] : []), - ]).then(() => undefined); - await this.withinShutdownBudget(shutdown, () => { - this.dreamController?.abort(); - for (const state of this.states.values()) state.requestController.abort(); - }); + try { + this.outbox.flush(); + await this.withinShutdownBudget( + Promise.all([this.drain(), this.dreamTask]).then(() => undefined), + () => { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.dreamController?.abort(); + }, + ); + } finally { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.outbox.close(); + this.closed = true; + } } - /** Return a content-free snapshot for the local DSH status page. */ + /** Return queue counts and bounded diagnostics without reply content. */ snapshot(): ReMeRuntimeSnapshot { const config = this.configSource(); - const tasksRunning = this.recentTasks.filter( - (task) => task.phase === "running", + const queued = this.outbox.stats(); + const running = [...this.memoryWorkers.values()].filter( + (worker) => worker.active !== undefined, ).length; - const tasksQueued = this.recentTasks.filter( - (task) => task.phase === "queued", - ).length; - const lastError = this.recentTasks.find( - (task) => task.phase === "failed" || task.phase === "cancelled", - )?.error; return { - phase: this.stopping ? "stopping" : this.started ? "running" : "stopped", + phase: this.closed + ? "stopped" + : this.stopping + ? "stopping" + : this.started + ? "running" + : "stopped", autoMemory: { enabled: config.autoMemoryEnabled, interval: config.autoMemoryInterval, activeSessions: this.states.size, - queuedTurns: [...this.states.values()].reduce( - (total, state) => - total + state.pendingTurns.length + state.unconfirmedTurns, - 0, - ), - tasksRunning, - tasksQueued, + queuedTurns: queued.turns, + tasksRunning: running, + tasksQueued: Math.max(0, queued.batches - running), recentTasks: this.recentTasks.map((task) => ({ ...task })), - ...(lastError ? { lastError } : {}), + ...(this.captureError || this.memoryError + ? { lastError: this.captureError || this.memoryError } + : {}), }, autoDream: { enabled: config.autoDreamEnabled, @@ -363,16 +431,18 @@ export class ReMeRuntime { private async withinShutdownBudget( task: Promise, abort: () => void, - ): Promise { + ): Promise { let timer: ReturnType | undefined; - const timeout = new Promise((resolve) => { - timer = setTimeout(() => { - abort(); - resolve(false); - }, this.configSource().shutdownTimeoutMs); - }); try { - return await Promise.race([task.then(() => true), timeout]); + await Promise.race([ + task, + new Promise((resolve) => { + timer = setTimeout(() => { + abort(); + resolve(); + }, this.configSource().shutdownTimeoutMs); + }), + ]); } finally { if (timer) clearTimeout(timer); } @@ -388,18 +458,23 @@ export class ReMeRuntime { } } -function resultSummary( - answer: unknown, - metadata: Record | undefined, -): string | undefined { - const path = metadata?.path; - if (typeof path === "string" && path.length > 0) return path; - if (typeof answer !== "string") return undefined; - const normalized = answer.trim(); - if (!normalized) return undefined; - return normalized.length > 240 ? `${normalized.slice(0, 237)}…` : normalized; -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } + +function endpointKey(endpoint: string): string { + return `endpoint:${endpoint}`; +} + +function sessionKey(batch: MemoryBatch): string { + return `session:${JSON.stringify([batch.endpoint, batch.sessionId])}`; +} + +function requestFailure(status = 0): boolean { + return ( + (status >= 200 && status < 300) || + (status >= 400 && + status < 500 && + ![401, 403, 404, 405, 408, 429].includes(status)) + ); +} diff --git a/integrations/dsh/src/types.ts b/integrations/dsh/src/types.ts index ea38c35a..13e870e4 100644 --- a/integrations/dsh/src/types.ts +++ b/integrations/dsh/src/types.ts @@ -7,6 +7,9 @@ export interface ReMeConfigInput { shutdownTimeoutMs?: number; autoMemoryEnabled?: boolean; autoMemoryInterval?: number; + autoMemoryQueueDir?: string; + autoMemoryRetryMs?: number; + autoMemoryRetryMaxMs?: number; autoDreamEnabled?: boolean; dreamCron?: string; dreamHint?: string; @@ -21,6 +24,9 @@ export interface ReMeConfig extends ReMeClientConfig { shutdownTimeoutMs: number; autoMemoryEnabled: boolean; autoMemoryInterval: number; + autoMemoryQueueDir: string; + autoMemoryRetryMs: number; + autoMemoryRetryMaxMs: number; autoDreamEnabled: boolean; dreamCron: string; dreamHint: string; @@ -32,7 +38,10 @@ export interface ReMeConfig extends ReMeClientConfig { } /** ReMe integration fields owned by the DSH user-settings document. */ -export type ReMeSettings = Omit; +export type ReMeSettings = Omit< + ReMeConfig, + "dreamIntervalMs" | "autoMemoryQueueDir" +>; export interface SessionEvent { type: string; diff --git a/integrations/dsh/tests/bundle.test.mjs b/integrations/dsh/tests/bundle.test.mjs index c7b3f2a9..39bd23ec 100644 --- a/integrations/dsh/tests/bundle.test.mjs +++ b/integrations/dsh/tests/bundle.test.mjs @@ -30,6 +30,7 @@ test("declares one installable DeepSeek Harness plugin", async () => { "0.2.0-rc.2", "0.2.1-alpha.2", "0.2.1-alpha.4", + "0.2.1-rc.2", ]) { assert.ok( semver.satisfies(version, manifest.peerDependencies[dependency]), diff --git a/integrations/dsh/tests/client-settings.test.mjs b/integrations/dsh/tests/client-settings.test.mjs index 93db8c85..d8406ec5 100644 --- a/integrations/dsh/tests/client-settings.test.mjs +++ b/integrations/dsh/tests/client-settings.test.mjs @@ -168,7 +168,7 @@ test("resets overrides in one mutation only after Save", async () => { await ui.button("save").props.onClick(); }); assert.equal(ui.calls.length, 1); - assert.equal(ui.calls[0].ops.length, 13); + assert.equal(ui.calls[0].ops.length, 15); assert.ok(ui.calls[0].ops.every((op) => op.op === "unset")); assert.equal(ui.calls[0].revision, 7); } finally { @@ -191,3 +191,30 @@ test("read-only configuration cannot be saved", async () => { ui.close(); } }); + +for (const endpoint of [ + "http://alice:SECRET@localhost:2333", + "http://localhost:2333?token=SECRET", + "http://localhost:2333#search", + "http://localhost:2333?", + "http://localhost:2333#", +]) { + test( + "rejects unsupported endpoint components before saving: " + + new URL(endpoint).hash + + new URL(endpoint).search, + async () => { + const ui = await editor(async () => true); + try { + ui.change(0, endpoint); + assert.equal(ui.button("save").props.disabled, true); + await act(async () => { + await ui.button("save").props.onClick(); + }); + assert.equal(ui.calls.length, 0); + } finally { + ui.close(); + } + }, + ); +} diff --git a/integrations/dsh/tests/client.test.mjs b/integrations/dsh/tests/client.test.mjs index 40424ab0..af3e839b 100644 --- a/integrations/dsh/tests/client.test.mjs +++ b/integrations/dsh/tests/client.test.mjs @@ -1,6 +1,75 @@ import assert from "node:assert/strict"; import test from "node:test"; import { ReMeClient } from "../dist/reme/client.js"; +import { safeError } from "../dist/endpoint.js"; + +test("rejects credentials before fetch and redacts URL credentials from service and transport errors", async () => { + const originalFetch = globalThis.fetch; + let calls = 0; + const secretUrl = "http://alice:SECRET-CANARY@localhost:2333/search"; + globalThis.fetch = async () => { + calls++; + return Response.json({ success: false, answer: `refused ${secretUrl}` }); + }; + try { + const client = new ReMeClient({ + endpoint: secretUrl, + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await client.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.equal(calls, 0); + const valid = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + globalThis.fetch = async () => { + throw new Error(`failed ${secretUrl}`); + }; + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.doesNotMatch( + safeError(new Error(`failed ${secretUrl}`)), + /alice|SECRET-CANARY/, + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("uses the persisted destination and request ID when sending a recovered batch", async () => { + const originalFetch = globalThis.fetch; + let request; + globalThis.fetch = async (url, init) => { + request = { url, body: JSON.parse(init.body) }; + return Response.json({ success: true }); + }; + try { + const client = new ReMeClient({ + endpoint: "http://new.test", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + await client.autoMemory([], "session", { + endpoint: "http://original.test", + requestId: "persisted-batch", + }); + assert.equal(request.url, "http://original.test/auto_memory"); + assert.equal(request.body.request_id, "persisted-batch"); + } finally { + globalThis.fetch = originalFetch; + } +}); test("calls ReMe jobs with their native request and response envelopes", async () => { const calls = []; diff --git a/integrations/dsh/tests/config.test.mjs b/integrations/dsh/tests/config.test.mjs index 8e2613eb..ce03debd 100644 --- a/integrations/dsh/tests/config.test.mjs +++ b/integrations/dsh/tests/config.test.mjs @@ -29,6 +29,11 @@ test("exports a Cordis schema that rejects invalid configuration", async () => { for (const input of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:bad-port" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://[:::]" }, @@ -142,6 +147,11 @@ test("Loader keeps the previous live values when an update fails schema validati await entry.fiber.await(); for (const patch of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://999.0.0.1" }, { dreamCron: "every night" }, diff --git a/integrations/dsh/tests/dsh-index.test.mjs b/integrations/dsh/tests/dsh-index.test.mjs index 7a346710..f19b3476 100644 --- a/integrations/dsh/tests/dsh-index.test.mjs +++ b/integrations/dsh/tests/dsh-index.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { apply, Config } from "../dist/index.js"; import { hasGuidance } from "../dist/guidance.js"; @@ -28,7 +31,7 @@ test("recognizes guidance already stored by the previous DSH plugin", () => { ); }); -test("composes root-agent guidance and reme_search on supported DSH releases", async () => { +test("composes root-agent guidance and reme_search on supported DSH releases", async (t) => { const handlers = new Map(); const tools = []; const cleanups = []; @@ -58,7 +61,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a handlers.set(name, handler); }, }; - apply(ctx, { + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false, language: "zh", @@ -97,7 +100,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a await Promise.all(cleanups.map((cleanup) => cleanup())); }); -test("keeps prompt injection and capture out of subagents by default", async () => { +test("keeps prompt injection and capture out of subagents by default", async (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -118,7 +121,7 @@ test("keeps prompt injection and capture out of subagents by default", async () handlers.set(name, handler); }, }; - apply(ctx, { autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoDreamEnabled: false }); let injected = false; handlers.get("agent/created")({ agent: { @@ -138,7 +141,7 @@ test("keeps prompt injection and capture out of subagents by default", async () assert.equal(injected, false); }); -test("reads live DSH configuration for new sessions", async () => { +test("reads live DSH configuration for new sessions", async (t) => { const handlers = new Map(); const ctx = { fiber: { state: 0 }, @@ -165,7 +168,7 @@ test("reads live DSH configuration for new sessions", async () => { language: "zh", }); assert.equal(config.issues, undefined); - apply(ctx, config.value); + applyPlugin(t, ctx, config.value); const injected = []; handlers.get("agent/created")({ agent: { @@ -204,7 +207,7 @@ test("reads live DSH configuration for new sessions", async () => { assert.match(injected[1].content[0].text, /Long-term Memory/); }); -test("session events read a validated configuration snapshot", () => { +test("session events read a validated configuration snapshot", (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -224,7 +227,7 @@ test("session events read a validated configuration snapshot", () => { handlers.set(name, handler); }, }; - apply(ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); const original = Intl.DateTimeFormat.prototype.formatToParts; let scans = 0; Intl.DateTimeFormat.prototype.formatToParts = function (...args) { @@ -243,3 +246,18 @@ test("session events read a validated configuration snapshot", () => { Intl.DateTimeFormat.prototype.formatToParts = original; } }); + +function applyPlugin(t, ctx, input) { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-plugin-")); + let runtime; + const provide = ctx.provide; + ctx.provide = function (name, value) { + if (name === "remeMemory") runtime = value; + return provide.call(this, name, value); + }; + t.after(async () => { + await runtime?.disposeAll(); + rmSync(directory, { recursive: true, force: true }); + }); + return apply(ctx, { ...input, autoMemoryQueueDir: directory }); +} diff --git a/integrations/dsh/tests/outbox.test.mjs b/integrations/dsh/tests/outbox.test.mjs new file mode 100644 index 00000000..2d102c41 --- /dev/null +++ b/integrations/dsh/tests/outbox.test.mjs @@ -0,0 +1,197 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { MemoryOutbox } from "../dist/outbox.js"; + +function directory(t) { + const path = mkdtempSync(join(tmpdir(), "reme-outbox-")); + t.after(() => rmSync(path, { recursive: true, force: true })); + return path; +} + +const messages = [ + { + id: "m1", + name: "user", + role: "user", + content: [{ type: "text", text: "private fact" }], + }, +]; + +test("recovers a partial batch after the owning process exits without cleanup", (t) => { + const path = directory(t); + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { MemoryOutbox } from ${JSON.stringify( + new URL("../dist/outbox.js", import.meta.url).href, + )}; + const queue = new MemoryOutbox(${JSON.stringify(path)}); + queue.append("http://localhost:2333", "session", "2026-10-21", ${JSON.stringify( + messages, + )}, 5); + `, + ], + { encoding: "utf8" }, + ); + assert.equal(child.status, 0, child.stderr); + const recovered = new MemoryOutbox(path); + try { + recovered.flush(); + assert.deepEqual(recovered.stats(), { turns: 1, batches: 1 }); + const { name, batch } = recovered.next(); + assert.deepEqual(batch.turns, [messages]); + if (process.platform !== "win32") + assert.equal(statSync(join(path, name)).mode & 0o777, 0o600); + recovered.acknowledge(name); + assert.deepEqual(recovered.stats(), { turns: 0, batches: 0 }); + } finally { + recovered.close(); + } +}); + +test("refuses another owner without changing the queue", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + assert.throws(() => new MemoryOutbox(path), /in use/); + assert.equal(queue.stats().turns, 1); + } finally { + queue.close(); + } +}); + +test("serializes concurrent stale-lock recovery without deleting a replacement owner", async (t) => { + const path = directory(t); + const moduleUrl = new URL("../dist/outbox.js", import.meta.url).href; + const seed = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + new MemoryOutbox(${JSON.stringify(path)});`, + ], + { encoding: "utf8" }, + ); + assert.equal(seed.status, 0, seed.stderr); + const staleOwner = readFileSync(join(path, ".lock"), "utf8"); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { existsSync, writeFileSync } from "node:fs"; + import { join } from "node:path"; + import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + const path = ${JSON.stringify(path)}; + const kill = process.kill.bind(process); + process.kill = (pid, signal) => { + writeFileSync(join(path, "observed-stale"), "ready"); + const deadline = Date.now() + 5000; + while (!existsSync(join(path, "release"))) { + if (Date.now() > deadline) throw new Error("Recovery probe timed out"); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + return kill(pid, signal); + }; + const queue = new MemoryOutbox(path); + process.on("message", () => { queue.close(); process.exit(0); }); + writeFileSync(join(path, "acquired"), "ready"); + `, + ], + { stdio: ["ignore", "ignore", "pipe", "ipc"] }, + ); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr += data; + }); + const exited = new Promise((resolve) => child.once("exit", resolve)); + async function waitFor(name) { + const deadline = Date.now() + 5000; + while (!existsSync(join(path, name))) { + assert.equal(child.exitCode, null, stderr); + assert.ok( + Date.now() < deadline, + `Timed out waiting for ${name}: ${stderr}`, + ); + await delay(5); + } + } + try { + await waitFor("observed-stale"); + assert.throws(() => new MemoryOutbox(path), /lock recovery is busy/); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), staleOwner); + writeFileSync(join(path, "release"), "ready"); + await waitFor("acquired"); + assert.equal( + Number(readFileSync(join(path, ".lock"), "utf8").split(":")[0]), + child.pid, + ); + assert.throws(() => new MemoryOutbox(path), /in use/); + } finally { + writeFileSync(join(path, "release"), "ready"); + if (child.connected) child.send("close"); + const timer = setTimeout(() => child.kill(), 1000); + await exited; + clearTimeout(timer); + } +}); + +test("fails closed on an interrupted recovery guard without touching the owner or records", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const owner = readFileSync(join(path, ".lock"), "utf8"); + mkdirSync(join(path, ".lock-recovery")); + assert.throws( + () => new MemoryOutbox(path), + /lock recovery is busy or interrupted/, + ); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), owner); + assert.equal(queue.stats().turns, 1); + assert.equal(existsSync(join(path, ".lock-recovery")), true); + } finally { + queue.close(); + } +}); + +test("preserves invalid records and keeps parser errors free of message content", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const name = readdirSync(path).find((file) => file.startsWith("ready-")); + const stored = JSON.parse(readFileSync(join(path, name), "utf8")); + stored.id = "../outside"; + writeFileSync(join(path, name), JSON.stringify(stored)); + assert.throws(() => queue.next(), /Invalid ReMe outbox record/); + writeFileSync(join(path, name), "PRIVATE-CONTENT"); + assert.throws( + () => queue.next(), + (error) => !error.message.includes("PRIVATE-CONTENT"), + ); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + } finally { + queue.close(); + } +}); diff --git a/integrations/dsh/tests/runtime.test.mjs b/integrations/dsh/tests/runtime.test.mjs index 9de94c9b..d0ee3ff5 100644 --- a/integrations/dsh/tests/runtime.test.mjs +++ b/integrations/dsh/tests/runtime.test.mjs @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate as settle } from "node:timers/promises"; import { ReMeRuntime } from "../dist/runtime.js"; const CONFIG = { + endpoint: "http://127.0.0.1:2333", autoMemoryEnabled: true, autoMemoryInterval: 2, + autoMemoryRetryMs: 10, + autoMemoryRetryMaxMs: 40, autoDreamEnabled: false, shutdownTimeoutMs: 50, dreamIntervalMs: 0, @@ -13,215 +20,491 @@ const CONFIG = { timezone: "Asia/Shanghai", }; -test("submits completed turns to auto-memory in background batches", async () => { +function makeRuntime( + t, + client, + config = CONFIG, + logger = silentLogger(), + directory, +) { + const ownsDirectory = directory === undefined; + directory ??= mkdtempSync(join(tmpdir(), "reme-dsh-runtime-")); + const current = typeof config === "function" ? config : () => config; + const runtime = new ReMeRuntime( + client, + () => ({ ...current(), autoMemoryQueueDir: directory }), + logger, + ); + t.after(async () => { + await runtime.disposeAll(); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + }); + return runtime; +} + +test("persists partial turns and submits one batch at the interval", async (t) => { const calls = []; - const client = { + const runtime = makeRuntime(t, { async autoMemory(messages, sessionId) { calls.push({ messages, sessionId }); return { ok: true }; }, - }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + }); const session = { id: "session-one" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 0); - + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); completeTurn(runtime, session, 2, 20); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.equal(calls[0].messages.length, 4); assert.match(calls[0].sessionId, /^dsh-[a-f0-9]{24}$/); - const status = runtime.snapshot(); - assert.equal(status.autoMemory.queuedTurns, 0); - assert.equal(status.autoMemory.recentTasks[0].phase, "completed"); - assert.equal(status.autoMemory.recentTasks[0].turns, 2); - assert.equal(status.autoMemory.recentTasks[0].messages, 4); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); }); -test("requeues failed auto-memory batches and flushes them on disposal", async () => { +test("backs off across new turns and drains every batch after recovery", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; let attempts = 0; - const client = { + const runtime = makeRuntime(t, { async autoMemory() { - attempts += 1; - return { ok: attempts > 1, error: "offline" }; + attempts++; + return { ok: available, error: "offline" }; }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retry-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); - - await runtime.dispose(session); - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("retries an in-flight failed batch before disposal completes", async () => { - let attempts = 0; - let markStarted; - let releaseFirst; - const started = new Promise((resolve) => { - markStarted = resolve; }); - const firstRequest = new Promise((resolve) => { - releaseFirst = resolve; - }); - const client = { - async autoMemory() { - attempts += 1; - if (attempts === 1) { - markStarted(); - await firstRequest; - return { ok: false, error: "offline" }; - } - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "in-flight-retry-session" }; - completeTurn(runtime, session, 1, 10); - await started; - - const disposal = runtime.dispose(session); - releaseFirst(); - await disposal; - - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("splits auto-memory batches at workspace date boundaries", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 1); - assert.equal(calls[0].options.date, "2026-08-19"); - await runtime.dispose(session); - assert.equal(calls.length, 2); - assert.equal(calls[1].options.date, "2026-08-20"); -}); - -test("retries a failed partial prior-day batch after later activity", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: calls.length > 1, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-retry-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - assert.equal(calls.length, 1); - assert.equal(runtime.stateFor(session).pendingTurns.length, 2); - - for (let turn = 3; turn <= 7; turn += 1) { - completeTurn( - runtime, - session, - turn, - turn * 10, - Date.parse(`2026-08-20T00:0${turn}:00Z`), - ); + const session = { id: "outage" }; + for (let turn = 1; turn <= 200; turn++) { + completeTurn(runtime, session, turn, turn * 10); + await runtime.drain(); } - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 3); - assert.equal(calls[1].options.date, "2026-08-19"); - assert.equal(calls[1].messages.length, 2); - assert.equal(calls[2].options.date, "2026-08-20"); - assert.equal(calls[2].messages.length, 10); - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); -}); - -test("bounds session disposal and aborts an unresponsive write", async () => { - let observedSignal; - const client = { - async autoMemory(_messages, _sessionId, options) { - observedSignal = options.signal; - return new Promise(() => {}); - }, - }; - const runtime = new ReMeRuntime( - client, - { - ...CONFIG, - autoMemoryInterval: 1, - shutdownTimeoutMs: 20, - }, - silentLogger(), - ); - const session = { id: "stuck-session" }; - completeTurn(runtime, session, 1, 10); - - const started = Date.now(); - await runtime.dispose(session); - - assert.equal(observedSignal.aborted, true); - assert.ok(Date.now() - started < 500); - assert.equal(runtime.states.has(session.id), true); -}); - -test("retains a failed final batch for a later plugin-shutdown retry", async () => { - let attempts = 0; - const client = { - async autoMemory() { - attempts += 1; - return { ok: attempts > 2, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retained-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - - await runtime.dispose(session); + assert.equal(attempts, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 200); + assert.equal(runtime.states.get(session.id).length, 0); + t.mock.timers.tick(10); + await runtime.drain(); assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), true); - - await runtime.disposeAll(); - assert.equal(attempts, 3); - assert.equal(runtime.states.has(session.id), false); + available = true; + t.mock.timers.tick(19); + await settle(); + assert.equal(attempts, 2); + t.mock.timers.tick(1); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + assert.equal(attempts, 102); }); -test("runs only one auto-dream task at a time", async () => { +for (const status of [413, 200]) { + test(`isolates a rejected batch (HTTP ${status}) while preserving its session's order`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout", "Date"] }); + let accepted = false; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _session, options) { + const text = messages[0].content[0].text; + calls.push({ text, requestId: options.requestId }); + return { ok: accepted || text !== "question 101", status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "rejected" }, 101, 10); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "rejected" }, 102, 20); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "healthy" }, 201, 30); + await runtime.drain(); + assert.deepEqual( + calls.map((call) => call.text), + ["question 101", "question 201"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + + t.mock.timers.tick(10); + await runtime.drain(); + assert.deepEqual(calls[2], calls[0]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + accepted = true; + t.mock.timers.tick(19); + await runtime.drain(); + assert.equal(calls.length, 3); + t.mock.timers.tick(1); + await runtime.drain(); + assert.deepEqual( + calls.slice(3).map((call) => call.text), + ["question 101", "question 102"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +for (const status of [429, 503]) { + test(`backs off the whole endpoint for HTTP ${status}`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: available, status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + completeTurn(runtime, { id: "second" }, 1, 10); + await runtime.drain(); + assert.equal(calls, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls, 3); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +test("retries ended sessions without another turn and releases session state", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + const runtime = makeRuntime(t, { + async autoMemory() { + return { ok: available, error: "offline" }; + }, + }); + const session = { id: "ended" }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(runtime.states.has(session.id), false); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("recovers queued and partial batches after restarting the Host", async (t) => { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-restart-")); + const original = makeRuntime( + t, + { + async autoMemory() { + return { ok: false, error: "offline" }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + completeTurn(original, { id: "restart" }, 1, 10); + await original.disposeAll(); + const calls = []; + const recovered = makeRuntime( + t, + { + async autoMemory(messages) { + calls.push(messages); + return { ok: true }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + recovered.start(); + await recovered.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].length, 2); + assert.equal(recovered.snapshot().autoMemory.queuedTurns, 0); + t.after(() => rmSync(directory, { recursive: true, force: true })); +}); + +test("keeps the same request ID and batch when retrying after a timeout", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + const calls = []; + const runtime = makeRuntime(t, { + async autoMemory(messages, sessionId, options) { + calls.push({ messages, sessionId, requestId: options.requestId }); + return { ok: calls.length > 1, error: "timeout" }; + }, + }); + const session = { id: "timeout" }; + completeTurn(runtime, session, 1, 10); + completeTurn(runtime, session, 2, 20); + await runtime.drain(); + await runtime.dispose(session); + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls.length, 2); + assert.deepEqual(calls[0], calls[1]); + const tasks = runtime.snapshot().autoMemory.recentTasks; + assert.equal(new Set(tasks.map((task) => task.id)).size, tasks.length); +}); + +test("keeps counts accurate beyond the twenty-entry diagnostic history", async (t) => { + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const runtime = makeRuntime( + t, + { + async autoMemory() { + await gate; + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + const session = { id: "many-batches" }; + for (let n = 1; n <= 25; n++) completeTurn(runtime, session, n, n * 10); + await settle(); + const status = runtime.snapshot().autoMemory; + assert.equal(status.tasksRunning, 1); + assert.equal(status.tasksQueued, 24); + assert.equal(status.queuedTurns, 25); + release(); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.recentTasks.length, 20); +}); + +test("does not put service reply text or file names into diagnostic snapshots", async (t) => { + const runtime = makeRuntime( + t, + { + async autoMemory() { + return { + ok: true, + answer: "PRIVATE-ANSWER", + metadata: { path: "PRIVATE-PATH" }, + }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "privacy" }, 1, 10); + await runtime.drain(); + assert.doesNotMatch(JSON.stringify(runtime.snapshot()), /PRIVATE/); +}); + +test("splits dates and flushes a partial previous day without a new interval", async (t) => { + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _id, options) { + calls.push({ messages, date: options.date }); + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 5 }, + ); + const session = { id: "midnight" }; + completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); + completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); + await runtime.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].date, "2026-08-19"); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(calls[1].date, "2026-08-20"); +}); + +test("aborts shutdown within its budget and leaves an unacknowledged batch on disk", async (t) => { + let signal; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _id, options) { + signal = options.signal; + return new Promise(() => {}); + }, + }, + { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }, + ); + completeTurn(runtime, { id: "stuck" }, 1, 10); + await settle(); + const started = Date.now(); + await runtime.disposeAll(); + assert.equal(signal.aborted, true); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.ok(Date.now() - started < 500); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("does not retry per ended session while its service is in backoff", async (t) => { + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: false }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + for (let n = 0; n < 5; n++) { + const session = { id: "ended-" + n }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + } + assert.equal(calls, 1); +}); + +test("pauses further sends while disabled and resumes persisted work when enabled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: true }; + }, + }, + () => config, + ); + runtime.capture({ id: "partial" }, { type: "turn/start" }); + runtime.capture( + { id: "partial" }, + { + type: "user/message", + seq: 1, + data: { content: "unfinished", source: { kind: "user" } }, + }, + ); + config = { ...config, autoMemoryEnabled: false }; + runtime.reconfigure(); + assert.equal(runtime.states.size, 0); + runtime.outbox.append( + config.endpoint, + "session", + "2026-10-21", + [{ id: "m", name: "user", role: "user", content: [] }], + 1, + ); + await runtime.drain(); + assert.equal(calls, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + config = { ...config, autoMemoryEnabled: true }; + runtime.reconfigure(); + await runtime.drain(); + assert.equal(calls, 1); +}); + +test("a failed old destination does not block a newly configured service", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + calls.push(options.endpoint); + return { ok: options.endpoint === "http://new.test" }; + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await runtime.drain(); + config = { ...config, endpoint: "http://new.test" }; + completeTurn(runtime, { id: "new" }, 1, 10); + await runtime.drain(); + assert.deepEqual(calls, [CONFIG.endpoint, "http://new.test"]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("drains a new destination while the old request remains stalled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const calls = []; + const active = new Map(); + const maximum = new Map(); + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + const endpoint = options.endpoint; + calls.push(endpoint); + active.set(endpoint, (active.get(endpoint) ?? 0) + 1); + maximum.set( + endpoint, + Math.max(maximum.get(endpoint) ?? 0, active.get(endpoint)), + ); + await (endpoint === CONFIG.endpoint ? gate : settle()); + active.set(endpoint, active.get(endpoint) - 1); + return { ok: true }; + }, + }, + () => config, + ); + try { + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + completeTurn(runtime, { id: "new" }, 2, 20); + await settle(); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 2); + await settle(); + await settle(); + assert.deepEqual(calls, [ + CONFIG.endpoint, + "http://new.test", + "http://new.test", + ]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 1); + assert.equal(runtime.snapshot().autoMemory.tasksQueued, 0); + assert.deepEqual([...maximum.values()], [1, 1]); + } finally { + release(); + await runtime.drain(); + } + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("aborts all destination workers at the shutdown deadline without discarding batches", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }; + const signals = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + signals.push(options.signal); + return new Promise(() => {}); + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + await settle(); + assert.equal(signals.length, 2); + await runtime.disposeAll(); + assert.ok(signals.every((signal) => signal.aborted)); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); +}); + +test("runs only one auto-dream task at a time", async (t) => { let calls = 0; let release; const client = { @@ -233,7 +516,7 @@ test("runs only one auto-dream task at a time", async () => { return { ok: true }; }, }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + const runtime = makeRuntime(t, client, CONFIG, silentLogger()); const first = runtime.runDream(); const second = runtime.runDream(); assert.equal(calls, 1); @@ -242,9 +525,10 @@ test("runs only one auto-dream task at a time", async () => { assert.equal(calls, 1); }); -test("contains unexpected auto-dream client failures", async () => { +test("contains unexpected auto-dream client failures", async (t) => { const warnings = []; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { throw new Error("broken transport"); @@ -266,10 +550,11 @@ test("contains unexpected auto-dream client failures", async () => { assert.match(runtime.snapshot().autoDream.lastError, /broken transport/); }); -test("applies changed batching and dream settings without replacing the runtime", async () => { +test("applies changed batching and dream settings without replacing the runtime", async (t) => { const calls = []; let config = { ...CONFIG, autoMemoryInterval: 5, autoDreamEnabled: false }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoMemory(messages) { calls.push(messages); @@ -294,7 +579,7 @@ test("applies changed batching and dream settings without replacing the runtime" dreamIntervalMs: 100000, }; runtime.reconfigure(); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.notEqual(runtime.dreamTimer, null); assert.equal(runtime.snapshot().autoDream.enabled, true); @@ -302,7 +587,7 @@ test("applies changed batching and dream settings without replacing the runtime" await runtime.disposeAll(); }); -test("keeps one auto-dream schedule when reconfigured during a run", async () => { +test("keeps one auto-dream schedule when reconfigured during a run", async (t) => { let calls = 0; let releaseFirst; const firstRequest = new Promise((resolve) => { @@ -314,7 +599,8 @@ test("keeps one auto-dream schedule when reconfigured during a run", async () => dreamIntervalMs: 20, shutdownTimeoutMs: 100, }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { calls += 1; diff --git a/reme/config/default.yaml b/reme/config/default.yaml index dbab8ac2..c57c0b3c 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -173,6 +173,9 @@ jobs: parameters: type: object properties: + request_id: + type: string + description: "Optional stable batch ID (1-128 characters); replay completed requests without rerunning the memory agent" messages: type: array description: "messages" diff --git a/reme/steps/evolve/auto_memory.py b/reme/steps/evolve/auto_memory.py index ee38fcff..5e513fab 100644 --- a/reme/steps/evolve/auto_memory.py +++ b/reme/steps/evolve/auto_memory.py @@ -3,6 +3,8 @@ import base64 import binascii import datetime +import hashlib +import json from pathlib import Path import re from urllib.parse import urlsplit @@ -21,6 +23,7 @@ from ..file_io import validate_filename_component, validate_session_id from ..file_io._path import IMAGE_MIME_BY_EXT, _check_path_permission, resolve_path from ..index import normalize_posix_path from ...components import R +from ...schema import Response from ...utils.wikilink_handler import WikilinkHandler _SESSION_ID_KEY = "session_id" @@ -458,8 +461,68 @@ class AutoMemoryStep(BaseStep): content=[images[part] if part in images else TextBlock(text=part) for part in parts if part], ) - # pylint: disable=too-many-return-statements async def execute(self): + """Replay completed request IDs without repeating the memory agent.""" + assert self.context is not None + request_id = self.context.get("request_id") + if request_id is None: + return await self._execute() + if not isinstance(request_id, str) or not 1 <= len(request_id) <= 128: + self.context.response.success = False + self.context.response.answer = "Error: request_id must be a string of 1 to 128 characters" + return + + payload = { + key: self.context.get(key) for key in ("session_id", "messages", "date", "memory_hint", "include_images") + } + if payload["messages"]: + # Keep caller-provided IDs/timestamps; parsing dictionaries would generate defaults on every retry. + payload["messages"] = [ + message.model_dump(mode="json") if isinstance(message, Msg) else message + for message in payload["messages"] + ] + fingerprint = hashlib.sha256( + json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode(), + ).hexdigest() + session_key = hashlib.sha256(str(payload["session_id"]).encode()).hexdigest() + request_key = hashlib.sha256(request_id.encode()).hexdigest() + directory = f"{self.config_value('metadata_dir')}/auto_memory_requests/{session_key}" + workspace = self.file_store.workspace_path + allowed = self.context.get("_allowed_paths") + lock_path = _checked_write_path(workspace, f"{directory}/.lock", allowed) + async with await get_path_lock(lock_path): + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + if path.exists(): + record = json.loads(path.read_text(encoding="utf-8")) + if record["fingerprint"] != fingerprint: + self.context.response.success = False + self.context.response.answer = "Error: request_id was already used for different input" + return + response = Response.model_validate(record["response"]) + self.context.response.answer = response.answer + self.context.response.success = response.success + self.context.response.metadata.update(response.metadata) + self.context["changes"] = record["changes"] + return + await self._execute() + if self.context.response.success: + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = _checked_write_path(workspace, f"{directory}/{request_key}.tmp", allowed) + await write_file_safe( + temporary, + json.dumps( + { + "fingerprint": fingerprint, + "response": self.context.response.model_dump(mode="json"), + "changes": self.context.get("changes") or [], + }, + ), + ) + temporary.replace(path) + + # pylint: disable=too-many-return-statements + async def _execute(self): assert self.context is not None self.context["changes"] = [] raw_messages = self.context.get("messages") or [] diff --git a/tests/unit/test_auto_memory_idempotency.py b/tests/unit/test_auto_memory_idempotency.py new file mode 100644 index 00000000..73bc1b5d --- /dev/null +++ b/tests/unit/test_auto_memory_idempotency.py @@ -0,0 +1,148 @@ +"""Completed automatic-memory requests can be replayed after a client timeout.""" + +# pylint: disable=protected-access,missing-function-docstring + +import asyncio +from types import SimpleNamespace + +import pytest +from agentscope.message import Msg + +from reme.components.runtime_context import RuntimeContext +from reme.steps.evolve.auto_memory import AutoMemoryStep + + +def _step(tmp_path, request_id="batch-1", **kwargs): + step = AutoMemoryStep() + step.file_store = SimpleNamespace(workspace_path=tmp_path) + step.context = RuntimeContext( + request_id=request_id, + session_id="session-1", + messages=[{"id": "m1", "content": "fact"}], + date="2026-10-21", + **kwargs, + ) + return step + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_is_msg", [False, True]) +@pytest.mark.parametrize("replay_is_msg", [False, True]) +async def test_message_objects_and_dicts_share_a_stable_request_fingerprint(tmp_path, first_is_msg, replay_is_msg): + message = Msg.model_validate( + { + "id": "stable-message", + "name": "user", + "role": "user", + "content": [{"type": "text", "text": "fact"}], + "created_at": "2026-10-21T12:00:00+08:00", + }, + ) + original = message.model_dump(mode="json") + first = _step(tmp_path) + first.context["messages"] = [message if first_is_msg else original] + + async def execute(): + first.context.response.answer = "recorded" + + first._execute = execute + await first.execute() + assert first.context.response.success + assert message.model_dump(mode="json") == original + replay = _step(tmp_path) + replay.context["messages"] = [Msg.model_validate(original) if replay_is_msg else original] + + async def unexpected(): + pytest.fail("replaying a supported message form must not invoke the agent") + + replay._execute = unexpected + await replay.execute() + assert replay.context.response == first.context.response + + +@pytest.mark.asyncio +async def test_completed_request_survives_restart_without_repeating_agent(tmp_path): + first = _step(tmp_path) + + async def execute(): + first.context.response.answer = "recorded" + first.context.response.metadata = {"path": "daily/note.md"} + first.context["changes"] = [{"change": "modified", "path": "daily/note.md"}] + + first._execute = execute + await first.execute() + replay = _step(tmp_path) + + async def unexpected(): + pytest.fail("completed request must not invoke the memory agent") + + replay._execute = unexpected + await replay.execute() + assert replay.context.response == first.context.response + assert replay.context["changes"] == first.context["changes"] + + +@pytest.mark.asyncio +async def test_concurrent_duplicate_waits_for_the_original_request(tmp_path): + first, duplicate = _step(tmp_path), _step(tmp_path) + started, release = asyncio.Event(), asyncio.Event() + calls = [] + + async def execute(): + calls.append("agent") + started.set() + await release.wait() + + first._execute = duplicate._execute = execute + original = asyncio.create_task(first.execute()) + await started.wait() + retry = asyncio.create_task(duplicate.execute()) + await asyncio.sleep(0) + assert calls == ["agent"] + release.set() + await asyncio.wait_for(asyncio.gather(original, retry), 2) + assert calls == ["agent"] + + +@pytest.mark.asyncio +async def test_failed_request_can_retry_but_reusing_a_completed_id_for_other_input_fails(tmp_path): + step = _step(tmp_path) + + async def fail(): + step.context.response.success = False + + step._execute = fail + await step.execute() + assert not list(tmp_path.rglob("*.json")) + + retry = _step(tmp_path) + + async def succeed(): + return None + + retry._execute = succeed + await retry.execute() + assert retry.context.response.success + conflicting = _step(tmp_path) + conflicting.context["messages"] = [{"id": "m2", "content": "different fact"}] + await conflicting.execute() + assert not conflicting.context.response.success + assert "different input" in conflicting.context.response.answer + + +@pytest.mark.asyncio +async def test_receipts_obey_workspace_permissions(tmp_path): + step = _step(tmp_path, _allowed_paths=["daily"]) + with pytest.raises(PermissionError): + await step.execute() + assert not list(tmp_path.rglob("*.json")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_id", ["", "x" * 129, 123]) +async def test_rejects_invalid_request_id_before_running_agent(tmp_path, request_id): + step = _step(tmp_path, request_id) + await step.execute() + assert not step.context.response.success + assert "request_id" in step.context.response.answer + assert not list(tmp_path.iterdir()) From 69e3db8deb22221c39afb6c0b1f24e7cccec81bd Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:15:27 +0800 Subject: [PATCH 06/19] refactor(embedding): simplify retry results and error classification (#656) --- .../embedding_store/_embedding_types.py | 109 +++++ .../embedding_store/local_embedding_store.py | 406 ++---------------- tests/unit/test_local_embedding_store.py | 146 ++++--- 3 files changed, 228 insertions(+), 433 deletions(-) create mode 100644 reme/components/embedding_store/_embedding_types.py diff --git a/reme/components/embedding_store/_embedding_types.py b/reme/components/embedding_store/_embedding_types.py new file mode 100644 index 00000000..69aebebe --- /dev/null +++ b/reme/components/embedding_store/_embedding_types.py @@ -0,0 +1,109 @@ +"""Internal embedding results and normalized provider errors.""" + +import re +from dataclasses import dataclass + +_PROVIDER_WIDE_STATUS_CODES = frozenset({401, 403, 404}) + + +# AgentScope's DashScope wrapper serializes the response into a RuntimeError. +_STATUS_IN_MESSAGE = re.compile(r'"status_code"\s*:\s*(\d{3})') + + +_CODE_IN_MESSAGE = re.compile(r'"code"\s*:\s*"([^"]+)"') + + +_INPUT_DEPENDENT_ERROR_TYPES = frozenset( + { + "ContextLengthError", + "ContextWindowExceededError", + "InputLengthError", + "TokenLimitError", + }, +) + + +_INPUT_DEPENDENT_MARKERS = ( + "context length", + "context_length", + "context window", + "input length", + "input is too long", + "maximum context", + "reduce the length", + "token limit", + "too many tokens", + "too long", +) + + +@dataclass(frozen=True) +class _EmbeddingResult: + """Keep recovered vectors together with the failure state that ends isolation.""" + + embeddings: list[list[float] | None] | None = None + terminal_failure: bool = False + + +@dataclass(frozen=True) +class _EmbeddingError: + """Normalize provider error shapes once before applying the retry policy.""" + + status: int | None + code: str | None + text: str + error_type: str + + @classmethod + def from_exception(cls, error: Exception) -> "_EmbeddingError": + """Read attributes, structured bodies, and serialized DashScope responses.""" + message = str(error) + body = getattr(error, "body", None) + status = None + for value in ( + getattr(error, "status_code", None), + getattr(error, "code", None), + getattr(getattr(error, "response", None), "status_code", None), + ): + # Integer codes are HTTP statuses; strings name business errors. + if isinstance(value, int) and not isinstance(value, bool): + status = value + break + if status is None and (match := _STATUS_IN_MESSAGE.search(message)): + status = int(match.group(1)) + + code = getattr(error, "code", None) + if not isinstance(code, str) or not code: + details = body.get("error", body) if isinstance(body, dict) else None + code = details.get("code") if isinstance(details, dict) else None + if not isinstance(code, str) or not code: + match = _CODE_IN_MESSAGE.search(message) + code = match.group(1) if match else None + text = f"{message} {body}" if isinstance(body, dict) else message + return cls(status, code, text.lower(), type(error).__name__) + + @property + def is_insufficient_quota(self) -> bool: + """Quota exhaustion takes precedence over a generic HTTP 429.""" + return self.code == "insufficient_quota" + + @property + def is_rate_limited(self) -> bool: + """Recognize rate limits while preserving the separate quota policy.""" + return not self.is_insufficient_quota and (self.code == "rate_limit_exceeded" or self.status == 429) + + @property + def is_provider_wide(self) -> bool: + """Credentials, missing routes, and server outages cannot be isolated per input.""" + return ( + self.status is not None and (self.status in _PROVIDER_WIDE_STATUS_CODES or 500 <= self.status < 600) + ) or self.error_type in ("AuthenticationError", "PermissionDeniedError", "NotFoundError", "InternalServerError") + + @property + def is_input_dependent(self) -> bool: + """Only input-length rejections may split; a 413 always requires a text signal.""" + if self.status not in (None, 400, 413): + return False + if self.status != 413 and self.error_type in _INPUT_DEPENDENT_ERROR_TYPES: + return True + return any(marker in self.text for marker in _INPUT_DEPENDENT_MARKERS) diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index abb08caa..99e59369 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -2,12 +2,12 @@ import asyncio import hashlib -import re from collections import OrderedDict from pathlib import Path import numpy as np +from ._embedding_types import _EmbeddingError, _EmbeddingResult from .base_embedding_store import BaseEmbeddingStore from ..component_registry import R from ..as_embedding import BaseAsEmbedding @@ -15,12 +15,8 @@ from ..as_embedding import BaseAsEmbedding Miss = tuple[int, str, str] # (result_index, text, cache_key) _MAX_VECTOR_SPACE_ATTEMPTS = 3 -_PROVIDER_WIDE_STATUS_CODES = frozenset({401, 403, 404}) -# Transport failures raised by the OpenAI SDK and httpx. They do not inherit from the built-in -# TimeoutError / ConnectionError / OSError, so the built-in check alone lets them slip into the -# generic handler and fan out into per-item requests. Matching on class names (across the MRO, -# so subclasses such as APITimeoutError are covered) keeps this module free of SDK imports. +# Match SDK/httpx transport exceptions without importing optional providers. _TRANSPORT_FAILURE_TYPES = frozenset( { "APIConnectionError", @@ -36,60 +32,9 @@ _TRANSPORT_FAILURE_TYPES = frozenset( }, ) -# Transport errors an SDK retries on its own. A positive numeric retry budget is not evidence: -# AgentScope's Ollama model keeps the inherited ``max_retries=3`` next to an empty policy, and -# the number never buys a retry. Only a client that both has a budget and retries this class of -# error owns the failure; otherwise the store keeps its backoff. +# A client retry budget only applies to the transport exceptions it retries. _SDK_TRANSPORT_RETRY_TYPES = frozenset({"APIConnectionError", "APITimeoutError"}) -# The status a provider reports can live in more than one place. ``status_code`` is the -# OpenAI-compatible attribute, Google GenAI exposes an integer ``code``, httpx-style wrappers -# keep it on ``response``, and AgentScope's DashScope implementation raises a plain -# ``RuntimeError`` whose message embeds the structured response, so the status has to be read -# back out of the message. A string ``code`` names an error (``insufficient_quota``, -# ``rate_limit_exceeded``, ``InvalidApiKey``) and is never a status. -_STATUS_IN_MESSAGE = re.compile(r'"status_code"\s*:\s*(\d{3})') - -# The business error code travels the same way the status does: a string ``code`` on the -# OpenAI-compatible SDKs, ``error.code`` inside a structured body, and a serialized -# ``"code"`` in the message for AgentScope's DashScope ``RuntimeError``. Reading only the -# attribute loses the DashScope case and misclassifies a quota error as a rate limit. -_CODE_IN_MESSAGE = re.compile(r'"code"\s*:\s*"([^"]+)"') - -# A rejected batch can only be paid back one item at a time when the provider blames an input. -# A context-length rejection is the one status an over-long item explains by itself, so it is -# the only error allowed to fan out into per-item requests. -_INPUT_DEPENDENT_STATUS_CODES = frozenset({400}) - -# A 413 is ambiguous. Providers such as Ollama v0.40.3 normalize an embedding input-limit -# rejection to 413, but a bare 413 can also describe an unrelated request-body limit that no -# single input explains. It therefore only qualifies as input-dependent when the error itself -# names the input length, which is why it is checked through the marker list instead of being -# accepted outright. -_INPUT_DEPENDENT_AMBIGUOUS_STATUS_CODES = frozenset({413}) - -_INPUT_DEPENDENT_ERROR_TYPES = frozenset( - { - "ContextLengthError", - "ContextWindowExceededError", - "InputLengthError", - "TokenLimitError", - }, -) - -_INPUT_DEPENDENT_MARKERS = ( - "context length", - "context_length", - "context window", - "input length", - "input is too long", - "maximum context", - "reduce the length", - "token limit", - "too many tokens", - "too long", -) - @R.register("local") class LocalEmbeddingStore(BaseEmbeddingStore): @@ -238,17 +183,8 @@ class LocalEmbeddingStore(BaseEmbeddingStore): async def _compute_batch(self, batch: list[Miss], **kwargs) -> list[tuple[int, str, np.ndarray]]: texts = [text for _, text, _ in batch] - # A provider-wide failure can end a request after some of its vectors were already - # recovered. Those vectors are still worth keeping, but they must not report the - # provider as healthy again: the unhealthy-to-healthy transition is what schedules - # missing-vector backfill, so a hidden outage would either trigger it while the - # provider is still unavailable or suppress a later genuine recovery. - terminal_failure: list[bool] = [] - embeddings = await self._call_with_retry( - texts, - terminal_failure=terminal_failure, - **kwargs, - ) + result = await self._call_with_retry(texts, **kwargs) + embeddings = result.embeddings if not embeddings or len(embeddings) != len(texts): return [] out: list[tuple[int, str, np.ndarray]] = [] @@ -264,221 +200,91 @@ class LocalEmbeddingStore(BaseEmbeddingStore): if bad_dims: details = ", ".join(f"{count} with dim {dim}" for dim, count in sorted(bad_dims.items())) self.logger.error(f"Embedding dimension mismatch in batch: expected {self.dimensions}; rejected {details}") - if out and not terminal_failure: - self.is_healthy = True - else: - self.is_healthy = False + # Retain partial vectors without hiding the outage that ended isolation. + self.is_healthy = bool(out) and not result.terminal_failure return out - async def _call_with_retry( - self, - texts: list[str], - *, - terminal_failure: list[bool] | None = None, - **kwargs, - ) -> list[list[float] | None] | None: - """Answer a batch, retrying transient failures and isolating input ones. - - Transport failures are provider-wide: no individual input can explain them, so the - request is never split per item. Which layer retries them depends on the provider, not - on where the request came from: when the embedding model below already retries the - transport exceptions it raises (AgentScope's OpenAI model does, and the OpenAI SDK - retries each of those attempts), the store answers once and leaves that budget there. - A provider whose model has no retry policy -- Ollama inherits an empty retryable tuple - and its SDK turns ``httpx.ConnectError`` into a plain ``ConnectionError`` -- gets the - store's backoff instead, because otherwise a single dropped connection would cost the - vector. - - ``terminal_failure`` is an optional out-parameter. When the caller passes a list, - it is set to ``[True]`` if this call gave up on a failure that no remaining item - can explain (a provider-wide error, exhausted quota, or exhausted transient - retries). The per-item isolation loop uses it to stop instead of asking for every - remaining item and multiplying the rejected requests, and it forwards the same - flag to its caller so a request that recovered some vectors before the failure is - still not reported as healthy. - """ - - def stop() -> None: - """Record that this failure ends the work for the whole batch.""" - if terminal_failure is not None: - terminal_failure[:] = [True] - + async def _call_with_retry(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Retry failures owned by the store and isolate input-length rejections.""" for attempt in range(self.max_retries): try: - result = await self.as_embedding(texts, **kwargs) - if result and len(result) == len(texts): - return result + embeddings = await self.as_embedding(texts, **kwargs) + if embeddings and len(embeddings) == len(texts): + return _EmbeddingResult(embeddings) except Exception as error: if self._is_transport_failure(error): - # A connection failure or a timeout describes the endpoint, not any single - # item, so this request is never split per item. Whether the store retries - # it is a property of the provider: when the model below has its own retry - # policy, that layer (and the SDK under it) already answers the failure -- - # a batch they answer after 12 attempts used to reach the provider 36 times - # with this store retrying on top of them, and an isolated item used to - # multiply the same budget a third time. + # Preserve recovery where the provider has no matching retry policy; + # otherwise leave its budget intact instead of adding another loop. if self._provider_owns_transport_retry(error): self.logger.exception( "Embedding request failed: transport error owned by the provider, not splitting", ) - self.is_healthy = False - stop() break if attempt < self.max_retries - 1: await asyncio.sleep(2**attempt) continue - self.logger.exception( - "Embedding request failed: transport error after exhausting retries", - ) - self.is_healthy = False - stop() + self.logger.exception("Embedding request failed: transport error after exhausting retries") break - if self._is_rate_limited(error): + + details = _EmbeddingError.from_exception(error) + if details.is_rate_limited: if attempt < self.max_retries - 1: delay = 2**attempt self.logger.warning(f"Embedding rate limited; retrying in {delay:.1f}s") await asyncio.sleep(delay) continue self.logger.exception("Embedding request failed after exhausting rate-limit retries") - self.is_healthy = False - stop() - return None - if ( - self.quota_retry_delay is not None - and self._is_insufficient_quota(error) - and attempt < self.max_retries - 1 - ): - self.logger.warning( - f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s", - ) - await asyncio.sleep(self.quota_retry_delay) - continue - if self._is_insufficient_quota(error): - # Quota is a service-level failure: no single item is to blame, so splitting the - # batch cannot isolate anything and would only multiply the rejected requests. - # This also covers the case where the quota retries above are exhausted. - self.logger.exception("Embedding request failed: quota exhausted") - self.is_healthy = False - stop() - return None - if self._is_provider_wide_failure(error): - # Credentials, permissions and server outages describe the endpoint, not any - # single item: isolation cannot fix them, so give up on the whole batch - # instead of multiplying the rejected requests. - # - # A 5xx is deliberately *not* retried here. The embedding wrapper and the - # provider SDK already retry transient server errors, and every extra layer - # that retries multiplies the attempts and the waiting time of the layers - # below it -- a batch the SDK answered after 12 attempts became 36 once this - # store retried on top of it. Retry ownership therefore stays with those - # layers: a caller that wants a longer budget configures it there instead of - # stacking another silent loop here. A transport failure is left to them for - # the same reason; it only skips the per-item split. - self.logger.exception("Embedding request failed: provider-wide error, not splitting") - stop() break - if self._is_input_dependent_failure(error): - # A rejection a single input explains by itself is the one case the per-item - # split can resolve: isolating it keeps the remaining vectors usable. + if details.is_insufficient_quota: + if self.quota_retry_delay is not None and attempt < self.max_retries - 1: + self.logger.warning(f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s") + await asyncio.sleep(self.quota_retry_delay) + continue + self.logger.exception("Embedding request failed: quota exhausted") + break + if details.is_provider_wide: + # Server-error retries belong to the wrapper/SDK, not to item isolation. + self.logger.exception("Embedding request failed: provider-wide error, not splitting") + break + if details.is_input_dependent: self.logger.exception("Embedding request failed: input-dependent error") self.is_healthy = False if len(texts) > 1: - return await self._embed_individually( - texts, - terminal_failure=terminal_failure, - **kwargs, - ) - return None - # Anything else counts as provider-wide. Splitting cannot recover a vector unless - # the failing input is identified, so an unrecognized error is never answered - # with one request per item. + return await self._embed_individually(texts, **kwargs) + return _EmbeddingResult() self.logger.exception("Embedding request failed: unrecognized error, not splitting") - stop() break self.is_healthy = False - stop() - return None + return _EmbeddingResult(terminal_failure=True) - async def _embed_individually( - self, - texts: list[str], - *, - terminal_failure: list[bool] | None = None, - **kwargs, - ) -> list[list[float] | None]: - """Retry a rejected batch one item at a time so one bad item cannot drop the rest. - - Provider-side request errors, such as a per-item context-length limit, reject the - whole batch and would otherwise cost every vector in it. Isolating each item bounds - the damage to the item that actually failed and keeps the remaining vectors usable. - Each single-item request goes back through ``_call_with_retry`` so that a rate limit and - a transient transport failure still get the retries that layer owns instead of losing - the vector; a provider-wide failure ends the work for the whole batch rather than being - asked for every remaining item, and a one-item request never splits again. - When the loop stops on a provider-wide failure, ``terminal_failure`` is set for the - caller so the vectors recovered before it are not read as a healthy provider. - """ - results: list[list[float] | None] = [] + async def _embed_individually(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Keep healthy siblings, stopping when a failure is no longer specific to an input.""" + embeddings: list[list[float] | None] = [] for index, text in enumerate(texts): - terminal: list[bool] = [] - single = await self._call_with_retry( - [text], - terminal_failure=terminal, - **kwargs, - ) - if single: - results.append(single[0]) + result = await self._call_with_retry([text], **kwargs) + if result.embeddings: + embeddings.append(result.embeddings[0]) continue self.logger.error(f"Embedding request failed for a single item (chars={len(text)})") - results.append(None) - if terminal and terminal[0]: - # No remaining item can explain a provider-wide failure, so asking for the - # rest would only multiply the rejected requests. Keep the vectors already - # recovered and leave the items that were never answered as None. The batch - # caller has to learn about it as well: a partial success must not report - # the provider as healthy again. - if terminal_failure is not None: - terminal_failure[:] = [True] + embeddings.append(None) + if result.terminal_failure: self.logger.error( f"Stopping per-item isolation after {index + 1}/{len(texts)} items: " "the failure is not specific to any input", ) - results.extend([None] * (len(texts) - index - 1)) - break - return results + embeddings.extend([None] * (len(texts) - index - 1)) + return _EmbeddingResult(embeddings, terminal_failure=True) + return _EmbeddingResult(embeddings) @staticmethod def _is_transport_failure(error: Exception) -> bool: - """Return whether an error describes a transport failure rather than a rejected input. - - Covers the built-in socket errors as well as the provider/httpx exceptions that describe - a connection failure or a timeout. Transport failures are provider-wide: no individual - input can be blamed, so the request is never split per item. Who retries them is decided - per provider by ``_provider_owns_transport_retry`` rather than per call site. - """ + """Recognize transport errors that must never trigger per-item isolation.""" if isinstance(error, (TimeoutError, ConnectionError, OSError)): return True return any(cls.__name__ in _TRANSPORT_FAILURE_TYPES for cls in type(error).__mro__) def _provider_owns_transport_retry(self, error: Exception) -> bool: - """Return whether a lower layer retries *this* transport failure. - - Retry ownership is a property of the provider and of the error, not of the call site. - A model whose ``_get_retryable_exceptions()`` covers the error owns the retry, and so - does an SDK that retries that class of error. Everything else must keep the store's - backoff, or one dropped connection costs the vector. - - Two shapes are deliberately not treated as ownership. AgentScope's Ollama embedding - model declares an empty policy but keeps the inherited ``max_retries=3``: its loop - re-raises every exception outside the empty tuple, so the number never buys a retry. - AgentScope's DashScope model declares ``(RuntimeError,)``, which does not cover a - connection error raised underneath it. A nonempty policy is therefore checked against - the error that actually occurred rather than trusted as a whole. - - A model that exposes no policy at all -- test doubles and custom providers -- is only - trusted to an SDK that has a positive retry budget *and* retries this class of - transport error. Otherwise the store answers, as the base implementation did. - """ + """Require both a matching exception policy and an enabled retry budget.""" model = getattr(self.as_embedding, "model", None) if model is None: return False @@ -510,13 +316,7 @@ class LocalEmbeddingStore(BaseEmbeddingStore): @staticmethod def _sdk_owns_transport_retry(model, error: Exception) -> bool: - """Return whether the SDK under the model retries this transport error itself. - - The numeric ``max_retries`` only counts once the client also identifies this error as - retryable. The OpenAI client retries ``APIConnectionError`` / ``APITimeoutError``; - Ollama's client has no retry budget at all, and AgentScope's DashScope path calls the - module directly, so neither has a client to consult. - """ + """Check the SDK's budget and supported transport errors independently of the model.""" if not LocalEmbeddingStore._is_transport_failure(error): return False client = getattr(model, "client", None) @@ -527,118 +327,6 @@ class LocalEmbeddingStore(BaseEmbeddingStore): return False return any(cls.__name__ in _SDK_TRANSPORT_RETRY_TYPES for cls in type(error).__mro__) - @staticmethod - def _http_status(error: Exception) -> int | None: - """Return the HTTP status an exception carries, across the provider SDK shapes. - - ``status_code`` covers the OpenAI-compatible SDKs, an integer ``code`` covers Google - GenAI, ``response.status_code`` covers httpx-style wrappers, and the structured - ``status_code`` embedded in the message covers AgentScope's DashScope ``RuntimeError``. - Only integers are accepted: a string ``code`` is a provider error name such as - ``insufficient_quota``, and reading it as a status would misclassify those errors. - """ - for attribute in ("status_code", "code"): - value = getattr(error, attribute, None) - if isinstance(value, int) and not isinstance(value, bool): - return value - value = getattr(getattr(error, "response", None), "status_code", None) - if isinstance(value, int) and not isinstance(value, bool): - return value - match = _STATUS_IN_MESSAGE.search(str(error)) - return int(match.group(1)) if match else None - - @staticmethod - def _is_provider_wide_failure(error: Exception) -> bool: - """Return whether an error describes the provider rather than a single input. - - Authentication, permission, missing model or route and server errors belong to the - endpoint, the credentials or the deployment: no individual input can be blamed, so - splitting the batch cannot isolate anything and would only multiply the rejected - requests. Rate limits and quota exhaustion have their own branches above and never - reach this classifier. A context-length ``400`` stays input-dependent on purpose, - because that is the one status an over-limit item can explain by itself. - """ - status = LocalEmbeddingStore._http_status(error) - if status is not None and (status in _PROVIDER_WIDE_STATUS_CODES or 500 <= status < 600): - return True - return type(error).__name__ in ( - "AuthenticationError", - "PermissionDeniedError", - "NotFoundError", - "InternalServerError", - ) - - @staticmethod - def _is_input_dependent_failure(error: Exception) -> bool: - """Return whether a single input can be blamed for the rejected request. - - Only a context-length rejection qualifies: it is the one error an over-long item - explains by itself, and therefore the only one a per-item retry can resolve. It can - arrive as a ``400`` or, from a provider that normalizes the rejection (Ollama reports - ``413``), as an ambiguous status that is accepted only when the error names the input - itself. Anything else stays on the provider-wide path, so a client never answers a - service failure with one request per item. - """ - status = LocalEmbeddingStore._http_status(error) - if status is not None and status not in _INPUT_DEPENDENT_STATUS_CODES: - if status not in _INPUT_DEPENDENT_AMBIGUOUS_STATUS_CODES: - return False - # The status alone does not name an input for an ambiguous code, so the error text - # has to carry the signal; otherwise this is a request-level rejection. - text = LocalEmbeddingStore._error_text(error) - return any(marker in text for marker in _INPUT_DEPENDENT_MARKERS) - if type(error).__name__ in _INPUT_DEPENDENT_ERROR_TYPES: - return True - text = LocalEmbeddingStore._error_text(error) - return any(marker in text for marker in _INPUT_DEPENDENT_MARKERS) - - @staticmethod - def _error_text(error: Exception) -> str: - """Flatten an exception and its OpenAI-compatible body into one lowercase string.""" - parts = [str(error)] - body = getattr(error, "body", None) - if isinstance(body, dict): - parts.append(str(body)) - return " ".join(parts).lower() - - @staticmethod - def _error_code(error: Exception) -> str | None: - """Return the provider's business error code, across the supported error shapes. - - ``code`` covers the OpenAI-compatible SDKs when the provider puts the error name - there, ``error.code`` inside ``body`` covers the structured OpenAI payload, and the - serialized message covers AgentScope's DashScope ``RuntimeError`` — the same shape - ``_http_status`` already reads the status back out of. Without the last case a - DashScope quota error is indistinguishable from an ordinary 429. - """ - code = getattr(error, "code", None) - if isinstance(code, str) and code: - return code - body = getattr(error, "body", None) - if isinstance(body, dict): - details = body.get("error", body) - if isinstance(details, dict): - inner = details.get("code") - if isinstance(inner, str) and inner: - return inner - match = _CODE_IN_MESSAGE.search(str(error)) - return match.group(1) if match else None - - @staticmethod - def _is_rate_limited(error: Exception) -> bool: - """Recognize an OpenAI-compatible 429 response without importing a provider SDK.""" - code = LocalEmbeddingStore._error_code(error) - if code == "insufficient_quota": - return False - if code == "rate_limit_exceeded": - return True - return LocalEmbeddingStore._http_status(error) == 429 - - @staticmethod - def _is_insufficient_quota(error: Exception) -> bool: - """Recognize OpenAI-compatible quota errors without importing a provider SDK.""" - return LocalEmbeddingStore._error_code(error) == "insufficient_quota" - def _validate_dim(self, emb: np.ndarray) -> bool: """Return whether an embedding exactly matches the configured dimension.""" return len(emb) == self.dimensions diff --git a/tests/unit/test_local_embedding_store.py b/tests/unit/test_local_embedding_store.py index 912c1789..23444703 100644 --- a/tests/unit/test_local_embedding_store.py +++ b/tests/unit/test_local_embedding_store.py @@ -11,6 +11,7 @@ import numpy as np import openai from reme.components.as_embedding import DashScopeAsEmbedding, OllamaAsEmbedding, OpenAIAsEmbedding +from reme.components.embedding_store._embedding_types import _EmbeddingError from reme.components.embedding_store.base_embedding_store import BaseEmbeddingStore from reme.components.embedding_store.local_embedding_store import LocalEmbeddingStore from reme.schema import EmbNode @@ -566,7 +567,7 @@ def test_insufficient_quota_waits_sixty_seconds_before_retry(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - result = await store._call_with_retry(["text"]) + result = (await store._call_with_retry(["text"])).embeddings assert result == [[1.0, 0.0]] assert embedding.calls == 2 @@ -589,7 +590,7 @@ def test_insufficient_quota_does_not_retry_without_opt_in(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - result = await store._call_with_retry(["text"]) + result = (await store._call_with_retry(["text"])).embeddings assert result is None assert embedding.calls == 1 @@ -612,7 +613,7 @@ def test_rate_limit_retries_with_backoff_without_opt_in(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - result = await store._call_with_retry(["text"]) + result = (await store._call_with_retry(["text"])).embeddings assert result == [[1.0, 0.0]] assert embedding.calls == 2 @@ -634,16 +635,16 @@ def test_is_rate_limited_recognizes_status_code_or_body_and_nothing_else(): body = {"error": {"code": "rate_limit_exceeded"}} - assert LocalEmbeddingStore._is_rate_limited(StatusCodeError()) - assert LocalEmbeddingStore._is_rate_limited(BodyCodeError()) - assert not LocalEmbeddingStore._is_rate_limited(ValueError("unrelated")) + assert _EmbeddingError.from_exception(StatusCodeError()).is_rate_limited + assert _EmbeddingError.from_exception(BodyCodeError()).is_rate_limited + assert not _EmbeddingError.from_exception(ValueError("unrelated")).is_rate_limited def test_is_rate_limited_defers_to_insufficient_quota_on_status_code_429(): """An insufficient_quota error carrying status_code=429 is not the generic rate-limit case.""" - assert not LocalEmbeddingStore._is_rate_limited(InsufficientQuotaError("quota exhausted")) - assert LocalEmbeddingStore._is_insufficient_quota(InsufficientQuotaError("quota exhausted")) + assert not _EmbeddingError.from_exception(InsufficientQuotaError("quota exhausted")).is_rate_limited + assert _EmbeddingError.from_exception(InsufficientQuotaError("quota exhausted")).is_insufficient_quota def test_rate_limit_exhausts_retries_and_reports_unhealthy(monkeypatch): @@ -660,7 +661,7 @@ def test_rate_limit_exhausts_retries_and_reports_unhealthy(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - result = await store._call_with_retry(["text"]) + result = (await store._call_with_retry(["text"])).embeddings assert result is None assert store.is_healthy is False @@ -1033,7 +1034,7 @@ def test_batch_failure_falls_back_to_per_item_embedding(): store = LocalEmbeddingStore(name="t_local_embedding_poison_batch", enable_cache=False) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [[1.0, 0.0], None, [1.0, 0.0]] assert provider.batch_sizes == [3, 1, 1, 1] @@ -1065,7 +1066,9 @@ def test_single_item_failure_still_reports_a_failed_batch(): store = LocalEmbeddingStore(name="t_local_embedding_single_poison", enable_cache=False) store.as_embedding = PerItemLimitAsEmbedding() - assert await store._call_with_retry(["x" * 64]) is None + result = await store._call_with_retry(["x" * 64]) + assert result.embeddings is None + assert result.terminal_failure is False run(go()) @@ -1084,7 +1087,7 @@ def test_rate_limited_batch_is_not_split_into_per_item_requests(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - assert await store._call_with_retry(["a", "b"]) is None + assert (await store._call_with_retry(["a", "b"])).embeddings is None assert embedding.calls == 2 assert sleeps == [1.0] @@ -1105,7 +1108,7 @@ def test_insufficient_quota_batch_is_not_split_into_per_item_requests(monkeypatc store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - assert await store._call_with_retry(["a", "b", "c"]) is None + assert (await store._call_with_retry(["a", "b", "c"])).embeddings is None assert embedding.batch_sizes == [3] assert store.is_healthy is False assert not sleeps @@ -1131,7 +1134,7 @@ def test_exhausted_quota_retries_still_do_not_split_the_batch(monkeypatch): store.as_embedding = embedding monkeypatch.setattr(asyncio, "sleep", fake_sleep) - assert await store._call_with_retry(["a", "b"]) is None + assert (await store._call_with_retry(["a", "b"])).embeddings is None assert embedding.batch_sizes == [2, 2, 2] assert sleeps == [5.0, 5.0] @@ -1163,7 +1166,7 @@ def test_per_item_fallback_keeps_a_rate_limited_vector(monkeypatch): store.as_embedding = provider monkeypatch.setattr(asyncio, "sleep", fake_sleep) - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [[1.0, 0.0], None, [1.0, 0.0]] assert provider.batch_sizes == [3, 1, 1, 1, 1] @@ -1185,7 +1188,7 @@ def test_provider_wide_credentials_errors_are_not_split_into_per_item_requests() store = LocalEmbeddingStore(name="t_local_embedding_provider_wide", max_retries=3) store.as_embedding = provider - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None assert provider.batch_sizes == [10] assert store.is_healthy is False @@ -1206,7 +1209,7 @@ def test_provider_wide_server_errors_are_not_retried_without_splitting(monkeypat store.as_embedding = provider monkeypatch.setattr(asyncio, "sleep", fake_sleep) - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None # One request only: the wrapper and the SDK already retry a 5xx, and retrying here # as well multiplied their attempts (12 -> 36 in the reported reproduction). assert provider.batch_sizes == [10] @@ -1237,7 +1240,7 @@ def test_sdk_transport_errors_are_provider_wide_and_not_retried_here(monkeypatch store = LocalEmbeddingStore(name="t_local_embedding_sdk_transport", max_retries=3) store.as_embedding = provider - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None # One request only: the wrapper and the SDK already retry a transport failure # (4 wrapper attempts, each retried by the SDK), so another loop here only # multiplies them -- a persistent timeout used to cost 36 attempts instead of 12. @@ -1279,7 +1282,7 @@ def test_a_provider_without_a_retry_policy_keeps_the_store_backoff(monkeypatch): store.as_embedding = provider monkeypatch.setattr(asyncio, "sleep", fake_sleep) - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [[1.0, 0.0], None, [1.0, 0.0]] assert provider.batch_sizes == [3, 1, 1, 1, 1] @@ -1306,7 +1309,7 @@ def test_a_persistent_transport_error_is_not_multiplied_by_the_store(monkeypatch ) store.as_embedding = provider - assert await store._call_with_retry([f"item{i}" for i in range(3)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(3)])).embeddings is None assert provider.calls == 1 assert endpoint.requests == 12 assert store.is_healthy is False @@ -1335,7 +1338,7 @@ def test_a_transient_transport_error_still_recovers_through_the_sdk(monkeypatch) ) store.as_embedding = provider - results = await store._call_with_retry([f"item{i}" for i in range(3)]) + results = (await store._call_with_retry([f"item{i}" for i in range(3)])).embeddings assert results == [[0.0, 1.0]] * 3 assert provider.calls == 1 @@ -1374,7 +1377,7 @@ def test_an_isolated_item_still_recovers_through_the_sdk(monkeypatch): ) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [[0.0, 1.0], None, [0.0, 1.0]] assert provider.calls == 4 @@ -1387,16 +1390,15 @@ def test_provider_wide_statuses_exclude_input_dependent_bad_request(): """404 is provider-wide, but an over-limit 400 must stay input-dependent.""" async def go(): - store = LocalEmbeddingStore(name="t_local_embedding_classifier") for error in ( AuthenticationError("bad key"), PermissionDeniedError("no access"), NotFoundError("model_not_found"), InternalServerError("boom"), ): - assert store._is_provider_wide_failure(error) is True + assert _EmbeddingError.from_exception(error).is_provider_wide is True over_limit = ContextLengthError("the input length exceeds the context length") - assert store._is_provider_wide_failure(over_limit) is False + assert _EmbeddingError.from_exception(over_limit).is_provider_wide is False run(go()) @@ -1415,7 +1417,7 @@ def test_provider_wide_not_found_errors_are_not_split(monkeypatch): store.as_embedding = provider monkeypatch.setattr(asyncio, "sleep", fake_sleep) - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None assert provider.batch_sizes == [10] assert not sleeps @@ -1489,26 +1491,22 @@ class DashScopeRuntimeError(RuntimeError): def test_http_status_reads_every_supported_exception_shape(): """Read the status from status_code, an integer code, response.status_code or the message.""" - store = LocalEmbeddingStore(name="t_local_embedding_status_shapes") - - assert store._http_status(AuthenticationError("bad key")) == 401 - assert store._http_status(NotFoundError("model_not_found")) == 404 - assert store._http_status(GenAiClientError(401)) == 401 - assert store._http_status(GenAiServerError(503)) == 503 - assert store._http_status(WrappedTransportError(404)) == 404 - assert store._http_status(DashScopeRuntimeError.for_status(401)) == 401 - assert store._http_status(ValueError("no status here")) is None + assert _EmbeddingError.from_exception(AuthenticationError("bad key")).status == 401 + assert _EmbeddingError.from_exception(NotFoundError("model_not_found")).status == 404 + assert _EmbeddingError.from_exception(GenAiClientError(401)).status == 401 + assert _EmbeddingError.from_exception(GenAiServerError(503)).status == 503 + assert _EmbeddingError.from_exception(WrappedTransportError(404)).status == 404 + assert _EmbeddingError.from_exception(DashScopeRuntimeError.for_status(401)).status == 401 + assert _EmbeddingError.from_exception(ValueError("no status here")).status is None def test_http_status_ignores_a_string_code(): """A string ``code`` names the error; reading it as a status would misclassify quota errors.""" - store = LocalEmbeddingStore(name="t_local_embedding_status_string_code") - - assert store._http_status(StringCodeError("invalid key")) is None - assert store._is_insufficient_quota(StringCodeError("invalid key")) is False - assert store._http_status(InsufficientQuotaError("quota exhausted")) == 429 - assert store._is_insufficient_quota(InsufficientQuotaError("quota exhausted")) is True + assert _EmbeddingError.from_exception(StringCodeError("invalid key")).status is None + assert _EmbeddingError.from_exception(StringCodeError("invalid key")).is_insufficient_quota is False + assert _EmbeddingError.from_exception(InsufficientQuotaError("quota exhausted")).status == 429 + assert _EmbeddingError.from_exception(InsufficientQuotaError("quota exhausted")).is_insufficient_quota is True def test_google_genai_service_failures_are_not_split_into_per_item_requests(monkeypatch): @@ -1541,7 +1539,7 @@ def test_google_genai_service_failures_are_not_split_into_per_item_requests(monk ) store.as_embedding = provider - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None assert provider.batch_sizes == expected_batches run(go()) @@ -1572,7 +1570,7 @@ def test_dashscope_runtime_errors_are_not_split_into_per_item_requests(monkeypat ) store.as_embedding = provider - assert await store._call_with_retry([f"item{i}" for i in range(10)]) is None + assert (await store._call_with_retry([f"item{i}" for i in range(10)])).embeddings is None assert provider.batch_sizes == expected_batches run(go()) @@ -1586,7 +1584,7 @@ def test_unrecognized_errors_are_not_split_into_per_item_requests(): store = LocalEmbeddingStore(name="t_local_embedding_unrecognized", max_retries=3) store.as_embedding = provider - assert await store._call_with_retry(["a", "b", "c"]) is None + assert (await store._call_with_retry(["a", "b", "c"])).embeddings is None assert provider.batch_sizes == [3] run(go()) @@ -1595,19 +1593,17 @@ def test_unrecognized_errors_are_not_split_into_per_item_requests(): def test_input_dependent_classifier_pins_both_halves_of_the_boundary(): """The isolation must keep working for the one error a single item can explain.""" - store = LocalEmbeddingStore(name="t_local_embedding_input_dependent") - assert ( - store._is_input_dependent_failure( + _EmbeddingError.from_exception( ContextLengthError("the input length exceeds the context length"), - ) + ).is_input_dependent is True ) - assert store._is_input_dependent_failure(ContextLengthError("too long")) is True - assert store._is_input_dependent_failure(AuthenticationError("bad key")) is False - assert store._is_input_dependent_failure(InternalServerError("boom")) is False - assert store._is_input_dependent_failure(GenAiClientError(400)) is False - assert store._is_input_dependent_failure(RuntimeError("provider said no")) is False + assert _EmbeddingError.from_exception(ContextLengthError("too long")).is_input_dependent is True + assert _EmbeddingError.from_exception(AuthenticationError("bad key")).is_input_dependent is False + assert _EmbeddingError.from_exception(InternalServerError("boom")).is_input_dependent is False + assert _EmbeddingError.from_exception(GenAiClientError(400)).is_input_dependent is False + assert _EmbeddingError.from_exception(RuntimeError("provider said no")).is_input_dependent is False def test_bad_request_without_an_input_length_signal_is_not_split(): @@ -1618,7 +1614,7 @@ def test_bad_request_without_an_input_length_signal_is_not_split(): store = LocalEmbeddingStore(name="t_local_embedding_bare_400", max_retries=2) store.as_embedding = provider - assert await store._call_with_retry(["a", "b", "c"]) is None + assert (await store._call_with_retry(["a", "b", "c"])).embeddings is None assert provider.batch_sizes == [3] run(go()) @@ -1640,9 +1636,10 @@ def test_isolation_stops_when_a_provider_wide_failure_hits(): ) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) + result = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) - assert results == [[1.0, 0.0], None, None, None] + assert result.embeddings == [[1.0, 0.0], None, None, None] + assert result.terminal_failure is True # One rejected batch, the recovered first item, the over-limit item, then the quota # failure. The fourth item is never requested once the quota failure is seen. assert provider.batch_sizes == [4, 1, 1, 1] @@ -1700,9 +1697,10 @@ def test_isolation_keeps_going_while_only_inputs_fail(): ) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) + result = await store._call_with_retry(["ok", "x" * 64, "also ok", "still ok"]) - assert results == [[1.0, 0.0], None, [1.0, 0.0], [1.0, 0.0]] + assert result.embeddings == [[1.0, 0.0], None, [1.0, 0.0], [1.0, 0.0]] + assert result.terminal_failure is False assert provider.batch_sizes == [4, 1, 1, 1, 1] run(go()) @@ -1725,8 +1723,8 @@ def test_dashscope_serialized_quota_errors_keep_their_quota_policy(monkeypatch): monkeypatch.setattr(asyncio, "sleep", fake_sleep) error = DashScopeRuntimeError.for_status(429, code="insufficient_quota") - assert LocalEmbeddingStore._is_insufficient_quota(error) is True - assert LocalEmbeddingStore._is_rate_limited(error) is False + assert _EmbeddingError.from_exception(error).is_insufficient_quota is True + assert _EmbeddingError.from_exception(error).is_rate_limited is False provider = AlwaysProviderWideFailureAsEmbedding(error) store = LocalEmbeddingStore( @@ -1737,7 +1735,7 @@ def test_dashscope_serialized_quota_errors_keep_their_quota_policy(monkeypatch): ) store.as_embedding = provider - assert await store._call_with_retry(["a", "b"]) is None + assert (await store._call_with_retry(["a", "b"])).embeddings is None assert provider.batch_sizes == [2, 2, 2] assert sleeps == [7.0, 7.0] @@ -1799,7 +1797,7 @@ def test_context_length_rejection_reported_as_http_413_is_isolated(): ) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [[1.0, 0.0], None, [1.0, 0.0]] assert provider.batch_sizes == [3, 1, 1, 1] @@ -1821,7 +1819,7 @@ def test_bare_http_413_without_an_input_signal_is_not_split(): store = LocalEmbeddingStore(name="t_local_embedding_bare_413", max_retries=2) store.as_embedding = provider - assert await store._call_with_retry(["a", "b", "c"]) is None + assert (await store._call_with_retry(["a", "b", "c"])).embeddings is None assert provider.batch_sizes == [3] run(go()) @@ -1830,18 +1828,18 @@ def test_bare_http_413_without_an_input_signal_is_not_split(): def test_input_dependent_classifier_accepts_a_413_only_with_an_input_signal(): """Pin both halves of the 413 boundary, and that the 400 behaviour is unchanged.""" - store = LocalEmbeddingStore(name="t_local_embedding_413_classifier") - assert ( - store._is_input_dependent_failure( + _EmbeddingError.from_exception( ContextLength413Error("the input length exceeds the context length"), - ) + ).is_input_dependent is True ) - assert store._is_input_dependent_failure(ContextLength413Error("too long")) is True - assert store._is_input_dependent_failure(RequestBodyTooLargeError("request entity too large")) is False - assert store._is_input_dependent_failure(ContextLengthError("too long")) is True - assert store._is_input_dependent_failure(GenAiClientError(400)) is False + assert _EmbeddingError.from_exception(ContextLength413Error("too long")).is_input_dependent is True + assert ( + _EmbeddingError.from_exception(RequestBodyTooLargeError("request entity too large")).is_input_dependent is False + ) + assert _EmbeddingError.from_exception(ContextLengthError("too long")).is_input_dependent is True + assert _EmbeddingError.from_exception(GenAiClientError(400)).is_input_dependent is False class _NoRetryPolicyModel: @@ -1904,7 +1902,7 @@ def test_a_provider_with_an_empty_retry_policy_recovers_a_connection_error(monke ) store.as_embedding = provider - results = await store._call_with_retry(["a", "b", "c"]) + results = (await store._call_with_retry(["a", "b", "c"])).embeddings assert results == [[1.0, 0.0]] * 3 assert provider.batch_sizes == [3, 3] @@ -1939,7 +1937,7 @@ def test_a_policy_that_does_not_cover_the_error_keeps_the_store_backoff(monkeypa ) store.as_embedding = provider - results = await store._call_with_retry(["a", "b", "c"]) + results = (await store._call_with_retry(["a", "b", "c"])).embeddings assert results == [[1.0, 0.0]] * 3 assert provider.batch_sizes == [3, 3] @@ -1976,7 +1974,7 @@ def test_an_isolated_item_does_not_multiply_sdk_transport_retries(monkeypatch): ) store.as_embedding = provider - results = await store._call_with_retry(["ok", "x" * 64, "also ok"]) + results = (await store._call_with_retry(["ok", "x" * 64, "also ok"])).embeddings assert results == [None, None, None] assert provider.calls == 2 From 55fdad94d05b2a564b510aa4700a37e4585014a6 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:21:37 +0800 Subject: [PATCH 07/19] fix(dsh): preserve outbox order and redact whitespace in credentials (#657) --- integrations/dsh/README.md | 2 ++ integrations/dsh/README_ZH.md | 2 ++ integrations/dsh/src/endpoint.ts | 3 +- integrations/dsh/src/outbox.ts | 15 ++++++++- integrations/dsh/tests/client.test.mjs | 29 +++++++++++++++++ integrations/dsh/tests/dsh-tools.test.mjs | 39 +++++++++++++++++++++++ integrations/dsh/tests/outbox.test.mjs | 39 +++++++++++++++++++++++ integrations/dsh/tests/runtime.test.mjs | 34 ++++++++++++++++++++ 8 files changed, 161 insertions(+), 2 deletions(-) diff --git a/integrations/dsh/README.md b/integrations/dsh/README.md index b3f4aa28..faac78cc 100644 --- a/integrations/dsh/README.md +++ b/integrations/dsh/README.md @@ -177,6 +177,8 @@ The outbox defaults to `$DSH_HOME/reme-memory/outbox` (`~/.dsh/reme-memory/outbo Owner-lock recovery is serialized. If startup reports a busy or interrupted `.lock-recovery`, retry first; if the error persists, stop all Hosts using that queue, inspect it, and remove only the empty `.lock-recovery` directory before restarting. Preserve `.lock` and all queued JSON files. +New batches receive strictly increasing ordering timestamps, recovered from queued files after restart, so equal wall-clock times or a clock rollback cannot reorder them. Older batches that already share a timestamp retain their existing file order. + Sealed batches keep a stable `request_id` across timeouts and restarts. The updated ReMe service stores completed request receipts under its configured metadata directory, rejects reuse with different input, and replays completed results without invoking the memory agent again. An interruption before a receipt is committed can still repeat an unfinished operation; receipts do not make arbitrary agent file edits transactional. ![Automatic-memory activity](./figures/reme-status-auto-memory.png) diff --git a/integrations/dsh/README_ZH.md b/integrations/dsh/README_ZH.md index 34222a7b..e7d7cd77 100644 --- a/integrations/dsh/README_ZH.md +++ b/integrations/dsh/README_ZH.md @@ -188,6 +188,8 @@ only on retrieved memory and cite the returned paths. 进程锁的回收会串行执行。若启动报告 `.lock-recovery` 正忙或恢复中断，先重试；错误持续时，停止所有使用该队列的 Host，检查队列后仅移除空的 `.lock-recovery` 目录，再重新启动。保留 `.lock` 和所有排队的 JSON 文件。 +新批次使用严格递增的排序时间戳，重启时从排队文件恢复，避免系统时间相同或回拨导致批次乱序。已有相同时间戳的旧批次仍沿用原有文件顺序。 + 已封存的批次在超时、重启后沿用相同的 `request_id`。更新后的 ReMe 服务在配置的 metadata 目录保存已完成请求的回执，拒绝把同一个 ID 用于不同输入，并直接重放已完成结果，避免重复调用记忆 Agent。回执提交前发生中断，未完成的操作仍可能重试；回执不会把任意 Agent 文件修改变成事务。 截图测试把间隔临时设为 1；运行记录中可以看到英文测试会话形成的已完成提交： diff --git a/integrations/dsh/src/endpoint.ts b/integrations/dsh/src/endpoint.ts index b7b8f99b..d50eb5f9 100644 --- a/integrations/dsh/src/endpoint.ts +++ b/integrations/dsh/src/endpoint.ts @@ -18,5 +18,6 @@ export function validEndpoint(value: string): boolean { /** Remove URL userinfo before errors cross model, browser, or log boundaries. */ export function safeError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); - return message.replace(/(https?:\/\/)[^\s/]+@/gi, "$1[redacted]@"); + // URL parsing encodes spaces and strips tabs/newlines in userinfo; do not treat them as boundaries. + return message.replace(/(https?:\/\/)[^/?#\\]+@/gi, "$1[redacted]@"); } diff --git a/integrations/dsh/src/outbox.ts b/integrations/dsh/src/outbox.ts index 3becfacd..1bc9e43c 100644 --- a/integrations/dsh/src/outbox.ts +++ b/integrations/dsh/src/outbox.ts @@ -30,6 +30,7 @@ export interface MemoryBatch { export class MemoryOutbox { private readonly lock: string; private readonly owner = `${process.pid}:${randomUUID()}`; + private latestCreatedAt = 0; constructor(readonly directory: string) { mkdirSync(directory, { recursive: true, mode: 0o700 }); @@ -68,6 +69,14 @@ export class MemoryOutbox { } finally { closeSync(fd); } + for (const name of this.files()) + this.latestCreatedAt = Math.max( + this.latestCreatedAt, + this.read(name).createdAt, + ); + } catch (error) { + this.close(); + throw error; } finally { rmdirSync(guard); } @@ -90,7 +99,11 @@ export class MemoryOutbox { : { version: 1, id: randomUUID(), - createdAt: Date.now(), + // Keep FIFO ordering when the clock repeats or moves backwards, including after restart. + createdAt: (this.latestCreatedAt = Math.max( + Date.now(), + this.latestCreatedAt + 1, + )), endpoint, sessionId, date, diff --git a/integrations/dsh/tests/client.test.mjs b/integrations/dsh/tests/client.test.mjs index af3e839b..21ed3d3c 100644 --- a/integrations/dsh/tests/client.test.mjs +++ b/integrations/dsh/tests/client.test.mjs @@ -3,6 +3,35 @@ import test from "node:test"; import { ReMeClient } from "../dist/reme/client.js"; import { safeError } from "../dist/endpoint.js"; +test("redacts URL credentials containing spaces or stripped URL whitespace", async () => { + const originalFetch = globalThis.fetch; + const client = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + try { + for (const password of [ + "SECRET CANARY", + "SECRET\tCANARY", + "SECRET\nCANARY", + ]) { + const url = `http://alice:${password}@localhost:2333/search`; + assert.ok(new URL(url).password); + const expected = "refused http://[redacted]@localhost:2333/search"; + globalThis.fetch = async () => + Response.json({ success: false, answer: `refused ${url}` }); + assert.equal((await client.search("fact")).error, expected); + globalThis.fetch = async () => { + throw new Error(`refused ${url}`); + }; + assert.equal((await client.search("fact")).error, expected); + } + } finally { + globalThis.fetch = originalFetch; + } +}); + test("rejects credentials before fetch and redacts URL credentials from service and transport errors", async () => { const originalFetch = globalThis.fetch; let calls = 0; diff --git a/integrations/dsh/tests/dsh-tools.test.mjs b/integrations/dsh/tests/dsh-tools.test.mjs index a21dd4a4..6c61a706 100644 --- a/integrations/dsh/tests/dsh-tools.test.mjs +++ b/integrations/dsh/tests/dsh-tools.test.mjs @@ -1,6 +1,45 @@ import assert from "node:assert/strict"; import test from "node:test"; import { registerReMeTools } from "../dist/tools.js"; +import { ReMeClient } from "../dist/reme/client.js"; + +test("reme_search does not expose whitespace-bearing URL credentials from service errors", async () => { + const originalFetch = globalThis.fetch; + const registered = []; + globalThis.fetch = async () => + Response.json({ + success: false, + answer: "refused http://alice:SECRET CANARY@localhost:2333/search", + }); + try { + registerReMeTools( + { + tools: { + register(tool) { + registered.push(tool); + return () => {}; + }, + }, + }, + new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }), + { searchLimit: 5 }, + ); + const result = await registered[0].execute( + { query: "history" }, + { signal: new AbortController().signal }, + ); + assert.equal( + result, + "ReMe search failed: refused http://[redacted]@localhost:2333/search", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); test("reme_search uses the ReMe search contract and renders model-facing text", async () => { const registered = []; diff --git a/integrations/dsh/tests/outbox.test.mjs b/integrations/dsh/tests/outbox.test.mjs index 2d102c41..9755c123 100644 --- a/integrations/dsh/tests/outbox.test.mjs +++ b/integrations/dsh/tests/outbox.test.mjs @@ -31,6 +31,41 @@ const messages = [ }, ]; +test("preserves batch order across equal timestamps, clock rollback, and restart", (t) => { + const now = Date.parse("2026-10-21T12:00:00Z"); + t.mock.timers.enable({ apis: ["Date"], now }); + const path = directory(t); + let queue = new MemoryOutbox(path); + const append = (id) => + queue.append( + "http://localhost:2333", + "session", + "2026-10-21", + [{ ...messages[0], id }], + 1, + ); + try { + append("first"); + append("second"); + const batches = readdirSync(path) + .filter((name) => name.startsWith("ready-")) + .map((name) => JSON.parse(readFileSync(join(path, name), "utf8"))); + assert.equal(new Set(batches.map((batch) => batch.createdAt)).size, 2); + queue.close(); + t.mock.timers.setTime(now - 1000); + queue = new MemoryOutbox(path); + append("third"); + const received = []; + for (let next = queue.next(); next; next = queue.next()) { + received.push(next.batch.turns[0][0].id); + queue.acknowledge(next.name); + } + assert.deepEqual(received, ["first", "second", "third"]); + } finally { + queue.close(); + } +}); + test("recovers a partial batch after the owning process exits without cleanup", (t) => { const path = directory(t); const child = spawnSync( @@ -191,6 +226,10 @@ test("preserves invalid records and keeps parser errors free of message content" (error) => !error.message.includes("PRIVATE-CONTENT"), ); assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + queue.close(); + assert.throws(() => new MemoryOutbox(path), /Invalid ReMe outbox record/); + assert.equal(existsSync(join(path, ".lock")), false); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); } finally { queue.close(); } diff --git a/integrations/dsh/tests/runtime.test.mjs b/integrations/dsh/tests/runtime.test.mjs index d0ee3ff5..7cb8136f 100644 --- a/integrations/dsh/tests/runtime.test.mjs +++ b/integrations/dsh/tests/runtime.test.mjs @@ -294,6 +294,40 @@ test("does not put service reply text or file names into diagnostic snapshots", assert.doesNotMatch(JSON.stringify(runtime.snapshot()), /PRIVATE/); }); +test("keeps whitespace-bearing URL credentials out of snapshots and logs", async (t) => { + const logs = []; + const error = "refused http://alice:SECRET CANARY\n@localhost:2333"; + const runtime = makeRuntime( + t, + { + async autoMemory() { + throw new Error(error); + }, + async autoDream() { + return { ok: false, error }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + { + debug() {}, + warn(...args) { + logs.push(args); + }, + }, + ); + completeTurn(runtime, { id: "credentials" }, 1, 10); + await runtime.drain(); + await runtime.runDream(); + const snapshot = runtime.snapshot(); + assert.equal(snapshot.autoMemory.recentTasks[0].phase, "failed"); + assert.equal(snapshot.autoDream.lastResult, "failed"); + assert.equal(logs.length, 2); + assert.doesNotMatch( + JSON.stringify({ snapshot, logs }), + /alice|SECRET|CANARY/, + ); +}); + test("splits dates and flushes a partial previous day without a new interval", async (t) => { const calls = []; const runtime = makeRuntime( From 925ef0dad17adf2250b866716df2be00b4cafdc1 Mon Sep 17 00:00:00 2001 From: DiegoCluv7 <161216254+iluv7@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:22:32 +0800 Subject: [PATCH 08/19] fix(auto-dream): preserve retries after invalid integration receipts (#654) --- reme/steps/evolve/dream/integrate.py | 20 +---- tests/unit/test_auto_dream.py | 72 ++++++++++++--- tests/unit/test_dream_checkpoint_versions.py | 95 +++++++++++++++++++- 3 files changed, 159 insertions(+), 28 deletions(-) diff --git a/reme/steps/evolve/dream/integrate.py b/reme/steps/evolve/dream/integrate.py index f43dbea7..f3946318 100644 --- a/reme/steps/evolve/dream/integrate.py +++ b/reme/steps/evolve/dream/integrate.py @@ -214,10 +214,10 @@ class DreamIntegrateStep(BaseStep): ) before = after continue - # After the bounded retry, checkpoint the source so malformed input cannot loop forever. - self._record_skipped(state, unit, bucket, paths, e) - self.logger.warning( - f"[{self.name}] unit {index}/{len(state.units)} skipped invalid receipt after retry: " + # Without a valid receipt or recoverable write, keep sources eligible for retry. + self._record_failure(state, unit, paths, e) + self.logger.error( + f"[{self.name}] unit {index}/{len(state.units)} failed invalid receipt after retry: " f"{type(e).__name__}: {e}", ) return @@ -339,18 +339,6 @@ class DreamIntegrateStep(BaseStep): note=f"Recovered from {type(error).__name__}: agent receipt unavailable", ) - @classmethod - def _record_skipped(cls, state, unit: dict, bucket: str, paths: list[str], error: Exception) -> None: - message = f"unit {unit.get('name', '')!r} skipped: unusable agent receipt ({type(error).__name__})" - key = cls._unit_key(unit, bucket, paths) - if not any( - cls._unit_key(item, str(item.get("bucket") or ""), [str(p) for p in item.get("paths") or []]) == key - for item in state.skipped_units - ): - state.skipped_units.append({**unit, "bucket": bucket, "paths": paths, "reason": message}) - if message not in state.warnings: - state.warnings.append(message) - @staticmethod def _record_failure(state, unit: dict, paths: list[str], error: Exception) -> None: message = f"{type(error).__name__}: {error}" diff --git a/tests/unit/test_auto_dream.py b/tests/unit/test_auto_dream.py index a808b615..788444fd 100644 --- a/tests/unit/test_auto_dream.py +++ b/tests/unit/test_auto_dream.py @@ -498,27 +498,71 @@ def test_extract_retries_one_unusable_receipt(tmp_path): asyncio.run(run()) -def test_integrate_invalid_receipt_without_file_change_is_skipped(tmp_path): - """An unusable integration receipt is a skipped unit, not a failed unit.""" +@pytest.mark.parametrize( + "receipt", + ["{}", "not a receipt", '{"action": "CREATE", "target_path": "digest/wiki/missing.md"}'], +) +def test_integrate_invalid_receipt_without_file_change_fails(tmp_path, receipt): + """An unusable integration receipt cannot acknowledge unprocessed source files.""" async def run(): unit = {"name": "unit", "bucket": "wiki", "summary": "summary", "paths": ["daily/a.md"]} state = DreamState(units=[unit]) step = DreamIntegrateStep() - step.agent_wrapper = _ReplyAgent() + step.agent_wrapper = _ReplyAgent({"result": receipt}) await step._integrate_one(state, unit, 1, tmp_path, "digest") # pylint: disable=protected-access assert state.integrate_results == [] - assert len(state.skipped_units) == 1 - assert state.failed_units == [] - assert not state.failed_paths - assert len(state.warnings) == 1 + assert state.skipped_units == [] + assert len(state.failed_units) == 1 + assert state.failed_units[0]["name"] == unit["name"] + assert state.failed_paths == unit["paths"] + assert len(state.errors) == 1 + assert state.warnings == [] assert step.agent_wrapper.calls == 2 asyncio.run(run()) +@pytest.mark.asyncio +@pytest.mark.parametrize("shared_source", [False, True]) +async def test_integrate_failure_only_blocks_affected_sources(tmp_path, monkeypatch, shared_source): + """A failed unit prevents checkpointing all its sources, even one used by a successful unit.""" + good, bad, target = "daily/good.md", "daily/bad.md", "digest/wiki/good.md" + observed = {rel: _touch(tmp_path / rel).stat().st_mtime for rel in (good, bad)} + failed_paths = [bad, good] if shared_source else [bad] + units = [ + {"name": "good", "bucket": "wiki", "summary": "good memory", "paths": [good]}, + {"name": "bad", "bucket": "wiki", "summary": "bad memory", "paths": failed_paths}, + ] + + def integrate_good(): + _touch(tmp_path / target, "integrated memory") + return {"result": json.dumps({"action": "CREATE", "target_path": target})} + + agent = _SequenceAgent(integrate_good, {"result": "{}"}, {"result": "{}"}) + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + state = DreamState(units=units, changed_paths=[good, bad], existing=observed, workspace=str(tmp_path)) + catalog = _Catalog() + context = RuntimeContext(dream=state.model_dump(), file_catalog=catalog, file_store=_FileStore(tmp_path)) + monkeypatch.setattr("reme.steps.evolve.dream.integrate.llm_available", lambda _step: True) + + response = await DreamIntegrateStep(app_context=app_context, agent_wrapper=agent)(context) + assert response.success is False + response = await DreamFinishStep(app_context=app_context)(context) + + dream = response.metadata["dream"] + assert response.success is False + assert len(dream["integrate_results"]) == 1 + assert len(dream["failed_units"]) == 1 + assert set(dream["failed_paths"]) == set(failed_paths) + assert dream["checkpoint_paths"] == ([] if shared_source else [good]) + assert {node.path for node in catalog.upserts} == (set() if shared_source else {good}) + assert (tmp_path / target).read_text(encoding="utf-8") == "integrated memory" + assert agent.calls == 3 + + def test_integrate_retries_one_invalid_receipt(tmp_path): """A transient invalid integration receipt gets one bounded retry.""" @@ -622,7 +666,10 @@ def test_integrate_does_not_recover_a_create_in_another_bucket(tmp_path): await step._integrate_one(state, unit, 1, tmp_path, "digest") # pylint: disable=protected-access assert state.integrate_results == [] - assert len(state.skipped_units) == 1 + assert state.skipped_units == [] + assert len(state.failed_units) == 1 + assert state.failed_paths == unit["paths"] + assert state.errors assert state.nodes_created == [] assert state.modified_paths == ["digest/procedure/unit.md"] @@ -762,13 +809,13 @@ async def test_integrate_clears_changes_when_skipping_or_missing_llm(tmp_path, m @pytest.mark.asyncio @pytest.mark.parametrize("job_name", ["auto_dream", "dream_cron"]) @pytest.mark.parametrize("tag_fails", [False, True]) -@pytest.mark.parametrize("integrate_fails", [False, True]) +@pytest.mark.parametrize("integration_failure", [None, "exception", "invalid_receipt"]) async def test_dream_jobs_tag_outputs_and_preserve_checkpoint_results( tmp_path, monkeypatch, job_name, tag_fails, - integrate_fails, + integration_failure, ): """Both configured pipelines tag durable outputs without changing dream success.""" app_context = ApplicationContext(workspace_dir=str(tmp_path)) @@ -798,8 +845,10 @@ async def test_dream_jobs_tag_outputs_and_preserve_checkpoint_results( for path in targets: previous = (tmp_path / path).read_text(encoding="utf-8") if (tmp_path / path).exists() else "" _touch(tmp_path / path, previous + "ReMe evidence\n") - if integrate_fails: + if integration_failure == "exception": raise RuntimeError("integration unavailable") + if integration_failure == "invalid_receipt": + return {"result": "{}"} return {"result": json.dumps({"action": "CREATE", "target_path": targets[0]})} agent = _ReplyAgent() @@ -821,6 +870,7 @@ async def test_dream_jobs_tag_outputs_and_preserve_checkpoint_results( finally: await job.close() + integrate_fails = bool(integration_failure) assert response.success is not integrate_fails assert response.answer.startswith("AutoDream failed" if integrate_fails else "AutoDream completed") assert response.metadata["modified"] is True diff --git a/tests/unit/test_dream_checkpoint_versions.py b/tests/unit/test_dream_checkpoint_versions.py index 38afe875..ec97e112 100644 --- a/tests/unit/test_dream_checkpoint_versions.py +++ b/tests/unit/test_dream_checkpoint_versions.py @@ -16,7 +16,7 @@ from reme.components.application_context import ApplicationContext from reme.components.file_catalog import LocalFileCatalog from reme.components.runtime_context import RuntimeContext from reme.config import resolve_app_config -from reme.schema import DreamState +from reme.schema import DreamState, FileNode from reme.steps.evolve.dream.finish import DreamFinishStep DAY = "2026-05-28" @@ -141,6 +141,99 @@ async def test_dream_reprocesses_concurrent_source_edits_after_restart(tmp_path, await app.close() +@pytest.mark.asyncio +@pytest.mark.parametrize("older_checkpoint", [False, True]) +@pytest.mark.parametrize("recover_on_attempt", [None, 1, 2]) +async def test_invalid_integration_preserves_retry_after_restart(tmp_path, older_checkpoint, recover_on_attempt): + """Unrecoverable integration preserves catalog versions; actual writes can still recover.""" + source = tmp_path / SOURCE + source.parent.mkdir(parents=True) + original = frontmatter.dumps(frontmatter.Post(OLD, name="deployment", description="Deployment policy")) + source.write_text(original) + index = f"daily/{DAY}.md" + (tmp_path / index).write_text("day index") + + def agent_for(app, write_attempt): + calls = {"extract": 0, "integrate": 0} + + async def reply(_message, **kwargs): + if "list_tags" in kwargs["job_tools"]: + return {"result": "No tags added by the controlled provider"} + if kwargs["job_tools"] == ["read"]: + calls["extract"] += 1 + unit = {"name": "deployment", "bucket": "wiki", "summary": OLD, "paths": [SOURCE, index]} + return {"result": json.dumps({"units": [unit]})} + calls["integrate"] += 1 + if calls["integrate"] == write_attempt: + written = await app.run_job("write", path=TARGET, content=OLD) + assert written.success, written.answer + return {"result": "{}"} + + agent = AsyncMock(spec=BaseAgentWrapper) + agent.reply.side_effect = reply + return agent, calls + + app = _app(tmp_path) + await app.start() + previous = {rel: (tmp_path / rel).stat().st_mtime - 2 for rel in (SOURCE, index)} if older_checkpoint else {} + try: + catalog = app.context.components["file_catalog"]["dream"] + await catalog.upsert([FileNode(path=rel, st_mtime=mtime) for rel, mtime in previous.items()]) + agent, calls = agent_for(app, recover_on_attempt) + response = await app.run_job( + "auto_dream", + date=DAY, + scan_days=1, + agent_wrapper=agent, + as_llm=Mock(spec=ChatModelBase), + ) + state = response.metadata["dream"] + assert response.success is (recover_on_attempt is not None) + assert state["skipped_units"] == [] + assert calls == {"extract": 1, "integrate": recover_on_attempt or 2} + assert source.read_text() == original + if recover_on_attempt is None: + assert response.answer.startswith("AutoDream failed") + assert len(state["failed_units"]) == 1 + assert set(state["failed_paths"]) == {SOURCE, index} + assert state["checkpoint_paths"] == [] + assert state["errors"] + assert {n.path: n.st_mtime for n in await catalog.get_nodes()} == previous + assert not (tmp_path / TARGET).exists() + else: + assert response.answer.startswith("AutoDream completed with warnings") + assert state["failed_units"] == [] + assert state["failed_paths"] == [] + assert not state["errors"] + assert set(state["checkpoint_paths"]) == {SOURCE, index} + assert state["integrate_results"][0]["target_path"] == TARGET + assert OLD in (tmp_path / TARGET).read_text() + finally: + await app.close() + + app = _app(tmp_path) + await app.start() + try: + catalog = app.context.components["file_catalog"]["dream"] + if recover_on_attempt is None: + assert {n.path: n.st_mtime for n in await catalog.get_nodes()} == previous + agent, calls = agent_for(app, 1) + response = await app.run_job( + "auto_dream", + date=DAY, + scan_days=1, + agent_wrapper=agent, + as_llm=Mock(spec=ChatModelBase), + ) + assert response.success, response.answer + assert calls == {"extract": int(recover_on_attempt is None), "integrate": int(recover_on_attempt is None)} + assert {SOURCE, index} <= {n.path for n in await catalog.get_nodes()} + assert OLD in (tmp_path / TARGET).read_text() + assert source.read_text() == original + finally: + await app.close() + + @pytest.mark.asyncio async def test_finish_keeps_scanned_versions_for_changed_sources_and_day_indexes(tmp_path): """Supplemental indexes obey the same snapshot and failed-source rules as notes.""" From ff2956a41f5e6e9db16cf41598f1ca4bf1777eb5 Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:35:21 +0800 Subject: [PATCH 09/19] refactor: simplify core, Studio, and unit test scaffolding (#658) * refactor: share backend search, file responses, and vector digests * refactor(studio): remove unused scaffolding and share workspace updates * test: simplify async tests and share deterministic store fixtures --- .github/workflows/_build-docs.yml | 2 +- .github/workflows/ci-reme-studio.yml | 2 +- example.env | 1 - .../file_store/faiss_local_file_store.py | 23 - .../components/file_store/local_file_store.py | 18 + .../file_store/zvec_local_file_store.py | 21 - reme/steps/file_io/_base.py | 13 + reme/steps/file_io/daily_write.py | 11 +- reme/steps/file_io/edit.py | 11 +- reme/steps/file_io/list.py | 12 +- reme/steps/file_io/load.py | 10 +- reme/steps/file_io/read_image.py | 11 +- reme/steps/file_io/save.py | 10 +- reme/steps/file_io/write.py | 11 +- reme/steps/index/_dedup.py | 29 +- reme/steps/index/_single_search.py | 68 + reme/steps/index/_tool_context.py | 23 + reme/steps/index/bm25_search.py | 65 +- reme/steps/index/draft.py | 11 +- reme/steps/index/search.py | 12 +- reme/steps/index/vector_search.py | 65 +- reme/utils/tushare.py | 17 - reme_studio/README.md | 6 +- reme_studio/README_ZH.md | 6 +- reme_studio/app/chatgpt-auth.ts | 90 - reme_studio/app/globals.css | 21 - reme_studio/app/open-file.ts | 21 +- reme_studio/app/store.ts | 331 +- reme_studio/app/workspace-chat.tsx | 296 ++ reme_studio/app/workspace-tabs.tsx | 136 + reme_studio/app/workspace-theme.tsx | 51 + reme_studio/app/workspace.tsx | 514 +-- reme_studio/cloudflare-env.d.ts | 5 - reme_studio/db/index.ts | 13 - reme_studio/db/schema.ts | 4 - reme_studio/drizzle.config.ts | 7 - reme_studio/drizzle/meta/_journal.json | 5 - .../examples/d1/app/api/notes/route.ts | 63 - reme_studio/examples/d1/db/schema.ts | 11 - reme_studio/package-lock.json | 1183 +------ reme_studio/package.json | 12 +- reme_studio/public/file.svg | 1 - reme_studio/public/globe.svg | 1 - reme_studio/public/window.svg | 1 - .../{ => distribution}/demo-build.test.mjs | 2 +- .../{ => distribution}/rendered-html.test.mjs | 4 +- .../{ => distribution}/static-build.test.mjs | 10 +- reme_studio/tooling/sites-vite-plugin.ts | 8 +- reme_studio/worker/index.ts | 1 - tests/unit/file_store_test_support.py | 79 + tests/unit/test_background_steps.py | 101 +- tests/unit/test_base_component.py | 28 - tests/unit/test_common_steps.py | 29 - tests/unit/test_component_registry.py | 17 - tests/unit/test_crud_steps.py | 124 +- tests/unit/test_daily_steps.py | 63 +- tests/unit/test_default_file_chunker.py | 33 - tests/unit/test_faiss_index_maintenance.py | 443 +-- tests/unit/test_file_catalog.py | 29 +- tests/unit/test_file_graph.py | 34 +- tests/unit/test_file_store_consistency.py | 3109 ++++++++--------- tests/unit/test_frontmatter_steps.py | 18 +- tests/unit/test_injected_job_kwargs.py | 4 - tests/unit/test_jsonl_file_chunker.py | 45 - tests/unit/test_keyword_index.py | 1543 ++++---- tests/unit/test_link_expansion.py | 37 +- tests/unit/test_local_embedding_store.py | 1508 ++++---- tests/unit/test_markdown_file_chunker.py | 68 +- tests/unit/test_path_permission.py | 22 +- tests/unit/test_prompt_handler.py | 36 - tests/unit/test_read_image_steps.py | 27 +- tests/unit/test_read_with_neighbors.py | 22 +- tests/unit/test_runtime_context.py | 22 - tests/unit/test_tokenizer.py | 23 - tests/unit/test_tushare.py | 21 - tests/unit/test_wikilink_utils.py | 54 +- tests/unit/test_write_metadata_lock.py | 23 +- tests/unit/test_zvec_file_store.py | 708 ++-- 78 files changed, 4171 insertions(+), 7348 deletions(-) create mode 100644 reme/steps/file_io/_base.py create mode 100644 reme/steps/index/_single_search.py create mode 100644 reme/steps/index/_tool_context.py delete mode 100644 reme/utils/tushare.py delete mode 100644 reme_studio/app/chatgpt-auth.ts create mode 100644 reme_studio/app/workspace-chat.tsx create mode 100644 reme_studio/app/workspace-tabs.tsx create mode 100644 reme_studio/app/workspace-theme.tsx delete mode 100644 reme_studio/cloudflare-env.d.ts delete mode 100644 reme_studio/db/index.ts delete mode 100644 reme_studio/db/schema.ts delete mode 100644 reme_studio/drizzle.config.ts delete mode 100644 reme_studio/drizzle/meta/_journal.json delete mode 100644 reme_studio/examples/d1/app/api/notes/route.ts delete mode 100644 reme_studio/examples/d1/db/schema.ts delete mode 100644 reme_studio/public/file.svg delete mode 100644 reme_studio/public/globe.svg delete mode 100644 reme_studio/public/window.svg rename reme_studio/tests/{ => distribution}/demo-build.test.mjs (93%) rename reme_studio/tests/{ => distribution}/rendered-html.test.mjs (88%) rename reme_studio/tests/{ => distribution}/static-build.test.mjs (74%) create mode 100644 tests/unit/file_store_test_support.py delete mode 100644 tests/unit/test_tushare.py diff --git a/.github/workflows/_build-docs.yml b/.github/workflows/_build-docs.yml index 1b3eb8f5..d2b8a577 100644 --- a/.github/workflows/_build-docs.yml +++ b/.github/workflows/_build-docs.yml @@ -67,7 +67,7 @@ jobs: - name: Verify browser demo bundle if: inputs.run_tests working-directory: reme_studio - run: node --test tests/demo-build.test.mjs + run: node --test tests/distribution/demo-build.test.mjs - name: Configure Pages if: inputs.upload_pages_artifact diff --git a/.github/workflows/ci-reme-studio.yml b/.github/workflows/ci-reme-studio.yml index fa8f5d5f..eb9ccf3f 100644 --- a/.github/workflows/ci-reme-studio.yml +++ b/.github/workflows/ci-reme-studio.yml @@ -66,7 +66,7 @@ jobs: - name: Verify npm package run: | - npm pack --pack-destination "${RUNNER_TEMP}" + npm pack --ignore-scripts --pack-destination "${RUNNER_TEMP}" tar -tzf "${RUNNER_TEMP}"/agentscope-ai-reme_studio-*.tgz | grep '^package/dist-static/index.html$' - name: Set up Python diff --git a/example.env b/example.env index 37c3aa76..b8e9e3d9 100644 --- a/example.env +++ b/example.env @@ -14,4 +14,3 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 # Optional data-source mirrors. Unset variables use the official services. # HF_MIRROR_URL=https://hf-mirror.com # ARXIV_MIRROR_URL=https://export.arxiv.org -# TUSHARE_MIRROR_URL=http://112.124.63.173:4000/tushare diff --git a/reme/components/file_store/faiss_local_file_store.py b/reme/components/file_store/faiss_local_file_store.py index 40dfd654..c78ae41d 100644 --- a/reme/components/file_store/faiss_local_file_store.py +++ b/reme/components/file_store/faiss_local_file_store.py @@ -1,7 +1,6 @@ """FAISS-backed file store: chunk JSONL stays authoritative; FAISS HNSW replaces the linear vector scan.""" import asyncio -import hashlib import json from collections.abc import Callable from contextlib import suppress @@ -389,28 +388,6 @@ class FaissLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory - (possibly float32 from the provider; assignment bypasses the pydantic - validator) or has round-tripped through the JSONL. Written into the - idmap sidecar at dump time and recomputed from ``self.file_chunks`` at - load time: a mismatch means the sidecar vectors belong to a different - chunk generation than the authoritative JSONL, which the live-id set - check alone cannot detect for same-ID in-place updates. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach FAISS state (sidecar or rebuild).""" await super().load() diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index dcbe6177..ee36674b 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -3,6 +3,7 @@ import asyncio import base64 import datetime +import hashlib import heapq import json import time @@ -107,6 +108,23 @@ class LocalFileStore(BaseFileStore): return 0 return max(0, int(getattr(self.embedding_store, "dimensions", 0))) + def _chunks_embedding_digest(self) -> str: + """Order-independent digest of the live (chunk_id, embedding) set. + + Float16 bytes match the authoritative chunk JSONL serialization even + before a fresh provider vector has round-tripped through persistence. + Vector backends use this digest to reject stale index sidecars. + """ + digest = hashlib.sha256() + eligible = sorted( + cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) + ) + for cid in eligible: + digest.update(cid.encode("utf-8")) + digest.update(b"\x00") + digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) + return digest.hexdigest() + # -- lifecycle ------------------------------------------------------------ async def _start(self) -> None: diff --git a/reme/components/file_store/zvec_local_file_store.py b/reme/components/file_store/zvec_local_file_store.py index 84db2dbd..8d81609d 100644 --- a/reme/components/file_store/zvec_local_file_store.py +++ b/reme/components/file_store/zvec_local_file_store.py @@ -1,6 +1,5 @@ """Zvec-backed file store: chunk JSONL stays authoritative; a zvec collection replaces the linear vector scan.""" -import hashlib import json import shutil import time @@ -227,26 +226,6 @@ class ZvecLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory or has - round-tripped through the JSONL. Written into the sidecar at dump time - and recomputed from ``self.file_chunks`` at load time: a mismatch means - the collection belongs to a different chunk generation than the - authoritative JSONL. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach the zvec collection (open or rebuild).""" await super().load() diff --git a/reme/steps/file_io/_base.py b/reme/steps/file_io/_base.py new file mode 100644 index 00000000..7c42bc81 --- /dev/null +++ b/reme/steps/file_io/_base.py @@ -0,0 +1,13 @@ +"""Common response handling for file operations.""" + +from ..base_step import BaseStep + + +class _FileIOStep(BaseStep): + """Preserve the file-operation failure envelope without changing its payload.""" + + def _fail(self, message: str, **meta) -> None: + assert self.context is not None + self.context.response.success = False + self.context.response.answer = f"Error: {message}" + self.context.response.metadata.update(meta) diff --git a/reme/steps/file_io/daily_write.py b/reme/steps/file_io/daily_write.py index 31ec050b..addff538 100644 --- a/reme/steps/file_io/daily_write.py +++ b/reme/steps/file_io/daily_write.py @@ -1,8 +1,8 @@ """Write a daily markdown note by dispatching the generic ``write`` step.""" +from ._base import _FileIOStep from ._daily_index import parse_daily_date, refresh_day_index, validate_session_id from ._path import validate_filename_component -from ..base_step import BaseStep from ..index import normalize_posix_path from ...components import R from ...steps.evolve import now @@ -11,16 +11,9 @@ _RESERVED_METADATA_KEYS = {"name", "description", "session_id", "source_conversa @R.register("daily_write_step") -class DailyWriteStep(BaseStep): +class DailyWriteStep(_FileIOStep): """Write ``daily//.md`` with conversation frontmatter.""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _session_dir(self) -> str: return normalize_posix_path(str(self.config_value("session_dir"))) diff --git a/reme/steps/file_io/edit.py b/reme/steps/file_io/edit.py index 2e412d54..b9a457d5 100644 --- a/reme/steps/file_io/edit.py +++ b/reme/steps/file_io/edit.py @@ -3,14 +3,14 @@ import frontmatter import yaml +from ._base import _FileIOStep from ._file_io import get_path_lock, read_file_safe, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("edit_step") -class EditStep(BaseStep): +class EditStep(_FileIOStep): """Replace every occurrence of ``old`` with ``new`` inside the file body. The YAML front matter block (if any) is parsed out, kept verbatim and @@ -24,13 +24,6 @@ class EditStep(BaseStep): read-modify-write cycle against the same file (multi-worker / multi- process safety is out of scope).""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/list.py b/reme/steps/file_io/list.py index df217e81..7f62185e 100644 --- a/reme/steps/file_io/list.py +++ b/reme/steps/file_io/list.py @@ -19,8 +19,8 @@ from pathlib import Path from stat import S_ISREG from typing import Iterable +from ._base import _FileIOStep from ._path import resolve_path -from ..base_step import BaseStep from ...components import R # Default cap on returned items so huge workspaces don't blow up the response. @@ -28,17 +28,9 @@ DEFAULT_LIMIT = 100 @R.register("list_step") -class ListStep(BaseStep): +class ListStep(_FileIOStep): """Enumerate files under a directory in the workspace.""" - def _fail(self, message: str, **meta) -> None: - """Set a failed response (matches the read/edit/... fail envelope).""" - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _collect_params(self) -> tuple[str, bool, int, str, frozenset[str]]: """Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively.""" assert self.context is not None diff --git a/reme/steps/file_io/load.py b/reme/steps/file_io/load.py index 94bc23fc..4403c72e 100644 --- a/reme/steps/file_io/load.py +++ b/reme/steps/file_io/load.py @@ -2,24 +2,18 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import read_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R DEFAULT_EDITOR_MAX_BYTES = 5 * 1024 * 1024 @R.register("load_step") -class LoadStep(BaseStep): +class LoadStep(_FileIOStep): """Return complete text without the truncation used by agent-facing reads.""" - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/read_image.py b/reme/steps/file_io/read_image.py index 67342e0c..84666eec 100644 --- a/reme/steps/file_io/read_image.py +++ b/reme/steps/file_io/read_image.py @@ -5,14 +5,14 @@ import base64 import aiofiles import aiofiles.os +from ._base import _FileIOStep from ._path import NON_IMAGE_WARNING, gate_image, resolve_path -from ..base_step import BaseStep from ...components import R from ...constants import DEFAULT_MAX_IMAGE_BYTES @R.register("read_image_step") -class ReadImageStep(BaseStep): +class ReadImageStep(_FileIOStep): """Read an image file under ``workspace_dir`` and return base64 in ``answer``. Step-level attributes (``kwargs``, configured in yaml under ``steps:`` — @@ -32,13 +32,6 @@ class ReadImageStep(BaseStep): base64; ``metadata.non_image_warning=True`` flags it. """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/save.py b/reme/steps/file_io/save.py index 0b77cf7a..9ec9845f 100644 --- a/reme/steps/file_io/save.py +++ b/reme/steps/file_io/save.py @@ -2,14 +2,14 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("save_step") -class SaveStep(BaseStep): +class SaveStep(_FileIOStep): """Persist editor content without interpreting or rebuilding frontmatter. ``expected_mtime`` is the ISO timestamp returned by ``stat``. When supplied, @@ -17,12 +17,6 @@ class SaveStep(BaseStep): filesystem authoritative when ReMe and another editor are used together. """ - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/write.py b/reme/steps/file_io/write.py index 44d1425d..ff5fd081 100644 --- a/reme/steps/file_io/write.py +++ b/reme/steps/file_io/write.py @@ -2,14 +2,14 @@ import frontmatter +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("write_step") -class WriteStep(BaseStep): +class WriteStep(_FileIOStep): """Write (create or overwrite) a markdown file. Frontmatter accepts two reserved string fields (``name`` / ``description``) @@ -26,13 +26,6 @@ class WriteStep(BaseStep): scope). """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/index/_dedup.py b/reme/steps/index/_dedup.py index 96929e7b..230aebfb 100644 --- a/reme/steps/index/_dedup.py +++ b/reme/steps/index/_dedup.py @@ -12,15 +12,13 @@ When ``app_context`` is absent the same structure is mirrored under """ import datetime -from typing import TYPE_CHECKING, Any, Callable, Final +from typing import Callable, Final +from ._tool_context import _ToolContextMixin from ...schema import FileChunk -if TYPE_CHECKING: - from ...components import ApplicationContext - -class _ToolContextDedupMixin: +class _ToolContextDedupMixin(_ToolContextMixin): """Mixin providing tool_context-scoped chunk dedup with TTL. Must be mixed into a ``BaseStep`` subclass (e.g. ``SearchStep``); it @@ -29,7 +27,6 @@ class _ToolContextDedupMixin: set by the host step's ``__init__`` (default 24h). """ - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_RANGES_KEY: Final[str] = "search_seen_chunk_ranges" def __init_subclass__(cls, **kwargs): @@ -53,25 +50,7 @@ class _ToolContextDedupMixin: ) return super().__new__(cls, *args, **kwargs) - if TYPE_CHECKING: - # Declared by BaseStep; repeated here so static analysis resolves - # attribute access on the mixin without inheriting BaseStep. - app_context: "ApplicationContext | None" - kwargs: dict[str, Any] - seen_ttl_hours: float - - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context. - - The bucket is created lazily on first access and lives at - ``metadata["tool_contexts"][tool_context_id]`` (or the same path under - ``kwargs`` when no ``app_context`` is available, e.g. in unit tests). - """ - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) + seen_ttl_hours: float @staticmethod def _now_ts() -> float: diff --git a/reme/steps/index/_single_search.py b/reme/steps/index/_single_search.py new file mode 100644 index 00000000..06973f02 --- /dev/null +++ b/reme/steps/index/_single_search.py @@ -0,0 +1,68 @@ +"""Shared execution for single-backend search steps.""" + +from abc import abstractmethod +from typing import Final + +from ._dedup import _ToolContextDedupMixin +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries +from ._source_format import merge_session_chunk_intervals, render_chunk_entries +from ..base_step import BaseStep +from ...schema import FileChunk + +_MAX_CANDIDATES: Final = 200 +_CANDIDATE_MULTIPLIER: Final = 10 + + +class _SingleSearchStep(_ToolContextDedupMixin, BaseStep): + """Retrieve candidates, then apply the common single-backend search contract.""" + + def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.seen_ttl_hours = seen_ttl_hours + self.include_source = include_source + + async def execute(self): + assert self.context is not None + raw_query = self.context.get("query") + query: str = "" if raw_query is None else str(raw_query).strip() + limit: int = int(self.context.get("limit") or 5) + min_score: float = float(self.context.get("min_score") or 0.0) + tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() + + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response + assert limit > 0, f"limit must be positive, got {limit}" + + candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) + results = await self._retrieve(query, candidates) + self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") + + if min_score > 0.0: + results = [chunk for chunk in results if chunk.score >= min_score] + + pre_dedup_count = 0 + if tool_context_id: + pre_dedup_count = len(results) + results, _ = self._dedupe_tool_context(results, tool_context_id, limit) + else: + results = results[:limit] + + session_dir = self.config_value("session_dir") + entries = render_chunk_entries( + merge_session_chunk_intervals(results, session_dir), + session_dir, + include_source=self.include_source, + ) + self.context.response.answer = join_chunk_entries(entries) + if not results: + self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE + self.context.response.metadata["results"] = [ + c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results + ] + return self.context.response + + @abstractmethod + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + """Fetch ranked candidates from the concrete backend.""" diff --git a/reme/steps/index/_tool_context.py b/reme/steps/index/_tool_context.py new file mode 100644 index 00000000..4b14a6d5 --- /dev/null +++ b/reme/steps/index/_tool_context.py @@ -0,0 +1,23 @@ +"""Shared access to application-lifetime tool context state.""" + +from typing import TYPE_CHECKING, Any, Final + +if TYPE_CHECKING: + from ...components import ApplicationContext + + +class _ToolContextMixin: + """Share storage access while leaving each step's state format independent.""" + + TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" + + if TYPE_CHECKING: + app_context: "ApplicationContext | None" + kwargs: dict[str, Any] + + def _tool_context_store(self, tool_context_id: str) -> dict: + if self.app_context is not None: + contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) + else: + contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) + return contexts.setdefault(tool_context_id, {}) diff --git a/reme/steps/index/bm25_search.py b/reme/steps/index/bm25_search.py index 99ca8a4d..59bd2460 100644 --- a/reme/steps/index/bm25_search.py +++ b/reme/steps/index/bm25_search.py @@ -1,64 +1,13 @@ -"""``bm25_search_step`` — plain BM25 keyword search with tool_context dedup.""" +"""Plain BM25 keyword search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("bm25_search_step") -class Bm25SearchStep(_ToolContextDedupMixin, BaseStep): - """BM25-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class Bm25SearchStep(_SingleSearchStep): + """Search the BM25 keyword backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.keyword_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.keyword_search(query, candidates, {}) diff --git a/reme/steps/index/draft.py b/reme/steps/index/draft.py index a33d3db2..5684e329 100644 --- a/reme/steps/index/draft.py +++ b/reme/steps/index/draft.py @@ -2,24 +2,17 @@ from typing import Final +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ...components import R @R.register("add_draft_step") -class AddDraftStep(BaseStep): +class AddDraftStep(_ToolContextMixin, BaseStep): """Append one draft text item to the current tool context.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" DRAFTS_KEY: Final[str] = "drafts" - def _tool_context_store(self, tool_context_id: str) -> dict: - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - async def execute(self): assert self.context is not None text = self.context.get("text", "") diff --git a/reme/steps/index/search.py b/reme/steps/index/search.py index 928587f5..77ff4a4c 100644 --- a/reme/steps/index/search.py +++ b/reme/steps/index/search.py @@ -5,6 +5,7 @@ import datetime import os from typing import Final +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ..file_io import extract_daily_date from ...components import R @@ -35,10 +36,9 @@ def _filter_values(value: object) -> set: @R.register("search_step") -class SearchStep(BaseStep): +class SearchStep(_ToolContextMixin, BaseStep): """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_KEY: Final[str] = "search_seen_chunk_ids" def __init__( @@ -98,14 +98,6 @@ class SearchStep(BaseStep): def _now_ts() -> float: return datetime.datetime.now().timestamp() - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context.""" - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - def _dedupe_tool_context( self, chunks: list[FileChunk], diff --git a/reme/steps/index/vector_search.py b/reme/steps/index/vector_search.py index 5de5aa94..858db2bb 100644 --- a/reme/steps/index/vector_search.py +++ b/reme/steps/index/vector_search.py @@ -1,64 +1,13 @@ -"""``vector_search_step`` — plain vector search with tool_context dedup.""" +"""Plain vector search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("vector_search_step") -class VectorSearchStep(_ToolContextDedupMixin, BaseStep): - """Vector-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class VectorSearchStep(_SingleSearchStep): + """Search the vector backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.vector_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.vector_search(query, candidates, {}) diff --git a/reme/utils/tushare.py b/reme/utils/tushare.py deleted file mode 100644 index 68218151..00000000 --- a/reme/utils/tushare.py +++ /dev/null @@ -1,17 +0,0 @@ -"""TuShare client construction with an optional mirror endpoint.""" - -import os - - -def create_tushare_api(token: str): - """Create a TuShare DataApi, using TUSHARE_MIRROR_URL when configured.""" - try: - import tushare as ts - except ImportError as exc: # pragma: no cover - optional core dependency. - raise RuntimeError("tushare is required for market-data research") from exc - - api = ts.pro_api(token) - api._DataApi__timeout = 600 # pylint: disable=protected-access - if mirror_url := os.getenv("TUSHARE_MIRROR_URL", "").strip(): - api._DataApi__http_url = mirror_url.rstrip("/") # pylint: disable=protected-access - return api diff --git a/reme_studio/README.md b/reme_studio/README.md index 8667cd91..2cbeea61 100644 --- a/reme_studio/README.md +++ b/reme_studio/README.md @@ -402,8 +402,10 @@ Run proportionate checks from `reme_studio/`: ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# Build and verify the Sites, static, and demo distributions: +npm run test:build +# Run both suites: npm test ``` diff --git a/reme_studio/README_ZH.md b/reme_studio/README_ZH.md index f3a11d59..3351d3a9 100644 --- a/reme_studio/README_ZH.md +++ b/reme_studio/README_ZH.md @@ -357,8 +357,10 @@ npm run build:demo ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# 构建并验证 Sites、静态和演示分发： +npm run test:build +# 运行以上全部检查： npm test ``` diff --git a/reme_studio/app/chatgpt-auth.ts b/reme_studio/app/chatgpt-auth.ts deleted file mode 100644 index a0ae2ed9..00000000 --- a/reme_studio/app/chatgpt-auth.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { headers } from "next/headers"; -import { redirect } from "next/navigation"; - -export type ChatGPTUser = { - userId: string; - displayName: string; - email: string; - fullName: string | null; -}; - -const USER_ID_HEADER = "oai-authenticated-user-id"; -const USER_EMAIL_HEADER = "oai-authenticated-user-email"; -const USER_FULL_NAME_HEADER = "oai-authenticated-user-full-name"; -const USER_FULL_NAME_ENCODING_HEADER = - "oai-authenticated-user-full-name-encoding"; -const PERCENT_ENCODED_UTF8 = "percent-encoded-utf-8"; -const SIGN_IN_PATH = "/signin-with-chatgpt"; -const SIGN_OUT_PATH = "/signout-with-chatgpt"; -const CALLBACK_PATH = "/callback"; - -export async function getChatGPTUser(): Promise { - const requestHeaders = await headers(); - const userId = requestHeaders.get(USER_ID_HEADER); - const email = requestHeaders.get(USER_EMAIL_HEADER); - if (!userId || !email) return null; - - const encodedFullName = requestHeaders.get(USER_FULL_NAME_HEADER); - const fullName = - encodedFullName && - requestHeaders.get(USER_FULL_NAME_ENCODING_HEADER) === PERCENT_ENCODED_UTF8 - ? safeDecodeURIComponent(encodedFullName) - : null; - - return { - userId, - displayName: fullName ?? email, - email, - fullName, - }; -} - -export async function requireChatGPTUser( - returnTo: string, -): Promise { - const user = await getChatGPTUser(); - if (user) return user; - - redirect(chatGPTSignInPath(returnTo)); -} - -export function chatGPTSignInPath(returnTo: string): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_IN_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -export function chatGPTSignOutPath(returnTo = "/"): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_OUT_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -function safeRelativeReturnPath(value: string): string { - if (!value.startsWith("/") || value.startsWith("//")) return "/"; - - let url: URL; - try { - url = new URL(value, "https://app.local"); - } catch { - return "/"; - } - if (url.origin !== "https://app.local") return "/"; - if (isReservedAuthPath(url.pathname)) return "/"; - - return `${url.pathname}${url.search}${url.hash}`; -} - -function isReservedAuthPath(pathname: string): boolean { - return ( - pathname === SIGN_IN_PATH || - pathname === SIGN_OUT_PATH || - pathname === CALLBACK_PATH - ); -} - -function safeDecodeURIComponent(value: string): string | null { - try { - return decodeURIComponent(value); - } catch { - return null; - } -} diff --git a/reme_studio/app/globals.css b/reme_studio/app/globals.css index 2fc41164..2621b818 100644 --- a/reme_studio/app/globals.css +++ b/reme_studio/app/globals.css @@ -1610,27 +1610,6 @@ textarea:focus-visible { margin: 8px 0 10px; font-size: 12px; } -.component-row { - min-height: 48px; - display: flex; - align-items: center; - justify-content: space-between; - gap: 16px; - border-top: 1px solid var(--line); -} -.component-row > span { - display: flex; - flex-direction: column; - font-size: 12px; -} -.component-row small { - margin-top: 2px; - color: var(--muted); - font-size: 10px; -} -.component-row strong { - font-size: 12px; -} .index-card { padding: 22px; } diff --git a/reme_studio/app/open-file.ts b/reme_studio/app/open-file.ts index 41b965c0..9fe34dab 100644 --- a/reme_studio/app/open-file.ts +++ b/reme_studio/app/open-file.ts @@ -9,13 +9,24 @@ export async function openWorkspaceFile(path: string) { ); const id = store.openMarkdown(path); if (existing) return; + await loadWorkspaceFile(id, path); +} + +/** Load both newly opened and restored tabs, respecting the caller's lifetime. */ +export async function loadWorkspaceFile( + id: string, + path: string, + shouldApply: () => boolean = () => true, +) { + const store = useWorkspaceStore.getState(); try { const file = await readWorkspaceFile(path); - store.hydrateMarkdown(id, file.content, file.stat.mtime); + if (shouldApply()) store.hydrateMarkdown(id, file.content, file.stat.mtime); } catch (error) { - store.failMarkdown( - id, - error instanceof Error ? error.message : "Failed to read file", - ); + if (shouldApply()) + store.failMarkdown( + id, + error instanceof Error ? error.message : "Failed to read file", + ); } } diff --git a/reme_studio/app/store.ts b/reme_studio/app/store.ts index 5f2fb64f..98ec968d 100644 --- a/reme_studio/app/store.ts +++ b/reme_studio/app/store.ts @@ -54,6 +54,19 @@ interface WorkspaceState { finishChat: (tabId: string, messageId: string, error?: string) => void; } +function updateChatMessage( + tab: Extract, + messageId: string, + update: (message: ChatMessage) => ChatMessage, +) { + return { + ...tab, + messages: tab.messages.map((message) => + message.id === messageId ? update(message) : message, + ), + }; +} + const fileTitle = (path: string) => path.split("/").pop() || path; // Preserve the original key so existing tabs, chats, and unsaved drafts survive the Studio rename. const WORKSPACE_STORAGE_KEY = IS_DEMO @@ -62,189 +75,153 @@ const WORKSPACE_STORAGE_KEY = IS_DEMO export const useWorkspaceStore = create()( persist( - (set, get) => ({ - tabs: [], - activeTabId: undefined, - openMarkdown: (path) => { - const existing = get().tabs.find( - (tab) => tab.type === "markdown" && tab.path === path, - ); - if (existing) { - set({ activeTabId: existing.id }); - return existing.id; - } - const id = `file:${path}`; + (set, get) => { + const updateTab = ( + id: string, + type: Type, + update: (tab: Extract) => WorkspaceTab, + ) => set((state) => ({ - tabs: [ - ...state.tabs, - { - id, - type: "markdown", - title: fileTitle(path), - path, - content: "", - savedContent: "", - loading: true, - }, - ], - activeTabId: id, + tabs: state.tabs.map((tab) => + tab.id === id && tab.type === type + ? update(tab as Extract) + : tab, + ), })); - return id; - }, - openAgent: () => { - const id = `agent:${crypto.randomUUID()}`; - set((state) => ({ - tabs: [...state.tabs, { id, type: "agent", title: "", messages: [] }], - activeTabId: id, - })); - return id; - }, - openGraph: (root) => { - const id = `graph:${root}`; - const existing = get().tabs.find( - (tab) => tab.id === id && tab.type === "graph", - ); - if (existing) { - set({ activeTabId: id }); + return { + tabs: [], + activeTabId: undefined, + openMarkdown: (path) => { + const existing = get().tabs.find( + (tab) => tab.type === "markdown" && tab.path === path, + ); + if (existing) { + set({ activeTabId: existing.id }); + return existing.id; + } + const id = `file:${path}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { + id, + type: "markdown", + title: fileTitle(path), + path, + content: "", + savedContent: "", + loading: true, + }, + ], + activeTabId: id, + })); return id; - } - set((state) => ({ - tabs: [ - ...state.tabs, - { id, type: "graph", title: `${root} graph`, root }, - ], - activeTabId: id, - })); - return id; - }, - closeTab: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, false).length - ) - return state; - const index = state.tabs.findIndex((tab) => tab.id === id); - const tabs = state.tabs.filter((tab) => tab.id !== id); - const activeTabId = - state.activeTabId === id - ? tabs[Math.max(0, index - 1)]?.id - : state.activeTabId; - return { tabs, activeTabId }; - }), - closeOtherTabs: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, true).length - ) - return state; - const tab = state.tabs.find((item) => item.id === id); - return tab ? { tabs: [tab], activeTabId: id } : state; - }), - setActiveTab: (activeTabId) => set({ activeTabId }), - hydrateMarkdown: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { - ...tab, - content, - savedContent: content, - mtime, - loading: false, - error: undefined, - } - : tab, - ), - })), - failMarkdown: (id, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, loading: false, error } - : tab, - ), - })), - updateMarkdown: (id, content) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, content } - : tab, - ), - })), - markSaved: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? markMarkdownContentSaved(tab, content, mtime) - : tab, - ), - })), - addChatTurn: (tabId, user, assistant) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - title: tab.messages.length - ? tab.title - : user.content.slice(0, 18), - messages: [...tab.messages, user, assistant], - streaming: true, - } - : tab, - ), - })), - applyChatChunk: (tabId, messageId, chunk) => - set((state) => ({ - tabs: state.tabs.map((tab) => { - if (tab.id !== tabId || tab.type !== "agent") return tab; - const messages = tab.messages.map((message) => - message.id === messageId - ? applyStreamChunk(message, chunk) - : message, - ); - return { - ...tab, - sessionId: chunk.session_id || tab.sessionId, - messages, - }; + }, + openAgent: () => { + const id = `agent:${crypto.randomUUID()}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "agent", title: "", messages: [] }, + ], + activeTabId: id, + })); + return id; + }, + openGraph: (root) => { + const id = `graph:${root}`; + const existing = get().tabs.find( + (tab) => tab.id === id && tab.type === "graph", + ); + if (existing) { + set({ activeTabId: id }); + return id; + } + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "graph", title: `${root} graph`, root }, + ], + activeTabId: id, + })); + return id; + }, + closeTab: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, false).length + ) + return state; + const index = state.tabs.findIndex((tab) => tab.id === id); + const tabs = state.tabs.filter((tab) => tab.id !== id); + const activeTabId = + state.activeTabId === id + ? tabs[Math.max(0, index - 1)]?.id + : state.activeTabId; + return { tabs, activeTabId }; }), - })), - toggleChatBlock: (tabId, messageId, blockId, expanded) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - messages: tab.messages.map((message) => - message.id === messageId - ? toggleChatBlock(message, blockId, expanded) - : message, - ), - } - : tab, + closeOtherTabs: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, true).length + ) + return state; + const tab = state.tabs.find((item) => item.id === id); + return tab ? { tabs: [tab], activeTabId: id } : state; + }), + setActiveTab: (activeTabId) => set({ activeTabId }), + hydrateMarkdown: (id, content, mtime) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + content, + savedContent: content, + mtime, + loading: false, + error: undefined, + })), + failMarkdown: (id, error) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + loading: false, + error, + })), + updateMarkdown: (id, content) => + updateTab(id, "markdown", (tab) => ({ ...tab, content })), + markSaved: (id, content, mtime) => + updateTab(id, "markdown", (tab) => + markMarkdownContentSaved(tab, content, mtime), ), - })), - finishChat: (tabId, messageId, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - streaming: false, - messages: tab.messages.map((message) => - message.id === messageId - ? finishChatMessage(message, error) - : message, - ), - } - : tab, + addChatTurn: (tabId, user, assistant) => + updateTab(tabId, "agent", (tab) => ({ + ...tab, + title: tab.messages.length ? tab.title : user.content.slice(0, 18), + messages: [...tab.messages, user, assistant], + streaming: true, + })), + applyChatChunk: (tabId, messageId, chunk) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + applyStreamChunk(message, chunk), + ), + sessionId: chunk.session_id || tab.sessionId, + })), + toggleChatBlock: (tabId, messageId, blockId, expanded) => + updateTab(tabId, "agent", (tab) => + updateChatMessage(tab, messageId, (message) => + toggleChatBlock(message, blockId, expanded), + ), ), - })), - }), + finishChat: (tabId, messageId, error) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + finishChatMessage(message, error), + ), + streaming: false, + })), + }; + }, { name: WORKSPACE_STORAGE_KEY, version: 1, diff --git a/reme_studio/app/workspace-chat.tsx b/reme_studio/app/workspace-chat.tsx new file mode 100644 index 00000000..c1b8d349 --- /dev/null +++ b/reme_studio/app/workspace-chat.tsx @@ -0,0 +1,296 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { Check, CircleAlert, LoaderCircle, Send } from "lucide-react"; +import WorkspaceMarkdown from "./files-workspace/WorkspaceMarkdown"; +import { streamChat } from "./api"; +import { chatStreamError, formatStreamPayloads } from "./chat-stream"; +import { useI18n } from "./i18n"; +import { useWorkspaceStore } from "./store"; +import { IS_DEMO } from "./studio-mode"; +import type { + ChatBlock, + ChatMessage, + DetailBlock, + WorkspaceTab, +} from "./types"; +import { + appendWorkspaceFileReference, + WORKSPACE_FILE_DRAG_TYPE, +} from "./workspace-drag"; +import { scenarios } from "../demo/scenarios"; + +function MarkdownView({ content }: { content: string }) { + return ( +
+ +
+ ); +} + +function ChunkStatus({ + active, + error = false, +}: { + active: boolean; + error?: boolean; +}) { + if (error) return ; + return active ? ( + + ) : ( + + ); +} + +function detailLabel( + block: DetailBlock, + t: ReturnType["t"], +): string { + if (block.type === "think") return t("thinking"); + if (block.type === "data") + return block.mediaType + ? `${t("dataChunk")} · ${block.mediaType}` + : t("dataChunk"); + if (block.type === "approval") return t("approvalChunk"); + if (block.type === "usage") return t("usageChunk"); + return t("unknownChunk", { type: block.sourceType }); +} + +function ChatBlockView({ + block, + onToggle, +}: { + block: ChatBlock; + onToggle: (blockId: string, expanded: boolean) => void; +}) { + const { t } = useI18n(); + if (block.type === "content") return ; + if (block.type === "error") + return ( +
+ + {block.text} +
+ ); + if (block.type === "usage") return null; + + const active = + block.status === "streaming" || + block.status === "calling" || + block.status === "running"; + const failed = block.status === "error"; + if (block.type === "tool") { + const call = formatStreamPayloads(block.callPayloads); + const result = formatStreamPayloads(block.resultPayloads); + return ( +
{ + if (event.currentTarget.open !== block.expanded) + onToggle(block.id, event.currentTarget.open); + }} + > + + + {block.name} + {active ? t("streaming") : t("completed")} + +
+ {call && ( +
+ {t("toolCall")} + 
{call}
+
+ )} + {result && ( +
+ {t("toolResult")} + 
{result}
+
+ )} +
+
+ ); + } + + const detail = formatStreamPayloads(block.payloads); + return ( +
{ + if (event.currentTarget.open !== block.expanded) + onToggle(block.id, event.currentTarget.open); + }} + > + + + {detailLabel(block, t)} + {active ? t("streaming") : t("completed")} + + {detail && ( +
+ 
{detail}
+
+ )} +
+ ); +} + +export default function Chat({ + tab, +}: { + tab: Extract; +}) { + const { language, t } = useI18n(); + const [input, setInput] = useState(""); + const [fileDragOver, setFileDragOver] = useState(false); + const endRef = useRef(null); + const controller = useRef(null); + const { addChatTurn, applyChatChunk, toggleChatBlock, finishChat } = + useWorkspaceStore(); + + useEffect(() => { + endRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [tab.messages]); + useEffect(() => () => controller.current?.abort(), []); + + const suggestions = ( +
+ {(IS_DEMO + ? scenarios[language].map((scenario) => scenario.question) + : [t("promptRecent"), t("promptTasks"), t("promptIdeas")] + ).map((question) => ( + + ))} +
+ ); + + const send = async () => { + const query = input.trim(); + if (!query || tab.streaming) return; + setInput(""); + const user: ChatMessage = { + id: crypto.randomUUID(), + role: "user", + content: query, + }; + const assistant: ChatMessage = { + id: crypto.randomUUID(), + role: "assistant", + content: "", + blocks: [], + }; + addChatTurn(tab.id, user, assistant); + const requestController = new AbortController(); + controller.current = requestController; + const error = await chatStreamError( + () => + streamChat(query, tab.sessionId, requestController.signal, (chunk) => + applyChatChunk(tab.id, assistant.id, chunk), + ), + requestController.signal, + t("chatFailed"), + ); + finishChat(tab.id, assistant.id, error); + if (controller.current === requestController) controller.current = null; + }; + + return ( +
+
+ {!tab.messages.length && ( +
+ + )} + {tab.messages.map((message) => ( +
+
+ {message.role === "user" ? ( + t("you") + ) : ( + + )} +
+
+ {message.role === "user" &&

{message.content}

} + {message.role === "assistant" && + message.blocks?.map((block) => ( + + toggleChatBlock(tab.id, message.id, blockId, expanded) + } + /> + ))} + {message.role === "assistant" && + tab.streaming && + !message.blocks?.some((block) => block.type !== "usage") && ( + + )} +
+
+ ))} + {IS_DEMO && tab.messages.length > 0 && suggestions} +
+
+
{ + if (!event.dataTransfer.types.includes(WORKSPACE_FILE_DRAG_TYPE)) + return; + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + setFileDragOver(true); + }} + onDragLeave={(event) => { + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) + setFileDragOver(false); + }} + onDrop={(event) => { + const path = event.dataTransfer.getData(WORKSPACE_FILE_DRAG_TYPE); + if (!path) return; + event.preventDefault(); + setFileDragOver(false); + setInput((current) => appendWorkspaceFileReference(current, path)); + }} + > +
+