diff --git a/.github/workflows/_build-docs.yml b/.github/workflows/_build-docs.yml index 1b3eb8f5..d2b8a577 100644 --- a/.github/workflows/_build-docs.yml +++ b/.github/workflows/_build-docs.yml @@ -67,7 +67,7 @@ jobs: - name: Verify browser demo bundle if: inputs.run_tests working-directory: reme_studio - run: node --test tests/demo-build.test.mjs + run: node --test tests/distribution/demo-build.test.mjs - name: Configure Pages if: inputs.upload_pages_artifact diff --git a/.github/workflows/ci-reme-studio.yml b/.github/workflows/ci-reme-studio.yml index fa8f5d5f..eb9ccf3f 100644 --- a/.github/workflows/ci-reme-studio.yml +++ b/.github/workflows/ci-reme-studio.yml @@ -66,7 +66,7 @@ jobs: - name: Verify npm package run: | - npm pack --pack-destination "${RUNNER_TEMP}" + npm pack --ignore-scripts --pack-destination "${RUNNER_TEMP}" tar -tzf "${RUNNER_TEMP}"/agentscope-ai-reme_studio-*.tgz | grep '^package/dist-static/index.html$' - name: Set up Python diff --git a/docs/en/auto_memory.md b/docs/en/auto_memory.md index 02064837..e9c4597f 100644 --- a/docs/en/auto_memory.md +++ b/docs/en/auto_memory.md @@ -63,6 +63,11 @@ retarget inbound wikilinks. To see what happened on a day, start with `YYYY-MM-D ## Preserving the Original Information +For retrying clients, `auto_memory` accepts an optional `request_id` of 1–128 characters. Keep the ID and input unchanged +when retrying a batch: completed results are replayed without another memory-agent call; reuse with different input is +rejected. Receipts live under the configured `metadata_dir/auto_memory_requests`. A process interruption before receipt +commit may repeat an unfinished operation; arbitrary Agent file edits are not transactional. + The distilled daily note is optimized for readability; a filtered source conversation record is retained for trust and verification. diff --git a/docs/en/blog_20261009.md b/docs/en/blog_20261009.md index aaec0870..86f41bf4 100644 --- a/docs/en/blog_20261009.md +++ b/docs/en/blog_20261009.md @@ -79,6 +79,18 @@ Image resources then participate in the existing retrieval workflow. A later sea Auto Resource supports PNG, JPG/JPEG, WebP, GIF, BMP, TIFF, and HEIC. Larger images may be resized or converted for the model request; the original resource files remain unchanged. HEIC requires the optional `reme-ai[image-heif]` package. +## Performance: Multimodal Long-History QA on MemLens-256K {#memlens-256k-results} + +On MemLens-256K, ReMe Image achieved LLM-Judge scores of **53.44%** with Qwen3.6-35B-A3B and **72.49%** with Qwen3.8-Max, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +![MemLens-256K LLM-Judge with Qwen3.6-35B-A3B: Full Context Image 47.09%, ReMe Text 41.27%, ReMe Image 53.44%, OpenViking 37.04%, Mem0 38.62%.](../figure/memlens-256k-qwen36-judge.svg) + +![MemLens-256K LLM-Judge with Qwen3.8-Max: Full Context Image 65.61%, ReMe Text 52.91%, ReMe Image 72.49%, OpenViking 42.33%, Mem0 51.32%.](../figure/memlens-256k-qwen38-judge.svg) + +The comparison uses the official LLM-Judge protocol to evaluate each complete system. Six cases triggered model-provider content moderation in at least one method, so all methods use the same filtered set of **189/195** cases. + +ReMe can retrieve original conversations, and Image can reopen source images; OpenViking retrieval in this evaluation is restricted to memory. See the [evaluation notes](./memlens_256k_evaluation.md) for experimental settings, exclusions and the reproduction comparison. + ## Getting Started Both paths require a vision model that supports image input and tool calling, along with a compatible AgentScope wrapper and formatter. The selected wrapper's `as_llm` binding determines which model is used. See [Configuration](./configuration.md) for credential setup. diff --git a/docs/en/context_compaction.md b/docs/en/context_compaction.md index 17ca97b3..c6677668 100644 --- a/docs/en/context_compaction.md +++ b/docs/en/context_compaction.md @@ -159,3 +159,20 @@ the entire gain cannot be attributed to Jev ranking itself. Agent continuation s [Complete experiment: design, results, expected answers and reproduction](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README.md) + +## Host-owned decision models + +Embedded hosts can reuse their existing AgentScope JEV instance, just as they inject an LLM: + +```python +# app contains an as_decision_model.compaction component with backend: agentscope. +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +The YAML `model` value remains the provider model name. The runtime `model` property accepts a callable instance +and does not construct a client on access. Inject before the first request; ReMe validates and converts JSON +questions normally, but does not require separate ReMe credentials or close the borrowed client. The host must +keep the instance alive until ReMe has finished its calls, then close it itself. Close the component before +replacing an internally constructed model or any model with active requests. Shutdown waits for active calls and +owned-client cleanup even when cancelled, then propagates cancellation. Component shutdown detaches the instance; +inject again before restarting a borrowed-model component. Standalone credential-based behavior is unchanged. diff --git a/docs/en/memlens_256k_evaluation.md b/docs/en/memlens_256k_evaluation.md new file mode 100644 index 00000000..938a75f3 --- /dev/null +++ b/docs/en/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K Evaluation Notes + +This report describes the experimental settings, data selection and results of the MemLens-256K evaluation in the [multimodal memory article](./blog_20261009.md#memlens-256k-results). + +## Results and Evaluation Set {#results} + +The evaluation uses the 195 questions in the official [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) `agent_subset_195.json`, each with its complete 256K history. After excluding the six questions listed below, all methods are evaluated on the same 189 questions. The official category/subtype-specific LLM-Judge protocol assigns each answer a score of 0 or 1; the reported percentage is `100 × correct / 189`. + +| Model | Method | Judge correct / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image scores **53.44%** and **72.49%** on the two models, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +## Method and Judge Settings {#evaluation-settings} + +| Method | Memory input and answer-time data access | +| --- | --- | +| Full Context Image | The answer model receives the complete conversation text and original images directly. | +| ReMe Text | Builds memory from the original conversation text. The answer Agent uses BM25 to search daily notes, digest notes and original sessions, then reads relevant text. | +| ReMe Image | Builds memory from the original conversation text and images. Retrieval covers the same sources as Text, with `read_image` available to reopen saved original images. | +| OpenViking | Uses native image descriptions and memory processing. Answer-time retrieval is restricted to memory with the default top 10. | +| Mem0 | Builds memory through Mem0 2.0.4's native vision interface. Answer-time retrieval uses FAISS to select the top 20 memories. | + +Each method is evaluated with Qwen3.6-35B-A3B and Qwen3.8-Max as the candidate models. ReMe Image and Mem0 both use original images. ReMe's memory and answer context limits are 200,000 and 400,000 tokens, with temperature 0 and thinking enabled. Memory/compression and answer output budgets are 65,536 and 16,384 tokens. Each question uses an independent answer Agent with at most 30 iterations and native context compression. The `read_image` adapter accesses saved attachments with a 50 MiB read limit and a 4,096-image retention limit. + +Full Context, OpenViking and Mem0 also use temperature 0, thinking enabled and a 16,384-token output budget for answering. Mem0's memory stage uses its native settings: a 2,000-token output budget, temperature 0.1 and top_p 0.1. The OpenViking evaluation adapter includes a JSON parsing correction and explicit error reporting for image-processing failures. + +Scoring follows the [official MemLens protocol](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2). All methods use **Qwen3.8-Max** as Judge, with temperature **0**, thinking **disabled** and a **256-token** output budget. The Judge runs independently of candidate memory applications; reference answers and question-type labels are used only for scoring. + +## Common Content-Moderation Filter {#content-moderation-filter} + +The method/question combinations below triggered content moderation when calling the Qwen model service. Taking the union of affected questions across all methods and both models gives six questions. We exclude this same set from every method and model, leaving **189/195** questions for evaluation. + +In the table, A3B denotes Qwen3.6-35B-A3B and Max denotes Qwen3.8-Max. + +| Question ID | Method / model that triggered moderation | Stage | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | Memory construction | +| `q_1d99a36c` | OpenViking / A3B | Memory construction | +| `q_3b01c02a` | ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B | Memory construction | +| `q_564a657b` | ReMe Image / Max | Answering | +| `q_600a79aa` | ReMe Image / Max | Memory construction | +| `q_c9f8e24d` | ReMe Text / Max | Answering | + +## Independent Reproduction Results {#independent-reproduction} + +We organized and integrated the MemLens benchmark code into ReMe, then independently reran ReMe Image using the same data inputs and scoring settings. See the [run instructions](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md), [method configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml), [Judge configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml) and [Judge scoring rules](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py) for the implementation and settings. + +The corresponding public evaluation code is available in [PR #640](https://github.com/agentscope-ai/ReMe/pull/640) at commit [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8). + +After applying the same six-question filter, one A3B question and two Max questions remain incomplete because conversation-history processing failed. The table compares the 188 and 187 questions completed in both experiments, respectively. + +| Model | Paired questions | Original ReMe Image correct | Independent reproduction correct | Original → reproduction LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +On these matched questions, the reproduced LLM-Judge scores for Qwen3.6-35B-A3B and Qwen3.8-Max are 2.13 and 0.53 percentage points lower than the original results, respectively. Overall, performance remains broadly consistent with the original experiment. diff --git a/docs/en/memory_search.md b/docs/en/memory_search.md index 9c63a725..c2a72d6b 100644 --- a/docs/en/memory_search.md +++ b/docs/en/memory_search.md @@ -276,6 +276,23 @@ connects to. If a result is worth pursuing, use `read path=...` to open the sour followed by outlinks and inlinks. - `response.metadata`: structured programmatic results containing `results`, `link_expansion`, and `counts`. +When the default `search` Job returns no results, `response.answer` remains `""` and `success` remains `true`. +The `results` list is empty and `counts.returned` is `0`. An optional `response.metadata.empty_result` explains why: + +| `reason` | Meaning | +|----------------|-----------------------------------------------------------------------------| +| `no_results` | No candidates matched or survived `min_score`. | +| `all_returned` | All eligible chunks were already returned within the same `tool_context_id`. | + +The field also contains a human-readable `message`. Programmatic consumers should use `reason` to distinguish the +outcomes. Normal hits omit `empty_result` and keep their existing answer format. + +To show the explanation through the running service's CLI client, enable the existing metadata display option: + +```bash +reme search query="recent discussions about indexing" show_metadata=true +``` + Typical text structure: ```text diff --git a/docs/en/reme-blog.md b/docs/en/reme-blog.md index 1e7f3cf1..6267b7e3 100644 --- a/docs/en/reme-blog.md +++ b/docs/en/reme-blog.md @@ -276,6 +276,8 @@ BEAM covers ten categories of tasks, including contradiction resolution, event o ReMe also uses $\pi$-Bench to evaluate the potential of multi-session reasoning to improve agent proactivity. The PROC score in $\pi$-Bench evaluates capabilities including directly fulfilling hidden intent, guiding targeted clarification, recovering cross-session preferences, reusing cross-session conventions, inferring cross-task dependencies, and advancing underspecified requests. Across five user personas, ReMe Agent achieved an average PROC score of 0.580, outperforming NanoBot by 2.4% under the same test-model configuration. See the [$\pi$-Bench paper](https://arxiv.org/abs/2605.14678) for details about the benchmark. +On **MemLens-256K**, ReMe Image achieved LLM-Judge scores of **53.44% / 72.49%** with Qwen3.6-35B-A3B / Qwen3.8-Max, exceeding Full Context Image by **6.35 / 6.88** percentage points. All methods use the same **189/195** cases after filtering six cases that triggered model-provider content moderation. See the [multimodal memory evaluation](./blog_20261009.md#memlens-256k-results) for the five-method comparison. + ## Who Is ReMe For? ### People Who Use Agents Directly diff --git a/docs/figure/memlens-256k-qwen36-judge.svg b/docs/figure/memlens-256k-qwen36-judge.svg new file mode 100644 index 00000000..75a02882 --- /dev/null +++ b/docs/figure/memlens-256k-qwen36-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.6-35B-A3B + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 47.09%; ReMe Text: 41.27%; ReMe Image: 53.44%; OpenViking: 37.04%; Mem0: 38.62%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.6-35B-A3B + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 47.09% + Full ContextImage + + + + 41.27% + ReMe Text + + + + 53.44% + ReMe Image + + + + 37.04% + OpenViking + + + + 38.62% + Mem0 + + diff --git a/docs/figure/memlens-256k-qwen38-judge.svg b/docs/figure/memlens-256k-qwen38-judge.svg new file mode 100644 index 00000000..3b5a0800 --- /dev/null +++ b/docs/figure/memlens-256k-qwen38-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.8-Max + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 65.61%; ReMe Text: 52.91%; ReMe Image: 72.49%; OpenViking: 42.33%; Mem0: 51.32%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.8-Max + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 65.61% + Full ContextImage + + + + 52.91% + ReMe Text + + + + 72.49% + ReMe Image + + + + 42.33% + OpenViking + + + + 51.32% + Mem0 + + diff --git a/docs/zh/auto_memory.md b/docs/zh/auto_memory.md index 7e946f15..aa16cf01 100644 --- a/docs/zh/auto_memory.md +++ b/docs/zh/auto_memory.md @@ -57,6 +57,10 @@ source_conversation: "[[session/dialog/session-a.jsonl]]" ## 同时保存原始信息 +需要重试的客户端可为 `auto_memory` 传入 1–128 字符的可选 `request_id`。重试同一批次时保持 ID 与输入一致： +已完成请求直接重放结果，不再调用记忆 Agent；同一个 ID 对应不同输入会报错。回执保存在配置的 +`metadata_dir/auto_memory_requests` 下。回执提交前发生进程中断，未完成的操作仍可能再次执行；任意 Agent 文件修改不具备事务性。 + 整理后的 daily note 负责“好读”，过滤后的对话来源记录负责“可信”。 Auto Memory 在生成记忆卡片的同时，也会保存对话来源消息： diff --git a/docs/zh/blog_20261009.md b/docs/zh/blog_20261009.md index 61b5a82d..49438192 100644 --- a/docs/zh/blog_20261009.md +++ b/docs/zh/blog_20261009.md @@ -78,6 +78,18 @@ source_resource: "[[resource/2026-10-09/system-overview.png]]" Auto Resource 支持 PNG、JPG/JPEG、WebP、GIF、BMP、TIFF 和 HEIC。较大的图片会在模型请求中按需缩小或转换格式，原始资源文件保持不变。HEIC 需要额外安装 `reme-ai[image-heif]`。 +## Performance：MemLens-256K 多模态长历史问答 {#memlens-256k-results} + +在 MemLens-256K 上，ReMe Image 的 LLM-Judge 得分在 Qwen3.6-35B-A3B 和 Qwen3.8-Max 下分别达到 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +![Qwen3.6-35B-A3B 的 MemLens-256K LLM-Judge 对比：Full Context Image 47.09%，ReMe Text 41.27%，ReMe Image 53.44%，OpenViking 37.04%，Mem0 38.62%。](../figure/memlens-256k-qwen36-judge.svg) + +![Qwen3.8-Max 的 MemLens-256K LLM-Judge 对比：Full Context Image 65.61%，ReMe Text 52.91%，ReMe Image 72.49%，OpenViking 42.33%，Mem0 51.32%。](../figure/memlens-256k-qwen38-judge.svg) + +评测使用官方 LLM-Judge 协议，比较各方法完整系统的表现。6 个 case 在至少一种方法中触发了模型服务商的内容安全审核，因此对所有方法统一过滤，汇报其余 **189/195** 个 case 的结果。 + +ReMe 可以检索原始会话，Image 还可重新读取原图；本次 OpenViking 的检索范围限定为 memory。实验设置、过滤清单和复现对比见[详细评测说明](./memlens_256k_evaluation.md)。 + ## 如何开始使用 两个入口都需要支持图片和工具调用的视觉模型，以及兼容的 AgentScope wrapper 和 formatter。模型由所选 wrapper 的 `as_llm` 绑定决定。凭证配置见[基础配置](./configuration.md)。 diff --git a/docs/zh/context_compaction.md b/docs/zh/context_compaction.md index 4de6352a..33accb2e 100644 --- a/docs/zh/context_compaction.md +++ b/docs/zh/context_compaction.md @@ -147,3 +147,18 @@ agent 继续执行的成功率尚未验证。 [完整实验：设计、结果、标准答案与复现](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README_ZH.md) + +## 注入宿主管理的决策模型 + +嵌入式宿主可以复用已有的 AgentScope JEV 实例，与注入 LLM 的方式一致： + +```python +# app 已配置 backend: agentscope 的 as_decision_model.compaction 组件。 +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +YAML 中的 `model` 仍表示供应商模型名称；运行时 `model` 属性接收可调用的模型实例，读取属性不会创建客户端。 +在首次请求之前注入。ReMe 继续校验并转换 JSON 问题，但不再要求额外的 ReMe 凭据，也不会关闭借用的客户端。 +宿主应等待 ReMe 调用结束后自行关闭实例。替换 ReMe 自建模型或存在进行中请求的实例之前，须先关闭组件。 +关闭过程即使被取消，也会等待活动请求和自建客户端清理完成，再传播取消信号。组件关闭时会解除实例引用； +重新启动使用外部模型的组件前需要再次注入。独立运行时按凭据创建模型的行为保持不变。 diff --git a/docs/zh/memlens_256k_evaluation.md b/docs/zh/memlens_256k_evaluation.md new file mode 100644 index 00000000..234bf50f --- /dev/null +++ b/docs/zh/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K 评测说明 + +本文介绍[多模态记忆文章](./blog_20261009.md#memlens-256k-results)中 MemLens-256K 评测的实验设置、数据筛选和结果。 + +## 结果与统计范围 {#results} + +评测使用 [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) 官方 `agent_subset_195.json` 中的 195 个问题，每题使用对应的完整 256K 历史。按下文说明过滤六个问题后，所有方法在相同的 189 题上评测。LLM-Judge 按官方类别／子类别评分协议为每题给出 0 或 1 分，得分为 `100 × 正确数 / 189`。 + +| 模型 | 方法 | Judge 正确数 / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image 在两个模型上的得分分别为 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +## 方法与 Judge 设置 {#evaluation-settings} + +| 方法 | 记忆输入与回答阶段的数据访问范围 | +| --- | --- | +| Full Context Image | 回答模型直接读取完整历史文字与原图。 | +| ReMe Text | 使用原始会话文字生成记忆；回答 Agent 通过 BM25 检索 daily、digest 和原始 session，并读取相关文本。 | +| ReMe Image | 使用原始会话文字和图片生成记忆；检索范围与 Text 相同，并可通过 `read_image` 重新读取保存的原图。 | +| OpenViking | 使用原生图片描述和记忆流程；回答阶段检索限定为 memory，采用默认 top 10。 | +| Mem0 | 使用 Mem0 2.0.4 原生视觉入口生成记忆；回答阶段通过 FAISS 检索 top 20 记忆。 | + +各方法分别使用 Qwen3.6-35B-A3B 和 Qwen3.8-Max 作为待测模型。ReMe Image 与 Mem0 均直接使用原图。ReMe 的记忆和回答上下文上限分别为 200,000 / 400,000 tokens，temperature 为 0，开启 thinking；记忆／压缩与回答输出预算分别为 65,536 / 16,384 tokens。每题使用独立回答 Agent，最多运行 30 轮，并采用原生上下文压缩。Image 的 `read_image` 支持读取保存的附件，读取上限为 50 MiB，图片保留上限为 4,096 张。 + +Full Context、OpenViking 和 Mem0 的回答阶段同样使用 temperature 0、thinking 开启、输出预算 16,384 tokens。Mem0 记忆阶段采用原生设置：输出预算 2,000 tokens，temperature 和 top_p 均为 0.1。OpenViking 的评测适配包含 JSON 解析修正，以及图片处理失败时的显式报错。 + +评分采用 [MemLens 官方协议](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2)，所有方法统一使用 **Qwen3.8-Max** 作为 Judge，temperature 为 **0**、thinking **关闭**、输出预算为 **256 tokens**。Judge 与待测方法的记忆应用独立运行；参考答案及题型标签仅用于评分。 + +## 统一内容审核过滤 {#content-moderation-filter} + +调用 Qwen 模型服务时，下表中的方法在处理相应问题时触发了内容安全审核（绿网协议）。我们对各方法、两个模型触发审核的问题取并集，共得到六题，并在所有方法和两个模型中统一过滤，最终统计相同的 **189/195** 题。 + +表中 A3B 指 Qwen3.6-35B-A3B，Max 指 Qwen3.8-Max。 + +| 问题 ID | 触发内容审核的方法 / 模型 | 阶段 | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | 记忆构建 | +| `q_1d99a36c` | OpenViking / A3B | 记忆构建 | +| `q_3b01c02a` | ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B | 记忆构建 | +| `q_564a657b` | ReMe Image / Max | 回答 | +| `q_600a79aa` | ReMe Image / Max | 记忆构建 | +| `q_c9f8e24d` | ReMe Text / Max | 回答 | + +## 独立复现结果 {#independent-reproduction} + +我们将 MemLens benchmark 代码整理并集成到 ReMe 项目中，在相同的数据输入和评分设置下，对 ReMe Image 进行了独立复现。运行方式见[评测说明](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md)，具体参数见[方法配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml)、[Judge 配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml)和[Judge 评分规则](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py)。 + +独立复现对应的公开评测代码见 [PR #640](https://github.com/agentscope-ai/ReMe/pull/640)，commit 为 [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8)。 + +应用相同的六题过滤后，复现实验中 A3B 有 1 题、Max 有 2 题因会话历史处理失败未得到完整结果。下表分别在两次实验均完成的 188 题和 187 题上进行配对比较。 + +| 模型 | 配对题数 | 原实验正确数 | 独立复现正确数 | 原实验 → 独立复现 LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +在上述配对题目上，Qwen3.6-35B-A3B 和 Qwen3.8-Max 的独立复现 LLM-Judge 得分分别比原实验低 2.13 和 0.53 个百分点，整体性能与原实验基本保持一致。 diff --git a/docs/zh/memory_search.md b/docs/zh/memory_search.md index 1c73257f..520990ba 100644 --- a/docs/zh/memory_search.md +++ b/docs/zh/memory_search.md @@ -253,6 +253,23 @@ Memory Search 的“渐进式”不是一次把全库内容塞进结果，而是 - `response.answer`：给人看的文本，每个命中块包含路径、行号、分数和 chunk 内容，后面跟 outlinks / inlinks。 - `response.metadata`：给程序看的结构化结果，包括 `results`、`link_expansion`、`counts`。 +默认 `search` Job 没有结果可返回时，`response.answer` 保持 `""`，`success` 保持 `true`。 +此时 `results` 为空列表，`counts.returned` 为 `0`，可选字段 `response.metadata.empty_result` 说明原因： + +| `reason` | 含义 | +|----------------|-----------------------------------------------------------------| +| `no_results` | 没有匹配结果，或所有候选都被 `min_score` 过滤。 | +| `all_returned` | 符合分数要求的内容在同一 `tool_context_id` 中都已返回过。 | + +该字段还包含便于阅读的 `message`。程序调用方应通过 `reason` 区分两种情况。 +正常命中不包含 `empty_result`，正文格式保持原样。 + +通过 CLI 客户端调用已启动的服务时，可以开启已有的元数据显示选项查看原因： + +```bash +reme search query="最近关于索引的讨论" show_metadata=true +``` + 典型文本结构： ```text diff --git a/docs/zh/reme-blog.md b/docs/zh/reme-blog.md index 0c4c3826..54ff919f 100644 --- a/docs/zh/reme-blog.md +++ b/docs/zh/reme-blog.md @@ -292,6 +292,8 @@ cases / 400 题上取得 66.1%，在 1M 设置下的 35 cases / 700 题上取得 Agent 在 5 种用户角色（User Persona）上平均取得 0.580 的 PROC 分数，超出相同测试模型配置的 NanoBot 2.4%。关于该基准的详细介绍见 [$\pi$-Bench 论文](https://arxiv.org/abs/2605.14678)。 +在 **MemLens-256K** 上，ReMe Image 的 LLM-Judge 在 Qwen3.6-35B-A3B / Qwen3.8-Max 下达到 **53.44% / 72.49%**，比 Full Context Image 高 **6.35 / 6.88** 个百分点。统一过滤 6 个触发模型服务商内容安全审核的 case 后，汇报 **189/195** 个 case；五种方法的对比见[多模态记忆评测](./blog_20261009.md#memlens-256k-results)。 + ## ReMe 能帮谁？ ### 直接使用 Agent 的人 diff --git a/example.env b/example.env index 37c3aa76..b8e9e3d9 100644 --- a/example.env +++ b/example.env @@ -14,4 +14,3 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 # Optional data-source mirrors. Unset variables use the official services. # HF_MIRROR_URL=https://hf-mirror.com # ARXIV_MIRROR_URL=https://export.arxiv.org -# TUSHARE_MIRROR_URL=http://112.124.63.173:4000/tushare diff --git a/github-pages/tests/memlens-evaluation.test.mjs b/github-pages/tests/memlens-evaluation.test.mjs new file mode 100644 index 00000000..a33406e9 --- /dev/null +++ b/github-pages/tests/memlens-evaluation.test.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const siteDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const generatedDir = path.join(siteDir, ".generated/site"); +const docsDir = path.resolve(siteDir, "../docs"); +const methods = ["Full Context Image", "ReMe Text", "ReMe Image", "OpenViking", "Mem0"]; +// Published binary verdict counts, not raw benchmark outputs or a second chart generator. +const models = [ + { name: "Qwen3.6-35B-A3B", figure: "qwen36", correct: [89, 78, 101, 70, 73] }, + { name: "Qwen3.8-Max", figure: "qwen38", correct: [124, 100, 137, 80, 97] }, +]; +const excluded = ["q_028cb832", "q_1d99a36c", "q_3b01c02a", "q_564a657b", "q_600a79aa", "q_c9f8e24d"]; +const publicRevision = "962d5cd8401c1d2be17f3bb95f8ac9c185672cd8"; +const reproductionLinks = [ + ...[ + "benchmark/memlens/README.md", + "plugins/memlens/src/reme_memlens/benchmark.yaml", + "plugins/memlens-judge/src/judge_memlens/plugin.yaml", + "plugins/memlens-judge/src/judge_memlens/protocol.py", + ].map((file) => `https://github.com/agentscope-ai/ReMe/blob/${publicRevision}/${file}`), + "https://github.com/agentscope-ai/ReMe/pull/640", + `https://github.com/agentscope-ai/ReMe/commit/${publicRevision}`, +]; + +test("MemLens chart labels and geometry match the published scores", async () => { + for (const model of models) { + const svg = await readFile(path.join(docsDir, "figure", `memlens-256k-${model.figure}-judge.svg`), "utf8"); + const bars = [...svg.matchAll(/\s*]*y="([\d.]+)"[^>]*height="([\d.]+)"[^>]*\/>\s*]*>([\d.]+)%<\/text>/g)]; + assert.equal(bars.length, methods.length); + for (const [index, method] of methods.entries()) { + const score = (100 * model.correct[index] / 189).toFixed(2); + assert.equal(bars[index][1], method); + assert.equal(bars[index][2], score); + assert.equal(bars[index][5], score, `${model.name}/${method}: visible score`); + assert.ok(Math.abs(Number(bars[index][4]) - Number(score) * 3) < 0.001); + assert.ok(Math.abs(Number(bars[index][3]) + Number(bars[index][4]) - 416) < 0.001); + } + } +}); + +test("MemLens bilingual notes and blogs match the published scores", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const rows = notes.split("\n").filter((line) => /^\| Qwen.*\| \d+ \/ 189 \|/.test(line)); + assert.equal(rows.length, 10); + for (const model of models) { + for (const [index, method] of methods.entries()) { + const correct = model.correct[index]; + const score = (100 * correct / 189).toFixed(2); + assert.ok(rows.includes(`| ${model.name} | ${method} | ${correct} / 189 | ${score} |`)); + assert.ok(blog.includes(`${method} ${score}%`), `${language}: alt text for ${model.name}/${method}`); + } + const gain = (100 * (model.correct[2] - model.correct[0]) / 189).toFixed(2); + assert.ok(blog.includes(`**${gain}**`)); + assert.ok(notes.includes(`**${gain}**`)); + } + } +}); + +test("MemLens notes disclose moderation filters, paired results and Judge settings", async () => { + const triggers = { + zh: [ + ["OpenViking / Max", "记忆构建"], + ["OpenViking / A3B", "记忆构建"], + ["ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B", "记忆构建"], + ["ReMe Image / Max", "回答"], + ["ReMe Image / Max", "记忆构建"], + ["ReMe Text / Max", "回答"], + ], + en: [ + ["OpenViking / Max", "Memory construction"], + ["OpenViking / A3B", "Memory construction"], + ["ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B", "Memory construction"], + ["ReMe Image / Max", "Answering"], + ["ReMe Image / Max", "Memory construction"], + ["ReMe Text / Max", "Answering"], + ], + }; + for (const language of ["en", "zh"]) { + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const filter = notes.split("{#content-moderation-filter}")[1].split("{#independent-reproduction}")[0]; + assert.deepEqual([...new Set(filter.match(/q_[0-9a-f]{8}/g))].sort(), excluded); + const rows = filter.split("\n").filter((line) => line.startsWith("| `q_")); + assert.deepEqual(rows, excluded.map((id, index) => { + const [method, stage] = triggers[language][index]; + return `| \`${id}\` | ${method} | ${stage} |`; + })); + assert.match(filter, /189\/195/); + assert.ok(notes.includes("| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 |")); + assert.ok(notes.includes("| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 |")); + const judge = notes.split("\n\n").find((paragraph) => /(?:as|作为) Judge/.test(paragraph)); + assert.ok(judge, `${language}: Judge settings must be documented`); + assert.match(judge, /\*\*Qwen3\.8-Max\*\*/); + assert.match(judge, /temperature (?:为 )?\*\*0\*\*/); + assert.match(judge, /thinking \*\*(?:disabled|关闭)\*\*/); + assert.match(judge, /\*\*256(?:-token| tokens)\*\*/); + assert.match(notes, /memory-only|restricted to memory|限定为 memory/); + } +}); + +test("MemLens notes pin the public evaluation revision without private audit details", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + assert.match(blog, /\]\(\.\/memlens_256k_evaluation\.md\)/); + assert.match(notes, /\]\(\.\/blog_20261009\.md#memlens-256k-results\)/); + assert.doesNotMatch(notes, /\/mnt\/|\/Users\/|docs\/_build|file:\/\//); + const reproduction = notes.split("{#independent-reproduction}")[1]; + const targets = [...reproduction.matchAll(/\]\(([^)\r\n]+)\)/g)].map(([, target]) => target); + assert.deepEqual(targets.sort(), [...reproductionLinks].sort()); + const prose = notes.replace(/\]\([^)]+\)/g, "]"); + assert.doesNotMatch(prose, /\b[a-f0-9]{40,64}\b/); + for (const anchor of ["results", "evaluation-settings", "content-moderation-filter", "independent-reproduction"]) { + assert.ok(notes.includes(`{#${anchor}}`)); + } + } +}); diff --git a/integrations/dsh/README.md b/integrations/dsh/README.md index d84c4bf6..faac78cc 100644 --- a/integrations/dsh/README.md +++ b/integrations/dsh/README.md @@ -4,7 +4,7 @@ This guide explains how to install, configure, and use `@agentscope-ai/reme-dsh-plugin` with DeepSeek Harness (DSH), including memory guidance injection, the `reme_search` tool, automatic memory, daily consolidation, and the ReMe Status page. -The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, and `0.2.1-alpha.4`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. +The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, and `0.2.1-rc.2`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. ## 1. How the plugin works @@ -26,7 +26,7 @@ The DSH adapter injects **usage guidance**, not every historical memory. Relevan ## 2. Requirements - ReMe is installed and its configuration exposes the `search`, `auto_memory`, and `auto_dream` jobs. -- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, or `0.2.1-alpha.4`. +- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, or `0.2.1-rc.2`. - Node.js `^22.19.0` or `>=24.0.0`, matching the current DSH engine range. - The browser running DSH can reach the configured ReMe HTTP endpoint. Cross-machine deployments must also allow the DSH browser origin. @@ -110,21 +110,23 @@ If an earlier upgrade already renamed `settings.yaml` to `settings.yaml.imported ![ReMe Memory plugin configuration](./figures/reme-memory-settings.png) -| UI meaning | Configuration key | Default | Description | -| ---------------------- | --------------------- | ----------------------- | --------------------------------------------------------------- | -| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute ReMe HTTP URL using `http` or `https`. | -| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | -| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | -| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | -| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | -| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | -| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | -| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | -| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | -| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | -| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | -| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | -| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| UI meaning | Configuration key | Default | Description | +| ---------------------- | ---------------------- | ----------------------- | --------------------------------------------------------------- | +| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute HTTP(S) URL without credentials, query, or fragment. | +| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | +| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | +| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | +| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | +| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | +| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | +| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | +| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | +| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | +| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | +| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | +| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| Retry delay | `autoMemoryRetryMs` | `1000` | Initial retry delay; doubles after each failure. | +| Maximum retry delay | `autoMemoryRetryMaxMs` | `60000` | Upper bound for retry delay. | Deployment configuration also supports `REME_URL`, or `REME_HOST` together with `REME_PORT`. The timer-only test option `dreamIntervalMs` is intentionally excluded from user settings. @@ -169,9 +171,19 @@ An empty successful response becomes `No relevant memory found.`. Service failur With `autoMemoryEnabled=true`, the plugin listens to DSH session events and collects completed user and assistant messages per session. When `autoMemoryInterval` is reached, the batch enters a background queue and calls ReMe `auto_memory`. Plugin-generated context and tool results are excluded from capture so they cannot be laundered back into long-term memory. +Each completed turn is saved in a private JSON outbox before submission. Failures use capped exponential backoff independently of new turns; recovery drains every ready batch, including ended sessions. Payload/job rejections, such as HTTP 413, pause the affected session while other sessions continue; network, authentication, rate-limit, and service failures pause the endpoint. Each endpoint drains independently and reads one batch at a time, so a stalled old destination cannot block a newly configured service. Disabling automatic memory pauses capture and further sends; queued files remain available when re-enabled. + +The outbox defaults to `$DSH_HOME/reme-memory/outbox` (`~/.dsh/reme-memory/outbox` without `DSH_HOME`). Set the deployment-only `autoMemoryQueueDir` field and restart to choose another directory; simultaneous Hosts must use separate directories. Batches retain their original service endpoint across settings changes. An exclusive owner lock prevents concurrent writes, and a restart recovers an exited owner's queue. Queue files contain conversation text: keep them private. Disk/permission failures are surfaced as errors; preserve DSH session logs for recovery. + +Owner-lock recovery is serialized. If startup reports a busy or interrupted `.lock-recovery`, retry first; if the error persists, stop all Hosts using that queue, inspect it, and remove only the empty `.lock-recovery` directory before restarting. Preserve `.lock` and all queued JSON files. + +New batches receive strictly increasing ordering timestamps, recovered from queued files after restart, so equal wall-clock times or a clock rollback cannot reorder them. Older batches that already share a timestamp retain their existing file order. + +Sealed batches keep a stable `request_id` across timeouts and restarts. The updated ReMe service stores completed request receipts under its configured metadata directory, rejects reuse with different input, and replays completed results without invoking the memory agent again. An interruption before a receipt is committed can still repeat an unfinished operation; receipts do not make arbitrary agent file edits transactional. + ![Automatic-memory activity](./figures/reme-status-auto-memory.png) -Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until running and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. +Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until queued turns, running tasks, and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. ## 8. ReMe Status tabs diff --git a/integrations/dsh/README_ZH.md b/integrations/dsh/README_ZH.md index 08262b95..e7d7cd77 100644 --- a/integrations/dsh/README_ZH.md +++ b/integrations/dsh/README_ZH.md @@ -26,7 +26,7 @@ DSH 启动新会话时，插件向根 Agent 注入一段“如何使用长期记 ## 2. 环境要求 - ReMe Python 服务已安装，且配置中提供 `search`、`auto_memory` 和 `auto_dream` Job。 -- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2` 或 `0.2.1-alpha.4`。 +- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4` 或 `0.2.1-rc.2`。 - Node.js `^22.19.0` 或 `>=24.0.0`，与当前 DSH 的 engine 范围一致。 - DSH 页面能够访问 ReMe HTTP 地址；跨机器部署时还要允许 DSH 页面所在的浏览器 Origin。 @@ -115,21 +115,23 @@ dsh web --no-open --port 3090 截图中的测试配置使用 `http://127.0.0.1:3457`、English 指引、默认搜索数量 5、搜索超时 10 秒，并启用了自动记忆和“Exclude subagents”。完整字段如下： -| 界面含义 | 配置键 | 默认值 | 说明 | -| -------------------- | --------------------- | ----------------------- | -------------------------------------------------------------------------- | -| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的 ReMe HTTP 服务绝对地址，只支持 `http`/`https`。 | -| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | -| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | -| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | -| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | -| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | -| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | -| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | -| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | -| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | -| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | -| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | -| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 界面含义 | 配置键 | 默认值 | 说明 | +| -------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------- | +| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的绝对 HTTP(S) 地址，不支持 URL 内嵌凭据、查询参数或片段。 | +| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | +| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | +| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | +| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | +| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | +| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | +| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | +| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | +| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | +| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | +| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | +| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 失败重试间隔（毫秒） | `autoMemoryRetryMs` | `1000` | 首次失败后的等待时间，连续失败时翻倍。 | +| 最大重试间隔（毫秒） | `autoMemoryRetryMaxMs` | `60000` | 重试等待时间的上限。 | 部署层还支持 `REME_URL`，或组合使用 `REME_HOST` 与 `REME_PORT`。仅供定时器测试的 `dreamIntervalMs` 不出现在用户设置中。 @@ -180,11 +182,21 @@ only on retrieved memory and cite the returned paths. 启用 `autoMemoryEnabled` 后，插件监听 DSH `session/event`，按会话收集完成的用户和助手消息。达到 `autoMemoryInterval` 后进入提交队列，后台调用 ReMe `auto_memory`。插件生成的上下文以及工具结果不会再次进入自动记忆，避免把指引或检索回显循环写回长期记忆。 +每个完成回合会先写入私有 JSON 提交队列。失败后按有上限的指数退避独立重试，不依赖下一轮对话；恢复后持续排空可发送批次，也包括已结束的会话。请求内容或 Job 被拒绝（例如 HTTP 413）时，仅暂停受影响的会话，其他会话继续处理；网络、认证、限流或服务故障会暂停对应服务地址。每个服务地址独立排空队列，每次读取一个批次，旧地址挂起不会阻塞新配置的服务。关闭自动记忆会暂停捕获及后续发送，重新启用后可以继续处理保留的队列。 + +队列默认位于 `$DSH_HOME/reme-memory/outbox`，未配置 `DSH_HOME` 时使用 `~/.dsh/reme-memory/outbox`。可在部署配置中设置 `autoMemoryQueueDir`，重启后生效；同时运行的 Host 必须分别使用不同目录。积压批次保留捕获时的服务地址，修改地址不会把旧内容发送到另一个服务。独占锁阻止并发写入，重启时可以恢复已退出进程的积压。队列包含对话正文，请保留私有权限。磁盘或权限错误会明确报告，原始 DSH 会话日志应保留用于恢复。 + +进程锁的回收会串行执行。若启动报告 `.lock-recovery` 正忙或恢复中断，先重试；错误持续时，停止所有使用该队列的 Host，检查队列后仅移除空的 `.lock-recovery` 目录，再重新启动。保留 `.lock` 和所有排队的 JSON 文件。 + +新批次使用严格递增的排序时间戳，重启时从排队文件恢复，避免系统时间相同或回拨导致批次乱序。已有相同时间戳的旧批次仍沿用原有文件顺序。 + +已封存的批次在超时、重启后沿用相同的 `request_id`。更新后的 ReMe 服务在配置的 metadata 目录保存已完成请求的回执，拒绝把同一个 ID 用于不同输入，并直接重放已完成结果，避免重复调用记忆 Agent。回执提交前发生中断，未完成的操作仍可能重试；回执不会把任意 Agent 文件修改变成事务。 + 截图测试把间隔临时设为 1；运行记录中可以看到英文测试会话形成的已完成提交： ![自动记忆运行记录](./figures/reme-status-auto-memory.png) -自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 +自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“待处理回合”“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 ## 8. ReMe 状态页六个标签 diff --git a/integrations/dsh/package-lock.json b/integrations/dsh/package-lock.json index 94cd94bc..f6796838 100644 --- a/integrations/dsh/package-lock.json +++ b/integrations/dsh/package-lock.json @@ -9,18 +9,18 @@ "version": "0.1.3", "license": "Apache-2.0", "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -328,21 +328,21 @@ } }, "node_modules/@deepseek-ai/cordis": { - "version": "4.0.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5-alpha.2.tgz", - "integrity": "sha512-rd5Qp6PNrMk+0saWVMLUaVv08s/WBwf2TXM84ST6tppFqlHS0Yl9l/NwPrA4oVOuS/1BKCYOGlZ6zJRj+JG6IQ==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5.tgz", + "integrity": "sha512-Sktq5axy0owk9PdnivBRNOJjHEVEibCt0+TDDZcv0+UH2KS1M910fX4kn5y45YFsrpZIi1mJf/1s5VzywMUnyg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" }, "bin": { "cordis": "bin.js" }, "peerDependencies": { - "@deepseek-ai/cordis-plugin-include": "~1.0.10-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "~1.0.6-alpha.2" + "@deepseek-ai/cordis-plugin-include": "~1.0.10", + "@deepseek-ai/cordis-plugin-loader": "~1.0.6" }, "peerDependenciesMeta": { "@deepseek-ai/cordis-plugin-include": { @@ -354,17 +354,17 @@ } }, "node_modules/@deepseek-ai/cordis-plugin-loader": { - "version": "1.0.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6-alpha.2.tgz", - "integrity": "sha512-ea4WjgWsrUxVJ0UYujTbGgaSIR/mV46myY/s6isYRTQRCmlkyVPGagkTxrRR2c/MqGwwbC4KXxusT4uM7H9BXg==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6.tgz", + "integrity": "sha512-fr011A782oEjkJXCY4pePt2qaPpkGvem7Rmst6Rhwgg6icgwI4eN9QBRb6S/q9XOYGVQUdvL0wXUWyNxO8WipA==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/cosmokit": "~1.8.6", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", + "@deepseek-ai/cordis": "~4.0.5", "node-addon-require-builtin": "^0.1.9" }, "peerDependenciesMeta": { @@ -374,28 +374,28 @@ } }, "node_modules/@deepseek-ai/cosmokit": { - "version": "1.8.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6-alpha.2.tgz", - "integrity": "sha512-FcxdMUal80CeFURh3lPYsw9TPDMVSGpovWnA3JtTGrBG15XFli4k38z1Egso1lgvifKpIW4Hx42X6UTelLEOAA==", + "version": "1.8.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6.tgz", + "integrity": "sha512-ZSD/fN52L498LBIOYwjIOlaLv5veY6ZEwI2yzvZUKQtctn0QAPqu8tKdJ+OF5+Tv6NBq1U+wJWH+QI6OGy5EvQ==", "dev": true, "license": "MIT" }, "node_modules/@deepseek-ai/dsh-agent": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-alpha.4.tgz", - "integrity": "sha512-QIL78OhM5csPWkrdGfhUJQr3zdyUd/DZyM4euwUnkMS0BA5Hp21hCWg7N69HY73bqvV20oyU10g5huPtqJHSyQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-rc.2.tgz", + "integrity": "sha512-e96Tnq8g/RNCDDUW7kAypY/4cho5QxjLHMhk86BGGIEu9b31XEkSFqwqQIwtYXQez3cDR5z7LZpAM8Uqx9Y8KA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/dsh-workspace": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/dsh-workspace": "0.2.1-rc.2" }, "peerDependenciesMeta": { "@deepseek-ai/dsh-workspace": { @@ -404,148 +404,148 @@ } }, "node_modules/@deepseek-ai/dsh-brand": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-alpha.4.tgz", - "integrity": "sha512-SlrYMisnHPNREhc9jsohRylcZmpQPuzcul7DxUCJNjK08fDUUuzcyrQI7uDjvbw1n34IwYHNNV6o4mrhIIForQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-rc.2.tgz", + "integrity": "sha512-T2kcygikRS8rv8cKjgho2x381EgGprv5yQmvHl4/iYbuHE19OW+SLGtHWbV27PGdSHz5U8CJxasYzFosKW4e2A==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-primitives": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-alpha.4.tgz", - "integrity": "sha512-nHzG2k7l8UuNG2rcIkzny1ko/fn/mQjiWrSh6wIe6X9d/qQi55Tdo87N3usmwce1akcZrvsXQ27v451tC1gzwA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-rc.2.tgz", + "integrity": "sha512-3K6W2ID4sbDUDHhZHs8cNO15XDXBWAYFnEzJ/iD8ZASqidazcJActxlEf8GO6NHmbbeX0lEq0P20Ftwhovt+Mw==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-settings": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-alpha.4.tgz", - "integrity": "sha512-SG6wzQyBmkNUuVt+5Da4tSnGeVVgNE8cJ6Qa6MG4zOJqWnEkJTCIRuOmsN1+7IGcWSNooFG76RGJpO+5uKkDdA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-rc.2.tgz", + "integrity": "sha512-oU8f/0+DsdbgF0XhM+Po7W4PDJmTbhifQVwIoAcKfj29JN/4tLxKqIKS0IRcMbIGD39BGFITne78KIMgrGJy3Q==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-fs": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-alpha.4.tgz", - "integrity": "sha512-K5TK7QxC3fRM8s1Uvfy1O3C4kaybSXOLGFMSb9WLwfTe1QGsq7JZN0v8jmiv3VFA0cFUkTPybz/PokyDtRWvRg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-rc.2.tgz", + "integrity": "sha512-Z6d2ZKsAiJc0FaiES8wC66vuUQMGohl9tCc2p3bAsPFYkJ0+L2zTViQ9Y5BArtRe9Br8qhNjNgLlN0jb/uwv1g==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-llm": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-alpha.4.tgz", - "integrity": "sha512-rjAT6x0iNUNLgCC3pCNaQxrtQxlcMAAdCh8x27C4PiGCXMaa7kNge8HnjPC3iaZoopSrr5XRBLcyIFmg46Cc+A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-rc.2.tgz", + "integrity": "sha512-3xfoMDHEpUJGcH2Lu+lNXUJYRBclYfBkdCG9Au02VzIjYAm9BrFmc/MNuaVTg0vwgvPpTsd6Fx1IeGtfmQFvlg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-timeout": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-crypto": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-timeout": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-crypto": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-ptc-runtime": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-alpha.4.tgz", - "integrity": "sha512-r+4GYR1UYbr0zyuu9tOO+iJuQvxYjtCJyTHr6YR9UZNJrfbwkrSeqlRIbR1cw5NaIh7dplZK1+a90FHURkrgWA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-rc.2.tgz", + "integrity": "sha512-e81rbNwp3YNIpzIbkFI4NsWQzTD3uAP8zXU16diL9wLmITUWjMS+kvrWEXtrkWAWp5cnK2Pgn4xCbqVZ6n4I8w==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-alpha.4.tgz", - "integrity": "sha512-PRCb2xDAvX609LagauDjLvYFmzYLBO3QYAYXcc7btfiqOdgfeDb7FgstzKjdbz2o31iVdvKUdDK1iFzXjVg6eg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-rc.2.tgz", + "integrity": "sha512-nnxNIkgi4A4tkPrTV8sbVMse3c32OT0U+mwYhLqDz/+cIm5tUGufQD+Ax0zqVeClyzXsxZo0/pVtcqDJrsp+7w==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox-policy": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-alpha.4.tgz", - "integrity": "sha512-xEfAiquD87cg1aOzdPtFoB82e0P4hzWkVEimwRk1s85ESb8YiV+DRzWLrIGprfi9fKhsxSFFIFnCKgzOKjURuA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-rc.2.tgz", + "integrity": "sha512-fBb8zQ13CLKM60ynl3EHvFmO8F0GARofQYf7CXUkqJEuBcjxXUG14rzDKRS/PhE8oJ023lEvd5HJjE6qUjslhw==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-scope": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-alpha.4.tgz", - "integrity": "sha512-oo1AraYsu5D3oYGdKim43HGg0Hf6NbYLmV5LZO8jiDnaXiE+xfNDqI8ARwq049lODJMxP0ONX2hTo5DTG5zjuQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-rc.2.tgz", + "integrity": "sha512-CVGPI4lzmUy7O3zg8X1LavUSmD3JKFkUdykfHN7VkyWXG5oAXSvig9s5PLnIb+sxRItmSpiqPXGnT/48X1T1qg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-session": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-alpha.4.tgz", - "integrity": "sha512-YMdG+ZdvM+jXFDkfzO+ksCn6YpUFuoN4YuMEIcbSqLnwNo0Q7ba0GJWJEH7oP5v0sCwGtMKQySKSgqBjkUyw+w==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-rc.2.tgz", + "integrity": "sha512-WDA4lIrXtG5nf1dn2wrA4UzEr7D6e8T3AdPpVSZ7eY4N4ICMDZ0RYuD4GkM+iSb7WkLojdO8ljr0etiIMEbqJg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-session-projection": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-alpha.4.tgz", - "integrity": "sha512-+Tm9xJd7ETbcLlolf9wx+Nm2z1O0SFqXermWU3GZ7ffQqo+tRKQ2oEa6NhSS5Mw1wVkY0xuxye5RRnVk3gbb4g==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-rc.2.tgz", + "integrity": "sha512-GBGgIAaC+Cw1rV7tpyGy9aaENUOkQCosP+LJHcmzqPWg64Ufts36mDEU3f0lC/nyGb0z66Ov+RbyaLLsCnZMwA==", "dev": true, "license": "MIT", "peer": true, @@ -553,144 +553,144 @@ "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-system-prompt": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-alpha.4.tgz", - "integrity": "sha512-oHk8b0F2ecwrcN58ZtkMkR8FIRTULNSMTkHB1KbOz1LpTYuD6M01VFhWG0FvzZw01xPeuDlbO9wNmYtYi6/RsA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-rc.2.tgz", + "integrity": "sha512-X1WG2FMyfZapAkcCYyTl0UO5vVqDcCsA9wECJY7eeDTLHJDsqrilQJg2DRgHQzM+2/7T01QyzFZhKd7hfALJxg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-timeout": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-alpha.4.tgz", - "integrity": "sha512-hXCp1YWwDZAaTrNNon3ypb/Z6ZgDNKi/3zdC2JLHb9gU2f7DdSexbvRs37xKfyXO8HhtXDKvp/FUCWvI6rpRlg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-rc.2.tgz", + "integrity": "sha512-R75r+GkYP+ZhtIsIfn8MMuRXPkugJr0OyQ1Y+rauoTae2sdVfnE8v5zvQORRCHdh7MiOcmh10dQN70KjHh7qhQ==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-tools": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-alpha.4.tgz", - "integrity": "sha512-oTurHJ91LmlirKbEMyU0K+ES8LYtfJW4MAGrLjSBSaRKrq8eERTWNJafcxlhEKNpTWeUNU7h3OHaX/rqcMmLyA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-rc.2.tgz", + "integrity": "sha512-K3gET3VPbtxM66pkeRsEGrZ+bQKiBqrTbNXiIreBKu0hP+qPvlSwhiSI5MTnFp2tMFK5rpuYMZosRIdbfrbJ/g==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-ptc-runtime": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox-policy": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-user-approval": "0.2.1-alpha.4", - "@deepseek-ai/dsh-working-directory": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-ptc-runtime": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox-policy": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-user-approval": "0.2.1-rc.2", + "@deepseek-ai/dsh-working-directory": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-typert-protocol": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-alpha.4.tgz", - "integrity": "sha512-LBNUgiwbRqhyRNPvaAca++eLAxJ1jmmFJnd3RJAa8pT+oULJgZvqPgr9jVnTfX/P5pQT0p7AMZeelTc7QpQ/OA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-rc.2.tgz", + "integrity": "sha512-rtsHJHGE9SWX+zOgtDbPUqz3h7r+qFm3h1WASzRJxDnULqo7JaBdIiGshNTFPmetsX+r1T69CKFdTZAJmHwMug==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-user-approval": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-alpha.4.tgz", - "integrity": "sha512-J6bcZV3rFuQiRDuMej5S/DalSeztfVIFngoGA15M31khb0E74kfrgWUIxvS7pRwy2j1MKz89wR8BbbDzufHTRw==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-rc.2.tgz", + "integrity": "sha512-pwhw1U1IZZm5tKyPayj2APZBq5I7/I0C0ad+ohd4vKuNb6twDIltt1GEHdCNt7I19IGHLRiUY+/Q9S/8+q5K0A==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-util-crypto": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-alpha.4.tgz", - "integrity": "sha512-BLMlmD3y+HhTZPGij4viPRFWx8DNdsPNMauUW5XxyE2Iq5KG9ohQ+6xULju/J2Ah9JldVr3XAku7ItDPDk8Mpg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-rc.2.tgz", + "integrity": "sha512-V4SxvXL4ZbY5RYJckVtmo3qz4YsA52pYKvDF13+8hsDCEmQIv9YfmLRMA61pjK/i05pGMe3p1pPhAB6L+NOIEA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-util-values": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-alpha.4.tgz", - "integrity": "sha512-RM3gAmdyUsldAb4AIsU++KYbBY98UvHRCskcHX2bojZ6kefGd1a8P1Yl/Fo+bUWIAHZj/VoA1pnAl7r5e5456A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-rc.2.tgz", + "integrity": "sha512-mgnByPadOgZtoBxbwMI184li494O6sNn66/NvdqN0RIrzAJmFGpFxpjFkQtQN/yetwWhlx7Ic17+6p7eKl8DNg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-working-directory": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-alpha.4.tgz", - "integrity": "sha512-VtgMMlvYJ4iOPtpasI+ey6ioBekJy5MLSw3ky6lvLdLnKPmj0zobVOFhldr7KEvZFRizKYN5ktU8D85qKZG9Cg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-rc.2.tgz", + "integrity": "sha512-zhPtvAaCxa9U0WUN9OEG37zVBq7GOXXXwnDx2Jds4DFksGY5Lmpl3ZK1QMGz+F/Aye4VHt9zoISI7m4wDA8DEg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-fs": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-fs": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/schemastery": { - "version": "3.18.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5-alpha.2.tgz", - "integrity": "sha512-W+2TSjdeKOEKqaii9BNpw7ev2Yf8C+J74i/BE/VBx56O7BX0zxV2zQKwRY4ADLTXD5O69iJ+/Skm91/ES7iMgw==", + "version": "3.18.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5.tgz", + "integrity": "sha512-Mnobvqf9rHsZYv4g5edj5ddxS1oF1kIdSlz0DunUgnY3LWG5F26MqvH3YOOsSaodCgyzJ16OMN9O99/s5Crtdw==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" } }, @@ -1488,9 +1488,9 @@ } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz", - "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==", + "version": "7.0.13", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.13.tgz", + "integrity": "sha512-2E8Q6rh+1RRSq/YdaGsNfXsEZy9FVqv6BSSrkmZJP7BnaoPnRfKL+YaofPvP+5rzdk144qSm6i4+hvPgUcbV4Q==", "dev": true, "license": "MIT", "engines": { @@ -1811,9 +1811,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.11.29", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.29.tgz", - "integrity": "sha512-YeesxhUmR4h5y5V8KF8iBCqJb8YFrBJ++XBTeGcWIaQOYXGN19JM0FoiOjAhwuDoln8owMgXcWBmgHtYSkky2w==", + "version": "2.11.30", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.30.tgz", + "integrity": "sha512-YJs6Pz17RMNv8HoyKREtCY38E3BV1nPkWVmIb+XN1uHC1Vqt3KZCH/uJH9ZKRoiKb/icZ6tvehyee6RZNbFfRA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1835,9 +1835,9 @@ } }, "node_modules/browserslist": { - "version": "4.29.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", - "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "version": "4.29.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.4.tgz", + "integrity": "sha512-2TcbbiwXEq+N0kD36tykpCfyyQigUXmuwNQbnPp4Xk4PX9JOqVIKjJlIDjpH7uY6tmRGoAd+apeT1p60T8wsdA==", "dev": true, "funding": [ { @@ -1855,11 +1855,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.11.26", - "caniuse-lite": "^1.0.30001813", - "electron-to-chromium": "^1.5.439", - "node-releases": "^2.0.57", - "update-browserslist-db": "^1.3.3" + "baseline-browser-mapping": "^2.11.29", + "caniuse-lite": "^1.0.30001816", + "electron-to-chromium": "^1.5.459", + "node-releases": "^2.0.59", + "update-browserslist-db": "^1.3.4" }, "bin": { "browserslist": "cli.js" @@ -1879,9 +1879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001816", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001816.tgz", - "integrity": "sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==", + "version": "1.0.30001817", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001817.tgz", + "integrity": "sha512-uZ1JG80BwhtMalIAGPxPfmNhHyd/B/nTdr3hITWlRamSyAdUStP1Gop01e7SD5q6J90ixJvWDbvmiFVfDgn4bw==", "dev": true, "funding": [ { @@ -1998,9 +1998,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.457", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.457.tgz", - "integrity": "sha512-RNKsgErRmkzRgC8nOV1wzFhSbfGlYJ6k4LT/aE7oFrDYaQgHhTh5oTp0ZgSbRRybKR8HlZIdCWc38Y+CDTk1Jw==", + "version": "1.5.463", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.463.tgz", + "integrity": "sha512-nVWNU6ltFjopvOdD44J4zoBkgO0M+WZiFKKFV7EjebOH9z0KNphk6bCEjplXtJofL21ARIuuwWOCsFkBVklm6w==", "dev": true, "license": "ISC" }, diff --git a/integrations/dsh/package.json b/integrations/dsh/package.json index 86388a31..0ec3eaa9 100644 --- a/integrations/dsh/package.json +++ b/integrations/dsh/package.json @@ -79,14 +79,14 @@ } }, "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -98,10 +98,10 @@ "react": "^18.2.0", "typescript": "^5.9.2", "typescript-eslint": "8.59.3", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", "react-test-renderer": "^18.3.1", "semver": "^7.8.5" }, diff --git a/integrations/dsh/scripts/test-package.mjs b/integrations/dsh/scripts/test-package.mjs index 67884661..3e8f3f46 100644 --- a/integrations/dsh/scripts/test-package.mjs +++ b/integrations/dsh/scripts/test-package.mjs @@ -37,6 +37,7 @@ const releases = [ loader: "1.0.6-alpha.2", schema: "3.18.5-alpha.2", }, + { dsh: "0.2.1-rc.2", cordis: "4.0.5", loader: "1.0.6", schema: "3.18.5" }, ]; async function run(command, args, cwd) { diff --git a/integrations/dsh/src/client/index.tsx b/integrations/dsh/src/client/index.tsx index f8674bd0..d14a0151 100644 --- a/integrations/dsh/src/client/index.tsx +++ b/integrations/dsh/src/client/index.tsx @@ -2,6 +2,7 @@ import { useState, useSyncExternalStore } from "react"; import type { ConfigForm } from "@deepseek-ai/dsh-client-ui-settings/client"; import type { ReMeSettings } from "../types.js"; +import { validEndpoint } from "../endpoint.js"; import { ReMeStatusPage, statusEn, @@ -19,6 +20,8 @@ const FIELDS = [ "requestTimeoutMs", "backgroundTimeoutMs", "shutdownTimeoutMs", + "autoMemoryRetryMs", + "autoMemoryRetryMaxMs", "autoMemoryEnabled", "autoMemoryInterval", "autoDreamEnabled", @@ -65,6 +68,8 @@ interface Draft { requestTimeoutMs: string; backgroundTimeoutMs: string; shutdownTimeoutMs: string; + autoMemoryRetryMs: string; + autoMemoryRetryMaxMs: string; autoMemoryEnabled: boolean; autoMemoryInterval: string; autoDreamEnabled: boolean; @@ -113,6 +118,8 @@ const en = { advanced: "Advanced", backgroundTimeout: "Background request timeout (ms)", shutdownTimeout: "Shutdown flush timeout (ms)", + retryDelay: "Retry delay (ms)", + retryMaxDelay: "Maximum retry delay (ms)", diagnostics: "Service diagnostics", processMemory: "Process memory", componentMemory: "Component memory", @@ -186,6 +193,8 @@ const zh: typeof en = { advanced: "高级设置", backgroundTimeout: "后台请求超时（毫秒）", shutdownTimeout: "关闭时写入等待（毫秒）", + retryDelay: "失败重试间隔（毫秒）", + retryMaxDelay: "最大重试间隔（毫秒）", diagnostics: "服务诊断", processMemory: "进程内存", componentMemory: "组件内存", @@ -568,6 +577,28 @@ function SettingsForm({ } /> + + + update("autoMemoryRetryMs", event.target.value) + } + /> + + + + update("autoMemoryRetryMaxMs", event.target.value) + } + /> + value === undefined) @@ -676,6 +721,8 @@ function parseDraft(draft: Draft): ReMeSettings | undefined { requestTimeoutMs: requestTimeoutMs!, backgroundTimeoutMs: backgroundTimeoutMs!, shutdownTimeoutMs: shutdownTimeoutMs!, + autoMemoryRetryMs: autoMemoryRetryMs!, + autoMemoryRetryMaxMs: autoMemoryRetryMaxMs!, autoMemoryEnabled: draft.autoMemoryEnabled, autoMemoryInterval: autoMemoryInterval!, autoDreamEnabled: draft.autoDreamEnabled, @@ -699,15 +746,6 @@ function boundedInteger( : undefined; } -function validEndpoint(value: string): boolean { - try { - const url = new URL(value.trim()); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - function validTimezone(value: string): boolean { try { new Intl.DateTimeFormat("en", { timeZone: value.trim() }).format(0); diff --git a/integrations/dsh/src/config.ts b/integrations/dsh/src/config.ts index ce4421bc..e517814c 100644 --- a/integrations/dsh/src/config.ts +++ b/integrations/dsh/src/config.ts @@ -1,4 +1,7 @@ import z from "@deepseek-ai/schemastery"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { validEndpoint } from "./endpoint.js"; import { validTimezone, validateDailyCron } from "./scheduling.js"; import type { ReMeConfig, ReMeConfigInput } from "./types.js"; @@ -26,6 +29,17 @@ export const Config = z.object({ shutdownTimeoutMs: z.natural().min(100).max(60000).default(5000).volatile(), autoMemoryEnabled: z.boolean().default(true).volatile(), autoMemoryInterval: z.natural().min(1).max(1000).default(5).volatile(), + autoMemoryQueueDir: z + .string() + .default("") + .description("Private persistent automatic-memory queue directory"), + autoMemoryRetryMs: z.natural().min(1000).max(60000).default(1000).volatile(), + autoMemoryRetryMaxMs: z + .natural() + .min(1000) + .max(3600000) + .default(60000) + .volatile(), autoDreamEnabled: z.boolean().default(true).volatile(), dreamCron: checkedString( DEFAULT_DREAM_CRON, @@ -77,6 +91,9 @@ const DEFAULT_CONFIG: Readonly = Object.freeze({ shutdownTimeoutMs: 5000, autoMemoryEnabled: true, autoMemoryInterval: 5, + autoMemoryQueueDir: "", + autoMemoryRetryMs: 1000, + autoMemoryRetryMaxMs: 60000, autoDreamEnabled: true, dreamCron: "0 23 * * *", dreamHint: "", @@ -121,6 +138,23 @@ export function resolveConfig( config.endpoint = stripTrailingSlashes(String(config.endpoint)); assertEndpoint(config.endpoint); + const home = env.DSH_HOME?.trim() || `${homedir()}/.dsh`; + const queueDir = config.autoMemoryQueueDir || `${home}/reme-memory/outbox`; + config.autoMemoryQueueDir = resolve( + queueDir.replace(/^~(?=\/|$)/, homedir()), + ); + config.autoMemoryRetryMs = integer( + config.autoMemoryRetryMs, + 1000, + 60000, + 1000, + ); + config.autoMemoryRetryMaxMs = integer( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs, + 3600000, + 60000, + ); config.requestTimeoutMs = integer( config.requestTimeoutMs, 1000, @@ -163,15 +197,10 @@ export function resolveConfig( } function assertEndpoint(value: string): void { - let endpoint: URL; - try { - endpoint = new URL(value); - } catch { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } - if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } + if (!validEndpoint(value)) + throw new TypeError( + "ReMe endpoint must be an absolute http(s) URL without credentials, query, or fragment", + ); } function stripTrailingSlashes(value: string): string { diff --git a/integrations/dsh/src/endpoint.ts b/integrations/dsh/src/endpoint.ts new file mode 100644 index 00000000..d50eb5f9 --- /dev/null +++ b/integrations/dsh/src/endpoint.ts @@ -0,0 +1,23 @@ +/** HTTP endpoints exclude credentials, query strings, and fragments. */ +export function validEndpoint(value: string): boolean { + try { + const url = new URL(value.trim()); + return ( + (url.protocol === "http:" || url.protocol === "https:") && + !url.username && + !url.password && + !url.search && + !url.hash && + !/[?#]/.test(value) + ); + } catch { + return false; + } +} + +/** Remove URL userinfo before errors cross model, browser, or log boundaries. */ +export function safeError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + // URL parsing encodes spaces and strips tabs/newlines in userinfo; do not treat them as boundaries. + return message.replace(/(https?:\/\/)[^/?#\\]+@/gi, "$1[redacted]@"); +} diff --git a/integrations/dsh/src/index.ts b/integrations/dsh/src/index.ts index fbdc3640..8a14eebd 100644 --- a/integrations/dsh/src/index.ts +++ b/integrations/dsh/src/index.ts @@ -20,6 +20,10 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { const current = () => config; const client = new ReMeClient(current); const runtime = new ReMeRuntime(client, current, ctx.logger); + ctx.effect(() => { + runtime.start(); + return () => runtime.disposeAll(); + }, "remeMemory.lifecycle()"); ctx.provide("remeMemory", runtime); void ctx.plugin(ReMeStatusGateway); ctx.effect( @@ -27,11 +31,6 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { "remeMemory.tools()", ); - ctx.effect(() => { - runtime.start(); - return () => runtime.disposeAll(); - }, "remeMemory.lifecycle()"); - ctx.on("loader/volatile-update", () => { config = resolveConfig(input); runtime.reconfigure(); diff --git a/integrations/dsh/src/outbox.ts b/integrations/dsh/src/outbox.ts new file mode 100644 index 00000000..1bc9e43c --- /dev/null +++ b/integrations/dsh/src/outbox.ts @@ -0,0 +1,244 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + opendirSync, + readFileSync, + renameSync, + rmdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import type { ReMeMessage } from "./reme/types.js"; +import { validEndpoint } from "./endpoint.js"; + +export interface MemoryBatch { + version: 1; + id: string; + createdAt: number; + endpoint: string; + sessionId: string; + date: string; + turns: ReMeMessage[][]; +} + +/** Private JSON outbox: seal batches before sending and remove only after acknowledgement. */ +export class MemoryOutbox { + private readonly lock: string; + private readonly owner = `${process.pid}:${randomUUID()}`; + private latestCreatedAt = 0; + + constructor(readonly directory: string) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + this.lock = join(directory, ".lock"); + // Serialize inspection, stale-owner removal, and replacement as one operation. + // An interrupted recovery fails closed; never reclaim this guard automatically. + const guard = join(directory, ".lock-recovery"); + try { + mkdirSync(guard, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + throw new Error( + "ReMe outbox lock recovery is busy or interrupted; retry, or inspect .lock-recovery after stopping all Hosts", + ); + } + try { + if (existsSync(this.lock)) { + const pid = Number(readFileSync(this.lock, "utf8").split(":")[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) + throw new Error( + "Invalid ReMe outbox lock; inspect the queue directory", + ); + try { + process.kill(pid, 0); + throw new Error( + "ReMe outbox is in use; configure a separate autoMemoryQueueDir for each Host", + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + unlinkSync(this.lock); + } + const fd = openSync(this.lock, "wx", 0o600); + try { + writeFileSync(fd, this.owner); + } finally { + closeSync(fd); + } + for (const name of this.files()) + this.latestCreatedAt = Math.max( + this.latestCreatedAt, + this.read(name).createdAt, + ); + } catch (error) { + this.close(); + throw error; + } finally { + rmdirSync(guard); + } + } + + append( + endpoint: string, + sessionId: string, + date: string, + messages: ReMeMessage[], + interval: number, + ): void { + const key = createHash("sha256") + .update(JSON.stringify([endpoint, sessionId, date])) + .digest("hex"); + const name = `partial-${key}.json`; + const path = join(this.directory, name); + const batch: MemoryBatch = existsSync(path) + ? this.read(name) + : { + version: 1, + id: randomUUID(), + // Keep FIFO ordering when the clock repeats or moves backwards, including after restart. + createdAt: (this.latestCreatedAt = Math.max( + Date.now(), + this.latestCreatedAt + 1, + )), + endpoint, + sessionId, + date, + turns: [], + }; + batch.turns.push(messages); + this.write(name, batch); + if (batch.turns.length >= interval) this.seal(name, batch); + // Finish the previous workspace day without combining dates in one request. + for (const file of this.files()) { + if (!file.startsWith("partial-") || file === name) continue; + const previous = this.read(file); + if ( + previous.endpoint === endpoint && + previous.sessionId === sessionId && + previous.date !== date + ) + this.seal(file, previous); + } + } + + flush(sessionId?: string): void { + for (const name of this.files()) { + if (!name.startsWith("partial-")) continue; + const batch = this.read(name); + if (sessionId === undefined || batch.sessionId === sessionId) + this.seal(name, batch); + } + } + + next( + accepts: (batch: MemoryBatch) => boolean = () => true, + ): { name: string; batch: MemoryBatch } | undefined { + let oldest: string | undefined; + for (const name of this.files()) { + if ( + name.startsWith("ready-") && + (oldest === undefined || name < oldest) && + accepts(this.read(name)) + ) + oldest = name; + } + return oldest === undefined + ? undefined + : { name: oldest, batch: this.read(oldest) }; + } + + acknowledge(name: string): void { + unlinkSync(join(this.directory, name)); + } + + stats(): { turns: number; batches: number } { + let turns = 0; + let batches = 0; + for (const name of this.files()) { + turns += this.read(name).turns.length; + if (name.startsWith("ready-")) batches += 1; + } + return { turns, batches }; + } + + close(): void { + if (existsSync(this.lock) && readFileSync(this.lock, "utf8") === this.owner) + unlinkSync(this.lock); + } + + private *files(): Generator { + const dir = opendirSync(this.directory); + try { + for (let entry = dir.readSync(); entry; entry = dir.readSync()) + if (entry.isFile() && /^(partial|ready)-.*\.json$/.test(entry.name)) + yield entry.name; + } finally { + dir.closeSync(); + } + } + + private read(name: string): MemoryBatch { + try { + const batch = JSON.parse( + readFileSync(join(this.directory, name), "utf8"), + ) as MemoryBatch; + if ( + batch.version !== 1 || + typeof batch.id !== "string" || + !/^[a-f0-9-]{36}$/.test(batch.id) || + !validEndpoint(batch.endpoint) || + typeof batch.sessionId !== "string" || + typeof batch.date !== "string" || + !Number.isFinite(batch.createdAt) || + !Array.isArray(batch.turns) || + !batch.turns.length || + !batch.turns.every( + (turn) => + Array.isArray(turn) && + turn.every( + (message) => + typeof message.id === "string" && + ["user", "assistant"].includes(message.role) && + Array.isArray(message.content) && + message.content.every( + (part) => + part.type === "text" && typeof part.text === "string", + ), + ), + ) + ) + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + return batch; + } catch { + // JSON parser errors can quote message content; leave the record untouched. + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + } + } + + private write(name: string, batch: MemoryBatch): void { + const temporary = join(this.directory, `${randomUUID()}.tmp`); + const fd = openSync(temporary, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify(batch)); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, join(this.directory, name)); + } + + private seal(name: string, batch: MemoryBatch): void { + renameSync( + join(this.directory, name), + join(this.directory, `ready-${batch.createdAt}-${batch.id}.json`), + ); + } +} diff --git a/integrations/dsh/src/reme/client.ts b/integrations/dsh/src/reme/client.ts index ff0bccbc..10d9050c 100644 --- a/integrations/dsh/src/reme/client.ts +++ b/integrations/dsh/src/reme/client.ts @@ -13,6 +13,8 @@ import type { SearchOptions, } from "./types.js"; +import { safeError, validEndpoint } from "../endpoint.js"; + interface ReMeResponseBody { success?: boolean; answer?: unknown; @@ -50,7 +52,10 @@ export class ReMeClient { sessionId: string, options: AutoMemoryOptions = {}, ): Promise { - const config = this.configSource(); + const config = { + ...this.configSource(), + ...(options.endpoint ? { endpoint: options.endpoint } : {}), + }; return this.request( "auto_memory", { @@ -58,6 +63,7 @@ export class ReMeClient { session_id: sessionId, memory_hint: options.memoryHint || "", date: options.date || "", + ...(options.requestId ? { request_id: options.requestId } : {}), }, config, config.backgroundTimeoutMs, @@ -189,6 +195,10 @@ export class ReMeClient { ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal; try { + if (!validEndpoint(config.endpoint)) + throw new TypeError( + "Invalid ReMe endpoint: credentials, query, and fragment are unsupported", + ); const response = await fetch(`${config.endpoint}/${job}`, { method: "POST", headers: { @@ -208,7 +218,7 @@ export class ReMeClient { metadata: body.metadata ?? {}, error: ok ? "" - : String(body.answer || body.detail || `HTTP ${response.status}`), + : safeError(body.answer || body.detail || `HTTP ${response.status}`), }; } catch (error) { return { @@ -216,7 +226,7 @@ export class ReMeClient { status: 0, answer: "", metadata: {}, - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }; } finally { clearTimeout(timer); diff --git a/integrations/dsh/src/reme/types.ts b/integrations/dsh/src/reme/types.ts index a6864637..cf32dcb2 100644 --- a/integrations/dsh/src/reme/types.ts +++ b/integrations/dsh/src/reme/types.ts @@ -95,6 +95,8 @@ export interface SearchOptions { /** Automatic-memory request controls supported by the plugin client. */ export interface AutoMemoryOptions { + endpoint?: string; + requestId?: string; date?: string; memoryHint?: string; signal?: AbortSignal; diff --git a/integrations/dsh/src/runtime.ts b/integrations/dsh/src/runtime.ts index 407ec3f4..986a86f7 100644 --- a/integrations/dsh/src/runtime.ts +++ b/integrations/dsh/src/runtime.ts @@ -1,35 +1,37 @@ import type { LoggerLike, ReMeClientLike, ReMeMessage } from "./reme/types.js"; - import { messagesDay, nextDailyRun } from "./scheduling.js"; import { captureMessage, remeSessionId } from "./messages.js"; +import { MemoryOutbox, type MemoryBatch } from "./outbox.js"; +import { safeError } from "./endpoint.js"; import type { DshSession, ReMeConfig, SessionEvent } from "./types.js"; import type { ReMeRuntimeSnapshot, ReMeRuntimeTask } from "./runtime-status.js"; -interface PendingTurn { - messages: ReMeMessage[]; - day: string; -} - -interface SessionState { - session: DshSession; - sessionId: string; - activeTurn: unknown; - activeMessages: ReMeMessage[]; - pendingTurns: PendingTurn[]; - unconfirmedTurns: number; - writes: Promise; - requestController: AbortController; +interface MemoryWorker { + controller: AbortController; + task: Promise; + active?: ReMeRuntimeTask; } export class ReMeRuntime { - readonly states = new Map(); + private readonly states = new Map(); private readonly configSource: () => ReMeConfig; + private readonly outbox: MemoryOutbox; + private memoryTask: Promise | null = null; + private readonly memoryWorkers = new Map(); + private readonly retryTimers = new Map< + string, + ReturnType + >(); + private readonly failures = new Map(); + private memoryError: string | undefined; + private captureError: string | undefined; private dreamTimer: ReturnType | null = null; private dreamScheduleGeneration = 0; private dreamTask: Promise | null = null; private dreamController: AbortController | null = null; private started = false; private stopping = false; + private closed = false; private taskSequence = 0; private readonly recentTasks: ReMeRuntimeTask[] = []; private nextDreamAt: string | undefined; @@ -44,148 +46,232 @@ export class ReMeRuntime { readonly logger: LoggerLike = console, ) { this.configSource = typeof config === "function" ? config : () => config; - } - - stateFor(session: DshSession): SessionState { - const existing = this.states.get(session.id); - if (existing) { - existing.session = session; - if (existing.requestController.signal.aborted) - existing.requestController = new AbortController(); - return existing; - } - const state: SessionState = { - session, - sessionId: remeSessionId(session.id), - activeTurn: null, - activeMessages: [], - pendingTurns: [], - unconfirmedTurns: 0, - writes: Promise.resolve(), - requestController: new AbortController(), - }; - this.states.set(session.id, state); - return state; + this.outbox = new MemoryOutbox(this.configSource().autoMemoryQueueDir); } capture(session: DshSession, event: SessionEvent): void { const config = this.configSource(); - if (!config.autoMemoryEnabled) return; - const state = this.stateFor(session); - const data = isRecord(event.data) ? event.data : undefined; + if (this.stopping || !config.autoMemoryEnabled) return; if (event.type === "turn/start") { - state.activeTurn = data?.turn ?? null; - state.activeMessages = []; + this.states.set(session.id, []); return; } + const messages = this.states.get(session.id); + if (!messages) return; const message = captureMessage(event, session.id); - if (message) state.activeMessages.push(message); + if (message) messages.push(message); if (event.type !== "turn/end") return; - - const reason = data?.reason; - const reasonKind = isRecord(reason) ? reason.kind : undefined; - const completed = reasonKind === "completed" || reasonKind === "max-tokens"; - const hasUser = state.activeMessages.some((item) => item.role === "user"); - const hasAssistant = state.activeMessages.some( - (item) => item.role === "assistant", - ); - if (completed && hasUser && hasAssistant) { - const day = messagesDay(state.activeMessages, config.timezone); - const previousDay = state.pendingTurns.at(-1)?.day; - if (previousDay && day && previousDay !== day) - this.scheduleAutoMemory(state, true); - state.pendingTurns.push({ messages: state.activeMessages, day }); + const data = isRecord(event.data) ? event.data : undefined; + const reason = isRecord(data?.reason) ? data.reason.kind : undefined; + if ( + (reason === "completed" || reason === "max-tokens") && + messages.some((item) => item.role === "user") && + messages.some((item) => item.role === "assistant") + ) { + try { + this.outbox.append( + config.endpoint, + remeSessionId(session.id), + messagesDay(messages, config.timezone), + messages, + config.autoMemoryInterval, + ); + } catch (error) { + this.captureError = safeError(error); + this.recordMemoryError(error); + } } - state.activeTurn = null; - state.activeMessages = []; - this.scheduleAutoMemory(state); + this.states.set(session.id, []); + void this.drain(); } - private scheduleAutoMemory(state: SessionState, force = false): void { - const interval = this.configSource().autoMemoryInterval; - const firstDay = state.pendingTurns[0]?.day; - const dayCount = state.pendingTurns.findIndex((turn) => - Boolean(firstDay && turn.day && turn.day !== firstDay), - ); - const available = dayCount === -1 ? state.pendingTurns.length : dayCount; - const crossesDayBoundary = dayCount !== -1; - if (!force && !crossesDayBoundary && available < interval) return; - const count = force || crossesDayBoundary ? available : interval; - if (count === 0) return; - const turns = state.pendingTurns.splice(0, count); - const messages = turns.flatMap((turn) => turn.messages); - const date = turns[0]?.day || ""; - const task: ReMeRuntimeTask = { - id: `auto-memory-${++this.taskSequence}`, - phase: "queued", - queuedAt: new Date().toISOString(), - turns: turns.length, - messages: messages.length, - }; - this.recentTasks.unshift(task); - this.recentTasks.length = Math.min(this.recentTasks.length, 20); - state.unconfirmedTurns += turns.length; - state.writes = state.writes.then(async () => { - task.phase = "running"; - task.startedAt = new Date().toISOString(); - try { - const result = await this.client.autoMemory(messages, state.sessionId, { - date, - signal: state.requestController.signal, - }); - if (result.ok) { - task.phase = "completed"; - task.result = resultSummary(result.answer, result.metadata); - this.log("debug", "auto_memory_complete", { - sessionId: state.sessionId, - turns: turns.length, + /** Drain each endpoint independently; failed batches stay on disk for retry. */ + drain(): Promise { + if (this.closed) return Promise.resolve(); + if (this.retryTimers.has("*") || !this.configSource().autoMemoryEnabled) + return this.memoryTask ?? Promise.resolve(); + try { + for (;;) { + const next = this.outbox.next( + (batch) => + !this.memoryWorkers.has(batch.endpoint) && !this.blocked(batch), + ); + if (!next) break; + const { endpoint } = next.batch; + const worker: MemoryWorker = { + controller: new AbortController(), + task: Promise.resolve(), + }; + this.memoryWorkers.set(endpoint, worker); + worker.task = Promise.resolve() + .then(() => this.drainEndpoint(endpoint, worker)) + .catch((error: unknown) => { + this.recordMemoryError(error); + this.scheduleRetry(); + }) + .finally(() => { + this.memoryWorkers.delete(endpoint); + // A capture may have arrived after this worker observed an empty queue. + if (!this.stopping) void this.drain(); }); + } + } catch (error) { + this.recordMemoryError(error); + this.scheduleRetry(); + } + if (this.memoryTask) return this.memoryTask; + if (!this.memoryWorkers.size) return Promise.resolve(); + // Share one completion promise instead of accumulating waiters on stalled requests. + this.memoryTask = Promise.resolve() + .then(async () => { + while (this.memoryWorkers.size) + await Promise.all( + [...this.memoryWorkers.values()].map((worker) => worker.task), + ); + }) + .finally(() => { + this.memoryTask = null; + if (!this.stopping) return this.drain(); + }); + return this.memoryTask; + } + + private blocked(batch: MemoryBatch): boolean { + return ( + this.retryTimers.has(endpointKey(batch.endpoint)) || + this.retryTimers.has(sessionKey(batch)) + ); + } + + private async drainEndpoint( + endpoint: string, + worker: MemoryWorker, + ): Promise { + const { signal } = worker.controller; + while ( + !signal.aborted && + !this.closed && + this.configSource().autoMemoryEnabled && + !this.retryTimers.has("*") + ) { + const next = this.outbox.next( + (batch) => batch.endpoint === endpoint && !this.blocked(batch), + ); + if (!next) return; + const { name, batch } = next; + const messages = batch.turns.flat(); + const task: ReMeRuntimeTask = { + id: `${batch.id}-${++this.taskSequence}`, + phase: "running", + queuedAt: new Date(batch.createdAt).toISOString(), + startedAt: new Date().toISOString(), + turns: batch.turns.length, + messages: messages.length, + }; + worker.active = task; + this.recentTasks.unshift(task); + this.recentTasks.length = Math.min(this.recentTasks.length, 20); + let retryKey = endpointKey(endpoint); + try { + const result = await this.client.autoMemory(messages, batch.sessionId, { + date: batch.date, + endpoint, + requestId: batch.id, + signal, + }); + if (signal.aborted) { + task.phase = "cancelled"; return; } - state.pendingTurns.unshift(...turns); - task.phase = "failed"; - task.error = - result.error || "ReMe rejected the automatic-memory request"; - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: result.error, - }); + if (!result.ok) { + // Payload/job errors affect this session; network, auth and service errors affect the endpoint. + if (requestFailure(result.status)) retryKey = sessionKey(batch); + throw new Error( + `ReMe automatic-memory request failed (HTTP ${result.status ?? 0})`, + ); + } + this.outbox.acknowledge(name); + task.phase = "completed"; + // Reply text and file names can contain personal information. + this.failures.delete(endpointKey(endpoint)); + this.failures.delete(sessionKey(batch)); + if (!this.retryTimers.size) this.memoryError = undefined; } catch (error) { - state.pendingTurns.unshift(...turns); - task.phase = state.requestController.signal.aborted - ? "cancelled" - : "failed"; - task.error = error instanceof Error ? error.message : String(error); - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: error instanceof Error ? error.message : String(error), - }); + task.phase = signal.aborted ? "cancelled" : "failed"; + task.error = safeError(error); + this.recordMemoryError(error); + if (this.stopping || signal.aborted) return; + this.scheduleRetry(retryKey); } finally { task.finishedAt = new Date().toISOString(); - state.unconfirmedTurns -= turns.length; + worker.active = undefined; } - }); + } + } + + private recordMemoryError(error: unknown): void { + this.memoryError = safeError(error); + this.log("warn", "auto_memory_failed", { error: this.memoryError }); + } + + private scheduleRetry(key = "*"): void { + if ( + this.stopping || + this.retryTimers.has(key) || + !this.configSource().autoMemoryEnabled + ) + return; + const config = this.configSource(); + const failures = this.failures.get(key) ?? 0; + this.failures.set(key, failures + 1); + const delay = Math.min( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs * 2 ** Math.min(failures, 20), + ); + const timer = setTimeout(() => { + this.retryTimers.delete(key); + void this.drain(); + }, delay); + timer.unref?.(); + this.retryTimers.set(key, timer); + } + + private clearRetries(): void { + for (const timer of this.retryTimers.values()) clearTimeout(timer); + this.retryTimers.clear(); + this.failures.clear(); } start(): void { + if (this.started || this.stopping) return; this.started = true; - if (!this.configSource().autoDreamEnabled || this.stopping) return; - this.scheduleDream(this.dreamScheduleGeneration); + // A previous Host may have stopped before the batching interval was reached. + try { + this.outbox.flush(); + } catch (error) { + this.outbox.close(); + throw error; + } + void this.drain(); + if (this.configSource().autoDreamEnabled) + this.scheduleDream(this.dreamScheduleGeneration); } - /** Apply a changed settings snapshot to pending batching and dream scheduling. */ + /** Apply settings without discarding captured work or replacing the runtime. */ reconfigure(): void { if (this.stopping) return; - const config = this.configSource(); - if (config.autoMemoryEnabled) { - for (const state of this.states.values()) this.scheduleAutoMemory(state); + this.clearRetries(); + if (!this.configSource().autoMemoryEnabled) this.states.clear(); + if (this.configSource().autoMemoryEnabled) { + this.outbox.flush(); + void this.drain(); } if (!this.started) return; this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); this.dreamTimer = null; this.nextDreamAt = undefined; - if (config.autoDreamEnabled) + if (this.configSource().autoDreamEnabled) this.scheduleDream(this.dreamScheduleGeneration); } @@ -206,7 +292,7 @@ export class ReMeRuntime { Date.now(); } catch (error) { this.log("warn", "auto_dream_schedule_invalid", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); return; } @@ -220,6 +306,7 @@ export class ReMeRuntime { } async runDream(): Promise { + if (this.stopping) return; if (this.dreamTask) return this.dreamTask; const config = this.configSource(); this.dreamController = new AbortController(); @@ -237,21 +324,20 @@ export class ReMeRuntime { result.ok ? "debug" : "warn", result.ok ? "auto_dream_complete" : "auto_dream_failed", { - error: result.ok ? undefined : result.error, + error: result.ok ? undefined : safeError(result.error), }, ); this.dreamLastResult = result.ok ? "completed" : "failed"; this.dreamLastError = result.ok ? undefined - : result.error || "ReMe rejected the Auto Dream request"; + : safeError(result.error || "ReMe rejected the Auto Dream request"); } catch (error) { this.dreamLastResult = this.dreamController?.signal.aborted ? "cancelled" : "failed"; - this.dreamLastError = - error instanceof Error ? error.message : String(error); + this.dreamLastError = safeError(error); this.log("warn", "auto_dream_failed", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); } })().finally(() => { @@ -263,84 +349,66 @@ export class ReMeRuntime { } async dispose(session: DshSession): Promise { - const state = this.states.get(session.id); - if (!state) return; - if (state.requestController.signal.aborted) - state.requestController = new AbortController(); - const flush = (async () => { - await state.writes; - const retryCount = state.pendingTurns.length; - let scheduled = 0; - while (state.pendingTurns.length && scheduled < retryCount) { - const before = state.pendingTurns.length; - this.scheduleAutoMemory(state, true); - scheduled += before - state.pendingTurns.length; - } - await state.writes; - })(); - const completed = await this.withinShutdownBudget(flush, () => - state.requestController.abort(), - ); - const unsentTurns = state.pendingTurns.length + state.unconfirmedTurns; - if (unsentTurns) { - this.log( - "warn", - completed ? "auto_memory_retained" : "auto_memory_shutdown_timeout", - { - sessionId: state.sessionId, - unsentTurns, - }, - ); - } else { - this.states.delete(session.id); - } + this.states.delete(session.id); + this.outbox.flush(remeSessionId(session.id)); + void this.drain(); } async disposeAll(): Promise { + if (this.stopping) return; this.stopping = true; this.started = false; + this.states.clear(); this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); + this.clearRetries(); this.dreamTimer = null; this.nextDreamAt = undefined; this.dreamController?.abort(); - const shutdown = Promise.all([ - ...[...this.states.values()].map((state) => this.dispose(state.session)), - ...(this.dreamTask ? [this.dreamTask] : []), - ]).then(() => undefined); - await this.withinShutdownBudget(shutdown, () => { - this.dreamController?.abort(); - for (const state of this.states.values()) state.requestController.abort(); - }); + try { + this.outbox.flush(); + await this.withinShutdownBudget( + Promise.all([this.drain(), this.dreamTask]).then(() => undefined), + () => { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.dreamController?.abort(); + }, + ); + } finally { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.outbox.close(); + this.closed = true; + } } - /** Return a content-free snapshot for the local DSH status page. */ + /** Return queue counts and bounded diagnostics without reply content. */ snapshot(): ReMeRuntimeSnapshot { const config = this.configSource(); - const tasksRunning = this.recentTasks.filter( - (task) => task.phase === "running", + const queued = this.outbox.stats(); + const running = [...this.memoryWorkers.values()].filter( + (worker) => worker.active !== undefined, ).length; - const tasksQueued = this.recentTasks.filter( - (task) => task.phase === "queued", - ).length; - const lastError = this.recentTasks.find( - (task) => task.phase === "failed" || task.phase === "cancelled", - )?.error; return { - phase: this.stopping ? "stopping" : this.started ? "running" : "stopped", + phase: this.closed + ? "stopped" + : this.stopping + ? "stopping" + : this.started + ? "running" + : "stopped", autoMemory: { enabled: config.autoMemoryEnabled, interval: config.autoMemoryInterval, activeSessions: this.states.size, - queuedTurns: [...this.states.values()].reduce( - (total, state) => - total + state.pendingTurns.length + state.unconfirmedTurns, - 0, - ), - tasksRunning, - tasksQueued, + queuedTurns: queued.turns, + tasksRunning: running, + tasksQueued: Math.max(0, queued.batches - running), recentTasks: this.recentTasks.map((task) => ({ ...task })), - ...(lastError ? { lastError } : {}), + ...(this.captureError || this.memoryError + ? { lastError: this.captureError || this.memoryError } + : {}), }, autoDream: { enabled: config.autoDreamEnabled, @@ -363,16 +431,18 @@ export class ReMeRuntime { private async withinShutdownBudget( task: Promise, abort: () => void, - ): Promise { + ): Promise { let timer: ReturnType | undefined; - const timeout = new Promise((resolve) => { - timer = setTimeout(() => { - abort(); - resolve(false); - }, this.configSource().shutdownTimeoutMs); - }); try { - return await Promise.race([task.then(() => true), timeout]); + await Promise.race([ + task, + new Promise((resolve) => { + timer = setTimeout(() => { + abort(); + resolve(); + }, this.configSource().shutdownTimeoutMs); + }), + ]); } finally { if (timer) clearTimeout(timer); } @@ -388,18 +458,23 @@ export class ReMeRuntime { } } -function resultSummary( - answer: unknown, - metadata: Record | undefined, -): string | undefined { - const path = metadata?.path; - if (typeof path === "string" && path.length > 0) return path; - if (typeof answer !== "string") return undefined; - const normalized = answer.trim(); - if (!normalized) return undefined; - return normalized.length > 240 ? `${normalized.slice(0, 237)}…` : normalized; -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } + +function endpointKey(endpoint: string): string { + return `endpoint:${endpoint}`; +} + +function sessionKey(batch: MemoryBatch): string { + return `session:${JSON.stringify([batch.endpoint, batch.sessionId])}`; +} + +function requestFailure(status = 0): boolean { + return ( + (status >= 200 && status < 300) || + (status >= 400 && + status < 500 && + ![401, 403, 404, 405, 408, 429].includes(status)) + ); +} diff --git a/integrations/dsh/src/types.ts b/integrations/dsh/src/types.ts index ea38c35a..13e870e4 100644 --- a/integrations/dsh/src/types.ts +++ b/integrations/dsh/src/types.ts @@ -7,6 +7,9 @@ export interface ReMeConfigInput { shutdownTimeoutMs?: number; autoMemoryEnabled?: boolean; autoMemoryInterval?: number; + autoMemoryQueueDir?: string; + autoMemoryRetryMs?: number; + autoMemoryRetryMaxMs?: number; autoDreamEnabled?: boolean; dreamCron?: string; dreamHint?: string; @@ -21,6 +24,9 @@ export interface ReMeConfig extends ReMeClientConfig { shutdownTimeoutMs: number; autoMemoryEnabled: boolean; autoMemoryInterval: number; + autoMemoryQueueDir: string; + autoMemoryRetryMs: number; + autoMemoryRetryMaxMs: number; autoDreamEnabled: boolean; dreamCron: string; dreamHint: string; @@ -32,7 +38,10 @@ export interface ReMeConfig extends ReMeClientConfig { } /** ReMe integration fields owned by the DSH user-settings document. */ -export type ReMeSettings = Omit; +export type ReMeSettings = Omit< + ReMeConfig, + "dreamIntervalMs" | "autoMemoryQueueDir" +>; export interface SessionEvent { type: string; diff --git a/integrations/dsh/tests/bundle.test.mjs b/integrations/dsh/tests/bundle.test.mjs index c7b3f2a9..39bd23ec 100644 --- a/integrations/dsh/tests/bundle.test.mjs +++ b/integrations/dsh/tests/bundle.test.mjs @@ -30,6 +30,7 @@ test("declares one installable DeepSeek Harness plugin", async () => { "0.2.0-rc.2", "0.2.1-alpha.2", "0.2.1-alpha.4", + "0.2.1-rc.2", ]) { assert.ok( semver.satisfies(version, manifest.peerDependencies[dependency]), diff --git a/integrations/dsh/tests/client-settings.test.mjs b/integrations/dsh/tests/client-settings.test.mjs index 93db8c85..d8406ec5 100644 --- a/integrations/dsh/tests/client-settings.test.mjs +++ b/integrations/dsh/tests/client-settings.test.mjs @@ -168,7 +168,7 @@ test("resets overrides in one mutation only after Save", async () => { await ui.button("save").props.onClick(); }); assert.equal(ui.calls.length, 1); - assert.equal(ui.calls[0].ops.length, 13); + assert.equal(ui.calls[0].ops.length, 15); assert.ok(ui.calls[0].ops.every((op) => op.op === "unset")); assert.equal(ui.calls[0].revision, 7); } finally { @@ -191,3 +191,30 @@ test("read-only configuration cannot be saved", async () => { ui.close(); } }); + +for (const endpoint of [ + "http://alice:SECRET@localhost:2333", + "http://localhost:2333?token=SECRET", + "http://localhost:2333#search", + "http://localhost:2333?", + "http://localhost:2333#", +]) { + test( + "rejects unsupported endpoint components before saving: " + + new URL(endpoint).hash + + new URL(endpoint).search, + async () => { + const ui = await editor(async () => true); + try { + ui.change(0, endpoint); + assert.equal(ui.button("save").props.disabled, true); + await act(async () => { + await ui.button("save").props.onClick(); + }); + assert.equal(ui.calls.length, 0); + } finally { + ui.close(); + } + }, + ); +} diff --git a/integrations/dsh/tests/client.test.mjs b/integrations/dsh/tests/client.test.mjs index 40424ab0..21ed3d3c 100644 --- a/integrations/dsh/tests/client.test.mjs +++ b/integrations/dsh/tests/client.test.mjs @@ -1,6 +1,104 @@ import assert from "node:assert/strict"; import test from "node:test"; import { ReMeClient } from "../dist/reme/client.js"; +import { safeError } from "../dist/endpoint.js"; + +test("redacts URL credentials containing spaces or stripped URL whitespace", async () => { + const originalFetch = globalThis.fetch; + const client = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + try { + for (const password of [ + "SECRET CANARY", + "SECRET\tCANARY", + "SECRET\nCANARY", + ]) { + const url = `http://alice:${password}@localhost:2333/search`; + assert.ok(new URL(url).password); + const expected = "refused http://[redacted]@localhost:2333/search"; + globalThis.fetch = async () => + Response.json({ success: false, answer: `refused ${url}` }); + assert.equal((await client.search("fact")).error, expected); + globalThis.fetch = async () => { + throw new Error(`refused ${url}`); + }; + assert.equal((await client.search("fact")).error, expected); + } + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("rejects credentials before fetch and redacts URL credentials from service and transport errors", async () => { + const originalFetch = globalThis.fetch; + let calls = 0; + const secretUrl = "http://alice:SECRET-CANARY@localhost:2333/search"; + globalThis.fetch = async () => { + calls++; + return Response.json({ success: false, answer: `refused ${secretUrl}` }); + }; + try { + const client = new ReMeClient({ + endpoint: secretUrl, + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await client.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.equal(calls, 0); + const valid = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + globalThis.fetch = async () => { + throw new Error(`failed ${secretUrl}`); + }; + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.doesNotMatch( + safeError(new Error(`failed ${secretUrl}`)), + /alice|SECRET-CANARY/, + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("uses the persisted destination and request ID when sending a recovered batch", async () => { + const originalFetch = globalThis.fetch; + let request; + globalThis.fetch = async (url, init) => { + request = { url, body: JSON.parse(init.body) }; + return Response.json({ success: true }); + }; + try { + const client = new ReMeClient({ + endpoint: "http://new.test", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + await client.autoMemory([], "session", { + endpoint: "http://original.test", + requestId: "persisted-batch", + }); + assert.equal(request.url, "http://original.test/auto_memory"); + assert.equal(request.body.request_id, "persisted-batch"); + } finally { + globalThis.fetch = originalFetch; + } +}); test("calls ReMe jobs with their native request and response envelopes", async () => { const calls = []; diff --git a/integrations/dsh/tests/config.test.mjs b/integrations/dsh/tests/config.test.mjs index 8e2613eb..ce03debd 100644 --- a/integrations/dsh/tests/config.test.mjs +++ b/integrations/dsh/tests/config.test.mjs @@ -29,6 +29,11 @@ test("exports a Cordis schema that rejects invalid configuration", async () => { for (const input of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:bad-port" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://[:::]" }, @@ -142,6 +147,11 @@ test("Loader keeps the previous live values when an update fails schema validati await entry.fiber.await(); for (const patch of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://999.0.0.1" }, { dreamCron: "every night" }, diff --git a/integrations/dsh/tests/dsh-index.test.mjs b/integrations/dsh/tests/dsh-index.test.mjs index 7a346710..f19b3476 100644 --- a/integrations/dsh/tests/dsh-index.test.mjs +++ b/integrations/dsh/tests/dsh-index.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { apply, Config } from "../dist/index.js"; import { hasGuidance } from "../dist/guidance.js"; @@ -28,7 +31,7 @@ test("recognizes guidance already stored by the previous DSH plugin", () => { ); }); -test("composes root-agent guidance and reme_search on supported DSH releases", async () => { +test("composes root-agent guidance and reme_search on supported DSH releases", async (t) => { const handlers = new Map(); const tools = []; const cleanups = []; @@ -58,7 +61,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a handlers.set(name, handler); }, }; - apply(ctx, { + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false, language: "zh", @@ -97,7 +100,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a await Promise.all(cleanups.map((cleanup) => cleanup())); }); -test("keeps prompt injection and capture out of subagents by default", async () => { +test("keeps prompt injection and capture out of subagents by default", async (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -118,7 +121,7 @@ test("keeps prompt injection and capture out of subagents by default", async () handlers.set(name, handler); }, }; - apply(ctx, { autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoDreamEnabled: false }); let injected = false; handlers.get("agent/created")({ agent: { @@ -138,7 +141,7 @@ test("keeps prompt injection and capture out of subagents by default", async () assert.equal(injected, false); }); -test("reads live DSH configuration for new sessions", async () => { +test("reads live DSH configuration for new sessions", async (t) => { const handlers = new Map(); const ctx = { fiber: { state: 0 }, @@ -165,7 +168,7 @@ test("reads live DSH configuration for new sessions", async () => { language: "zh", }); assert.equal(config.issues, undefined); - apply(ctx, config.value); + applyPlugin(t, ctx, config.value); const injected = []; handlers.get("agent/created")({ agent: { @@ -204,7 +207,7 @@ test("reads live DSH configuration for new sessions", async () => { assert.match(injected[1].content[0].text, /Long-term Memory/); }); -test("session events read a validated configuration snapshot", () => { +test("session events read a validated configuration snapshot", (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -224,7 +227,7 @@ test("session events read a validated configuration snapshot", () => { handlers.set(name, handler); }, }; - apply(ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); const original = Intl.DateTimeFormat.prototype.formatToParts; let scans = 0; Intl.DateTimeFormat.prototype.formatToParts = function (...args) { @@ -243,3 +246,18 @@ test("session events read a validated configuration snapshot", () => { Intl.DateTimeFormat.prototype.formatToParts = original; } }); + +function applyPlugin(t, ctx, input) { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-plugin-")); + let runtime; + const provide = ctx.provide; + ctx.provide = function (name, value) { + if (name === "remeMemory") runtime = value; + return provide.call(this, name, value); + }; + t.after(async () => { + await runtime?.disposeAll(); + rmSync(directory, { recursive: true, force: true }); + }); + return apply(ctx, { ...input, autoMemoryQueueDir: directory }); +} diff --git a/integrations/dsh/tests/dsh-tools.test.mjs b/integrations/dsh/tests/dsh-tools.test.mjs index a21dd4a4..6c61a706 100644 --- a/integrations/dsh/tests/dsh-tools.test.mjs +++ b/integrations/dsh/tests/dsh-tools.test.mjs @@ -1,6 +1,45 @@ import assert from "node:assert/strict"; import test from "node:test"; import { registerReMeTools } from "../dist/tools.js"; +import { ReMeClient } from "../dist/reme/client.js"; + +test("reme_search does not expose whitespace-bearing URL credentials from service errors", async () => { + const originalFetch = globalThis.fetch; + const registered = []; + globalThis.fetch = async () => + Response.json({ + success: false, + answer: "refused http://alice:SECRET CANARY@localhost:2333/search", + }); + try { + registerReMeTools( + { + tools: { + register(tool) { + registered.push(tool); + return () => {}; + }, + }, + }, + new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }), + { searchLimit: 5 }, + ); + const result = await registered[0].execute( + { query: "history" }, + { signal: new AbortController().signal }, + ); + assert.equal( + result, + "ReMe search failed: refused http://[redacted]@localhost:2333/search", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); test("reme_search uses the ReMe search contract and renders model-facing text", async () => { const registered = []; diff --git a/integrations/dsh/tests/outbox.test.mjs b/integrations/dsh/tests/outbox.test.mjs new file mode 100644 index 00000000..9755c123 --- /dev/null +++ b/integrations/dsh/tests/outbox.test.mjs @@ -0,0 +1,236 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { MemoryOutbox } from "../dist/outbox.js"; + +function directory(t) { + const path = mkdtempSync(join(tmpdir(), "reme-outbox-")); + t.after(() => rmSync(path, { recursive: true, force: true })); + return path; +} + +const messages = [ + { + id: "m1", + name: "user", + role: "user", + content: [{ type: "text", text: "private fact" }], + }, +]; + +test("preserves batch order across equal timestamps, clock rollback, and restart", (t) => { + const now = Date.parse("2026-10-21T12:00:00Z"); + t.mock.timers.enable({ apis: ["Date"], now }); + const path = directory(t); + let queue = new MemoryOutbox(path); + const append = (id) => + queue.append( + "http://localhost:2333", + "session", + "2026-10-21", + [{ ...messages[0], id }], + 1, + ); + try { + append("first"); + append("second"); + const batches = readdirSync(path) + .filter((name) => name.startsWith("ready-")) + .map((name) => JSON.parse(readFileSync(join(path, name), "utf8"))); + assert.equal(new Set(batches.map((batch) => batch.createdAt)).size, 2); + queue.close(); + t.mock.timers.setTime(now - 1000); + queue = new MemoryOutbox(path); + append("third"); + const received = []; + for (let next = queue.next(); next; next = queue.next()) { + received.push(next.batch.turns[0][0].id); + queue.acknowledge(next.name); + } + assert.deepEqual(received, ["first", "second", "third"]); + } finally { + queue.close(); + } +}); + +test("recovers a partial batch after the owning process exits without cleanup", (t) => { + const path = directory(t); + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { MemoryOutbox } from ${JSON.stringify( + new URL("../dist/outbox.js", import.meta.url).href, + )}; + const queue = new MemoryOutbox(${JSON.stringify(path)}); + queue.append("http://localhost:2333", "session", "2026-10-21", ${JSON.stringify( + messages, + )}, 5); + `, + ], + { encoding: "utf8" }, + ); + assert.equal(child.status, 0, child.stderr); + const recovered = new MemoryOutbox(path); + try { + recovered.flush(); + assert.deepEqual(recovered.stats(), { turns: 1, batches: 1 }); + const { name, batch } = recovered.next(); + assert.deepEqual(batch.turns, [messages]); + if (process.platform !== "win32") + assert.equal(statSync(join(path, name)).mode & 0o777, 0o600); + recovered.acknowledge(name); + assert.deepEqual(recovered.stats(), { turns: 0, batches: 0 }); + } finally { + recovered.close(); + } +}); + +test("refuses another owner without changing the queue", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + assert.throws(() => new MemoryOutbox(path), /in use/); + assert.equal(queue.stats().turns, 1); + } finally { + queue.close(); + } +}); + +test("serializes concurrent stale-lock recovery without deleting a replacement owner", async (t) => { + const path = directory(t); + const moduleUrl = new URL("../dist/outbox.js", import.meta.url).href; + const seed = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + new MemoryOutbox(${JSON.stringify(path)});`, + ], + { encoding: "utf8" }, + ); + assert.equal(seed.status, 0, seed.stderr); + const staleOwner = readFileSync(join(path, ".lock"), "utf8"); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { existsSync, writeFileSync } from "node:fs"; + import { join } from "node:path"; + import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + const path = ${JSON.stringify(path)}; + const kill = process.kill.bind(process); + process.kill = (pid, signal) => { + writeFileSync(join(path, "observed-stale"), "ready"); + const deadline = Date.now() + 5000; + while (!existsSync(join(path, "release"))) { + if (Date.now() > deadline) throw new Error("Recovery probe timed out"); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + return kill(pid, signal); + }; + const queue = new MemoryOutbox(path); + process.on("message", () => { queue.close(); process.exit(0); }); + writeFileSync(join(path, "acquired"), "ready"); + `, + ], + { stdio: ["ignore", "ignore", "pipe", "ipc"] }, + ); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr += data; + }); + const exited = new Promise((resolve) => child.once("exit", resolve)); + async function waitFor(name) { + const deadline = Date.now() + 5000; + while (!existsSync(join(path, name))) { + assert.equal(child.exitCode, null, stderr); + assert.ok( + Date.now() < deadline, + `Timed out waiting for ${name}: ${stderr}`, + ); + await delay(5); + } + } + try { + await waitFor("observed-stale"); + assert.throws(() => new MemoryOutbox(path), /lock recovery is busy/); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), staleOwner); + writeFileSync(join(path, "release"), "ready"); + await waitFor("acquired"); + assert.equal( + Number(readFileSync(join(path, ".lock"), "utf8").split(":")[0]), + child.pid, + ); + assert.throws(() => new MemoryOutbox(path), /in use/); + } finally { + writeFileSync(join(path, "release"), "ready"); + if (child.connected) child.send("close"); + const timer = setTimeout(() => child.kill(), 1000); + await exited; + clearTimeout(timer); + } +}); + +test("fails closed on an interrupted recovery guard without touching the owner or records", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const owner = readFileSync(join(path, ".lock"), "utf8"); + mkdirSync(join(path, ".lock-recovery")); + assert.throws( + () => new MemoryOutbox(path), + /lock recovery is busy or interrupted/, + ); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), owner); + assert.equal(queue.stats().turns, 1); + assert.equal(existsSync(join(path, ".lock-recovery")), true); + } finally { + queue.close(); + } +}); + +test("preserves invalid records and keeps parser errors free of message content", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const name = readdirSync(path).find((file) => file.startsWith("ready-")); + const stored = JSON.parse(readFileSync(join(path, name), "utf8")); + stored.id = "../outside"; + writeFileSync(join(path, name), JSON.stringify(stored)); + assert.throws(() => queue.next(), /Invalid ReMe outbox record/); + writeFileSync(join(path, name), "PRIVATE-CONTENT"); + assert.throws( + () => queue.next(), + (error) => !error.message.includes("PRIVATE-CONTENT"), + ); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + queue.close(); + assert.throws(() => new MemoryOutbox(path), /Invalid ReMe outbox record/); + assert.equal(existsSync(join(path, ".lock")), false); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + } finally { + queue.close(); + } +}); diff --git a/integrations/dsh/tests/runtime.test.mjs b/integrations/dsh/tests/runtime.test.mjs index 9de94c9b..7cb8136f 100644 --- a/integrations/dsh/tests/runtime.test.mjs +++ b/integrations/dsh/tests/runtime.test.mjs @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate as settle } from "node:timers/promises"; import { ReMeRuntime } from "../dist/runtime.js"; const CONFIG = { + endpoint: "http://127.0.0.1:2333", autoMemoryEnabled: true, autoMemoryInterval: 2, + autoMemoryRetryMs: 10, + autoMemoryRetryMaxMs: 40, autoDreamEnabled: false, shutdownTimeoutMs: 50, dreamIntervalMs: 0, @@ -13,215 +20,525 @@ const CONFIG = { timezone: "Asia/Shanghai", }; -test("submits completed turns to auto-memory in background batches", async () => { +function makeRuntime( + t, + client, + config = CONFIG, + logger = silentLogger(), + directory, +) { + const ownsDirectory = directory === undefined; + directory ??= mkdtempSync(join(tmpdir(), "reme-dsh-runtime-")); + const current = typeof config === "function" ? config : () => config; + const runtime = new ReMeRuntime( + client, + () => ({ ...current(), autoMemoryQueueDir: directory }), + logger, + ); + t.after(async () => { + await runtime.disposeAll(); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + }); + return runtime; +} + +test("persists partial turns and submits one batch at the interval", async (t) => { const calls = []; - const client = { + const runtime = makeRuntime(t, { async autoMemory(messages, sessionId) { calls.push({ messages, sessionId }); return { ok: true }; }, - }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + }); const session = { id: "session-one" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 0); - + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); completeTurn(runtime, session, 2, 20); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.equal(calls[0].messages.length, 4); assert.match(calls[0].sessionId, /^dsh-[a-f0-9]{24}$/); - const status = runtime.snapshot(); - assert.equal(status.autoMemory.queuedTurns, 0); - assert.equal(status.autoMemory.recentTasks[0].phase, "completed"); - assert.equal(status.autoMemory.recentTasks[0].turns, 2); - assert.equal(status.autoMemory.recentTasks[0].messages, 4); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); }); -test("requeues failed auto-memory batches and flushes them on disposal", async () => { +test("backs off across new turns and drains every batch after recovery", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; let attempts = 0; - const client = { + const runtime = makeRuntime(t, { async autoMemory() { - attempts += 1; - return { ok: attempts > 1, error: "offline" }; + attempts++; + return { ok: available, error: "offline" }; }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retry-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); - - await runtime.dispose(session); - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("retries an in-flight failed batch before disposal completes", async () => { - let attempts = 0; - let markStarted; - let releaseFirst; - const started = new Promise((resolve) => { - markStarted = resolve; }); - const firstRequest = new Promise((resolve) => { - releaseFirst = resolve; - }); - const client = { - async autoMemory() { - attempts += 1; - if (attempts === 1) { - markStarted(); - await firstRequest; - return { ok: false, error: "offline" }; - } - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "in-flight-retry-session" }; - completeTurn(runtime, session, 1, 10); - await started; - - const disposal = runtime.dispose(session); - releaseFirst(); - await disposal; - - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("splits auto-memory batches at workspace date boundaries", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 1); - assert.equal(calls[0].options.date, "2026-08-19"); - await runtime.dispose(session); - assert.equal(calls.length, 2); - assert.equal(calls[1].options.date, "2026-08-20"); -}); - -test("retries a failed partial prior-day batch after later activity", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: calls.length > 1, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-retry-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - assert.equal(calls.length, 1); - assert.equal(runtime.stateFor(session).pendingTurns.length, 2); - - for (let turn = 3; turn <= 7; turn += 1) { - completeTurn( - runtime, - session, - turn, - turn * 10, - Date.parse(`2026-08-20T00:0${turn}:00Z`), - ); + const session = { id: "outage" }; + for (let turn = 1; turn <= 200; turn++) { + completeTurn(runtime, session, turn, turn * 10); + await runtime.drain(); } - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 3); - assert.equal(calls[1].options.date, "2026-08-19"); - assert.equal(calls[1].messages.length, 2); - assert.equal(calls[2].options.date, "2026-08-20"); - assert.equal(calls[2].messages.length, 10); - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); -}); - -test("bounds session disposal and aborts an unresponsive write", async () => { - let observedSignal; - const client = { - async autoMemory(_messages, _sessionId, options) { - observedSignal = options.signal; - return new Promise(() => {}); - }, - }; - const runtime = new ReMeRuntime( - client, - { - ...CONFIG, - autoMemoryInterval: 1, - shutdownTimeoutMs: 20, - }, - silentLogger(), - ); - const session = { id: "stuck-session" }; - completeTurn(runtime, session, 1, 10); - - const started = Date.now(); - await runtime.dispose(session); - - assert.equal(observedSignal.aborted, true); - assert.ok(Date.now() - started < 500); - assert.equal(runtime.states.has(session.id), true); -}); - -test("retains a failed final batch for a later plugin-shutdown retry", async () => { - let attempts = 0; - const client = { - async autoMemory() { - attempts += 1; - return { ok: attempts > 2, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retained-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - - await runtime.dispose(session); + assert.equal(attempts, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 200); + assert.equal(runtime.states.get(session.id).length, 0); + t.mock.timers.tick(10); + await runtime.drain(); assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), true); - - await runtime.disposeAll(); - assert.equal(attempts, 3); - assert.equal(runtime.states.has(session.id), false); + available = true; + t.mock.timers.tick(19); + await settle(); + assert.equal(attempts, 2); + t.mock.timers.tick(1); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + assert.equal(attempts, 102); }); -test("runs only one auto-dream task at a time", async () => { +for (const status of [413, 200]) { + test(`isolates a rejected batch (HTTP ${status}) while preserving its session's order`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout", "Date"] }); + let accepted = false; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _session, options) { + const text = messages[0].content[0].text; + calls.push({ text, requestId: options.requestId }); + return { ok: accepted || text !== "question 101", status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "rejected" }, 101, 10); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "rejected" }, 102, 20); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "healthy" }, 201, 30); + await runtime.drain(); + assert.deepEqual( + calls.map((call) => call.text), + ["question 101", "question 201"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + + t.mock.timers.tick(10); + await runtime.drain(); + assert.deepEqual(calls[2], calls[0]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + accepted = true; + t.mock.timers.tick(19); + await runtime.drain(); + assert.equal(calls.length, 3); + t.mock.timers.tick(1); + await runtime.drain(); + assert.deepEqual( + calls.slice(3).map((call) => call.text), + ["question 101", "question 102"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +for (const status of [429, 503]) { + test(`backs off the whole endpoint for HTTP ${status}`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: available, status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + completeTurn(runtime, { id: "second" }, 1, 10); + await runtime.drain(); + assert.equal(calls, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls, 3); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +test("retries ended sessions without another turn and releases session state", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + const runtime = makeRuntime(t, { + async autoMemory() { + return { ok: available, error: "offline" }; + }, + }); + const session = { id: "ended" }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(runtime.states.has(session.id), false); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("recovers queued and partial batches after restarting the Host", async (t) => { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-restart-")); + const original = makeRuntime( + t, + { + async autoMemory() { + return { ok: false, error: "offline" }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + completeTurn(original, { id: "restart" }, 1, 10); + await original.disposeAll(); + const calls = []; + const recovered = makeRuntime( + t, + { + async autoMemory(messages) { + calls.push(messages); + return { ok: true }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + recovered.start(); + await recovered.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].length, 2); + assert.equal(recovered.snapshot().autoMemory.queuedTurns, 0); + t.after(() => rmSync(directory, { recursive: true, force: true })); +}); + +test("keeps the same request ID and batch when retrying after a timeout", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + const calls = []; + const runtime = makeRuntime(t, { + async autoMemory(messages, sessionId, options) { + calls.push({ messages, sessionId, requestId: options.requestId }); + return { ok: calls.length > 1, error: "timeout" }; + }, + }); + const session = { id: "timeout" }; + completeTurn(runtime, session, 1, 10); + completeTurn(runtime, session, 2, 20); + await runtime.drain(); + await runtime.dispose(session); + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls.length, 2); + assert.deepEqual(calls[0], calls[1]); + const tasks = runtime.snapshot().autoMemory.recentTasks; + assert.equal(new Set(tasks.map((task) => task.id)).size, tasks.length); +}); + +test("keeps counts accurate beyond the twenty-entry diagnostic history", async (t) => { + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const runtime = makeRuntime( + t, + { + async autoMemory() { + await gate; + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + const session = { id: "many-batches" }; + for (let n = 1; n <= 25; n++) completeTurn(runtime, session, n, n * 10); + await settle(); + const status = runtime.snapshot().autoMemory; + assert.equal(status.tasksRunning, 1); + assert.equal(status.tasksQueued, 24); + assert.equal(status.queuedTurns, 25); + release(); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.recentTasks.length, 20); +}); + +test("does not put service reply text or file names into diagnostic snapshots", async (t) => { + const runtime = makeRuntime( + t, + { + async autoMemory() { + return { + ok: true, + answer: "PRIVATE-ANSWER", + metadata: { path: "PRIVATE-PATH" }, + }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "privacy" }, 1, 10); + await runtime.drain(); + assert.doesNotMatch(JSON.stringify(runtime.snapshot()), /PRIVATE/); +}); + +test("keeps whitespace-bearing URL credentials out of snapshots and logs", async (t) => { + const logs = []; + const error = "refused http://alice:SECRET CANARY\n@localhost:2333"; + const runtime = makeRuntime( + t, + { + async autoMemory() { + throw new Error(error); + }, + async autoDream() { + return { ok: false, error }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + { + debug() {}, + warn(...args) { + logs.push(args); + }, + }, + ); + completeTurn(runtime, { id: "credentials" }, 1, 10); + await runtime.drain(); + await runtime.runDream(); + const snapshot = runtime.snapshot(); + assert.equal(snapshot.autoMemory.recentTasks[0].phase, "failed"); + assert.equal(snapshot.autoDream.lastResult, "failed"); + assert.equal(logs.length, 2); + assert.doesNotMatch( + JSON.stringify({ snapshot, logs }), + /alice|SECRET|CANARY/, + ); +}); + +test("splits dates and flushes a partial previous day without a new interval", async (t) => { + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _id, options) { + calls.push({ messages, date: options.date }); + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 5 }, + ); + const session = { id: "midnight" }; + completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); + completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); + await runtime.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].date, "2026-08-19"); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(calls[1].date, "2026-08-20"); +}); + +test("aborts shutdown within its budget and leaves an unacknowledged batch on disk", async (t) => { + let signal; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _id, options) { + signal = options.signal; + return new Promise(() => {}); + }, + }, + { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }, + ); + completeTurn(runtime, { id: "stuck" }, 1, 10); + await settle(); + const started = Date.now(); + await runtime.disposeAll(); + assert.equal(signal.aborted, true); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.ok(Date.now() - started < 500); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("does not retry per ended session while its service is in backoff", async (t) => { + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: false }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + for (let n = 0; n < 5; n++) { + const session = { id: "ended-" + n }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + } + assert.equal(calls, 1); +}); + +test("pauses further sends while disabled and resumes persisted work when enabled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: true }; + }, + }, + () => config, + ); + runtime.capture({ id: "partial" }, { type: "turn/start" }); + runtime.capture( + { id: "partial" }, + { + type: "user/message", + seq: 1, + data: { content: "unfinished", source: { kind: "user" } }, + }, + ); + config = { ...config, autoMemoryEnabled: false }; + runtime.reconfigure(); + assert.equal(runtime.states.size, 0); + runtime.outbox.append( + config.endpoint, + "session", + "2026-10-21", + [{ id: "m", name: "user", role: "user", content: [] }], + 1, + ); + await runtime.drain(); + assert.equal(calls, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + config = { ...config, autoMemoryEnabled: true }; + runtime.reconfigure(); + await runtime.drain(); + assert.equal(calls, 1); +}); + +test("a failed old destination does not block a newly configured service", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + calls.push(options.endpoint); + return { ok: options.endpoint === "http://new.test" }; + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await runtime.drain(); + config = { ...config, endpoint: "http://new.test" }; + completeTurn(runtime, { id: "new" }, 1, 10); + await runtime.drain(); + assert.deepEqual(calls, [CONFIG.endpoint, "http://new.test"]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("drains a new destination while the old request remains stalled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const calls = []; + const active = new Map(); + const maximum = new Map(); + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + const endpoint = options.endpoint; + calls.push(endpoint); + active.set(endpoint, (active.get(endpoint) ?? 0) + 1); + maximum.set( + endpoint, + Math.max(maximum.get(endpoint) ?? 0, active.get(endpoint)), + ); + await (endpoint === CONFIG.endpoint ? gate : settle()); + active.set(endpoint, active.get(endpoint) - 1); + return { ok: true }; + }, + }, + () => config, + ); + try { + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + completeTurn(runtime, { id: "new" }, 2, 20); + await settle(); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 2); + await settle(); + await settle(); + assert.deepEqual(calls, [ + CONFIG.endpoint, + "http://new.test", + "http://new.test", + ]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 1); + assert.equal(runtime.snapshot().autoMemory.tasksQueued, 0); + assert.deepEqual([...maximum.values()], [1, 1]); + } finally { + release(); + await runtime.drain(); + } + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("aborts all destination workers at the shutdown deadline without discarding batches", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }; + const signals = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + signals.push(options.signal); + return new Promise(() => {}); + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + await settle(); + assert.equal(signals.length, 2); + await runtime.disposeAll(); + assert.ok(signals.every((signal) => signal.aborted)); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); +}); + +test("runs only one auto-dream task at a time", async (t) => { let calls = 0; let release; const client = { @@ -233,7 +550,7 @@ test("runs only one auto-dream task at a time", async () => { return { ok: true }; }, }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + const runtime = makeRuntime(t, client, CONFIG, silentLogger()); const first = runtime.runDream(); const second = runtime.runDream(); assert.equal(calls, 1); @@ -242,9 +559,10 @@ test("runs only one auto-dream task at a time", async () => { assert.equal(calls, 1); }); -test("contains unexpected auto-dream client failures", async () => { +test("contains unexpected auto-dream client failures", async (t) => { const warnings = []; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { throw new Error("broken transport"); @@ -266,10 +584,11 @@ test("contains unexpected auto-dream client failures", async () => { assert.match(runtime.snapshot().autoDream.lastError, /broken transport/); }); -test("applies changed batching and dream settings without replacing the runtime", async () => { +test("applies changed batching and dream settings without replacing the runtime", async (t) => { const calls = []; let config = { ...CONFIG, autoMemoryInterval: 5, autoDreamEnabled: false }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoMemory(messages) { calls.push(messages); @@ -294,7 +613,7 @@ test("applies changed batching and dream settings without replacing the runtime" dreamIntervalMs: 100000, }; runtime.reconfigure(); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.notEqual(runtime.dreamTimer, null); assert.equal(runtime.snapshot().autoDream.enabled, true); @@ -302,7 +621,7 @@ test("applies changed batching and dream settings without replacing the runtime" await runtime.disposeAll(); }); -test("keeps one auto-dream schedule when reconfigured during a run", async () => { +test("keeps one auto-dream schedule when reconfigured during a run", async (t) => { let calls = 0; let releaseFirst; const firstRequest = new Promise((resolve) => { @@ -314,7 +633,8 @@ test("keeps one auto-dream schedule when reconfigured during a run", async () => dreamIntervalMs: 20, shutdownTimeoutMs: 100, }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { calls += 1; diff --git a/reme/components/as_decision_model/agentscope.py b/reme/components/as_decision_model/agentscope.py index 2e8ff025..640e3a93 100644 --- a/reme/components/as_decision_model/agentscope.py +++ b/reme/components/as_decision_model/agentscope.py @@ -31,8 +31,8 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): """Wrap AgentScope's TypeSafe decision model with lazy provider loading. ``questions`` accepts AgentScope question objects or JSON dictionaries with - a ``type`` discriminator (``binary``, ``choice``, or ``score``). Only this - component's configuration supplies credentials and transport options. + a ``type`` discriminator (``binary``, ``choice``, or ``score``). Hosts may inject a borrowed model + through ``model``; otherwise this component constructs and owns its client. """ def __init__( @@ -53,10 +53,31 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): retry_delay=retry_delay, ) self._model: "ClassifierModelBase | None" = None + self._owns_model = False self._active_calls = 0 self._idle = asyncio.Event() self._idle.set() + @property + def model(self) -> "ClassifierModelBase | None": + """Return the current instance without constructing a provider client.""" + return self._model + + @model.setter + def model(self, value: "ClassifierModelBase | None") -> None: + """Borrow a host-owned instance, including via Application.update_component. + + Close the component before replacing an internally owned client. An + active request must finish before any replacement or detachment. + """ + if value is self._model: + return + if value is not None and not callable(value): + raise TypeError("An injected decision model must be callable.") + if self._active_calls or self._owns_model: + raise RuntimeError("Close the decision component before replacing its active or owned model.") + self._model = value + @staticmethod def _load_decision_model(): """Load the optional AgentScope adapter on the first decision request.""" @@ -102,6 +123,7 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): credential=credential_cls(**credential), **self.options.model_dump(exclude_none=True), ) + self._owns_model = True except ImportError as exc: raise ImportError( "Decision model inference requires a compatible typesafe-sdk. " @@ -120,10 +142,24 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): self._idle.set() async def _close(self) -> None: + """Finish draining and cleanup before propagating shutdown cancellation.""" + cleanup = asyncio.create_task(self._release_model()) + cancellation = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as exc: + # Keep the lifecycle lock until cleanup finishes, even after + # repeated cancellation. No cleanup task outlives close(). + cancellation = exc + cleanup.result() + if cancellation is not None: + raise cancellation + + async def _release_model(self) -> None: """Release the owned TypeSafe client and allow a later fresh start.""" - try: - await self._idle.wait() - finally: - model, self._model = self._model, None - if model is not None: - await model.client.aclose() + await self._idle.wait() + model, owned = self._model, self._owns_model + self._model, self._owns_model = None, False + if model is not None and owned: + await model.client.aclose() diff --git a/reme/components/embedding_store/_embedding_types.py b/reme/components/embedding_store/_embedding_types.py new file mode 100644 index 00000000..69aebebe --- /dev/null +++ b/reme/components/embedding_store/_embedding_types.py @@ -0,0 +1,109 @@ +"""Internal embedding results and normalized provider errors.""" + +import re +from dataclasses import dataclass + +_PROVIDER_WIDE_STATUS_CODES = frozenset({401, 403, 404}) + + +# AgentScope's DashScope wrapper serializes the response into a RuntimeError. +_STATUS_IN_MESSAGE = re.compile(r'"status_code"\s*:\s*(\d{3})') + + +_CODE_IN_MESSAGE = re.compile(r'"code"\s*:\s*"([^"]+)"') + + +_INPUT_DEPENDENT_ERROR_TYPES = frozenset( + { + "ContextLengthError", + "ContextWindowExceededError", + "InputLengthError", + "TokenLimitError", + }, +) + + +_INPUT_DEPENDENT_MARKERS = ( + "context length", + "context_length", + "context window", + "input length", + "input is too long", + "maximum context", + "reduce the length", + "token limit", + "too many tokens", + "too long", +) + + +@dataclass(frozen=True) +class _EmbeddingResult: + """Keep recovered vectors together with the failure state that ends isolation.""" + + embeddings: list[list[float] | None] | None = None + terminal_failure: bool = False + + +@dataclass(frozen=True) +class _EmbeddingError: + """Normalize provider error shapes once before applying the retry policy.""" + + status: int | None + code: str | None + text: str + error_type: str + + @classmethod + def from_exception(cls, error: Exception) -> "_EmbeddingError": + """Read attributes, structured bodies, and serialized DashScope responses.""" + message = str(error) + body = getattr(error, "body", None) + status = None + for value in ( + getattr(error, "status_code", None), + getattr(error, "code", None), + getattr(getattr(error, "response", None), "status_code", None), + ): + # Integer codes are HTTP statuses; strings name business errors. + if isinstance(value, int) and not isinstance(value, bool): + status = value + break + if status is None and (match := _STATUS_IN_MESSAGE.search(message)): + status = int(match.group(1)) + + code = getattr(error, "code", None) + if not isinstance(code, str) or not code: + details = body.get("error", body) if isinstance(body, dict) else None + code = details.get("code") if isinstance(details, dict) else None + if not isinstance(code, str) or not code: + match = _CODE_IN_MESSAGE.search(message) + code = match.group(1) if match else None + text = f"{message} {body}" if isinstance(body, dict) else message + return cls(status, code, text.lower(), type(error).__name__) + + @property + def is_insufficient_quota(self) -> bool: + """Quota exhaustion takes precedence over a generic HTTP 429.""" + return self.code == "insufficient_quota" + + @property + def is_rate_limited(self) -> bool: + """Recognize rate limits while preserving the separate quota policy.""" + return not self.is_insufficient_quota and (self.code == "rate_limit_exceeded" or self.status == 429) + + @property + def is_provider_wide(self) -> bool: + """Credentials, missing routes, and server outages cannot be isolated per input.""" + return ( + self.status is not None and (self.status in _PROVIDER_WIDE_STATUS_CODES or 500 <= self.status < 600) + ) or self.error_type in ("AuthenticationError", "PermissionDeniedError", "NotFoundError", "InternalServerError") + + @property + def is_input_dependent(self) -> bool: + """Only input-length rejections may split; a 413 always requires a text signal.""" + if self.status not in (None, 400, 413): + return False + if self.status != 413 and self.error_type in _INPUT_DEPENDENT_ERROR_TYPES: + return True + return any(marker in self.text for marker in _INPUT_DEPENDENT_MARKERS) diff --git a/reme/components/embedding_store/base_embedding_store.py b/reme/components/embedding_store/base_embedding_store.py index bb7703e2..f499b0eb 100644 --- a/reme/components/embedding_store/base_embedding_store.py +++ b/reme/components/embedding_store/base_embedding_store.py @@ -36,7 +36,7 @@ class BaseEmbeddingStore(BaseComponent): self.health_check_timeout = health_check_timeout self.is_healthy: bool = True - def _truncate(self, text: str) -> str: + def _truncate(self, text: str, *, max_length: int | None = None) -> str: """Truncate text using a CJK-aware character budget. ASCII text keeps its historical character limit. For non-ASCII text, @@ -44,8 +44,12 @@ class BaseEmbeddingStore(BaseComponent): characters cost 1.5 units because they commonly consume more embedding tokens. The estimate reserves a 5% safety margin, and integer half-units avoid floating-point boundary errors. + + ``max_length`` overrides the configured ``max_input_length`` for this call only, so a + caller can re-prepare a rejected input under a smaller budget without mutating shared + configuration or changing the vector space. """ - limit = max(0, self.max_input_length) + limit = max(0, self.max_input_length if max_length is None else max_length) if text.isascii(): return text[:limit] diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index 4fad8361..14e5cc9c 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np +from ._embedding_types import _EmbeddingError, _EmbeddingResult from .base_embedding_store import BaseEmbeddingStore from ..component_registry import R from ..as_embedding import BaseAsEmbedding @@ -15,6 +16,26 @@ Miss = tuple[int, str, str] # (result_index, text, cache_key) _MAX_VECTOR_SPACE_ATTEMPTS = 3 +# Match SDK/httpx transport exceptions without importing optional providers. +_TRANSPORT_FAILURE_TYPES = frozenset( + { + "APIConnectionError", + "APITimeoutError", + "ConnectError", + "ConnectTimeout", + "PoolTimeout", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "WriteError", + "WriteTimeout", + }, +) + +# A client retry budget only applies to the transport exceptions it retries. +_SDK_TRANSPORT_RETRY_TYPES = frozenset({"APIConnectionError", "APITimeoutError"}) + + @R.register("local") class LocalEmbeddingStore(BaseEmbeddingStore): """Embedding store with LRU cache, disk persistence, and serial batching. @@ -162,7 +183,8 @@ class LocalEmbeddingStore(BaseEmbeddingStore): async def _compute_batch(self, batch: list[Miss], **kwargs) -> list[tuple[int, str, np.ndarray]]: texts = [text for _, text, _ in batch] - embeddings = await self._call_with_retry(texts, **kwargs) + result = await self._call_with_retry(texts, **kwargs) + embeddings = result.embeddings if not embeddings or len(embeddings) != len(texts): return [] out: list[tuple[int, str, np.ndarray]] = [] @@ -178,70 +200,146 @@ class LocalEmbeddingStore(BaseEmbeddingStore): if bad_dims: details = ", ".join(f"{count} with dim {dim}" for dim, count in sorted(bad_dims.items())) self.logger.error(f"Embedding dimension mismatch in batch: expected {self.dimensions}; rejected {details}") - if out: - self.is_healthy = True - else: - self.is_healthy = False + # Retain partial vectors without hiding the outage that ended isolation. + self.is_healthy = bool(out) and not result.terminal_failure return out - async def _call_with_retry(self, texts: list[str], **kwargs) -> list[list[float] | None] | None: + async def _call_with_retry(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Retry failures owned by the store and isolate input-length rejections.""" + shortened = False for attempt in range(self.max_retries): try: - result = await self.as_embedding(texts, **kwargs) - if result and len(result) == len(texts): - return result - except (TimeoutError, ConnectionError, OSError): - if attempt < self.max_retries - 1: - await asyncio.sleep(2**attempt) + embeddings = await self.as_embedding(texts, **kwargs) + if embeddings and len(embeddings) == len(texts): + return _EmbeddingResult(embeddings) except Exception as error: - if self._is_rate_limited(error): + if self._is_transport_failure(error): + # Preserve recovery where the provider has no matching retry policy; + # otherwise leave its budget intact instead of adding another loop. + if self._provider_owns_transport_retry(error): + self.logger.exception( + "Embedding request failed: transport error owned by the provider, not splitting", + ) + break + if attempt < self.max_retries - 1: + await asyncio.sleep(2**attempt) + continue + self.logger.exception("Embedding request failed: transport error after exhausting retries") + break + + details = _EmbeddingError.from_exception(error) + if details.is_rate_limited: if attempt < self.max_retries - 1: delay = 2**attempt self.logger.warning(f"Embedding rate limited; retrying in {delay:.1f}s") await asyncio.sleep(delay) continue self.logger.exception("Embedding request failed after exhausting rate-limit retries") + break + if details.is_insufficient_quota: + if self.quota_retry_delay is not None and attempt < self.max_retries - 1: + self.logger.warning(f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s") + await asyncio.sleep(self.quota_retry_delay) + continue + self.logger.exception("Embedding request failed: quota exhausted") + break + if details.is_provider_wide: + # Server-error retries belong to the wrapper/SDK, not to item isolation. + self.logger.exception("Embedding request failed: provider-wide error, not splitting") + break + if details.is_input_dependent: + # Retry one item with half the character budget without changing its source text. + # Require a remaining attempt so exhausted input failures stay non-terminal. + if not shortened and len(texts) == 1 and attempt < self.max_retries - 1: + shorter = self._truncate(texts[0], max_length=max(1, self.max_input_length // 2)) + if shorter != texts[0]: + self.logger.warning( + f"Embedding input rejected as too long; retrying with a smaller budget " + f"(chars={len(texts[0])} -> {len(shorter)})", + ) + texts = [shorter] + shortened = True + await asyncio.sleep(2**attempt) + continue + self.logger.exception("Embedding request failed: input-dependent error") self.is_healthy = False - return None - if ( - self.quota_retry_delay is not None - and self._is_insufficient_quota(error) - and attempt < self.max_retries - 1 - ): - self.logger.warning( - f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s", - ) - await asyncio.sleep(self.quota_retry_delay) - continue - self.logger.exception("Embedding request failed") - self.is_healthy = False - return None + if len(texts) > 1: + return await self._embed_individually(texts, **kwargs) + return _EmbeddingResult() + self.logger.exception("Embedding request failed: unrecognized error, not splitting") + break self.is_healthy = False - return None + return _EmbeddingResult(terminal_failure=True) + + async def _embed_individually(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Keep healthy siblings, stopping when a failure is no longer specific to an input.""" + embeddings: list[list[float] | None] = [] + for index, text in enumerate(texts): + result = await self._call_with_retry([text], **kwargs) + if result.embeddings: + embeddings.append(result.embeddings[0]) + continue + self.logger.error(f"Embedding request failed for a single item (chars={len(text)})") + embeddings.append(None) + if result.terminal_failure: + self.logger.error( + f"Stopping per-item isolation after {index + 1}/{len(texts)} items: " + "the failure is not specific to any input", + ) + embeddings.extend([None] * (len(texts) - index - 1)) + return _EmbeddingResult(embeddings, terminal_failure=True) + return _EmbeddingResult(embeddings) @staticmethod - def _is_rate_limited(error: Exception) -> bool: - """Recognize an OpenAI-compatible 429 response without importing a provider SDK.""" - if LocalEmbeddingStore._is_insufficient_quota(error): - return False - if getattr(error, "status_code", None) == 429: + def _is_transport_failure(error: Exception) -> bool: + """Recognize transport errors that must never trigger per-item isolation.""" + if isinstance(error, (TimeoutError, ConnectionError, OSError)): return True - body = getattr(error, "body", None) - if not isinstance(body, dict): + return any(cls.__name__ in _TRANSPORT_FAILURE_TYPES for cls in type(error).__mro__) + + def _provider_owns_transport_retry(self, error: Exception) -> bool: + """Require both a matching exception policy and an enabled retry budget.""" + model = getattr(self.as_embedding, "model", None) + if model is None: return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "rate_limit_exceeded" + policy = self._declared_retry_policy(model) + if policy: + max_retries = getattr(model, "max_retries", None) + if isinstance(max_retries, int) and max_retries > 0 and self._policy_covers(policy, error): + return True + return self._sdk_owns_transport_retry(model, error) @staticmethod - def _is_insufficient_quota(error: Exception) -> bool: - """Recognize OpenAI-compatible quota errors without importing a provider SDK.""" - if getattr(error, "code", None) == "insufficient_quota": - return True - body = getattr(error, "body", None) - if not isinstance(body, dict): + def _declared_retry_policy(model) -> tuple[type[Exception], ...] | None: + """Return the model's declared retryable types, or ``None`` when it declares none.""" + retryable = getattr(model, "_get_retryable_exceptions", None) + if not callable(retryable): + return None + try: + return tuple(retryable()) # pylint: disable=not-callable + except Exception: # pragma: no cover - a broken policy owns nothing + return None + + @staticmethod + def _policy_covers(policy: tuple[type[Exception], ...], error: Exception) -> bool: + """Return whether the declared policy actually retries this exception.""" + try: + return isinstance(error, policy) + except TypeError: # pragma: no cover - a malformed policy covers nothing return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "insufficient_quota" + + @staticmethod + def _sdk_owns_transport_retry(model, error: Exception) -> bool: + """Check the SDK's budget and supported transport errors independently of the model.""" + if not LocalEmbeddingStore._is_transport_failure(error): + return False + client = getattr(model, "client", None) + if client is None: + return False + max_retries = getattr(client, "max_retries", None) + if not isinstance(max_retries, int) or max_retries <= 0: + return False + return any(cls.__name__ in _SDK_TRANSPORT_RETRY_TYPES for cls in type(error).__mro__) def _validate_dim(self, emb: np.ndarray) -> bool: """Return whether an embedding exactly matches the configured dimension.""" diff --git a/reme/components/file_store/faiss_local_file_store.py b/reme/components/file_store/faiss_local_file_store.py index 40dfd654..c78ae41d 100644 --- a/reme/components/file_store/faiss_local_file_store.py +++ b/reme/components/file_store/faiss_local_file_store.py @@ -1,7 +1,6 @@ """FAISS-backed file store: chunk JSONL stays authoritative; FAISS HNSW replaces the linear vector scan.""" import asyncio -import hashlib import json from collections.abc import Callable from contextlib import suppress @@ -389,28 +388,6 @@ class FaissLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory - (possibly float32 from the provider; assignment bypasses the pydantic - validator) or has round-tripped through the JSONL. Written into the - idmap sidecar at dump time and recomputed from ``self.file_chunks`` at - load time: a mismatch means the sidecar vectors belong to a different - chunk generation than the authoritative JSONL, which the live-id set - check alone cannot detect for same-ID in-place updates. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach FAISS state (sidecar or rebuild).""" await super().load() diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index e5448f65..ee36674b 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -3,6 +3,7 @@ import asyncio import base64 import datetime +import hashlib import heapq import json import time @@ -43,6 +44,7 @@ _SEARCH_FILTER_RESERVED_KEYS = frozenset( "strict_date_filter", }, ) +_FAILED_CHUNK_PREVIEW = 5 @R.register("local") @@ -106,6 +108,23 @@ class LocalFileStore(BaseFileStore): return 0 return max(0, int(getattr(self.embedding_store, "dimensions", 0))) + def _chunks_embedding_digest(self) -> str: + """Order-independent digest of the live (chunk_id, embedding) set. + + Float16 bytes match the authoritative chunk JSONL serialization even + before a fresh provider vector has round-tripped through persistence. + Vector backends use this digest to reject stale index sidecars. + """ + digest = hashlib.sha256() + eligible = sorted( + cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) + ) + for cid in eligible: + digest.update(cid.encode("utf-8")) + digest.update(b"\x00") + digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) + return digest.hexdigest() + # -- lifecycle ------------------------------------------------------------ async def _start(self) -> None: @@ -440,7 +459,12 @@ class LocalFileStore(BaseFileStore): if chunk.text and not self._embedding_dim_matches(chunk.embedding) ] if missing: - raise RuntimeError(f"embedding reindex incomplete: {len(missing)} chunks failed") + preview = ", ".join(missing[:_FAILED_CHUNK_PREVIEW]) + if len(missing) > _FAILED_CHUNK_PREVIEW: + preview += f" (+{len(missing) - _FAILED_CHUNK_PREVIEW} more)" + raise RuntimeError( + f"embedding reindex incomplete: {len(missing)} chunks failed: {preview}", + ) await self._finalize_embedding_reindex() await self._dump_owned_state() if self.embedding_store is not None: @@ -544,6 +568,7 @@ class LocalFileStore(BaseFileStore): return processed = 0 + filled = 0 batch_count = 0 embedding_started_at = time.monotonic() next_percent = _PROGRESS_LOG_PERCENT_STEP @@ -552,10 +577,11 @@ class LocalFileStore(BaseFileStore): await self.embedding_store.get_node_embeddings(batch) self._drop_stale_embeddings(batch, "backfill") processed += len(batch) + filled += sum(1 for chunk in batch if chunk.embedding is not None) batch_count += 1 next_percent = self._log_progress("embedding backfill", processed, total, next_percent) self.logger.info( - f"{self.name}: embedding batches complete: processed={processed}/{total}, " + f"{self.name}: embedding batches complete: processed={processed}/{total}, filled={filled}, " f"batches={batch_count}, elapsed={time.monotonic() - embedding_started_at:.3f}s", ) except asyncio.CancelledError: diff --git a/reme/components/file_store/zvec_local_file_store.py b/reme/components/file_store/zvec_local_file_store.py index 84db2dbd..8d81609d 100644 --- a/reme/components/file_store/zvec_local_file_store.py +++ b/reme/components/file_store/zvec_local_file_store.py @@ -1,6 +1,5 @@ """Zvec-backed file store: chunk JSONL stays authoritative; a zvec collection replaces the linear vector scan.""" -import hashlib import json import shutil import time @@ -227,26 +226,6 @@ class ZvecLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory or has - round-tripped through the JSONL. Written into the sidecar at dump time - and recomputed from ``self.file_chunks`` at load time: a mismatch means - the collection belongs to a different chunk generation than the - authoritative JSONL. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach the zvec collection (open or rebuild).""" await super().load() diff --git a/reme/config/default.yaml b/reme/config/default.yaml index dbab8ac2..c57c0b3c 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -173,6 +173,9 @@ jobs: parameters: type: object properties: + request_id: + type: string + description: "Optional stable batch ID (1-128 characters); replay completed requests without rerunning the memory agent" messages: type: array description: "messages" diff --git a/reme/steps/evolve/auto_memory.py b/reme/steps/evolve/auto_memory.py index ee38fcff..5e513fab 100644 --- a/reme/steps/evolve/auto_memory.py +++ b/reme/steps/evolve/auto_memory.py @@ -3,6 +3,8 @@ import base64 import binascii import datetime +import hashlib +import json from pathlib import Path import re from urllib.parse import urlsplit @@ -21,6 +23,7 @@ from ..file_io import validate_filename_component, validate_session_id from ..file_io._path import IMAGE_MIME_BY_EXT, _check_path_permission, resolve_path from ..index import normalize_posix_path from ...components import R +from ...schema import Response from ...utils.wikilink_handler import WikilinkHandler _SESSION_ID_KEY = "session_id" @@ -458,8 +461,68 @@ class AutoMemoryStep(BaseStep): content=[images[part] if part in images else TextBlock(text=part) for part in parts if part], ) - # pylint: disable=too-many-return-statements async def execute(self): + """Replay completed request IDs without repeating the memory agent.""" + assert self.context is not None + request_id = self.context.get("request_id") + if request_id is None: + return await self._execute() + if not isinstance(request_id, str) or not 1 <= len(request_id) <= 128: + self.context.response.success = False + self.context.response.answer = "Error: request_id must be a string of 1 to 128 characters" + return + + payload = { + key: self.context.get(key) for key in ("session_id", "messages", "date", "memory_hint", "include_images") + } + if payload["messages"]: + # Keep caller-provided IDs/timestamps; parsing dictionaries would generate defaults on every retry. + payload["messages"] = [ + message.model_dump(mode="json") if isinstance(message, Msg) else message + for message in payload["messages"] + ] + fingerprint = hashlib.sha256( + json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode(), + ).hexdigest() + session_key = hashlib.sha256(str(payload["session_id"]).encode()).hexdigest() + request_key = hashlib.sha256(request_id.encode()).hexdigest() + directory = f"{self.config_value('metadata_dir')}/auto_memory_requests/{session_key}" + workspace = self.file_store.workspace_path + allowed = self.context.get("_allowed_paths") + lock_path = _checked_write_path(workspace, f"{directory}/.lock", allowed) + async with await get_path_lock(lock_path): + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + if path.exists(): + record = json.loads(path.read_text(encoding="utf-8")) + if record["fingerprint"] != fingerprint: + self.context.response.success = False + self.context.response.answer = "Error: request_id was already used for different input" + return + response = Response.model_validate(record["response"]) + self.context.response.answer = response.answer + self.context.response.success = response.success + self.context.response.metadata.update(response.metadata) + self.context["changes"] = record["changes"] + return + await self._execute() + if self.context.response.success: + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = _checked_write_path(workspace, f"{directory}/{request_key}.tmp", allowed) + await write_file_safe( + temporary, + json.dumps( + { + "fingerprint": fingerprint, + "response": self.context.response.model_dump(mode="json"), + "changes": self.context.get("changes") or [], + }, + ), + ) + temporary.replace(path) + + # pylint: disable=too-many-return-statements + async def _execute(self): assert self.context is not None self.context["changes"] = [] raw_messages = self.context.get("messages") or [] diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index 59f6b9c9..d3d4c3b7 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -7,7 +7,7 @@ from ...file_io import refresh_day_index from .._evolve import agent_reply_result_text from ....components import R from ....enumeration import DreamBucketEnum -from ....schema import DreamState +from ....schema import DreamExtractOutput, DreamState from .utils import ( clean_paths, daily_dir, @@ -113,32 +113,43 @@ class DreamExtractStep(BaseStep): return self._finish(state, False, state.errors[-1]) self.logger.info(f"[{self.name}] agent start changed={len(changed)} dates={len(dates)}") - raw_result, meta = "", {} + user_message = self.prompt_format( + "extract_user_message", + date=day, + dates_json=json.dumps(dates, ensure_ascii=False, indent=2), + hint=hint or "(none)", + max_units=max_units, + changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), + material_blob=pack_paths(workspace, changed), + ) + retry_feedback = "" + output_schema = DreamExtractOutput.model_json_schema() + output_schema["required"] = ["units"] for attempt in range(2): try: result = await self.agent_wrapper.reply( - self.prompt_format( - "extract_user_message", - date=day, - dates_json=json.dumps(dates, ensure_ascii=False, indent=2), - hint=hint or "(none)", - max_units=max_units, - changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), - material_blob=pack_paths(workspace, changed), - ), + user_message + retry_feedback, system_prompt=self.prompt_format( "extract_system_prompt", workspace_dir=str(workspace), buckets=", ".join(bucket.value for bucket in DreamBucketEnum), max_units=max_units, + output_schema_json=json.dumps(output_schema, ensure_ascii=False, indent=2), ), job_tools=list(_TOOLS), ) self.logger.info(f"[{self.name}] agent done has_result={bool(result.get('result'))}") raw_result = agent_reply_result_text(result) - meta = parse_structured_reply(raw_result) + state.extract_summary = raw_result + meta = parse_structured_reply(raw_result, strict=True) + self.clean_output(state, meta, max_units=max_units) + break except Exception as e: # noqa: BLE001 if attempt == 0: + retry_feedback = "\n\n" + self.prompt_format( + "extract_retry_message", + error=f"{type(e).__name__}: {e}", + ) self.logger.warning( f"[{self.name}] extract attempt 1 returned no usable receipt; retrying once: " f"{type(e).__name__}: {e}", @@ -150,20 +161,7 @@ class DreamExtractStep(BaseStep): self.logger.error(f"[{self.name}] {error}") return self._finish(state, False, error) - units = meta.get("units") if "units" in meta else meta.get("memory_units") - if isinstance(units, list): - break - if attempt == 0: - self.logger.warning(f"[{self.name}] extract attempt 1 returned an unusable receipt; retrying once") - continue - # Keep the warning-only result checkpointable after one retry so a bad source cannot loop forever. - warning = "dream extract skipped unusable agent receipt after retry; expected a units list" - state.warnings.append(warning) - self.logger.warning(f"[{self.name}] {warning}") - self.logger.info(f"[{self.name}] parse done keys={','.join(sorted(meta.keys())) if meta else '(none)'}") - self.clean_output(state, meta, max_units=max_units) - state.extract_summary = raw_result answer = f"Extracted {len(state.units)} unit(s) from {len(changed)} changed file(s) across {len(dates)} day(s)" return self._finish(state, True, answer) @@ -177,25 +175,34 @@ class DreamExtractStep(BaseStep): return out def clean_output(self, state: DreamState, meta: dict, max_units: int | None = None) -> None: - """Clean up output""" + """Validate the whole receipt before accepting any units or applying the cap.""" + units = meta.get("units") if "units" in meta else meta.get("memory_units") + if not isinstance(units, list): + raise ValueError("expected a units list (use units: [] only when nothing is worth remembering)") + normalized = [] + for raw in units: + if isinstance(raw, dict) and isinstance(raw.get("bucket"), str): + raw = {**raw, "bucket": raw["bucket"].strip()} + if ( + isinstance(raw, dict) + and "bucket" in raw + and raw["bucket"] not in {bucket.value for bucket in DreamBucketEnum} + ): + self.logger.warning( + f"[{self.name}] unit {raw.get('name')!r} emitted an unknown bucket; routing to wiki", + ) + raw = {**raw, "bucket": DreamBucketEnum.WIKI.value} + normalized.append(raw) + output = DreamExtractOutput.model_validate({"units": normalized}) allowed = set(state.changed_paths) - for raw in meta.get("units") or meta.get("memory_units") or []: - if max_units is not None and len(state.units) >= max_units: - break - if not isinstance(raw, dict): - continue - name = str(raw.get("name") or "").strip() - summary = str(raw.get("summary") or "").strip() - raw_bucket = str(raw.get("bucket") or "").strip() - paths = clean_paths(raw.get("paths"), allowed) + cleaned = [] + for index, unit in enumerate(output.units): + name, summary = unit.name.strip(), unit.summary.strip() + paths = clean_paths(unit.paths, allowed) if not name or not summary or not paths: - continue - try: - bucket = DreamBucketEnum(raw_bucket).value - except ValueError: - self.logger.warning(f"[{self.name}] unit {name!r} emitted bucket {raw_bucket!r}; routing to wiki") - bucket = DreamBucketEnum.WIKI.value - state.units.append({"name": name, "bucket": bucket, "summary": summary, "paths": paths}) + raise ValueError(f"units[{index}] requires non-empty name, summary, and paths from changed_paths") + cleaned.append({"name": name, "bucket": unit.bucket.value, "summary": summary, "paths": paths}) + state.units.extend(cleaned if max_units is None else cleaned[: max(max_units - len(state.units), 0)]) def _finish(self, state: DreamState, success: bool, answer: str): assert self.context is not None diff --git a/reme/steps/evolve/dream/extract.yaml b/reme/steps/evolve/dream/extract.yaml index b35847fe..2cdcb942 100644 --- a/reme/steps/evolve/dream/extract.yaml +++ b/reme/steps/evolve/dream/extract.yaml @@ -61,12 +61,20 @@ extract_system_prompt: | ## Output Format - Return only one YAML or JSON object with this exact shape: - units: - - name: - bucket: procedure|personal|wiki - summary: - paths: [, ...] + Return only one valid JSON object, without Markdown fences or surrounding prose. + The units field is required and must be a list. Every unit must contain name, + bucket, summary, and paths. name and summary must be non-empty strings; paths + must be a non-empty list of strings from changed_paths. + Double-quote all keys and string values, and escape embedded double quotes, + backslashes, and newlines according to JSON rules. Code-like text containing + `key: value` must stay inside a quoted string; never emit YAML plain scalars. + Return {{"units": []}} only when there is no reusable memory to extract. + + JSON schema: + {output_schema_json} + + Valid example (replace paths with actual changed_paths): + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout needs `submodules: recursive` to fetch nested dependencies.", "paths": ["daily/2026-10-20/ci.md"]}}]}} extract_system_prompt_zh: | 你是 dream 全局抽取 agent。一起阅读所有发生变化的 daily 文件，输出一个由合并记忆 unit 组成的精简跨文件计划。 @@ -118,12 +126,39 @@ extract_system_prompt_zh: | ## 输出格式 - 只返回一个 YAML 或 JSON object，结构必须严格如下： - units: - - name: <短名称> - bucket: procedure|personal|wiki - summary: <基于证据的可复用抽象> - paths: [, ...] + 只返回一个合法的 JSON object，不要使用 Markdown 代码围栏或添加解释文字。 + 必须包含 units 字段，且它必须是 list。每个 unit 必须包含 name、bucket、summary 和 paths。 + name 和 summary 必须是非空字符串；paths 必须是非空字符串列表，且只能使用 changed_paths 中的路径。 + 所有键和字符串值都必须使用双引号；内部双引号、反斜杠和换行必须按 JSON 规则转义。 + 含有 `key: value` 的代码文本必须放在带引号的字符串内，禁止输出 YAML 无引号标量。 + 只有在没有可复用记忆时才返回 {{"units": []}}。 + + JSON schema： + {output_schema_json} + + 合法示例（路径须替换为实际 changed_paths）： + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout 需要 `submodules: recursive`，以获取嵌套依赖。", "paths": ["daily/2026-10-20/ci.md"]}}]}} + +extract_retry_message: | + The previous extraction failed parsing or validation: + {error} + + Retry extraction from the changed material above and return a complete corrected + JSON object following the system prompt's schema. Include the units list and all + required unit fields. Double-quote every key and string value, including any + `key: value` text, and escape quotes, backslashes, and newlines. Do not use YAML, + Markdown fences, or explanatory prose. A format error does not mean there are no + reusable memories; use {{"units": []}} only when the material warrants it. + +extract_retry_message_zh: | + 上次抽取发生解析或校验错误： + {error} + + 请根据上面的变化内容重新抽取，按系统提示的 schema 返回完整、修正后的 JSON object。 + 必须包含 units 列表和每个 unit 的必填字段。所有键和字符串值都要加双引号， + 含 `key: value` 的文本也不例外；双引号、反斜杠和换行须转义。 + 不要使用 YAML、Markdown 代码围栏或解释文字。格式错误不代表没有可复用记忆； + 只有材料确实不值得记忆时才返回 {{"units": []}}。 extract_user_message: | date: {date} diff --git a/reme/steps/evolve/dream/finish.py b/reme/steps/evolve/dream/finish.py index 243a449a..980f93be 100644 --- a/reme/steps/evolve/dream/finish.py +++ b/reme/steps/evolve/dream/finish.py @@ -74,8 +74,11 @@ class DreamFinishStep(BaseStep): def render_summary(state: DreamState) -> str: """Render a concise user-facing summary.""" dates = ", ".join(state.dates or [state.date]) + title = "AutoDream failed" if state.errors else "AutoDream completed" + if state.warnings and not state.errors: + title += " with warnings" lines = [ - ("AutoDream completed with warnings" if state.warnings else "AutoDream completed"), + title, "", f"- Date: {state.date}", f"- Scan window: {dates}", diff --git a/reme/steps/evolve/dream/integrate.py b/reme/steps/evolve/dream/integrate.py index 12569e7e..f3946318 100644 --- a/reme/steps/evolve/dream/integrate.py +++ b/reme/steps/evolve/dream/integrate.py @@ -61,7 +61,7 @@ class DreamIntegrateStep(BaseStep): state = state_from_context(self) if not state.units: self.logger.info(f"[{self.name}] skip no units") - return self._finish(state, True, "No dream units to integrate") + return self._finish(state, not state.errors, "No dream units to integrate") if not llm_available(self): err = "no llm configured; dream integrate requires an LLM" state.errors.append(err) @@ -214,10 +214,10 @@ class DreamIntegrateStep(BaseStep): ) before = after continue - # After the bounded retry, checkpoint the source so malformed input cannot loop forever. - self._record_skipped(state, unit, bucket, paths, e) - self.logger.warning( - f"[{self.name}] unit {index}/{len(state.units)} skipped invalid receipt after retry: " + # Without a valid receipt or recoverable write, keep sources eligible for retry. + self._record_failure(state, unit, paths, e) + self.logger.error( + f"[{self.name}] unit {index}/{len(state.units)} failed invalid receipt after retry: " f"{type(e).__name__}: {e}", ) return @@ -339,18 +339,6 @@ class DreamIntegrateStep(BaseStep): note=f"Recovered from {type(error).__name__}: agent receipt unavailable", ) - @classmethod - def _record_skipped(cls, state, unit: dict, bucket: str, paths: list[str], error: Exception) -> None: - message = f"unit {unit.get('name', '')!r} skipped: unusable agent receipt ({type(error).__name__})" - key = cls._unit_key(unit, bucket, paths) - if not any( - cls._unit_key(item, str(item.get("bucket") or ""), [str(p) for p in item.get("paths") or []]) == key - for item in state.skipped_units - ): - state.skipped_units.append({**unit, "bucket": bucket, "paths": paths, "reason": message}) - if message not in state.warnings: - state.warnings.append(message) - @staticmethod def _record_failure(state, unit: dict, paths: list[str], error: Exception) -> None: message = f"{type(error).__name__}: {error}" diff --git a/reme/steps/evolve/dream/utils.py b/reme/steps/evolve/dream/utils.py index 2f41d27d..bce5c135 100644 --- a/reme/steps/evolve/dream/utils.py +++ b/reme/steps/evolve/dream/utils.py @@ -134,19 +134,25 @@ def previous_dates(day: str, n_days: int) -> list[str]: return [(base - dt.timedelta(days=i)).isoformat() for i in range(1, max(n_days, 0) + 1)] -def parse_structured_reply(text: str) -> dict: - """Parse a JSON/YAML object from an agent reply, including fenced blocks.""" +def parse_structured_reply(text: str, *, strict: bool = False) -> dict: + """Parse a JSON/YAML object; strict mode rejects lossy fallbacks with diagnostics.""" candidates = [text.strip()] candidates.extend(m.group(1).strip() for m in re.finditer(r"```(?:json|ya?ml)?\s*(.*?)```", text, re.S | re.I)) + error = "expected a JSON/YAML object" for raw in candidates: if not raw: continue try: data = yaml.safe_load(raw) - except yaml.YAMLError: + except yaml.YAMLError as exc: + error = f"{type(exc).__name__}: {exc}" + if strict: + continue data = _parse_scalar_mapping(raw) - if isinstance(data, dict) and data: + if isinstance(data, dict) and (data or strict): return data + if strict: + raise ValueError(f"Invalid structured reply: {error}") return {} diff --git a/reme/steps/file_io/_base.py b/reme/steps/file_io/_base.py new file mode 100644 index 00000000..7c42bc81 --- /dev/null +++ b/reme/steps/file_io/_base.py @@ -0,0 +1,13 @@ +"""Common response handling for file operations.""" + +from ..base_step import BaseStep + + +class _FileIOStep(BaseStep): + """Preserve the file-operation failure envelope without changing its payload.""" + + def _fail(self, message: str, **meta) -> None: + assert self.context is not None + self.context.response.success = False + self.context.response.answer = f"Error: {message}" + self.context.response.metadata.update(meta) diff --git a/reme/steps/file_io/daily_write.py b/reme/steps/file_io/daily_write.py index 31ec050b..addff538 100644 --- a/reme/steps/file_io/daily_write.py +++ b/reme/steps/file_io/daily_write.py @@ -1,8 +1,8 @@ """Write a daily markdown note by dispatching the generic ``write`` step.""" +from ._base import _FileIOStep from ._daily_index import parse_daily_date, refresh_day_index, validate_session_id from ._path import validate_filename_component -from ..base_step import BaseStep from ..index import normalize_posix_path from ...components import R from ...steps.evolve import now @@ -11,16 +11,9 @@ _RESERVED_METADATA_KEYS = {"name", "description", "session_id", "source_conversa @R.register("daily_write_step") -class DailyWriteStep(BaseStep): +class DailyWriteStep(_FileIOStep): """Write ``daily//.md`` with conversation frontmatter.""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _session_dir(self) -> str: return normalize_posix_path(str(self.config_value("session_dir"))) diff --git a/reme/steps/file_io/edit.py b/reme/steps/file_io/edit.py index 2e412d54..b9a457d5 100644 --- a/reme/steps/file_io/edit.py +++ b/reme/steps/file_io/edit.py @@ -3,14 +3,14 @@ import frontmatter import yaml +from ._base import _FileIOStep from ._file_io import get_path_lock, read_file_safe, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("edit_step") -class EditStep(BaseStep): +class EditStep(_FileIOStep): """Replace every occurrence of ``old`` with ``new`` inside the file body. The YAML front matter block (if any) is parsed out, kept verbatim and @@ -24,13 +24,6 @@ class EditStep(BaseStep): read-modify-write cycle against the same file (multi-worker / multi- process safety is out of scope).""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/list.py b/reme/steps/file_io/list.py index df217e81..7f62185e 100644 --- a/reme/steps/file_io/list.py +++ b/reme/steps/file_io/list.py @@ -19,8 +19,8 @@ from pathlib import Path from stat import S_ISREG from typing import Iterable +from ._base import _FileIOStep from ._path import resolve_path -from ..base_step import BaseStep from ...components import R # Default cap on returned items so huge workspaces don't blow up the response. @@ -28,17 +28,9 @@ DEFAULT_LIMIT = 100 @R.register("list_step") -class ListStep(BaseStep): +class ListStep(_FileIOStep): """Enumerate files under a directory in the workspace.""" - def _fail(self, message: str, **meta) -> None: - """Set a failed response (matches the read/edit/... fail envelope).""" - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _collect_params(self) -> tuple[str, bool, int, str, frozenset[str]]: """Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively.""" assert self.context is not None diff --git a/reme/steps/file_io/load.py b/reme/steps/file_io/load.py index 94bc23fc..4403c72e 100644 --- a/reme/steps/file_io/load.py +++ b/reme/steps/file_io/load.py @@ -2,24 +2,18 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import read_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R DEFAULT_EDITOR_MAX_BYTES = 5 * 1024 * 1024 @R.register("load_step") -class LoadStep(BaseStep): +class LoadStep(_FileIOStep): """Return complete text without the truncation used by agent-facing reads.""" - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/read.py b/reme/steps/file_io/read.py index 48f53aea..268f5455 100644 --- a/reme/steps/file_io/read.py +++ b/reme/steps/file_io/read.py @@ -2,9 +2,9 @@ from pathlib import Path +from ._base import _FileIOStep from ._file_io import read_file_lines_safe, read_file_safe, truncate_session_output, truncate_text_output from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ..index._source_format import is_session_path from ...components import R from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES @@ -12,7 +12,7 @@ from ...utils import expand_links, render_expansion_lines @R.register("read_step") -class ReadStep(BaseStep): +class ReadStep(_FileIOStep): """Read a markdown file. Optional `start_line`/`end_line` for ranged reads. Step-level attributes (``kwargs``, configured in yaml under ``steps:`` — @@ -37,12 +37,8 @@ class ReadStep(BaseStep): def _fail(self, message: str, **meta) -> None: """Mark the response failed and stash a human-readable error.""" - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" + super()._fail(message, **meta) self.logger.warning(f"[{self.name}] FAILED: {message} | meta={meta}") - if meta: - self.context.response.metadata.update(meta) def _resolve_target(self, raw: str) -> Path | None: """Resolve ``raw`` under workspace and gate the markdown suffix. diff --git a/reme/steps/file_io/read_image.py b/reme/steps/file_io/read_image.py index 67342e0c..84666eec 100644 --- a/reme/steps/file_io/read_image.py +++ b/reme/steps/file_io/read_image.py @@ -5,14 +5,14 @@ import base64 import aiofiles import aiofiles.os +from ._base import _FileIOStep from ._path import NON_IMAGE_WARNING, gate_image, resolve_path -from ..base_step import BaseStep from ...components import R from ...constants import DEFAULT_MAX_IMAGE_BYTES @R.register("read_image_step") -class ReadImageStep(BaseStep): +class ReadImageStep(_FileIOStep): """Read an image file under ``workspace_dir`` and return base64 in ``answer``. Step-level attributes (``kwargs``, configured in yaml under ``steps:`` — @@ -32,13 +32,6 @@ class ReadImageStep(BaseStep): base64; ``metadata.non_image_warning=True`` flags it. """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/save.py b/reme/steps/file_io/save.py index 0b77cf7a..9ec9845f 100644 --- a/reme/steps/file_io/save.py +++ b/reme/steps/file_io/save.py @@ -2,14 +2,14 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("save_step") -class SaveStep(BaseStep): +class SaveStep(_FileIOStep): """Persist editor content without interpreting or rebuilding frontmatter. ``expected_mtime`` is the ISO timestamp returned by ``stat``. When supplied, @@ -17,12 +17,6 @@ class SaveStep(BaseStep): filesystem authoritative when ReMe and another editor are used together. """ - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/write.py b/reme/steps/file_io/write.py index 44d1425d..ff5fd081 100644 --- a/reme/steps/file_io/write.py +++ b/reme/steps/file_io/write.py @@ -2,14 +2,14 @@ import frontmatter +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("write_step") -class WriteStep(BaseStep): +class WriteStep(_FileIOStep): """Write (create or overwrite) a markdown file. Frontmatter accepts two reserved string fields (``name`` / ``description``) @@ -26,13 +26,6 @@ class WriteStep(BaseStep): scope). """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/index/_change_batch.py b/reme/steps/index/_change_batch.py index 0d67057f..e1457603 100644 --- a/reme/steps/index/_change_batch.py +++ b/reme/steps/index/_change_batch.py @@ -7,7 +7,7 @@ from pathlib import Path from watchfiles import Change -def _normalize_change(raw) -> Change | None: +def _normalize_change(raw: object) -> Change | None: if isinstance(raw, Change): return raw if isinstance(raw, str): diff --git a/reme/steps/index/_dedup.py b/reme/steps/index/_dedup.py index 96929e7b..230aebfb 100644 --- a/reme/steps/index/_dedup.py +++ b/reme/steps/index/_dedup.py @@ -12,15 +12,13 @@ When ``app_context`` is absent the same structure is mirrored under """ import datetime -from typing import TYPE_CHECKING, Any, Callable, Final +from typing import Callable, Final +from ._tool_context import _ToolContextMixin from ...schema import FileChunk -if TYPE_CHECKING: - from ...components import ApplicationContext - -class _ToolContextDedupMixin: +class _ToolContextDedupMixin(_ToolContextMixin): """Mixin providing tool_context-scoped chunk dedup with TTL. Must be mixed into a ``BaseStep`` subclass (e.g. ``SearchStep``); it @@ -29,7 +27,6 @@ class _ToolContextDedupMixin: set by the host step's ``__init__`` (default 24h). """ - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_RANGES_KEY: Final[str] = "search_seen_chunk_ranges" def __init_subclass__(cls, **kwargs): @@ -53,25 +50,7 @@ class _ToolContextDedupMixin: ) return super().__new__(cls, *args, **kwargs) - if TYPE_CHECKING: - # Declared by BaseStep; repeated here so static analysis resolves - # attribute access on the mixin without inheriting BaseStep. - app_context: "ApplicationContext | None" - kwargs: dict[str, Any] - seen_ttl_hours: float - - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context. - - The bucket is created lazily on first access and lives at - ``metadata["tool_contexts"][tool_context_id]`` (or the same path under - ``kwargs`` when no ``app_context`` is available, e.g. in unit tests). - """ - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) + seen_ttl_hours: float @staticmethod def _now_ts() -> float: diff --git a/reme/steps/index/_single_search.py b/reme/steps/index/_single_search.py new file mode 100644 index 00000000..06973f02 --- /dev/null +++ b/reme/steps/index/_single_search.py @@ -0,0 +1,68 @@ +"""Shared execution for single-backend search steps.""" + +from abc import abstractmethod +from typing import Final + +from ._dedup import _ToolContextDedupMixin +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries +from ._source_format import merge_session_chunk_intervals, render_chunk_entries +from ..base_step import BaseStep +from ...schema import FileChunk + +_MAX_CANDIDATES: Final = 200 +_CANDIDATE_MULTIPLIER: Final = 10 + + +class _SingleSearchStep(_ToolContextDedupMixin, BaseStep): + """Retrieve candidates, then apply the common single-backend search contract.""" + + def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.seen_ttl_hours = seen_ttl_hours + self.include_source = include_source + + async def execute(self): + assert self.context is not None + raw_query = self.context.get("query") + query: str = "" if raw_query is None else str(raw_query).strip() + limit: int = int(self.context.get("limit") or 5) + min_score: float = float(self.context.get("min_score") or 0.0) + tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() + + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response + assert limit > 0, f"limit must be positive, got {limit}" + + candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) + results = await self._retrieve(query, candidates) + self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") + + if min_score > 0.0: + results = [chunk for chunk in results if chunk.score >= min_score] + + pre_dedup_count = 0 + if tool_context_id: + pre_dedup_count = len(results) + results, _ = self._dedupe_tool_context(results, tool_context_id, limit) + else: + results = results[:limit] + + session_dir = self.config_value("session_dir") + entries = render_chunk_entries( + merge_session_chunk_intervals(results, session_dir), + session_dir, + include_source=self.include_source, + ) + self.context.response.answer = join_chunk_entries(entries) + if not results: + self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE + self.context.response.metadata["results"] = [ + c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results + ] + return self.context.response + + @abstractmethod + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + """Fetch ranked candidates from the concrete backend.""" diff --git a/reme/steps/index/_tool_context.py b/reme/steps/index/_tool_context.py new file mode 100644 index 00000000..4b14a6d5 --- /dev/null +++ b/reme/steps/index/_tool_context.py @@ -0,0 +1,23 @@ +"""Shared access to application-lifetime tool context state.""" + +from typing import TYPE_CHECKING, Any, Final + +if TYPE_CHECKING: + from ...components import ApplicationContext + + +class _ToolContextMixin: + """Share storage access while leaving each step's state format independent.""" + + TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" + + if TYPE_CHECKING: + app_context: "ApplicationContext | None" + kwargs: dict[str, Any] + + def _tool_context_store(self, tool_context_id: str) -> dict: + if self.app_context is not None: + contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) + else: + contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) + return contexts.setdefault(tool_context_id, {}) diff --git a/reme/steps/index/bm25_search.py b/reme/steps/index/bm25_search.py index 99ca8a4d..59bd2460 100644 --- a/reme/steps/index/bm25_search.py +++ b/reme/steps/index/bm25_search.py @@ -1,64 +1,13 @@ -"""``bm25_search_step`` — plain BM25 keyword search with tool_context dedup.""" +"""Plain BM25 keyword search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("bm25_search_step") -class Bm25SearchStep(_ToolContextDedupMixin, BaseStep): - """BM25-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class Bm25SearchStep(_SingleSearchStep): + """Search the BM25 keyword backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.keyword_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.keyword_search(query, candidates, {}) diff --git a/reme/steps/index/draft.py b/reme/steps/index/draft.py index a33d3db2..5684e329 100644 --- a/reme/steps/index/draft.py +++ b/reme/steps/index/draft.py @@ -2,24 +2,17 @@ from typing import Final +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ...components import R @R.register("add_draft_step") -class AddDraftStep(BaseStep): +class AddDraftStep(_ToolContextMixin, BaseStep): """Append one draft text item to the current tool context.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" DRAFTS_KEY: Final[str] = "drafts" - def _tool_context_store(self, tool_context_id: str) -> dict: - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - async def execute(self): assert self.context is not None text = self.context.get("text", "") diff --git a/reme/steps/index/search.py b/reme/steps/index/search.py index 928587f5..f4e20b3c 100644 --- a/reme/steps/index/search.py +++ b/reme/steps/index/search.py @@ -5,6 +5,8 @@ import datetime import os from typing import Final +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ..file_io import extract_daily_date from ...components import R @@ -35,10 +37,9 @@ def _filter_values(value: object) -> set: @R.register("search_step") -class SearchStep(BaseStep): +class SearchStep(_ToolContextMixin, BaseStep): """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_KEY: Final[str] = "search_seen_chunk_ids" def __init__( @@ -98,14 +99,6 @@ class SearchStep(BaseStep): def _now_ts() -> float: return datetime.datetime.now().timestamp() - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context.""" - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - def _dedupe_tool_context( self, chunks: list[FileChunk], @@ -203,6 +196,7 @@ class SearchStep(BaseStep): async def execute(self): assert self.context is not None + self.context.response.metadata.pop("empty_result", None) raw_query = self.context.get("query") query: str = "" if raw_query is None else str(raw_query).strip() limit: int = int(self.context.get("limit") or _default_limit()) @@ -349,6 +343,7 @@ class SearchStep(BaseStep): ) fused = [c for c in fused if c.score >= min_score] + pre_dedup_count = len(fused) dedup: dict | None = None if tool_context_id: fused, dedup = self._dedupe_tool_context(fused, tool_context_id, limit) @@ -369,6 +364,11 @@ class SearchStep(BaseStep): answer_lines.extend(render_expansion_lines(link_expansion.get(c.path, {}))) self.context.response.answer = "\n".join(answer_lines) + if not fused: + self.context.response.metadata["empty_result"] = { + "reason": "all_returned" if pre_dedup_count else "no_results", + "message": ALL_RETURNED_MESSAGE if pre_dedup_count else NO_RESULTS_MESSAGE, + } self.context.response.metadata["results"] = [ c.model_dump(exclude_none=True, exclude={"embedding"}) for c in fused ] diff --git a/reme/steps/index/vector_search.py b/reme/steps/index/vector_search.py index 5de5aa94..858db2bb 100644 --- a/reme/steps/index/vector_search.py +++ b/reme/steps/index/vector_search.py @@ -1,64 +1,13 @@ -"""``vector_search_step`` — plain vector search with tool_context dedup.""" +"""Plain vector search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("vector_search_step") -class VectorSearchStep(_ToolContextDedupMixin, BaseStep): - """Vector-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class VectorSearchStep(_SingleSearchStep): + """Search the vector backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.vector_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.vector_search(query, candidates, {}) diff --git a/reme/utils/tushare.py b/reme/utils/tushare.py deleted file mode 100644 index 68218151..00000000 --- a/reme/utils/tushare.py +++ /dev/null @@ -1,17 +0,0 @@ -"""TuShare client construction with an optional mirror endpoint.""" - -import os - - -def create_tushare_api(token: str): - """Create a TuShare DataApi, using TUSHARE_MIRROR_URL when configured.""" - try: - import tushare as ts - except ImportError as exc: # pragma: no cover - optional core dependency. - raise RuntimeError("tushare is required for market-data research") from exc - - api = ts.pro_api(token) - api._DataApi__timeout = 600 # pylint: disable=protected-access - if mirror_url := os.getenv("TUSHARE_MIRROR_URL", "").strip(): - api._DataApi__http_url = mirror_url.rstrip("/") # pylint: disable=protected-access - return api diff --git a/reme_studio/README.md b/reme_studio/README.md index 8667cd91..2cbeea61 100644 --- a/reme_studio/README.md +++ b/reme_studio/README.md @@ -402,8 +402,10 @@ Run proportionate checks from `reme_studio/`: ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# Build and verify the Sites, static, and demo distributions: +npm run test:build +# Run both suites: npm test ``` diff --git a/reme_studio/README_ZH.md b/reme_studio/README_ZH.md index f3a11d59..3351d3a9 100644 --- a/reme_studio/README_ZH.md +++ b/reme_studio/README_ZH.md @@ -357,8 +357,10 @@ npm run build:demo ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# 构建并验证 Sites、静态和演示分发： +npm run test:build +# 运行以上全部检查： npm test ``` diff --git a/reme_studio/app/chatgpt-auth.ts b/reme_studio/app/chatgpt-auth.ts deleted file mode 100644 index a0ae2ed9..00000000 --- a/reme_studio/app/chatgpt-auth.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { headers } from "next/headers"; -import { redirect } from "next/navigation"; - -export type ChatGPTUser = { - userId: string; - displayName: string; - email: string; - fullName: string | null; -}; - -const USER_ID_HEADER = "oai-authenticated-user-id"; -const USER_EMAIL_HEADER = "oai-authenticated-user-email"; -const USER_FULL_NAME_HEADER = "oai-authenticated-user-full-name"; -const USER_FULL_NAME_ENCODING_HEADER = - "oai-authenticated-user-full-name-encoding"; -const PERCENT_ENCODED_UTF8 = "percent-encoded-utf-8"; -const SIGN_IN_PATH = "/signin-with-chatgpt"; -const SIGN_OUT_PATH = "/signout-with-chatgpt"; -const CALLBACK_PATH = "/callback"; - -export async function getChatGPTUser(): Promise { - const requestHeaders = await headers(); - const userId = requestHeaders.get(USER_ID_HEADER); - const email = requestHeaders.get(USER_EMAIL_HEADER); - if (!userId || !email) return null; - - const encodedFullName = requestHeaders.get(USER_FULL_NAME_HEADER); - const fullName = - encodedFullName && - requestHeaders.get(USER_FULL_NAME_ENCODING_HEADER) === PERCENT_ENCODED_UTF8 - ? safeDecodeURIComponent(encodedFullName) - : null; - - return { - userId, - displayName: fullName ?? email, - email, - fullName, - }; -} - -export async function requireChatGPTUser( - returnTo: string, -): Promise { - const user = await getChatGPTUser(); - if (user) return user; - - redirect(chatGPTSignInPath(returnTo)); -} - -export function chatGPTSignInPath(returnTo: string): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_IN_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -export function chatGPTSignOutPath(returnTo = "/"): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_OUT_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -function safeRelativeReturnPath(value: string): string { - if (!value.startsWith("/") || value.startsWith("//")) return "/"; - - let url: URL; - try { - url = new URL(value, "https://app.local"); - } catch { - return "/"; - } - if (url.origin !== "https://app.local") return "/"; - if (isReservedAuthPath(url.pathname)) return "/"; - - return `${url.pathname}${url.search}${url.hash}`; -} - -function isReservedAuthPath(pathname: string): boolean { - return ( - pathname === SIGN_IN_PATH || - pathname === SIGN_OUT_PATH || - pathname === CALLBACK_PATH - ); -} - -function safeDecodeURIComponent(value: string): string | null { - try { - return decodeURIComponent(value); - } catch { - return null; - } -} diff --git a/reme_studio/app/globals.css b/reme_studio/app/globals.css index 2fc41164..2621b818 100644 --- a/reme_studio/app/globals.css +++ b/reme_studio/app/globals.css @@ -1610,27 +1610,6 @@ textarea:focus-visible { margin: 8px 0 10px; font-size: 12px; } -.component-row { - min-height: 48px; - display: flex; - align-items: center; - justify-content: space-between; - gap: 16px; - border-top: 1px solid var(--line); -} -.component-row > span { - display: flex; - flex-direction: column; - font-size: 12px; -} -.component-row small { - margin-top: 2px; - color: var(--muted); - font-size: 10px; -} -.component-row strong { - font-size: 12px; -} .index-card { padding: 22px; } diff --git a/reme_studio/app/open-file.ts b/reme_studio/app/open-file.ts index 41b965c0..9fe34dab 100644 --- a/reme_studio/app/open-file.ts +++ b/reme_studio/app/open-file.ts @@ -9,13 +9,24 @@ export async function openWorkspaceFile(path: string) { ); const id = store.openMarkdown(path); if (existing) return; + await loadWorkspaceFile(id, path); +} + +/** Load both newly opened and restored tabs, respecting the caller's lifetime. */ +export async function loadWorkspaceFile( + id: string, + path: string, + shouldApply: () => boolean = () => true, +) { + const store = useWorkspaceStore.getState(); try { const file = await readWorkspaceFile(path); - store.hydrateMarkdown(id, file.content, file.stat.mtime); + if (shouldApply()) store.hydrateMarkdown(id, file.content, file.stat.mtime); } catch (error) { - store.failMarkdown( - id, - error instanceof Error ? error.message : "Failed to read file", - ); + if (shouldApply()) + store.failMarkdown( + id, + error instanceof Error ? error.message : "Failed to read file", + ); } } diff --git a/reme_studio/app/store.ts b/reme_studio/app/store.ts index 5f2fb64f..98ec968d 100644 --- a/reme_studio/app/store.ts +++ b/reme_studio/app/store.ts @@ -54,6 +54,19 @@ interface WorkspaceState { finishChat: (tabId: string, messageId: string, error?: string) => void; } +function updateChatMessage( + tab: Extract, + messageId: string, + update: (message: ChatMessage) => ChatMessage, +) { + return { + ...tab, + messages: tab.messages.map((message) => + message.id === messageId ? update(message) : message, + ), + }; +} + const fileTitle = (path: string) => path.split("/").pop() || path; // Preserve the original key so existing tabs, chats, and unsaved drafts survive the Studio rename. const WORKSPACE_STORAGE_KEY = IS_DEMO @@ -62,189 +75,153 @@ const WORKSPACE_STORAGE_KEY = IS_DEMO export const useWorkspaceStore = create()( persist( - (set, get) => ({ - tabs: [], - activeTabId: undefined, - openMarkdown: (path) => { - const existing = get().tabs.find( - (tab) => tab.type === "markdown" && tab.path === path, - ); - if (existing) { - set({ activeTabId: existing.id }); - return existing.id; - } - const id = `file:${path}`; + (set, get) => { + const updateTab = ( + id: string, + type: Type, + update: (tab: Extract) => WorkspaceTab, + ) => set((state) => ({ - tabs: [ - ...state.tabs, - { - id, - type: "markdown", - title: fileTitle(path), - path, - content: "", - savedContent: "", - loading: true, - }, - ], - activeTabId: id, + tabs: state.tabs.map((tab) => + tab.id === id && tab.type === type + ? update(tab as Extract) + : tab, + ), })); - return id; - }, - openAgent: () => { - const id = `agent:${crypto.randomUUID()}`; - set((state) => ({ - tabs: [...state.tabs, { id, type: "agent", title: "", messages: [] }], - activeTabId: id, - })); - return id; - }, - openGraph: (root) => { - const id = `graph:${root}`; - const existing = get().tabs.find( - (tab) => tab.id === id && tab.type === "graph", - ); - if (existing) { - set({ activeTabId: id }); + return { + tabs: [], + activeTabId: undefined, + openMarkdown: (path) => { + const existing = get().tabs.find( + (tab) => tab.type === "markdown" && tab.path === path, + ); + if (existing) { + set({ activeTabId: existing.id }); + return existing.id; + } + const id = `file:${path}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { + id, + type: "markdown", + title: fileTitle(path), + path, + content: "", + savedContent: "", + loading: true, + }, + ], + activeTabId: id, + })); return id; - } - set((state) => ({ - tabs: [ - ...state.tabs, - { id, type: "graph", title: `${root} graph`, root }, - ], - activeTabId: id, - })); - return id; - }, - closeTab: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, false).length - ) - return state; - const index = state.tabs.findIndex((tab) => tab.id === id); - const tabs = state.tabs.filter((tab) => tab.id !== id); - const activeTabId = - state.activeTabId === id - ? tabs[Math.max(0, index - 1)]?.id - : state.activeTabId; - return { tabs, activeTabId }; - }), - closeOtherTabs: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, true).length - ) - return state; - const tab = state.tabs.find((item) => item.id === id); - return tab ? { tabs: [tab], activeTabId: id } : state; - }), - setActiveTab: (activeTabId) => set({ activeTabId }), - hydrateMarkdown: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { - ...tab, - content, - savedContent: content, - mtime, - loading: false, - error: undefined, - } - : tab, - ), - })), - failMarkdown: (id, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, loading: false, error } - : tab, - ), - })), - updateMarkdown: (id, content) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, content } - : tab, - ), - })), - markSaved: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? markMarkdownContentSaved(tab, content, mtime) - : tab, - ), - })), - addChatTurn: (tabId, user, assistant) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - title: tab.messages.length - ? tab.title - : user.content.slice(0, 18), - messages: [...tab.messages, user, assistant], - streaming: true, - } - : tab, - ), - })), - applyChatChunk: (tabId, messageId, chunk) => - set((state) => ({ - tabs: state.tabs.map((tab) => { - if (tab.id !== tabId || tab.type !== "agent") return tab; - const messages = tab.messages.map((message) => - message.id === messageId - ? applyStreamChunk(message, chunk) - : message, - ); - return { - ...tab, - sessionId: chunk.session_id || tab.sessionId, - messages, - }; + }, + openAgent: () => { + const id = `agent:${crypto.randomUUID()}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "agent", title: "", messages: [] }, + ], + activeTabId: id, + })); + return id; + }, + openGraph: (root) => { + const id = `graph:${root}`; + const existing = get().tabs.find( + (tab) => tab.id === id && tab.type === "graph", + ); + if (existing) { + set({ activeTabId: id }); + return id; + } + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "graph", title: `${root} graph`, root }, + ], + activeTabId: id, + })); + return id; + }, + closeTab: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, false).length + ) + return state; + const index = state.tabs.findIndex((tab) => tab.id === id); + const tabs = state.tabs.filter((tab) => tab.id !== id); + const activeTabId = + state.activeTabId === id + ? tabs[Math.max(0, index - 1)]?.id + : state.activeTabId; + return { tabs, activeTabId }; }), - })), - toggleChatBlock: (tabId, messageId, blockId, expanded) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - messages: tab.messages.map((message) => - message.id === messageId - ? toggleChatBlock(message, blockId, expanded) - : message, - ), - } - : tab, + closeOtherTabs: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, true).length + ) + return state; + const tab = state.tabs.find((item) => item.id === id); + return tab ? { tabs: [tab], activeTabId: id } : state; + }), + setActiveTab: (activeTabId) => set({ activeTabId }), + hydrateMarkdown: (id, content, mtime) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + content, + savedContent: content, + mtime, + loading: false, + error: undefined, + })), + failMarkdown: (id, error) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + loading: false, + error, + })), + updateMarkdown: (id, content) => + updateTab(id, "markdown", (tab) => ({ ...tab, content })), + markSaved: (id, content, mtime) => + updateTab(id, "markdown", (tab) => + markMarkdownContentSaved(tab, content, mtime), ), - })), - finishChat: (tabId, messageId, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - streaming: false, - messages: tab.messages.map((message) => - message.id === messageId - ? finishChatMessage(message, error) - : message, - ), - } - : tab, + addChatTurn: (tabId, user, assistant) => + updateTab(tabId, "agent", (tab) => ({ + ...tab, + title: tab.messages.length ? tab.title : user.content.slice(0, 18), + messages: [...tab.messages, user, assistant], + streaming: true, + })), + applyChatChunk: (tabId, messageId, chunk) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + applyStreamChunk(message, chunk), + ), + sessionId: chunk.session_id || tab.sessionId, + })), + toggleChatBlock: (tabId, messageId, blockId, expanded) => + updateTab(tabId, "agent", (tab) => + updateChatMessage(tab, messageId, (message) => + toggleChatBlock(message, blockId, expanded), + ), ), - })), - }), + finishChat: (tabId, messageId, error) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + finishChatMessage(message, error), + ), + streaming: false, + })), + }; + }, { name: WORKSPACE_STORAGE_KEY, version: 1, diff --git a/reme_studio/app/workspace-chat.tsx b/reme_studio/app/workspace-chat.tsx new file mode 100644 index 00000000..6a6475ae --- /dev/null +++ b/reme_studio/app/workspace-chat.tsx @@ -0,0 +1,279 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { Check, CircleAlert, LoaderCircle, Send } from "lucide-react"; +import WorkspaceMarkdown from "./files-workspace/WorkspaceMarkdown"; +import { streamChat } from "./api"; +import { chatStreamError, formatStreamPayloads } from "./chat-stream"; +import { useI18n } from "./i18n"; +import { useWorkspaceStore } from "./store"; +import { IS_DEMO } from "./studio-mode"; +import type { + ChatBlock, + ChatMessage, + DetailBlock, + WorkspaceTab, +} from "./types"; +import { + appendWorkspaceFileReference, + WORKSPACE_FILE_DRAG_TYPE, +} from "./workspace-drag"; +import { scenarios } from "../demo/scenarios"; + +function MarkdownView({ content }: { content: string }) { + return ( +
+ +
+ ); +} + +function ChunkStatus({ + active, + error = false, +}: { + active: boolean; + error?: boolean; +}) { + if (error) return ; + return active ? ( + + ) : ( + + ); +} + +function detailLabel( + block: DetailBlock, + t: ReturnType["t"], +): string { + if (block.type === "think") return t("thinking"); + if (block.type === "data") + return block.mediaType + ? `${t("dataChunk")} · ${block.mediaType}` + : t("dataChunk"); + if (block.type === "approval") return t("approvalChunk"); + if (block.type === "usage") return t("usageChunk"); + return t("unknownChunk", { type: block.sourceType }); +} + +function ChatBlockView({ + block, + onToggle, +}: { + block: ChatBlock; + onToggle: (blockId: string, expanded: boolean) => void; +}) { + const { t } = useI18n(); + if (block.type === "content") return ; + if (block.type === "error") + return ( +
+ + {block.text} +
+ ); + if (block.type === "usage") return null; + + const active = + block.status === "streaming" || + block.status === "calling" || + block.status === "running"; + const failed = block.status === "error"; + const tool = block.type === "tool"; + const call = tool ? formatStreamPayloads(block.callPayloads) : ""; + const result = tool ? formatStreamPayloads(block.resultPayloads) : ""; + const detail = tool ? "" : formatStreamPayloads(block.payloads); + return ( +
{ + if (event.currentTarget.open !== block.expanded) + onToggle(block.id, event.currentTarget.open); + }} + > + + + {tool ? block.name : detailLabel(block, t)} + {active ? t("streaming") : t("completed")} + + {tool ? ( +
+ {call && ( +
+ {t("toolCall")} + 
{call}
+
+ )} + {result && ( +
+ {t("toolResult")} + 
{result}
+
+ )} +
+ ) : detail ? ( +
+ 
{detail}
+
+ ) : null} +
+ ); +} + +export default function Chat({ + tab, +}: { + tab: Extract; +}) { + const { language, t } = useI18n(); + const [input, setInput] = useState(""); + const [fileDragOver, setFileDragOver] = useState(false); + const endRef = useRef(null); + const controller = useRef(null); + const { addChatTurn, applyChatChunk, toggleChatBlock, finishChat } = + useWorkspaceStore(); + + useEffect(() => { + endRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [tab.messages]); + useEffect(() => () => controller.current?.abort(), []); + + const suggestions = ( +
+ {(IS_DEMO + ? scenarios[language].map((scenario) => scenario.question) + : [t("promptRecent"), t("promptTasks"), t("promptIdeas")] + ).map((question) => ( + + ))} +
+ ); + + const send = async () => { + const query = input.trim(); + if (!query || tab.streaming) return; + setInput(""); + const user: ChatMessage = { + id: crypto.randomUUID(), + role: "user", + content: query, + }; + const assistant: ChatMessage = { + id: crypto.randomUUID(), + role: "assistant", + content: "", + blocks: [], + }; + addChatTurn(tab.id, user, assistant); + const requestController = new AbortController(); + controller.current = requestController; + const error = await chatStreamError( + () => + streamChat(query, tab.sessionId, requestController.signal, (chunk) => + applyChatChunk(tab.id, assistant.id, chunk), + ), + requestController.signal, + t("chatFailed"), + ); + finishChat(tab.id, assistant.id, error); + if (controller.current === requestController) controller.current = null; + }; + + return ( +
+
+ {!tab.messages.length && ( +
+ + )} + {tab.messages.map((message) => ( +
+
+ {message.role === "user" ? ( + t("you") + ) : ( + + )} +
+
+ {message.role === "user" &&

{message.content}

} + {message.role === "assistant" && + message.blocks?.map((block) => ( + + toggleChatBlock(tab.id, message.id, blockId, expanded) + } + /> + ))} + {message.role === "assistant" && + tab.streaming && + !message.blocks?.some((block) => block.type !== "usage") && ( + + )} +
+
+ ))} + {IS_DEMO && tab.messages.length > 0 && suggestions} +
+
+
{ + if (!event.dataTransfer.types.includes(WORKSPACE_FILE_DRAG_TYPE)) + return; + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + setFileDragOver(true); + }} + onDragLeave={(event) => { + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) + setFileDragOver(false); + }} + onDrop={(event) => { + const path = event.dataTransfer.getData(WORKSPACE_FILE_DRAG_TYPE); + if (!path) return; + event.preventDefault(); + setFileDragOver(false); + setInput((current) => appendWorkspaceFileReference(current, path)); + }} + > +
+