diff --git a/.github/workflows/_build-docs.yml b/.github/workflows/_build-docs.yml index 1b3eb8f5..d2b8a577 100644 --- a/.github/workflows/_build-docs.yml +++ b/.github/workflows/_build-docs.yml @@ -67,7 +67,7 @@ jobs: - name: Verify browser demo bundle if: inputs.run_tests working-directory: reme_studio - run: node --test tests/demo-build.test.mjs + run: node --test tests/distribution/demo-build.test.mjs - name: Configure Pages if: inputs.upload_pages_artifact diff --git a/.github/workflows/ci-reme-studio.yml b/.github/workflows/ci-reme-studio.yml index fa8f5d5f..eb9ccf3f 100644 --- a/.github/workflows/ci-reme-studio.yml +++ b/.github/workflows/ci-reme-studio.yml @@ -66,7 +66,7 @@ jobs: - name: Verify npm package run: | - npm pack --pack-destination "${RUNNER_TEMP}" + npm pack --ignore-scripts --pack-destination "${RUNNER_TEMP}" tar -tzf "${RUNNER_TEMP}"/agentscope-ai-reme_studio-*.tgz | grep '^package/dist-static/index.html$' - name: Set up Python diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ce45e3a4..e6a1e69b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -118,7 +118,7 @@ jobs: - name: Log in to GHCR if: env.PUBLISH_IMAGE == 'true' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -177,7 +177,7 @@ jobs: - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-auto-fin.yml b/.github/workflows/release-auto-fin.yml index 31d294d1..ccb9f52e 100644 --- a/.github/workflows/release-auto-fin.yml +++ b/.github/workflows/release-auto-fin.yml @@ -133,7 +133,8 @@ jobs: assert set(manifest.backends) == { "auto_fin_data_step", "auto_fin_topic_step", - "auto_fin_merge_step", + "auto_fin_research_step", + "auto_fin_digest_step", } assert set(manifest.application_defaults["jobs"]) == { "auto_fin", @@ -165,6 +166,6 @@ jobs: path: dist/auto-fin - name: Publish reme-auto-fin - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: packages-dir: dist/auto-fin diff --git a/.github/workflows/release-daily-paper.yml b/.github/workflows/release-daily-paper.yml index 7d51dc81..0d20c1f5 100644 --- a/.github/workflows/release-daily-paper.yml +++ b/.github/workflows/release-daily-paper.yml @@ -165,6 +165,6 @@ jobs: path: dist/daily-paper - name: Publish reme-daily-paper - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: packages-dir: dist/daily-paper diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index d0769b91..5144b383 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -42,7 +42,7 @@ jobs: path: dist/reme - name: Publish ReMe - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: packages-dir: dist/reme skip-existing: true diff --git a/.github/workflows/release-reme-studio.yml b/.github/workflows/release-reme-studio.yml index 01e8d328..fa6c539f 100644 --- a/.github/workflows/release-reme-studio.yml +++ b/.github/workflows/release-reme-studio.yml @@ -137,7 +137,7 @@ jobs: path: dist - name: Publish ReMe Studio to PyPI - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 with: packages-dir: dist/studio-python skip-existing: true diff --git a/.github/workflows/security-codeql.yml b/.github/workflows/security-codeql.yml index 92c2d46d..e29819bf 100644 --- a/.github/workflows/security-codeql.yml +++ b/.github/workflows/security-codeql.yml @@ -34,12 +34,12 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4 with: languages: ${{ matrix.language }} build-mode: none - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4 with: category: /language:${{ matrix.language }} diff --git a/benchmark/memlens/README.md b/benchmark/memlens/README.md new file mode 100644 index 00000000..73dcebfa --- /dev/null +++ b/benchmark/memlens/README.md @@ -0,0 +1,107 @@ +# ReMe Image on MemLens-128K / 256K + +Evaluate ReMe Image on MemLens's official 195-question memory-agent subset. +Each length uses its own full histories; every question has an isolated workspace. + +## Run + +```bash +pip install -e ".[as]" -e plugins/memlens -e plugins/memlens-judge +export MEMLENS_DATA=/path/to/MEMLENS +export EXPERIMENT_DIR=/path/to/new-experiment +python -m benchmark.memlens.run --model qwen3.8-max --context-length 256k +``` + +Supply `LLM_API_KEY` and `LLM_BASE_URL` through the environment, outside Git. +The dataset directory contains `dataset_128k.json`, `dataset_256k.json`, +`agent_subset_195.json` and `release_images/`. Select 128K with +`--context-length 128k`; `--limit-scenes 1 --output /path/to/separate-experiment` +runs one complete history. Concurrency is configured in `config.yaml`. + +## Evaluation + +Original text, timestamps, Image IDs and interleaved image bytes go to native +Auto Memory. Supplied BLIP captions and answer annotations are not inputs. +BM25 indexes daily notes, digest notes and original session transcripts. +Each question starts a fresh 30-iteration Agent with `search`, `read` and +`read_image`; no embedding or auto-tagging is used. + +| Setting | Memory / compression | Answer | +| --- | --- | --- | +| Model | Selected candidate | Same candidate | +| Thinking / temperature / streaming | On / 0 / off | On / 0 / off | +| Output budget | 65,536 | 16,384 | +| Model context size | 200,000 | 400,000 | + +The [candidate preset](../../plugins/memlens/src/reme_memlens/benchmark.yaml) +retains 30 Agent iterations, context trigger/reserve ratios 0.8/0.1, a +50,000-token tool-result limit and 4,096-image retention. Native requests use +three retries, a 60-second delay and a 600-second HTTP timeout; nested retries +are disabled. The runner does not repair model outputs. + +The independent [MemLens Judge](../../plugins/memlens-judge/README.md) uses the +official category/subtype-specific binary accuracy protocol, Qwen3.8-Max by default, +thinking off, temperature 0 and 256 output tokens. SubEM and F1 follow the +official code. Reference answers, question categories and subtypes enter only scoring. +`JUDGE_MODEL_NAME` overrides the judge model independently of the candidate. +`settings.json` records the effective judge model, generation parameters and +protocol after plugin defaults and environment expansion, without credentials. + +## Results + +`/memlens-//` contains settings and summary JSON. +`reme_image/scenes//` retains ingestion responses, a native workspace, +answers, judgments, metrics and coverage status. The workspace's `mem_session/` +contains native Agent JSONL trajectories. Existing output groups are not overwritten. + +New runs use usage policy `successful-http-requests-v2`: usage includes observed +successful HTTP requests in memory/answer scopes, including compression and requests +completed before a later Job or history failure. Each scope is counted once. +Judge requests and unsuccessful HTTP attempts are excluded. This is measured request +usage, not a full-sample cost estimate or provider billing total. Unknown counters +remain `null`; `records` and per-counter `coverage` show how many requests were measured. +Expected, scored and unscored question counts make coverage gaps explicit. + +Technical failures are coverage gaps; valid empty or iteration-limit answers are scored. +Failure artifacts retain the failed Response before validation and record the stage, +scene/session/question IDs, sanitized error chain and traceback locations without locals. +These artifacts omit credentials, request headers, raw configuration and image data. +Failed responses use separate `diagnostics/*-response.json` files; `answer.json` continues to +indicate a successful answer Job and retains the original prediction. Per-scope +`diagnostics/*-usage.json` receipts survive direct exceptions and cancellation, while +`status.json.memory_complete` records successful ingestion and indexing independently. +Cleanup errors retain scores and usage, increment `cleanup_error_count`, and cause +a nonzero CLI exit even when all questions were scored. +Historical reports using selected successful Job/history usage retain their original +values and scope; they are not rewritten under the new policy. + +## Sources and license + +[Dataset](https://huggingface.co/datasets/xiyuRenBill/MEMLENS); +[official code](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2). +The dataset retains its own data/image terms. Adapted query and metric code carries +the following upstream notice: + +```text +MIT License + +Copyright (c) 2026 Xiyu Ren, Zhaowei Wang, and the MemLens Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/benchmark/memlens/README_ZH.md b/benchmark/memlens/README_ZH.md new file mode 100644 index 00000000..5a384859 --- /dev/null +++ b/benchmark/memlens/README_ZH.md @@ -0,0 +1,69 @@ +# 在 MemLens-128K / 256K 上评测 ReMe Image + +使用官方固定的 195 题 memory-agent 子集。128K、256K 分别读取自己的完整历史， +每题使用独立 workspace。 + +## 运行 + +```bash +pip install -e ".[as]" -e plugins/memlens -e plugins/memlens-judge +export MEMLENS_DATA=/path/to/MEMLENS +export EXPERIMENT_DIR=/path/to/new-experiment +python -m benchmark.memlens.run --model qwen3.8-max --context-length 256k +``` + +通过环境变量提供 `LLM_API_KEY` 和 `LLM_BASE_URL`，不要提交凭据。 +数据目录包含 `dataset_128k.json`、`dataset_256k.json`、 +`agent_subset_195.json` 和 `release_images/`。使用 +`--context-length 128k` 切换长度；用 `--limit-scenes 1` 和独立的 +`--output` 路径小批量验证，不裁剪场景历史。并发配置在 `config.yaml`。 + +## 评测设置 + +原始文本、时间、Image ID 和交错的原图输入原生 Auto Memory； +不使用附带的 BLIP caption 或答案标注。BM25 索引 daily、digest 和原始 +session 正文。每题新建 30 轮回答 Agent，工具为 `search`、`read`、 +`read_image`，不启用 embedding 或 auto-tag。 + +| 设置 | 记忆／压缩 | 回答 | +| --- | --- | --- | +| 模型 | 选定的候选模型 | 同一候选模型 | +| Thinking／temperature／stream | 开启／0／关闭 | 开启／0／关闭 | +| 输出预算 | 65,536 | 16,384 | +| 模型 context size | 200,000 | 400,000 | + +[候选插件配置](../../plugins/memlens/src/reme_memlens/benchmark.yaml) 保留 +30 轮预算、0.8／0.1 上下文触发／预留比例、50,000 token 的工具结果限制和 +4,096 张图片保留上限。原生请求最多重试 3 次，间隔 60 秒，HTTP timeout +为 600 秒；不叠加 SDK／Agent 重试，也不修补模型输出。 + +独立的 [MemLens Judge](../../plugins/memlens-judge/README.md) 使用官方按 +category／subtype 区分的二元评分：默认 Qwen3.8-Max、thinking 关闭、temperature +0、输出预算 256。辅助指标为官方 SubEM、F1。答案与题型字段仅进入评分。 +`JUDGE_MODEL_NAME` 可独立覆盖 Judge 模型；`settings.json` 记录插件默认值和 +环境变量展开后实际生效的 Judge 模型、生成参数与协议，不保存凭据或服务端点。 + +## 结果 + +结果保存到 `/memlens-//`。该目录的 JSON 记录配置与汇总； +`reme_image/scenes//` 保存摄入响应、原生 workspace、回答、judge、 +指标及完成状态。workspace 下的 `mem_session/` 保留原生 Agent JSONL 轨迹。 +已有结果目录不会被覆盖。 + +新运行使用 `successful-http-requests-v2` 口径：统计记忆／回答作用域中观测到的 +成功 HTTP 请求，包括压缩，以及所属 Job 或历史处理随后失败前已成功的请求； +每个作用域只计一次。不计 Judge 请求及未成功的 HTTP 尝试。这是实测请求用量， +并非完整样本成本估计或供应商账单。未知计数保持 `null`，`records` 与各计数的 +`coverage` 标示观测覆盖；预期、已评分、未评分题数分别记录。 + +技术失败计为覆盖缺口，合法空答案与达到轮数上限的回答仍评分。失败诊断记录阶段、 +场景／session／题目 ID、脱敏错误链和不含局部变量的堆栈位置，不保存凭据、 +请求头、原始配置或图片数据。失败 Response 在检查前保存到独立的 +`diagnostics/*-response.json`；`answer.json` 仍只表示成功回答，保留原始预测。 +`diagnostics/*-usage.json` 在直接异常或取消时也保留已观测用量； +`status.json.memory_complete` 独立表示历史摄入和索引完成。 +清理失败保留评分及用量、增加 `cleanup_error_count`，并使 CLI 返回非零状态。 + +历史报告沿用其原有的选定成功 Job／历史统计口径，不按新口径重写数据。 + +数据来源、上游版本和完整授权文本见[英文 README](README.md#sources-and-license)。 diff --git a/benchmark/memlens/artifacts.py b/benchmark/memlens/artifacts.py new file mode 100644 index 00000000..18c48749 --- /dev/null +++ b/benchmark/memlens/artifacts.py @@ -0,0 +1,163 @@ +"""Small, credential-free snapshots and diagnostics for benchmark artifacts.""" + +import os +from pathlib import Path +import re +import traceback +from urllib.parse import urlsplit, urlunsplit + +_PRIVATE_KEY = re.compile( + r"credential|api[_-]?key|private[_-]?key|authorization|password|secret|cookie|access[_-]?token|refresh[_-]?token|" + r"(^|[_-])token$|headers|request[_-]?body|^environment$|^config(?:uration)?$|^messages$", + re.IGNORECASE, +) +_ASSIGNMENT = re.compile( + r"(?i)([\"']?(?:api[_-]?key|private[_-]?key|authorization|password|secret|(?:access|refresh|id)[_-]?token|token)" + r"[\"']?\s*[:=]\s*)(?:\"[^\"]*\"|'[^']*'|[^\s,;}]+)", +) +_MODEL_FIELDS = ( + "backend", + "model", + "stream", + "context_size", + "max_retries", + "retry_delay", +) +_MODEL_OPTIONS = { + "parameters": ( + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "thinking_enable", + ), + "extra_body": ("enable_thinking",), + "client_kwargs": ("timeout", "max_retries"), +} +_USAGE_FIELDS = { + "usage", + "coverage", + "records", + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "noncached_input_tokens", + "reasoning_tokens", +} + + +def _safe_text(value: str) -> str: + for key, secret in sorted(os.environ.items(), key=lambda item: len(item[1]), reverse=True): + if secret and _PRIVATE_KEY.search(key): + if len(secret) < 12: + value = re.sub(r"(?]+", safe_url, value) + value = re.sub(r"(?s)-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", "[REDACTED_KEY]", value) + value = re.sub(r"(?im)(\b(?:set[-_])?cookie[\"']?\s*[:=]\s*)[^\r\n]+", r"\1[REDACTED]", value) + value = re.sub(r"(?i)\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", "[REDACTED_AUTH]", value) + value = _ASSIGNMENT.sub(r"\1[REDACTED]", value) + value = re.sub(r"(?i)data:[^;,\s]+;base64,[A-Za-z0-9+/=\s]+", "[REDACTED_IMAGE]", value) + value = re.sub(r"\b[A-Za-z0-9+/]{80,}={0,2}", "[REDACTED_DATA]", value) + # Exceptions sometimes embed complete request/config dictionaries in their text. + return re.sub( + r"(?is)(\b(?:config|configuration|request_headers|headers|request_body|messages)\b[\"']?\s*[:=]).*", + r"\1[REDACTED]", + value, + ) + + +def safe_value(value, *, fixed_keys=()): + """Redact diagnostic data recursively without serializing arbitrary objects.""" + if isinstance(value, dict): + return { + (key if key in fixed_keys else _safe_text(str(key))): ( + "[REDACTED]" + if _PRIVATE_KEY.search(str(key)) or (key == "data" and value.get("type") == "base64") + else safe_value(item, fixed_keys=fixed_keys) + ) + for key, item in value.items() + } + if isinstance(value, (list, tuple)): + return [safe_value(item, fixed_keys=fixed_keys) for item in value] + if isinstance(value, str): + return _safe_text(value) + if value is None or isinstance(value, (bool, int, float)): + return value + return f"<{type(value).__name__}>" + + +def status_record(status: dict) -> dict: + """Keep runner-owned schema/enums/counters intact; errors are already sanitized.""" + return { + key: safe_value(value) if key in {"scene_id", "session_id", "question_id"} else value + for key, value in status.items() + } + + +def response_record(response) -> dict: + """Keep the Response schema while sanitizing its untrusted payload and extras.""" + raw = response.model_dump() + return { + **safe_value({key: value for key, value in raw.items() if key not in {"success", "answer", "metadata"}}), + "success": raw["success"], + "answer": safe_value(raw["answer"]), + "metadata": safe_value(raw["metadata"], fixed_keys=_USAGE_FIELDS), + } + + +def error_details(exc: BaseException, seen: set[int] | None = None) -> dict: + """Keep the error chain and frame locations, never frame locals or source text.""" + seen = set() if seen is None else seen + seen.add(id(exc)) + result = { + "type": type(exc).__name__, + "message": _safe_text(str(exc)), + "traceback": [ + { + "file": _safe_text(Path(frame.filename).name), + "line": frame.lineno, + "function": _safe_text(frame.name), + } + for frame in traceback.extract_tb(exc.__traceback__) + ], + } + cause = exc.__cause__ or (None if exc.__suppress_context__ else exc.__context__) + if cause is not None and id(cause) not in seen: + result["cause"] = error_details(cause, seen) + return result + + +def model_settings(config) -> dict: + """Whitelist reproducibility fields from a validated ApplicationConfig.""" + models = {} + for name, component in config.components.get("as_llm", {}).items(): + spec = component.model_dump() + snapshot = {key: safe_value(spec[key]) for key in _MODEL_FIELDS if key in spec} + for key, allowed in _MODEL_OPTIONS.items(): + options = spec.get(key) or {} + snapshot[key] = {name: safe_value(options[name]) for name in allowed if name in options} + models[_safe_text(name)] = snapshot + return models + + +def judge_settings(app) -> dict: + """Record the actual plugin/env-expanded judge model and official protocol.""" + from judge_memlens.protocol import PROTOCOL_ID, SOURCE_REVISION + + return { + "protocol_id": PROTOCOL_ID, + "source_revision": SOURCE_REVISION, + "models": model_settings(app.config), + } diff --git a/benchmark/memlens/config.yaml b/benchmark/memlens/config.yaml new file mode 100644 index 00000000..77d961c8 --- /dev/null +++ b/benchmark/memlens/config.yaml @@ -0,0 +1,4 @@ +dataset: ${MEMLENS_DATA} +output: ${EXPERIMENT_DIR} +scenario_concurrency: 4 +query_concurrency: 16 diff --git a/benchmark/memlens/run.py b/benchmark/memlens/run.py new file mode 100644 index 00000000..433270fc --- /dev/null +++ b/benchmark/memlens/run.py @@ -0,0 +1,621 @@ +"""Evaluate ReMe Image on the official MemLens memory-agent subset.""" + +import argparse +import base64 +from collections import Counter +from contextlib import contextmanager +from contextvars import ContextVar +from datetime import datetime +import io +import asyncio +import json +import os +from pathlib import Path +import re +import subprocess +import string +import sys + +import yaml + +from benchmark.memlens.artifacts import ( + error_details, + judge_settings, + model_settings, + response_record, + safe_value, + status_record, +) + +# Adapted query/metric code retains its upstream MIT notice in README.md. +SYSTEM_PROMPT = ( + "Provide answers based on the given conversation history. " + "If the question cannot be answered based on the given conversation, " + 'respond with "Insufficient information".' +) +INSTRUCTION = "Directly output the answer with no extra output." + + +def _image_path(root: Path, source: str) -> Path: + """Resolve only original images inside the released image directory.""" + path = (root / source).resolve() + if Path(source).is_absolute() or "\\" in source or not path.is_relative_to(root): + raise ValueError("MemLens image must stay inside release_images") + return path + + +def load_memlens(root: Path, context_length: str = "128k") -> list[dict]: + """Select the official 195 IDs in dataset order, with each question's own history.""" + root = root.expanduser().resolve() + source = root / f"dataset_{context_length}.json" + selected = json.loads((root / "agent_subset_195.json").read_text(encoding="utf-8"))["question_ids"] + if len(selected) != 195 or len(set(selected)) != 195: + raise ValueError("MemLens requires its official 195-question agent subset") + selected = set(selected) + images = (root / "release_images").resolve() + if not images.is_relative_to(root): + raise ValueError("MemLens images must stay inside the dataset directory") + result = [] + for row in json.loads(source.read_text(encoding="utf-8")): + if row["question_id"] not in selected: + continue + sessions = [] + for turns, date, session_id in zip( + row["haystack_sessions"], + row["haystack_dates"], + row["haystack_session_ids"], + strict=True, + ): + for turn in turns: + for image in turn.get("images", []): + _image_path(images, image["file"]) + sessions.append( + { + "id": session_id, + "timestamp": datetime.strptime(date, "%Y/%m/%d (%a) %H:%M").isoformat(timespec="minutes"), + "turns": turns, + "question_id": row["question_id"], + "image_root": images, + }, + ) + question = { + "id": row["question_id"], + "text": row["question"], + "question_date": row["question_date"], + "answer": row["answer"], + "category": row["question_type"], + "subtype": row.get("question_subtype", ""), + "old_answer": row.get("old_answer") or row.get("question_content", {}).get("old_answer", ""), + } + result.append({"id": row["question_id"], "sessions": sessions, "questions": [question]}) + if len(result) != 195 or {scene["id"] for scene in result} != selected: + raise ValueError("MemLens dataset must contain every selected question exactly once") + return result + + +def image_block(path: Path) -> dict: + """Encode untouched image bytes with their actual decoded MIME.""" + from PIL import Image + + if path.stat().st_size > 50 * 1024 * 1024: + raise ValueError(f"Oversized original image: {path.name}") + data = path.read_bytes() + if len(data) > 50 * 1024 * 1024: + raise ValueError(f"Oversized original image: {path.name}") + with Image.open(io.BytesIO(data)) as decoded: + if decoded.width * decoded.height > 50_000_000: + raise ValueError(f"Oversized original image dimensions: {path.name}") + media_type = decoded.get_format_mimetype() + if not media_type or not media_type.startswith("image/"): + raise ValueError(f"Missing image MIME: {path.name}") + decoded.load() + return { + "type": "data", + "source": { + "type": "base64", + "data": base64.b64encode(data).decode("ascii"), + "media_type": media_type, + }, + } + + +def session_messages(session: dict) -> list[dict]: + """Keep source roles, IDs, timestamps and the official placement.""" + messages = [] + for index, turn in enumerate(session["turns"]): + text, images = turn.get("content", ""), turn.get("images", []) + if not images: + stripped = text.replace("", "").strip() + parts = [stripped] if stripped else [] + elif "" not in text: + parts = [*images, *([text.strip()] if text.strip() else [])] + else: + parts, image_index = [], 0 + for part in re.split(r"()", text): + if part == "": + if image_index < len(images): + parts.append(images[image_index]) + image_index += 1 + elif part.strip(): + parts.append(part) + parts.extend(images[image_index:]) + content = [] + for part in parts: + if isinstance(part, str): + content.append({"type": "text", "text": part}) + else: + source = part["file"] + content.extend( + [ + {"type": "text", "text": f"[Image ID: {Path(source).stem}]"}, + image_block(_image_path(session["image_root"], source)), + ], + ) + messages.append( + { + "id": f"{session['question_id']}:{session['id']}:turn:{index}", + "name": turn["role"], + "role": turn["role"], + "content": content, + "created_at": session["timestamp"], + }, + ) + return messages + + +def query_prompt(question: dict, context: str) -> str: + """Keep the official dated query template unchanged.""" + return ( + f"{SYSTEM_PROMPT}\nConversation:\n{context}\n\n" + f"{INSTRUCTION}\nQuestion Date: {question['question_date']}\nQuestion: {question['text']}\n" + ) + + +def _normalize(text: str) -> str: + """Native SubEM/F1 normalization, preserving decimal points and date slashes.""" + text = text.lower() + text = "".join( + ( + char + if char not in string.punctuation + or (char in "./" and 0 < index < len(text) - 1 and text[index - 1].isdigit() and text[index + 1].isdigit()) + else " " + ) + for index, char in enumerate(text) + ) + text = re.sub(r"\ba(?=\s+[a-z])", " ", text) + text = re.sub(r"\b(an|the)\b", " ", text) + return " ".join(text.split()) + + +def score_answer(question: dict, prediction: str) -> dict: + """Official inline auxiliary SubEM/F1; primary accuracy comes from the LLM judge.""" + from judge_memlens.protocol import parse_model_output + + predicted, expected = _normalize(parse_model_output(prediction)), _normalize(question["answer"]) + prediction_tokens, answer_tokens = predicted.split(), expected.split() + overlap = sum((Counter(prediction_tokens) & Counter(answer_tokens)).values()) + return { + "sub_em": float(bool(predicted and expected) and expected in predicted), + "f1": 2 * overlap / (len(prediction_tokens) + len(answer_tokens)) if overlap else 0.0, + } + + +_CURRENT = ContextVar("memlens_usage", default=None) +_KEYS = ( + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "noncached_input_tokens", + "reasoning_tokens", +) + + +def normalized_usage(raw: dict) -> dict: + """Keep unknown counters unknown; cache/reasoning are subsets, not extra tokens.""" + incoming = raw.get("input_tokens", raw.get("prompt_tokens")) + outgoing = raw.get("output_tokens", raw.get("completion_tokens")) + total = raw.get("total_tokens") + if total is None and incoming is not None and outgoing is not None: + total = incoming + outgoing + cached = raw.get( + "cached_input_tokens", + raw.get("cached_tokens", (raw.get("prompt_tokens_details") or {}).get("cached_tokens")), + ) + reasoning = raw.get("reasoning_tokens", (raw.get("completion_tokens_details") or {}).get("reasoning_tokens")) + return { + "input_tokens": incoming, + "output_tokens": outgoing, + "total_tokens": total, + "cached_input_tokens": cached, + "noncached_input_tokens": incoming - cached if incoming is not None and cached is not None else None, + "reasoning_tokens": reasoning, + } + + +def total_usage(records: list[dict]) -> dict: + """Aggregate measured requests, recording coverage of optional counters.""" + return { + key: ( + sum(row[key] for row in records if row.get(key) is not None) + if any(row.get(key) is not None for row in records) + else None + ) + for key in _KEYS + } | { + "records": len(records), + "coverage": {key: sum(row.get(key) is not None for row in records) for key in _KEYS}, + } + + +@contextmanager +def collect_usage(target: list | None = None, receipt: Path | None = None): + """Keep concurrent calls' usage separate and include their compression calls.""" + records = [] + token = _CURRENT.set(records) + try: + yield records + finally: + _CURRENT.reset(token) + if target is not None: + target.extend(records) + if receipt is not None: + save(receipt, total_usage(records)) + + +def observe_usage(app) -> None: + """Observe owned clients and distinguish native interrupted replies from answers.""" + for wrapper in app.context.components.get("agent_wrapper", {}).values(): + original_reply = wrapper.reply + + async def reply(*args, _reply=original_reply, **kwargs): + result = await _reply(*args, **kwargs) + last = result.get("last_message") or {} + if last.get("finished_reason") == "interrupted" or last.get("error") is not None: + raise RuntimeError("Native Agent was interrupted; see the application log.") + return result + + wrapper.reply = reply + for component in app.context.components.get("as_llm", {}).values(): + client = getattr(getattr(component.model, "client", None), "_client", None) + if client is None or getattr(client, "_memlens_usage_observer", False): + continue + original = client.send + + async def send(request, *args, _send=original, **kwargs): + response = await _send(request, *args, **kwargs) + records = _CURRENT.get() + if records is not None and response.status_code == 200 and response.is_stream_consumed: + try: + payload = response.json() + except (ValueError, UnicodeError): + payload = {} + records.append(normalized_usage(payload.get("usage") or {})) + return response + + client.send = send + client._memlens_usage_observer = True # pylint: disable=protected-access + + +def save(path: Path, value) -> None: + """Save source/results unchanged; callers sanitize diagnostics and config snapshots.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8") + + +def app_config(workspace: Path, model: str, *, judge: bool = False) -> dict: + """Resolve standard plugin settings through ReMe's configuration parser.""" + from reme.config import resolve_app_config + + values = resolve_app_config( + config="benchmark" if judge else "memlens-benchmark", + plugins=["memlens-judge" if judge else "memlens"], + workspace_dir=str(workspace), + log_config=False, + enable_logo=False, + log_to_console=False, + log_to_file=True, + ) + # BM25-only: do not start unused embedding clients. + values["components"].pop("as_embedding", None) + values["components"].pop("embedding_store", None) + values["components"]["file_store"]["default"]["embedding_store"] = None + if not judge: + for name in ("default", "bench", "compressor"): + values["components"]["as_llm"][name]["model"] = model + else: + values["jobs"] = {"answer_judge": {}} + values["components"] = {} + return values + + +def check_response(response, diagnostic_path: Path) -> None: + """Save failed responses before validation; keep execution failures unscored.""" + if not response.success: + save(diagnostic_path, {"response": response_record(response)}) + if asyncio.current_task().cancelling(): + raise asyncio.CancelledError + if not response.success: + detail = response.answer or response.metadata.get("error") or "No failure detail provided." + raise RuntimeError(f"ReMe Job failed; response detail: {safe_value(detail)}") + + +async def close_application(app) -> None: + """Finish owned cleanup even when cancellation arrives during close.""" + closing = asyncio.create_task(app.close()) + cancellation = None + while True: + try: + await asyncio.shield(closing) + break + except asyncio.CancelledError as exc: + if closing.done(): + raise + cancellation = exc + except Exception as exc: + if cancellation is not None: + raise cancellation from exc + raise + if cancellation is not None: + raise cancellation + + +async def evaluate_scene(scene, number, model, group, judge, slots): + """Accumulate one isolated history, then answer and judge its question.""" + from reme import Application + + label = re.sub(r"[^\w.-]+", "_", scene["id"])[:100] + case = group / "reme_image" / "scenes" / f"{number:04d}-{label}" + app = None + memory, answers, results = [], [], [] + status = {"complete": False, "memory_complete": False, "stage": "initialize", "scene_id": scene["id"]} + try: + save(case / "source.json", {"id": scene["id"], "sessions": len(scene["sessions"])}) + app = Application(**app_config(case / "ingestion/attempt-001/workspace", model)) + status["stage"] = "start" + await app.start() + observe_usage(app) + for ordinal, session in enumerate(scene["sessions"]): + directory = case / "ingestion/attempt-001" + status.update(stage="preprocess", session_id=session["id"]) + async with slots: + with collect_usage(memory, directory / "diagnostics" / f"{ordinal:05d}-usage.json") as requests: + messages = await asyncio.to_thread(session_messages, session) + status["stage"] = "auto_memory" + response = await app.run_job( + "auto_memory", + messages=messages, + session_id=session["id"], + date=session["timestamp"][:10], + include_images=True, + include_source_sessions=True, + ) + check_response(response, directory / "diagnostics" / f"{ordinal:05d}-response.json") + save( + directory / "sessions" / f"{ordinal:05d}.json", + { + "session_id": session["id"], + "response": response.model_dump(), + "usage": total_usage(requests), + }, + ) + status.pop("session_id", None) + status["stage"] = "index_update" + response = await app.run_job("index_update") + check_response(response, case / "ingestion/attempt-001/diagnostics/index-response.json") + status["memory_complete"] = True + for index, question in enumerate(scene["questions"]): + directory = case / "questions" / f"{index:04d}" + status.update(stage="agentic_answer", question_id=question["id"]) + async with slots: + with collect_usage(answers, directory / "diagnostics/answer-usage.json") as requests: + response = await app.run_job( + "agentic_answer", + query=query_prompt(question, "Use the available tools to retrieve relevant evidence."), + query_time=question["question_date"], + query_images=[], + scene_id=scene["id"], + include_images=True, + include_source_sessions=True, + max_iters=30, + ) + check_response(response, directory / "diagnostics/answer-response.json") + save( + directory / "answer.json", + { + "question_id": question["id"], + "prediction": response.answer, + "usage": total_usage(requests), + "agent": response.metadata, + }, + ) + status["stage"] = "judge" + async with slots: + judgment = await judge.run_job( + "answer_judge", + question=question["text"], + reference=question["answer"], + prediction=response.answer, + category=question["category"], + subtype=question.get("subtype", ""), + old_answer=question.get("old_answer", ""), + ) + check_response(judgment, directory / "diagnostics/judge-response.json") + status["stage"] = "score" + metrics = score_answer(question, response.answer) + metrics["llm_judge"] = judgment.metadata["llm_judge_score"] + save(directory / "judge.json", judgment.metadata) + record = { + "question_id": question["id"], + "category": question["category"], + "question": question["text"], + "reference": question["answer"], + "prediction": response.answer, + "metrics": metrics, + } + save(directory / "result.json", record) + results.append(record) + status.update(complete=True, stage="complete") + except asyncio.CancelledError as exc: + status["error"] = error_details(exc) + raise + except Exception as exc: # pylint: disable=broad-exception-caught + status["error"] = error_details(exc) + finally: + try: + if app is not None: + await close_application(app) + except (Exception, asyncio.CancelledError) as exc: # pylint: disable=broad-exception-caught + if "error" in status: + status["cleanup_error"] = error_details(exc) + else: + status.update(complete=False, stage="cleanup", error=error_details(exc)) + if isinstance(exc, asyncio.CancelledError): + raise + finally: + status.update(memory_usage=total_usage(memory), answer_usage=total_usage(answers)) + if "error" in status: + status["error_type"] = status["error"]["type"] + save(case / "status.json", status_record(status)) + return results, memory, answers, status + + +async def run(settings: dict, model: str, length: str, limit: int | None = None) -> dict: + """Run one model in a new output group; existing runs are never overwritten.""" + from reme import Application + from reme.schema import ApplicationConfig + + group = Path(settings["output"]).expanduser().resolve() / f"memlens-{length}" / model + group.mkdir(parents=True, exist_ok=False) + judge = None + status = {"complete": False, "stage": "preprocess"} + try: + scenes = load_memlens(Path(settings["dataset"]), length) + if limit is not None: + scenes = scenes[:limit] + status["stage"] = "configure" + effective = ApplicationConfig(**app_config(group / "unused-workspace", model)) + judge = Application(**app_config(group / "judge-workspace", model, judge=True)) + save( + group / "settings.json", + { + "method": "reme_image", + "model": model, + "context_length": length, + "dataset": str(Path(settings["dataset"]).expanduser().resolve()), + "limit_scenes": limit, + "include_source_sessions": True, + "include_images": True, + "answer_max_iters": 30, + "concurrency": {key: settings[key] for key in ("scenario_concurrency", "query_concurrency")}, + "models": model_settings(effective), + "judge": judge_settings(judge), + "usage_policy_version": "successful-http-requests-v2", + "reme_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + }, + ) + slots = asyncio.Semaphore(settings["query_concurrency"]) + scene_slots = asyncio.Semaphore(settings["scenario_concurrency"]) + status["stage"] = "judge_start" + await judge.start() + status["stage"] = "evaluate" + + async def evaluate(number, scene): + async with scene_slots: + print(f"Starting {length} {model} {scene['id']}", flush=True) + result = await evaluate_scene(scene, number, model, group, judge, slots) + print(f"Finished {scene['id']}: {len(result[0])}/{len(scene['questions'])} scored", flush=True) + return result + + tasks = [asyncio.create_task(evaluate(index, scene)) for index, scene in enumerate(scenes)] + try: + outputs = await asyncio.gather(*tasks) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + status["stage"] = "summary" + results = [row for output in outputs for row in output[0]] + memory = [row for output in outputs for row in output[1]] + answers = [row for output in outputs for row in output[2]] + expected = sum(len(scene["questions"]) for scene in scenes) + statuses = [output[3] for output in outputs] + cleanup_errors = sum("cleanup_error" in row or row["stage"] == "cleanup" for row in statuses) + summary = { + "method": "reme_image", + "model": model, + "context_length": length, + "expected_scenes": len(scenes), + "completed_scenes": sum(row["complete"] for row in statuses), + "memory_complete_scenes": sum(row["memory_complete"] for row in statuses), + "cleanup_error_count": cleanup_errors, + "expected_questions": expected, + "scored_questions": len(results), + "unscored_questions": expected - len(results), + "metrics": { + key: (sum(row["metrics"][key] for row in results) / len(results) if results else None) + for key in ("llm_judge", "sub_em", "f1") + }, + "memory_usage": total_usage(memory), + "answer_usage": total_usage(answers), + "token_usage": total_usage(memory + answers), + "usage_policy_version": "successful-http-requests-v2", + "usage_policy": ( + "observed successful HTTP requests in memory/answer scopes, including compression and requests " + "before Job/history failure; judge excluded; not full-sample or billed cost" + ), + } + save(group / "summary.json", summary) + status.update(complete=len(results) == expected and not cleanup_errors, stage="complete") + except (Exception, asyncio.CancelledError) as exc: + status["error"] = error_details(exc) + raise + finally: + try: + if judge is not None: + await close_application(judge) + except (Exception, asyncio.CancelledError) as exc: + if "error" in status: + status["cleanup_error"] = error_details(exc) + if isinstance(exc, asyncio.CancelledError): + raise + else: + status.update(complete=False, stage="judge_cleanup", error=error_details(exc)) + raise + finally: + save(group / "status.json", status_record(status)) + return summary + + +def main(): + """Run one model at the requested MemLens history length.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=Path(__file__).with_name("config.yaml")) + parser.add_argument("--model", default="qwen3.8-max") + parser.add_argument("--context-length", choices=("256k", "128k"), default="256k") + parser.add_argument("--limit-scenes", type=int) + parser.add_argument("--output", type=Path) + arguments = parser.parse_args() + try: + settings = yaml.safe_load(os.path.expandvars(arguments.config.read_text(encoding="utf-8"))) + if arguments.output: + settings["output"] = str(arguments.output) + if arguments.limit_scenes is not None and arguments.limit_scenes < 1: + parser.error("--limit-scenes must be positive") + for key in ("query_concurrency", "scenario_concurrency"): + if not isinstance(settings[key], int) or settings[key] < 1: + parser.error(f"{key} must be positive") + summary = asyncio.run(run(settings, arguments.model, arguments.context_length, arguments.limit_scenes)) + except (Exception, asyncio.CancelledError, KeyboardInterrupt) as exc: # pylint: disable=broad-exception-caught + print(json.dumps({"complete": False, "error": error_details(exc)}, ensure_ascii=False), file=sys.stderr) + return 130 if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt)) else 2 + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return ( + 0 if summary["scored_questions"] == summary["expected_questions"] and not summary["cleanup_error_count"] else 2 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/mobilemem/README.md b/benchmark/mobilemem/README.md new file mode 100644 index 00000000..ca10e49f --- /dev/null +++ b/benchmark/mobilemem/README.md @@ -0,0 +1,96 @@ +# MobileMem-Omni Visual Reasoning + +Evaluate ReMe Image on the published `visual_reasoning` questions using each user's complete chronological +dialogue history. Original dialogue text, including any descriptions already in it, is retained alongside images. +Separate summaries/captions, reference answers, evidence annotations and question types are not added to candidate Jobs. +Native `multi_visual_reasoning` questions are not relabelled or included. + +## Run + +Install both plugins and extract the official archive without changing its images: + +```bash +pip install -e ".[as]" -e plugins/mobilemem -e plugins/mobilemem-judge +mkdir -p /path/to/omni/image +python -m zipfile -e /path/to/omni/image.zip /path/to/omni/image +export LLM_API_KEY="..." +export LLM_BASE_URL="..." +python -m benchmark.mobilemem.run --dataset /path/to/omni --output /path/to/exp-xxxx +``` + +[config.yaml](config.yaml) supplies defaults: `qwen3.8-max`, four concurrent users and 16 concurrent questions. +Override them with `--model`, `--user-concurrency` and `--query-concurrency`. +Every run creates `/mobilemem/` and rejects an existing directory. + +## Evaluation + +Each user has a fresh workspace. All source sessions are ingested chronologically through multimodal +`auto_memory`, then indexed once. Every question starts a fresh 30-iteration `agentic_answer` Agent with +BM25 `search`, `read` and `read_image`. Search covers daily notes, digest notes and original session text. +Original Image IDs accompany source images; saved native attachments can be reopened by the answer Agent. + +The [candidate preset](../../plugins/mobilemem/src/reme_mobilemem/benchmark.yaml) uses the same model for memory, +answering and compression: temperature 0, thinking enabled, 65,536 output tokens, 1,000,000-token context and +at most 4,096 images. Native context ratios remain 0.8/0.1. Input images are not resized; MPO inputs retain +the existing primary-picture PNG conversion. Published path aliases are resolved by unique image basename. + +The official QA template wraps the question with evidence-tool instructions. Only the isolated +[MobileMem Judge](../../plugins/mobilemem-judge/README.md) receives references and evidence. Its default is the +MemBase executable-default `mobilemem-omni-judge` rubric, `qwen3.8-max`, temperature 0, thinking disabled and +4,096 output tokens. Scores report binary LLM-Judge accuracy and the official token-overlap F1 and BLEU-1. + +## Results + +Each user's `workspace/` retains notes, source JSONL, images and native AgentScope trajectories. +`sessions/*.json` and `questions/*/` retain responses, judgments, metrics and technical errors. +`summary.json` reports completed/expected questions and scores over successfully scored results only. +Native interrupted replies and failed Jobs remain unscored technical gaps, not zero-score answers. +The runner adds no fallback or retry queue. + +Usage policy version 2 counts observed successful HTTP requests, including compression and requests made before +a candidate history or answer later fails. Memory cost comes from `sessions/*.json` and `index-response.json`; +answer cost comes only from `answer-response.json`, including failed responses or `response: null` when a call raises. +Successful and failed answers use this single response file; its `response.success` determines whether scoring proceeds. +Earlier runs' `answer.json` copies, result files and response metadata are not counted again. `memory_usage.json` remains a +completion marker: it is written only after the full history and index update succeed. +Judge usage is excluded; absent counters stay unknown and each counter includes its observation coverage. +The `mean_tokens_per_user` / `mean_tokens_per_question` fields amortize this run's observed cost over scheduled users / +expected questions, with explicit denominators and attempt/completion counts. They are not successful-sample averages. +Earlier selected-success-only reports retain their original accounting policy. + +Errors retain the failing stage, user/session/question identifiers, safe image paths, sanitized reasons and stack +locations; credentials, request headers, image payloads and frame locals are omitted. Cleanup errors are separate; +`cleanup_error_count` makes the run exit with status 2 without discarding already scored results. +`judge-settings.json` records a credential-free whitelist of the validated Judge model, parameters and protocol, +including an optional `JUDGE_MODEL_NAME` override, so scoring can be traced to the effective configuration. + +Sources: [MobileMem evaluation code](https://github.com/zjunlp/MobileMem/tree/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval) +and [MemBase judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py). +Dataset revision: `989ba00900bddf4e0d4b0c7dd52f0840a7ee3a54`. + +## Upstream code license + +The QA template and overlap metrics in `run.py` are adapted from ZJUNLP MobileMem's evaluation code at +`fa5cd13c9c659699df9caf650987ba2bea8ca085`, under the following license. + +MIT License + +Copyright (c) 2026 ZJUNLP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/benchmark/mobilemem/README_ZH.md b/benchmark/mobilemem/README_ZH.md new file mode 100644 index 00000000..5b1c01a7 --- /dev/null +++ b/benchmark/mobilemem/README_ZH.md @@ -0,0 +1,61 @@ +# MobileMem-Omni 视觉推理评测 + +使用每个用户完整、按时间排序的原始对话，评测 ReMe Image 在官方 `visual_reasoning` 题目上的表现。 +记忆与回答 Job 仅接收原始文本和图像，不接收数据集摘要、caption、标准答案、证据标注或题型。 +官方 `multi_visual_reasoning` 不被重命名，也不纳入这个评测范围。 + +## 运行 + +安装两个插件，并按原目录结构解压官方图片： + +```bash +pip install -e ".[as]" -e plugins/mobilemem -e plugins/mobilemem-judge +mkdir -p /path/to/omni/image +python -m zipfile -e /path/to/omni/image.zip /path/to/omni/image +export LLM_API_KEY="..." +export LLM_BASE_URL="..." +python -m benchmark.mobilemem.run --dataset /path/to/omni --output /path/to/exp-xxxx +``` + +[config.yaml](config.yaml) 默认使用 `qwen3.8-max`、4 个用户并发、16 个题目并发； +可通过 `--model`、`--user-concurrency`、`--query-concurrency` 覆盖。 +每次运行创建 `/mobilemem/`，已有目录不会被覆盖。 + +## 评测设置 + +每个用户使用独立的新工作空间，按时间顺序通过多模态 `auto_memory` 摄入全部 session，然后更新一次索引。 +每题使用新的 `agentic_answer` Agent，最多 30 轮，工具为 BM25 `search`、`read` 和 `read_image`。 +检索范围包含 daily、digest 笔记及原始 session 正文；保存的原生图像附件与 Image ID 可供回答 Agent 重新读图。 + +[候选模型配置](../../plugins/mobilemem/src/reme_mobilemem/benchmark.yaml) 在记忆、回答和压缩阶段使用同一模型： +temperature 为 0，开启 thinking，输出预算 65,536，上下文为 1,000,000 token，最多 4,096 张图。 +原生上下文比例保持 0.8/0.1。输入图片不缩放；MPO 保持既有的主图 PNG 转换。 +数据集图片路径别名通过唯一 basename 对应原件。 + +问题使用官方 QA 模板，并提示通过证据工具作答。标准答案与证据仅交给独立的 +[MobileMem Judge](../../plugins/mobilemem-judge/README.md)。默认采用 MemBase 可执行默认 +`mobilemem-omni-judge` 评分模板，模型为 `qwen3.8-max`，temperature 为 0，关闭 thinking，输出预算 4,096。 +报告包含二元 LLM-Judge 准确率及官方 token-overlap F1、BLEU-1。 + +## 结果 + +每个用户的 `workspace/` 保存笔记、原始 JSONL、图像及原生 AgentScope 轨迹。 +`sessions/*.json` 与 `questions/*/` 保存响应、评分、指标和技术错误。 +`summary.json` 汇总完成/应测题数，指标只对成功评分的结果求均值。 +原生中断与失败 Job 保留为未评分的技术缺口，不作为零分答案；runner 不额外加入 fallback 或重试队列。 + +usage policy 第 2 版统计已观测的成功 HTTP 请求，包括压缩调用及摄入/回答最终失败前已经产生的请求。 +记忆成本只汇总 `sessions/*.json` 和 `index-response.json`；回答成本只汇总 `answer-response.json`， +包括失败响应及直接异常时的 `response: null`。成功与失败的回答统一保存在这个文件中， +根据 `response.success` 决定是否进入评分；旧运行的 `answer.json` 副本、result 和 response metadata 不重复计费。 +`memory_usage.json` 仍仅在完整历史摄入及索引更新成功后生成，不能将它误作部分摄入标记。 +Judge 用量不计入方法成本；缺失字段保持未知，并记录每个字段的观测覆盖数。 +`mean_tokens_per_user` / `mean_tokens_per_question` 是本次已观测成本按计划用户数/应测题数均摊， +summary 明确记录分母和尝试/完成数，不代表成功样本平均成本。旧的 selected-success-only 报告不改写口径。 + +错误记录包含失败阶段、用户/会话/题号、安全的图片相对路径、脱敏原因和调用栈位置，不保存凭据、请求 headers、 +图片载荷或 frame locals；清理错误单独保留，`cleanup_error_count` 使本轮以状态码 2 退出，但不丢弃已评分结果。 +`judge-settings.json` 从实际校验后的配置中保存 Judge 模型、参数和协议白名单， +包括可选的 `JUDGE_MODEL_NAME` 覆盖值，不导出凭据。 + +协议来源、数据版本和改编代码的完整 MIT 许可证见 [英文说明](README.md)。 diff --git a/benchmark/mobilemem/_audit.py b/benchmark/mobilemem/_audit.py new file mode 100644 index 00000000..5bb1ba2f --- /dev/null +++ b/benchmark/mobilemem/_audit.py @@ -0,0 +1,182 @@ +"""Small, credential-safe diagnostics and accounting helpers for this runner.""" + +import os +from pathlib import Path, PurePosixPath +import re +import traceback + +USAGE_POLICY = { + "version": 2, + "scope": "observed successful HTTP requests, including failed candidate flows and compression; judge excluded", + "memory_sources": ["users/*/sessions/*.json", "users/*/index-response.json"], + "answer_source": "users/*/questions/*/answer-response.json", + "means": "observed run cost amortized over scheduled users / expected questions, not successful samples", +} +_SENSITIVE = re.compile( + r"authorization|credential|secret|password|api.?key|private[_-]?key|cookie|headers|" + r"(?:^|[_-])(?:access[_-]?token|refresh[_-]?token|id[_-]?token|token)$", + re.I, +) +_USAGE_FIELDS = { + "usage", + "coverage", + "records", + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "noncached_input_tokens", + "reasoning_tokens", +} + + +def safe_text(value, limit: int = 1200) -> str: + """Retain short diagnostic text, without credentials, URLs or binary payloads.""" + text = str(value) + for key, secret in os.environ.items(): + if secret and (_SENSITIVE.search(key) or re.search(r"(?:^|_)TOKEN(?:$|_)", key.upper())): + if len(secret) < 12: + text = re.sub(r"(?]+", "[url redacted]", text, flags=re.I) + text = re.sub(r"(? 6: + return "[nested data omitted]" + if isinstance(value, dict): + return { + (key if key in fixed_keys else safe_text(key, 100)): ( + "[redacted]" + if _SENSITIVE.search(str(key)) or str(key).lower() in {"base64", "data", "body", "request", "traceback"} + else safe_value(item, depth + 1, fixed_keys=fixed_keys) + ) + for key, item in list(value.items())[:50] + } + if isinstance(value, (list, tuple)): + return [safe_value(item, depth + 1, fixed_keys=fixed_keys) for item in value[:20]] + if value is None or isinstance(value, (bool, int, float)): + return value + return safe_text(value) if isinstance(value, str) else f"[{type(value).__name__} omitted]" + + +def response_record(response): + """Keep successful response artifacts compatible; sanitize failed envelopes.""" + if response is None: + return None + value = response.model_dump() + if response.success: + return value + return { + **safe_value({key: item for key, item in value.items() if key not in {"success", "answer", "metadata"}}), + "success": value["success"], + "answer": safe_value(value["answer"]), + "metadata": safe_value(value["metadata"], fixed_keys=_USAGE_FIELDS), + } + + +def safe_image_path(source) -> str: + """Only retain bounded dataset-relative image identifiers in diagnostics.""" + source = str(source) + path = PurePosixPath(source) + if not source or len(source) > 300 or path.is_absolute() or ".." in path.parts: + return "[invalid image path]" + if any(character in source for character in ("\\", ":", "\n", "\r")): + return "[invalid image path]" + return safe_text(source, 300) + + +class ImageInputError(ValueError): + """Attach only the dataset-relative image identifier to a preprocessing error.""" + + def __init__(self, source, error): + super().__init__(safe_text(error)) + self.image_path = safe_image_path(source) + + +def error_record(error, *, stage: str, user_id=None, session_id=None, question_id=None, response=None) -> dict: + """Preserve stage, a bounded cause chain and stack locations, never frame locals.""" + chain, seen = [], set() + current = error + image_path = None + while current is not None and id(current) not in seen and len(chain) < 5: + seen.add(id(current)) + image_path = image_path or getattr(current, "image_path", None) + chain.append( + { + "type": type(current).__name__, + "reason": safe_text(current), + "traceback": [ + {"file": Path(frame.filename).name, "line": frame.lineno, "function": frame.name} + for frame in traceback.extract_tb(current.__traceback__)[-12:] + ], + }, + ) + current = current.__cause__ or (None if current.__suppress_context__ else current.__context__) + reason = response.answer if response is not None and not response.success else str(error) + return { + "stage": stage, + "user_id": safe_text(user_id, 120) if user_id is not None else None, + "session_id": safe_text(session_id, 120) if session_id is not None else None, + "question_id": safe_text(question_id, 120) if question_id is not None else None, + "image_path": image_path, + "error_type": type(error).__name__, + "reason": safe_value(reason), + "chain": chain, + } + + +def mean_tokens(total: dict, denominator: int) -> dict: + """Amortize observed counters over the explicit scheduled population.""" + return { + key: total[key] / denominator if denominator and total[key] is not None else None + for key in ("input_tokens", "output_tokens", "total_tokens", "cached_input_tokens", "noncached_input_tokens") + } + + +def judge_settings(app) -> dict: + """Export a whitelist from the validated application, not unresolved YAML/env text.""" + from judge_mobilemem.protocol import MEMBASE_REVISION, SOURCE_REVISION, protocol_id + + model = app.config.components["as_llm"]["judge"].model_dump() + mode = app.config.jobs["answer_judge"].model_dump().get("judge_protocol", "membase_default") + fields = ("backend", "model", "stream", "context_size", "max_retries", "retry_delay") + return { + **{key: safe_value(model[key]) for key in fields if key in model}, + "parameters": { + key: safe_value(value) + for key, value in (model.get("parameters") or {}).items() + if key in {"temperature", "max_tokens", "thinking_enable", "top_p"} + }, + "extra_body": {"enable_thinking": (model.get("extra_body") or {}).get("enable_thinking")}, + "client_kwargs": { + key: value for key, value in (model.get("client_kwargs") or {}).items() if key in {"timeout", "max_retries"} + }, + "protocol": protocol_id(mode), + "protocol_mode": mode, + "mobilemem_revision": SOURCE_REVISION, + "membase_revision": MEMBASE_REVISION, + } diff --git a/benchmark/mobilemem/config.yaml b/benchmark/mobilemem/config.yaml new file mode 100644 index 00000000..f6ff1e6d --- /dev/null +++ b/benchmark/mobilemem/config.yaml @@ -0,0 +1,3 @@ +model: qwen3.8-max +user_concurrency: 4 +query_concurrency: 16 diff --git a/benchmark/mobilemem/run.py b/benchmark/mobilemem/run.py new file mode 100644 index 00000000..28962ab4 --- /dev/null +++ b/benchmark/mobilemem/run.py @@ -0,0 +1,621 @@ +"""Evaluate ReMe Image on MobileMem-Omni Visual Reasoning. + +Official QA and overlap metrics: Copyright (c) 2026 ZJUNLP, MIT License. +The complete upstream notice is retained in README.md. +""" + +import argparse +import asyncio +import base64 +from collections import Counter +from contextlib import contextmanager +from contextvars import ContextVar +from datetime import datetime +import io +import json +import math +import mimetypes +from pathlib import Path, PurePosixPath +import re + +from PIL import Image, ImageOps, JpegImagePlugin +import yaml + +from reme import Application +from reme.config import resolve_app_config + +from benchmark.mobilemem._audit import ( + ImageInputError, + USAGE_POLICY, + error_record, + judge_settings, + mean_tokens, + response_record, +) + + +def timestamp(value: str) -> str: + """Preserve published start-time precision in AgentScope ISO timestamps.""" + return datetime.fromisoformat(value.replace("Z", "+00:00")).isoformat() + + +def load_mobilemem(root: Path) -> list[dict]: + """Keep complete raw trajectories, selecting only the native visual category.""" + root = root.expanduser().resolve() + records = [] + for name in ("data.jsonl", "questions.jsonl"): + rows = {} + with (root / name).open(encoding="utf-8") as stream: + for line in stream: + if line.strip(): + row = json.loads(line) + identity = str(row["uuid"]) + if identity in rows: + raise ValueError(f"Duplicate MobileMem user: {identity}") + rows[identity] = row + records.append(rows) + histories, questions = records + if set(histories) != set(questions): + raise ValueError("Every MobileMem user must have a trajectory and question record") + return [ + { + "id": f"uuid_{identity}", + "image_root": root / "image", + "sessions": sorted( + row["sessions"], + key=lambda session: datetime.fromisoformat(timestamp(session["event_start_time"])), + ), + "questions": [ + question + for question in questions[identity]["questions"] + if question["question_type"] == "visual_reasoning" + ], + } + for identity, row in histories.items() + ] + + +def resolve_image(root: Path, source: str, aliases: dict) -> Path: + """Resolve exact paths or unique published basename aliases inside image/.""" + root = root.resolve() + relative = PurePosixPath(source) + if relative.is_absolute() or ".." in relative.parts or "\\" in source or ":" in source: + raise ValueError(f"Invalid MobileMem image path: {source}") + parts = relative.parts[1:] if relative.parts[0] == "image" else relative.parts + target = (root / Path(*parts)).resolve() + if not target.is_relative_to(root): + raise ValueError(f"MobileMem image escapes image/: {source}") + if target.is_file(): + return target + if not aliases: + for path in root.rglob("*"): + if path.is_file() and path.resolve().is_relative_to(root): + if (mimetypes.guess_type(path)[0] or "").startswith("image/"): + aliases.setdefault(path.name, []).append(path.resolve()) + matches = aliases.get(relative.name, []) + if len(matches) != 1: + raise ValueError(f"Missing or ambiguous MobileMem image: {source}") + return matches[0] + + +def image_block(path: Path) -> dict: + """Use original bytes, or the existing lossless MPO primary-picture conversion.""" + data = path.read_bytes() + if len(data) > 50 * 1024 * 1024: + raise ValueError(f"Oversized MobileMem image: {path.name}") + opener = JpegImagePlugin.JpegImageFile if data.startswith(b"\xff\xd8\xff") else Image.open + with opener(io.BytesIO(data)) as image: + mime, mpo = image.get_format_mimetype(), "mp" in image.info + if not mime or not mime.startswith("image/") or image.width * image.height > 50_000_000: + raise ValueError(f"Invalid or oversized MobileMem image: {path.name}") + image.verify() + with opener(io.BytesIO(data)) as image: + image.load() + if mpo: + with ImageOps.exif_transpose(image) as oriented, oriented.convert("RGB") as picture: + buffer = io.BytesIO() + picture.save(buffer, format="PNG") + data, mime = buffer.getvalue(), "image/png" + return { + "type": "data", + "source": { + "type": "base64", + "data": base64.b64encode(data).decode("ascii"), + "media_type": mime, + }, + } + + +def session_messages(session: dict, image_root: Path, aliases=None) -> list[dict]: + """Convert original dialogue directly; annotations never enter the candidate.""" + aliases = {} if aliases is None else aliases + messages = [] + for turn in session["dialogue"]: + content = [{"type": "text", "text": turn.get("content", "")}] + refs = turn.get("image_inline") or [] + refs = [refs] if isinstance(refs, str) else refs + if turn["content_type"] == "image" and not refs: + raise ValueError("MobileMem image turn has no image reference") + for source in refs: + try: + image = image_block(resolve_image(image_root, source, aliases)) + except Exception as error: # pylint: disable=broad-exception-caught + raise ImageInputError(source, error) from error + content.extend( + [ + {"type": "text", "text": f"[Image ID: {source}]"}, + image, + ], + ) + messages.append( + { + "id": f"{session['session_id']}:{turn['turn']}", + "name": turn["role"], + "role": turn["role"], + "content": content, + "created_at": timestamp(session["event_start_time"]), + }, + ) + return messages + + +def evidence_values(question: dict) -> list[str]: + """Keep the official Raw2Locomo/MemBase evidence-string conversion.""" + return [ + str(item["explanation"]) if isinstance(item, dict) else item + for item in question.get("evidence", []) + if (isinstance(item, dict) and item.get("explanation")) or isinstance(item, str) + ] + + +ANSWER_RULES = ( + "Answer Rules:\n" + "1. If options are provided in the question, use the original option text as your answer.\n" + "2. For multiple-choice questions (multiple-select), select all correct options.\n" + "3. For open-ended questions, respond freely." +) + + +def query_prompt(question: str, context: str) -> str: + """Use the published Text Memory QA template verbatim.""" + return ( + f"Question:\n{question}\nPlease answer the question based on the following memories:\n" + f"{context}\n{ANSWER_RULES}" + ) + + +def tokenize(text: str) -> list[str]: + """Official lowercase ASCII word and individual CJK-character tokens.""" + text = re.sub(r"[\u0000-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]", " ", str(text).lower()) + return re.findall(r"[\u4e00-\u9fff]|[a-z0-9]+", text) + + +def score_answer(reference: str, prediction: str) -> dict[str, float]: + """Official token-overlap F1 and BLEU-1.""" + pt, rt = tokenize(prediction), tokenize(reference) + overlap = sum((Counter(pt) & Counter(rt)).values()) + f1 = 1.0 if not pt and not rt else 0.0 + if pt and rt and overlap: + precision, recall = overlap / len(pt), overlap / len(rt) + f1 = 2 * precision * recall / (precision + recall) + bleu = overlap / len(pt) * (math.exp(1 - len(rt) / len(pt)) if len(pt) < len(rt) else 1) if pt else 0.0 + return {"f1": f1, "bleu_1": bleu} + + +_CURRENT = ContextVar("mobilemem_usage", default=None) +_KEYS = ( + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "noncached_input_tokens", + "reasoning_tokens", +) + + +def normalized_usage(raw: dict) -> dict: + """Keep absent cache fields unknown and avoid double-counting cache subsets.""" + incoming = raw.get("input_tokens", raw.get("prompt_tokens")) + outgoing = raw.get("output_tokens", raw.get("completion_tokens")) + total = raw.get("total_tokens") + if total is None and incoming is not None and outgoing is not None: + total = incoming + outgoing + cached = raw.get( + "cached_input_tokens", + raw.get("cached_tokens", (raw.get("prompt_tokens_details") or {}).get("cached_tokens")), + ) + return { + "input_tokens": incoming, + "output_tokens": outgoing, + "total_tokens": total, + "cached_input_tokens": cached, + "noncached_input_tokens": incoming - cached if incoming is not None and cached is not None else None, + "reasoning_tokens": raw.get( + "reasoning_tokens", + (raw.get("completion_tokens_details") or {}).get("reasoning_tokens"), + ), + } + + +def aggregate_usage(records: list[dict]) -> dict: + """Return totals and coverage; unavailable counters remain unknown.""" + return { + key: ( + sum(row[key] for row in records if row.get(key) is not None) + if any(row.get(key) is not None for row in records) + else None + ) + for key in _KEYS + } | { + "records": sum(row.get("records", 1) for row in records), + "coverage": { + key: sum(row.get("coverage", {}).get(key, int(row.get(key) is not None)) for row in records) + for key in _KEYS + }, + } + + +@contextmanager +def usage_scope(): + """Attribute each successful request to its invoking async task.""" + records = [] + token = _CURRENT.set(records) + try: + yield records + finally: + _CURRENT.reset(token) + + +def observe_usage(app) -> None: + """Observe lifecycle-owned non-streaming clients, including the compressor.""" + for component in app.context.components.get("as_llm", {}).values(): + client = getattr(getattr(component.model, "client", None), "_client", None) + if client is None or getattr(client, "_mobilemem_usage_observer", False): + continue + original = client.send + + async def send(request, *args, _send=original, **kwargs): + response = await _send(request, *args, **kwargs) + records = _CURRENT.get() + if records is not None and response.status_code == 200 and response.is_stream_consumed: + try: + raw = response.json() + except (ValueError, UnicodeError): + raw = {} + usage = raw.get("usage") if isinstance(raw, dict) else None + records.append(normalized_usage(usage if isinstance(usage, dict) else {})) + return response + + client.send = send + client._mobilemem_usage_observer = True # pylint: disable=protected-access + + +def observe_completion(app) -> None: + """Keep native interrupted replies unscored; successful replies are unchanged.""" + for wrapper in app.context.components.get("agent_wrapper", {}).values(): + original = wrapper.reply + + async def reply(inputs, *args, _reply=original, **kwargs): + result = await _reply(inputs, *args, **kwargs) + last = result.get("last_message") or {} + if str(last.get("finished_reason")) == "interrupted" or last.get("error") is not None: + raise RuntimeError("Native Agent was interrupted; its state remains saved.") + return result + + wrapper.reply = reply + + +def save(path: Path, value) -> None: + """Write an evaluation artifact under the newly created output directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8") + + +def candidate_config(workspace: Path, model: str) -> dict: + """Select one candidate model; behavioral defaults belong to the plugin preset.""" + config = resolve_app_config( + config="mobilemem-benchmark", + plugins=["mobilemem"], + workspace_dir=str(workspace), + log_config=False, + enable_logo=False, + log_to_console=False, + log_to_file=True, + components={"as_llm": {name: {"model": model} for name in ("default", "bench", "compressor")}}, + ) + config["components"].pop("as_embedding", None) + config["components"].pop("embedding_store", None) + return config + + +def judge_config(workspace: Path) -> dict: + """Keep scoring separate from candidate memory, model and evidence tools.""" + return { + "plugins": ["mobilemem-judge"], + "service": {"backend": "http"}, + "workspace_dir": str(workspace), + "enable_logo": False, + "log_to_console": False, + "log_to_file": True, + } + + +def check_response(response): + """Keep failed Jobs as technical errors rather than incorrect predictions.""" + if response.metadata.get("cancelled"): + raise asyncio.CancelledError("ReMe Job cancelled") + if not response.success: + raise RuntimeError("ReMe Job failed; see the saved response artifact.") + return response + + +async def close_application(app, directory, *, user_id=None): + """Finish owned cleanup; keep its diagnostics separate from the primary failure.""" + if app is None: + return + closing = asyncio.create_task(app.close()) + try: + await asyncio.shield(closing) + except asyncio.CancelledError as cancellation: + if closing.cancelled(): + save(directory / "cleanup-error.json", error_record(cancellation, stage="cleanup", user_id=user_id)) + return + try: + await closing + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id)) + raise + except Exception as error: # pylint: disable=broad-exception-caught + save(directory / "cleanup-error.json", error_record(error, stage="cleanup", user_id=user_id)) + + +async def gather_tasks(coroutines) -> None: + """Wait for owned tasks, cancelling and joining siblings on failure or cancellation.""" + tasks = [asyncio.create_task(coroutine) for coroutine in coroutines] + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def evaluate_user(scene, directory: Path, model: str, judge, query_slots) -> None: + """Ingest the complete chronological history into one fresh user workspace.""" + app, response, session_id = None, None, None + stage = "startup" + completed = 0 + memory_usage, aliases = [], {} + try: + app = Application(**candidate_config(directory / "workspace", model)) + await app.start() + observe_usage(app) + observe_completion(app) + for ordinal, session in enumerate(scene["sessions"]): + session_id, response = session["session_id"], None + stage = "preprocessing" + with usage_scope() as measured: + try: + messages = await asyncio.to_thread(session_messages, session, scene["image_root"], aliases) + stage = "memory" + response = await app.run_job( + "auto_memory", + messages=messages, + session_id=session_id, + date=timestamp(session["event_start_time"])[:10], + include_images=True, + include_source_sessions=True, + ) + check_response(response) + finally: + save( + directory / "sessions" / f"{ordinal:05d}.json", + { + "session_id": session_id, + "response": response_record(response), + "usage": aggregate_usage(measured), + }, + ) + memory_usage.extend(measured) + print(f"{scene['id']}: session {ordinal + 1}/{len(scene['sessions'])}", flush=True) + stage, session_id, response = "index", None, None + with usage_scope() as measured: + try: + response = await app.run_job("index_update") + check_response(response) + finally: + save( + directory / "index-response.json", + {"response": response_record(response), "usage": aggregate_usage(measured)}, + ) + memory_usage.extend(measured) + # Compatibility marker: never write this for a partial history or failed index. + save(directory / "memory_usage.json", aggregate_usage(memory_usage)) + + async def answer_question(ordinal, question): + nonlocal completed + question_dir = directory / "questions" / f"{ordinal:04d}" + answer_stage, answer_response = "answer", None + try: + async with query_slots: + with usage_scope() as measured: + try: + answer_response = await app.run_job( + "agentic_answer", + query=query_prompt(question["question"], "Use the available evidence tools."), + include_images=True, + include_source_sessions=True, + max_iters=30, + ) + check_response(answer_response) + finally: + answer_record = { + "question_id": question["question_id"], + "response": response_record(answer_response), + "usage": aggregate_usage(measured), + } + save(question_dir / "answer-response.json", answer_record) + prediction = answer_response.answer + answer_stage, answer_response = "judge", None + try: + answer_response = await judge.run_job( + "answer_judge", + question=question["question"], + reference=question["answer"], + prediction=prediction, + evidence=evidence_values(question), + ) + check_response(answer_response) + finally: + save(question_dir / "judge.json", response_record(answer_response)) + answer_stage = "metrics" + metrics = score_answer(question["answer"], prediction) + metrics["llm_judge"] = answer_response.metadata["llm_judge_score"] + result = { + "question_id": question["question_id"], + "category": question["question_type"], + "question": question["question"], + "reference": question["answer"], + "prediction": prediction, + "metrics": metrics, + "usage": aggregate_usage(measured), + "judge_parse_default_zero": answer_response.metadata["parse_default_zero"], + } + save(question_dir / "result.json", result) + completed += 1 + print(f"{scene['id']}: {question['question_id']} completed", flush=True) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save( + question_dir / "error.json", + error_record( + error, + stage=answer_stage, + user_id=scene["id"], + question_id=question["question_id"], + response=answer_response, + ), + ) + if isinstance(error, asyncio.CancelledError): + raise + print(f"{scene['id']}: {question['question_id']} failed ({type(error).__name__})", flush=True) + + stage, response = "answers", None + await gather_tasks(answer_question(i, question) for i, question in enumerate(scene["questions"])) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save( + directory / "error.json", + error_record(error, stage=stage, user_id=scene["id"], session_id=session_id, response=response), + ) + if isinstance(error, asyncio.CancelledError): + raise + print(f"{scene['id']}: {stage} failed ({type(error).__name__})", flush=True) + finally: + try: + await close_application(app, directory, user_id=scene["id"]) + finally: + save(directory / "summary.json", {"completed": completed, "expected": len(scene["questions"])}) + + +def summarize_run(scenes, output: Path, model: str) -> dict: + """Read each canonical receipt/result once, including a run interrupted after partial progress.""" + expected = sum(len(scene["questions"]) for scene in scenes) + rows = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/result.json")] + sessions = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/sessions/*.json")] + indexes = [json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/index-response.json")] + memory = [row["usage"] for row in sessions + indexes] + answers = [ + json.loads(path.read_text(encoding="utf-8")) for path in output.glob("users/*/questions/*/answer-response.json") + ] + answer_usage = [row["usage"] for row in answers] + total = aggregate_usage(memory + answer_usage) + summary = { + "model": model, + "completed": len(rows), + "expected": expected, + "metrics": { + key: sum(row["metrics"][key] for row in rows) / len(rows) if rows else None + for key in ("f1", "bleu_1", "llm_judge") + }, + "memory_usage": aggregate_usage(memory), + "answer_usage": aggregate_usage(answer_usage), + "token_usage": total, + "usage_policy": USAGE_POLICY, + "cleanup_error_count": len(list(output.glob("users/*/cleanup-error.json"))) + + int((output / "cleanup-error.json").exists()), + "usage_denominators": {"scheduled_users": len(scenes), "expected_questions": expected}, + "usage_counts": { + "attempted_users": len(list(output.glob("users/*/summary.json"))), + "attempted_sessions": len(sessions), + "successful_sessions": sum(bool((row.get("response") or {}).get("success")) for row in sessions), + "completed_ingestion_users": len(list(output.glob("users/*/memory_usage.json"))), + "attempted_answers": len(answers), + "successful_answers": sum(bool((row.get("response") or {}).get("success")) for row in answers), + "scored_questions": len(rows), + }, + "mean_tokens_per_user": mean_tokens(total, len(scenes)), + "mean_tokens_per_question": mean_tokens(total, expected), + "judge_parse_default_zero": sum(row["judge_parse_default_zero"] for row in rows), + } + save(output / "summary.json", summary) + return summary + + +async def run(scenes, output: Path, model: str, user_concurrency: int, query_concurrency: int) -> bool: + """Run independent users concurrently and save official metrics plus method cost.""" + judge = None + user_slots, query_slots = asyncio.Semaphore(user_concurrency), asyncio.Semaphore(query_concurrency) + try: + judge = Application(**judge_config(output / "judge-workspace")) + save(output / "judge-settings.json", judge_settings(judge)) + await judge.start() + + async def user_job(scene): + async with user_slots: + await evaluate_user(scene, output / "users" / scene["id"], model, judge, query_slots) + + await gather_tasks(user_job(scene) for scene in scenes) + except (Exception, asyncio.CancelledError) as error: # pylint: disable=broad-exception-caught + save(output / "error.json", error_record(error, stage="run")) + raise + finally: + try: + await close_application(judge, output) + finally: + summary = summarize_run(scenes, output, model) + print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True) + return summary["completed"] == summary["expected"] and summary["cleanup_error_count"] == 0 + + +def main() -> int: + """Create a fresh evaluation; the PR reproduces ReMe Image only.""" + defaults = yaml.safe_load(Path(__file__).with_name("config.yaml").read_text(encoding="utf-8")) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--model", default=defaults["model"]) + parser.add_argument("--user-concurrency", type=int, default=defaults["user_concurrency"]) + parser.add_argument("--query-concurrency", type=int, default=defaults["query_concurrency"]) + args = parser.parse_args() + if min(args.user_concurrency, args.query_concurrency) < 1: + parser.error("Concurrency must be positive") + scenes = load_mobilemem(args.dataset) + output = args.output.expanduser().resolve() / "mobilemem" / args.model + output.mkdir(parents=True, exist_ok=False) + save( + output / "settings.json", + { + "dataset": str(args.dataset.expanduser().resolve()), + "model": args.model, + "candidate": "mobilemem-benchmark", + "judge": "mobilemem-judge", + "question_type": "visual_reasoning", + "user_concurrency": args.user_concurrency, + "query_concurrency": args.query_concurrency, + }, + ) + return 0 if asyncio.run(run(scenes, output, args.model, args.user_concurrency, args.query_concurrency)) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/en/auto_memory.md b/docs/en/auto_memory.md index 02064837..e9c4597f 100644 --- a/docs/en/auto_memory.md +++ b/docs/en/auto_memory.md @@ -63,6 +63,11 @@ retarget inbound wikilinks. To see what happened on a day, start with `YYYY-MM-D ## Preserving the Original Information +For retrying clients, `auto_memory` accepts an optional `request_id` of 1–128 characters. Keep the ID and input unchanged +when retrying a batch: completed results are replayed without another memory-agent call; reuse with different input is +rejected. Receipts live under the configured `metadata_dir/auto_memory_requests`. A process interruption before receipt +commit may repeat an unfinished operation; arbitrary Agent file edits are not transactional. + The distilled daily note is optimized for readability; a filtered source conversation record is retained for trust and verification. diff --git a/docs/en/blog_20261009.md b/docs/en/blog_20261009.md index aaec0870..86f41bf4 100644 --- a/docs/en/blog_20261009.md +++ b/docs/en/blog_20261009.md @@ -79,6 +79,18 @@ Image resources then participate in the existing retrieval workflow. A later sea Auto Resource supports PNG, JPG/JPEG, WebP, GIF, BMP, TIFF, and HEIC. Larger images may be resized or converted for the model request; the original resource files remain unchanged. HEIC requires the optional `reme-ai[image-heif]` package. +## Performance: Multimodal Long-History QA on MemLens-256K {#memlens-256k-results} + +On MemLens-256K, ReMe Image achieved LLM-Judge scores of **53.44%** with Qwen3.6-35B-A3B and **72.49%** with Qwen3.8-Max, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +![MemLens-256K LLM-Judge with Qwen3.6-35B-A3B: Full Context Image 47.09%, ReMe Text 41.27%, ReMe Image 53.44%, OpenViking 37.04%, Mem0 38.62%.](../figure/memlens-256k-qwen36-judge.svg) + +![MemLens-256K LLM-Judge with Qwen3.8-Max: Full Context Image 65.61%, ReMe Text 52.91%, ReMe Image 72.49%, OpenViking 42.33%, Mem0 51.32%.](../figure/memlens-256k-qwen38-judge.svg) + +The comparison uses the official LLM-Judge protocol to evaluate each complete system. Six cases triggered model-provider content moderation in at least one method, so all methods use the same filtered set of **189/195** cases. + +ReMe can retrieve original conversations, and Image can reopen source images; OpenViking retrieval in this evaluation is restricted to memory. See the [evaluation notes](./memlens_256k_evaluation.md) for experimental settings, exclusions and the reproduction comparison. + ## Getting Started Both paths require a vision model that supports image input and tool calling, along with a compatible AgentScope wrapper and formatter. The selected wrapper's `as_llm` binding determines which model is used. See [Configuration](./configuration.md) for credential setup. diff --git a/docs/en/context_compaction.md b/docs/en/context_compaction.md index 17ca97b3..48e705be 100644 --- a/docs/en/context_compaction.md +++ b/docs/en/context_compaction.md @@ -159,3 +159,28 @@ the entire gain cannot be attributed to Jev ranking itself. Agent continuation s [Complete experiment: design, results, expected answers and reproduction](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README.md) + +## Host-owned decision models + +Embedded hosts can reuse their existing AgentScope JEV instance, just as they inject an LLM: + +```python +# app contains an as_decision_model.compaction component with backend: agentscope. +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +The YAML `model` value remains the provider model name. The runtime `model` property accepts a callable instance +and does not construct a client on access. Inject before the first request; ReMe validates and converts JSON +questions normally, but does not require separate ReMe credentials or close the borrowed client. The host must +keep the instance alive until ReMe has finished its calls, then close it itself. Close the component before +replacing an internally constructed model or any model with active requests. Shutdown waits for active calls and +owned-client cleanup even when cancelled, then propagates cancellation. Component shutdown detaches the instance; +inject again before restarting a borrowed-model component. Standalone credential-based behavior is unchanged. + +Owned clients initialize in a worker thread so certificate loading does not block the event loop. Cancelling a request +during construction waits for initialization to finish and leaves the client owned by the component until shutdown; +the cancelled request is not sent. + +Hosts can import `CompactContextStep`, `ToolPair`, `collect_pairs`, and `preview_result` from `reme.steps.compaction`. +The pairing and preview helpers share the Step's rules for validating proposals. Archiving, resolving references, +and committing proposals remain host responsibilities. diff --git a/docs/en/memlens_256k_evaluation.md b/docs/en/memlens_256k_evaluation.md new file mode 100644 index 00000000..938a75f3 --- /dev/null +++ b/docs/en/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K Evaluation Notes + +This report describes the experimental settings, data selection and results of the MemLens-256K evaluation in the [multimodal memory article](./blog_20261009.md#memlens-256k-results). + +## Results and Evaluation Set {#results} + +The evaluation uses the 195 questions in the official [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) `agent_subset_195.json`, each with its complete 256K history. After excluding the six questions listed below, all methods are evaluated on the same 189 questions. The official category/subtype-specific LLM-Judge protocol assigns each answer a score of 0 or 1; the reported percentage is `100 × correct / 189`. + +| Model | Method | Judge correct / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image scores **53.44%** and **72.49%** on the two models, exceeding Full Context Image by **6.35** and **6.88** percentage points, respectively. + +## Method and Judge Settings {#evaluation-settings} + +| Method | Memory input and answer-time data access | +| --- | --- | +| Full Context Image | The answer model receives the complete conversation text and original images directly. | +| ReMe Text | Builds memory from the original conversation text. The answer Agent uses BM25 to search daily notes, digest notes and original sessions, then reads relevant text. | +| ReMe Image | Builds memory from the original conversation text and images. Retrieval covers the same sources as Text, with `read_image` available to reopen saved original images. | +| OpenViking | Uses native image descriptions and memory processing. Answer-time retrieval is restricted to memory with the default top 10. | +| Mem0 | Builds memory through Mem0 2.0.4's native vision interface. Answer-time retrieval uses FAISS to select the top 20 memories. | + +Each method is evaluated with Qwen3.6-35B-A3B and Qwen3.8-Max as the candidate models. ReMe Image and Mem0 both use original images. ReMe's memory and answer context limits are 200,000 and 400,000 tokens, with temperature 0 and thinking enabled. Memory/compression and answer output budgets are 65,536 and 16,384 tokens. Each question uses an independent answer Agent with at most 30 iterations and native context compression. The `read_image` adapter accesses saved attachments with a 50 MiB read limit and a 4,096-image retention limit. + +Full Context, OpenViking and Mem0 also use temperature 0, thinking enabled and a 16,384-token output budget for answering. Mem0's memory stage uses its native settings: a 2,000-token output budget, temperature 0.1 and top_p 0.1. The OpenViking evaluation adapter includes a JSON parsing correction and explicit error reporting for image-processing failures. + +Scoring follows the [official MemLens protocol](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2). All methods use **Qwen3.8-Max** as Judge, with temperature **0**, thinking **disabled** and a **256-token** output budget. The Judge runs independently of candidate memory applications; reference answers and question-type labels are used only for scoring. + +## Common Content-Moderation Filter {#content-moderation-filter} + +The method/question combinations below triggered content moderation when calling the Qwen model service. Taking the union of affected questions across all methods and both models gives six questions. We exclude this same set from every method and model, leaving **189/195** questions for evaluation. + +In the table, A3B denotes Qwen3.6-35B-A3B and Max denotes Qwen3.8-Max. + +| Question ID | Method / model that triggered moderation | Stage | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | Memory construction | +| `q_1d99a36c` | OpenViking / A3B | Memory construction | +| `q_3b01c02a` | ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B | Memory construction | +| `q_564a657b` | ReMe Image / Max | Answering | +| `q_600a79aa` | ReMe Image / Max | Memory construction | +| `q_c9f8e24d` | ReMe Text / Max | Answering | + +## Independent Reproduction Results {#independent-reproduction} + +We organized and integrated the MemLens benchmark code into ReMe, then independently reran ReMe Image using the same data inputs and scoring settings. See the [run instructions](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md), [method configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml), [Judge configuration](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml) and [Judge scoring rules](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py) for the implementation and settings. + +The corresponding public evaluation code is available in [PR #640](https://github.com/agentscope-ai/ReMe/pull/640) at commit [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8). + +After applying the same six-question filter, one A3B question and two Max questions remain incomplete because conversation-history processing failed. The table compares the 188 and 187 questions completed in both experiments, respectively. + +| Model | Paired questions | Original ReMe Image correct | Independent reproduction correct | Original → reproduction LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +On these matched questions, the reproduced LLM-Judge scores for Qwen3.6-35B-A3B and Qwen3.8-Max are 2.13 and 0.53 percentage points lower than the original results, respectively. Overall, performance remains broadly consistent with the original experiment. diff --git a/docs/en/memory_search.md b/docs/en/memory_search.md index 9c63a725..c2a72d6b 100644 --- a/docs/en/memory_search.md +++ b/docs/en/memory_search.md @@ -276,6 +276,23 @@ connects to. If a result is worth pursuing, use `read path=...` to open the sour followed by outlinks and inlinks. - `response.metadata`: structured programmatic results containing `results`, `link_expansion`, and `counts`. +When the default `search` Job returns no results, `response.answer` remains `""` and `success` remains `true`. +The `results` list is empty and `counts.returned` is `0`. An optional `response.metadata.empty_result` explains why: + +| `reason` | Meaning | +|----------------|-----------------------------------------------------------------------------| +| `no_results` | No candidates matched or survived `min_score`. | +| `all_returned` | All eligible chunks were already returned within the same `tool_context_id`. | + +The field also contains a human-readable `message`. Programmatic consumers should use `reason` to distinguish the +outcomes. Normal hits omit `empty_result` and keep their existing answer format. + +To show the explanation through the running service's CLI client, enable the existing metadata display option: + +```bash +reme search query="recent discussions about indexing" show_metadata=true +``` + Typical text structure: ```text diff --git a/docs/en/reme-blog.md b/docs/en/reme-blog.md index 1e7f3cf1..6267b7e3 100644 --- a/docs/en/reme-blog.md +++ b/docs/en/reme-blog.md @@ -276,6 +276,8 @@ BEAM covers ten categories of tasks, including contradiction resolution, event o ReMe also uses $\pi$-Bench to evaluate the potential of multi-session reasoning to improve agent proactivity. The PROC score in $\pi$-Bench evaluates capabilities including directly fulfilling hidden intent, guiding targeted clarification, recovering cross-session preferences, reusing cross-session conventions, inferring cross-task dependencies, and advancing underspecified requests. Across five user personas, ReMe Agent achieved an average PROC score of 0.580, outperforming NanoBot by 2.4% under the same test-model configuration. See the [$\pi$-Bench paper](https://arxiv.org/abs/2605.14678) for details about the benchmark. +On **MemLens-256K**, ReMe Image achieved LLM-Judge scores of **53.44% / 72.49%** with Qwen3.6-35B-A3B / Qwen3.8-Max, exceeding Full Context Image by **6.35 / 6.88** percentage points. All methods use the same **189/195** cases after filtering six cases that triggered model-provider content moderation. See the [multimodal memory evaluation](./blog_20261009.md#memlens-256k-results) for the five-method comparison. + ## Who Is ReMe For? ### People Who Use Agents Directly diff --git a/docs/figure/memlens-256k-qwen36-judge.svg b/docs/figure/memlens-256k-qwen36-judge.svg new file mode 100644 index 00000000..75a02882 --- /dev/null +++ b/docs/figure/memlens-256k-qwen36-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.6-35B-A3B + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 47.09%; ReMe Text: 41.27%; ReMe Image: 53.44%; OpenViking: 37.04%; Mem0: 38.62%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.6-35B-A3B + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 47.09% + Full ContextImage + + + + 41.27% + ReMe Text + + + + 53.44% + ReMe Image + + + + 37.04% + OpenViking + + + + 38.62% + Mem0 + + diff --git a/docs/figure/memlens-256k-qwen38-judge.svg b/docs/figure/memlens-256k-qwen38-judge.svg new file mode 100644 index 00000000..3b5a0800 --- /dev/null +++ b/docs/figure/memlens-256k-qwen38-judge.svg @@ -0,0 +1,56 @@ + + MemLens-256K: Qwen3.8-Max + LLM-Judge scores on the same 189 of 195 cases. Full Context Image: 65.61%; ReMe Text: 52.91%; ReMe Image: 72.49%; OpenViking: 42.33%; Mem0: 51.32%. Six cases with provider content-moderation evidence are excluded across all methods. + + + Qwen3.8-Max + MemLens-256K · LLM-Judge (%) ↑ + + 0% + + 20% + + 40% + + 60% + + 80% + + 100% + + + + 65.61% + Full ContextImage + + + + 52.91% + ReMe Text + + + + 72.49% + ReMe Image + + + + 42.33% + OpenViking + + + + 51.32% + Mem0 + + diff --git a/docs/zh/auto_memory.md b/docs/zh/auto_memory.md index 7e946f15..aa16cf01 100644 --- a/docs/zh/auto_memory.md +++ b/docs/zh/auto_memory.md @@ -57,6 +57,10 @@ source_conversation: "[[session/dialog/session-a.jsonl]]" ## 同时保存原始信息 +需要重试的客户端可为 `auto_memory` 传入 1–128 字符的可选 `request_id`。重试同一批次时保持 ID 与输入一致： +已完成请求直接重放结果，不再调用记忆 Agent；同一个 ID 对应不同输入会报错。回执保存在配置的 +`metadata_dir/auto_memory_requests` 下。回执提交前发生进程中断，未完成的操作仍可能再次执行；任意 Agent 文件修改不具备事务性。 + 整理后的 daily note 负责“好读”，过滤后的对话来源记录负责“可信”。 Auto Memory 在生成记忆卡片的同时，也会保存对话来源消息： diff --git a/docs/zh/blog_20261009.md b/docs/zh/blog_20261009.md index 61b5a82d..49438192 100644 --- a/docs/zh/blog_20261009.md +++ b/docs/zh/blog_20261009.md @@ -78,6 +78,18 @@ source_resource: "[[resource/2026-10-09/system-overview.png]]" Auto Resource 支持 PNG、JPG/JPEG、WebP、GIF、BMP、TIFF 和 HEIC。较大的图片会在模型请求中按需缩小或转换格式，原始资源文件保持不变。HEIC 需要额外安装 `reme-ai[image-heif]`。 +## Performance：MemLens-256K 多模态长历史问答 {#memlens-256k-results} + +在 MemLens-256K 上，ReMe Image 的 LLM-Judge 得分在 Qwen3.6-35B-A3B 和 Qwen3.8-Max 下分别达到 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +![Qwen3.6-35B-A3B 的 MemLens-256K LLM-Judge 对比：Full Context Image 47.09%，ReMe Text 41.27%，ReMe Image 53.44%，OpenViking 37.04%，Mem0 38.62%。](../figure/memlens-256k-qwen36-judge.svg) + +![Qwen3.8-Max 的 MemLens-256K LLM-Judge 对比：Full Context Image 65.61%，ReMe Text 52.91%，ReMe Image 72.49%，OpenViking 42.33%，Mem0 51.32%。](../figure/memlens-256k-qwen38-judge.svg) + +评测使用官方 LLM-Judge 协议，比较各方法完整系统的表现。6 个 case 在至少一种方法中触发了模型服务商的内容安全审核，因此对所有方法统一过滤，汇报其余 **189/195** 个 case 的结果。 + +ReMe 可以检索原始会话，Image 还可重新读取原图；本次 OpenViking 的检索范围限定为 memory。实验设置、过滤清单和复现对比见[详细评测说明](./memlens_256k_evaluation.md)。 + ## 如何开始使用 两个入口都需要支持图片和工具调用的视觉模型，以及兼容的 AgentScope wrapper 和 formatter。模型由所选 wrapper 的 `as_llm` 绑定决定。凭证配置见[基础配置](./configuration.md)。 diff --git a/docs/zh/context_compaction.md b/docs/zh/context_compaction.md index 4de6352a..78328411 100644 --- a/docs/zh/context_compaction.md +++ b/docs/zh/context_compaction.md @@ -147,3 +147,24 @@ agent 继续执行的成功率尚未验证。 [完整实验：设计、结果、标准答案与复现](https://github.com/jinliyl/ReMe/blob/74e1251ddcd212205788f21ccd6b5ebd82ade7c3/benchmark/compaction/README_ZH.md) + +## 注入宿主管理的决策模型 + +嵌入式宿主可以复用已有的 AgentScope JEV 实例，与注入 LLM 的方式一致： + +```python +# app 已配置 backend: agentscope 的 as_decision_model.compaction 组件。 +await app.update_component("as_decision_model", "compaction", model=host_jev) +``` + +YAML 中的 `model` 仍表示供应商模型名称；运行时 `model` 属性接收可调用的模型实例，读取属性不会创建客户端。 +在首次请求之前注入。ReMe 继续校验并转换 JSON 问题，但不再要求额外的 ReMe 凭据，也不会关闭借用的客户端。 +宿主应等待 ReMe 调用结束后自行关闭实例。替换 ReMe 自建模型或存在进行中请求的实例之前，须先关闭组件。 +关闭过程即使被取消，也会等待活动请求和自建客户端清理完成，再传播取消信号。组件关闭时会解除实例引用； +重新启动使用外部模型的组件前需要再次注入。独立运行时按凭据创建模型的行为保持不变。 + +自建客户端在线程中初始化，证书加载不会阻塞事件循环。构造期间取消请求时，组件会等待构造完成并保留客户端所有权， +由后续组件关闭负责释放；此时不会发送该请求。 + +宿主可从 `reme.steps.compaction` 导入 `CompactContextStep`、`ToolPair`、`collect_pairs` 和 `preview_result`。 +配对识别与预览构造函数和压缩 Step 共享相同规则，供宿主校验提案使用；归档、引用解析和提交仍由宿主负责。 diff --git a/docs/zh/memlens_256k_evaluation.md b/docs/zh/memlens_256k_evaluation.md new file mode 100644 index 00000000..234bf50f --- /dev/null +++ b/docs/zh/memlens_256k_evaluation.md @@ -0,0 +1,68 @@ +# MemLens-256K 评测说明 + +本文介绍[多模态记忆文章](./blog_20261009.md#memlens-256k-results)中 MemLens-256K 评测的实验设置、数据筛选和结果。 + +## 结果与统计范围 {#results} + +评测使用 [MemLens](https://huggingface.co/datasets/xiyuRenBill/MEMLENS) 官方 `agent_subset_195.json` 中的 195 个问题，每题使用对应的完整 256K 历史。按下文说明过滤六个问题后，所有方法在相同的 189 题上评测。LLM-Judge 按官方类别／子类别评分协议为每题给出 0 或 1 分，得分为 `100 × 正确数 / 189`。 + +| 模型 | 方法 | Judge 正确数 / 189 | LLM-Judge (%) | +| --- | --- | ---: | ---: | +| Qwen3.6-35B-A3B | Full Context Image | 89 / 189 | 47.09 | +| Qwen3.6-35B-A3B | ReMe Text | 78 / 189 | 41.27 | +| Qwen3.6-35B-A3B | ReMe Image | 101 / 189 | 53.44 | +| Qwen3.6-35B-A3B | OpenViking | 70 / 189 | 37.04 | +| Qwen3.6-35B-A3B | Mem0 | 73 / 189 | 38.62 | +| Qwen3.8-Max | Full Context Image | 124 / 189 | 65.61 | +| Qwen3.8-Max | ReMe Text | 100 / 189 | 52.91 | +| Qwen3.8-Max | ReMe Image | 137 / 189 | 72.49 | +| Qwen3.8-Max | OpenViking | 80 / 189 | 42.33 | +| Qwen3.8-Max | Mem0 | 97 / 189 | 51.32 | + +ReMe Image 在两个模型上的得分分别为 **53.44%** 和 **72.49%**，比 Full Context Image 高 **6.35** 和 **6.88** 个百分点。 + +## 方法与 Judge 设置 {#evaluation-settings} + +| 方法 | 记忆输入与回答阶段的数据访问范围 | +| --- | --- | +| Full Context Image | 回答模型直接读取完整历史文字与原图。 | +| ReMe Text | 使用原始会话文字生成记忆；回答 Agent 通过 BM25 检索 daily、digest 和原始 session，并读取相关文本。 | +| ReMe Image | 使用原始会话文字和图片生成记忆；检索范围与 Text 相同，并可通过 `read_image` 重新读取保存的原图。 | +| OpenViking | 使用原生图片描述和记忆流程；回答阶段检索限定为 memory，采用默认 top 10。 | +| Mem0 | 使用 Mem0 2.0.4 原生视觉入口生成记忆；回答阶段通过 FAISS 检索 top 20 记忆。 | + +各方法分别使用 Qwen3.6-35B-A3B 和 Qwen3.8-Max 作为待测模型。ReMe Image 与 Mem0 均直接使用原图。ReMe 的记忆和回答上下文上限分别为 200,000 / 400,000 tokens，temperature 为 0，开启 thinking；记忆／压缩与回答输出预算分别为 65,536 / 16,384 tokens。每题使用独立回答 Agent，最多运行 30 轮，并采用原生上下文压缩。Image 的 `read_image` 支持读取保存的附件，读取上限为 50 MiB，图片保留上限为 4,096 张。 + +Full Context、OpenViking 和 Mem0 的回答阶段同样使用 temperature 0、thinking 开启、输出预算 16,384 tokens。Mem0 记忆阶段采用原生设置：输出预算 2,000 tokens，temperature 和 top_p 均为 0.1。OpenViking 的评测适配包含 JSON 解析修正，以及图片处理失败时的显式报错。 + +评分采用 [MemLens 官方协议](https://github.com/xrenaf/MEMLENS/tree/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2)，所有方法统一使用 **Qwen3.8-Max** 作为 Judge，temperature 为 **0**、thinking **关闭**、输出预算为 **256 tokens**。Judge 与待测方法的记忆应用独立运行；参考答案及题型标签仅用于评分。 + +## 统一内容审核过滤 {#content-moderation-filter} + +调用 Qwen 模型服务时，下表中的方法在处理相应问题时触发了内容安全审核（绿网协议）。我们对各方法、两个模型触发审核的问题取并集，共得到六题，并在所有方法和两个模型中统一过滤，最终统计相同的 **189/195** 题。 + +表中 A3B 指 Qwen3.6-35B-A3B，Max 指 Qwen3.8-Max。 + +| 问题 ID | 触发内容审核的方法 / 模型 | 阶段 | +| --- | --- | --- | +| `q_028cb832` | OpenViking / Max | 记忆构建 | +| `q_1d99a36c` | OpenViking / A3B | 记忆构建 | +| `q_3b01c02a` | ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B | 记忆构建 | +| `q_564a657b` | ReMe Image / Max | 回答 | +| `q_600a79aa` | ReMe Image / Max | 记忆构建 | +| `q_c9f8e24d` | ReMe Text / Max | 回答 | + +## 独立复现结果 {#independent-reproduction} + +我们将 MemLens benchmark 代码整理并集成到 ReMe 项目中，在相同的数据输入和评分设置下，对 ReMe Image 进行了独立复现。运行方式见[评测说明](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/benchmark/memlens/README.md)，具体参数见[方法配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens/src/reme_memlens/benchmark.yaml)、[Judge 配置](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/plugin.yaml)和[Judge 评分规则](https://github.com/agentscope-ai/ReMe/blob/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8/plugins/memlens-judge/src/judge_memlens/protocol.py)。 + +独立复现对应的公开评测代码见 [PR #640](https://github.com/agentscope-ai/ReMe/pull/640)，commit 为 [962d5cd](https://github.com/agentscope-ai/ReMe/commit/962d5cd8401c1d2be17f3bb95f8ac9c185672cd8)。 + +应用相同的六题过滤后，复现实验中 A3B 有 1 题、Max 有 2 题因会话历史处理失败未得到完整结果。下表分别在两次实验均完成的 188 题和 187 题上进行配对比较。 + +| 模型 | 配对题数 | 原实验正确数 | 独立复现正确数 | 原实验 → 独立复现 LLM-Judge (%) | +| --- | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 | +| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 | + +在上述配对题目上，Qwen3.6-35B-A3B 和 Qwen3.8-Max 的独立复现 LLM-Judge 得分分别比原实验低 2.13 和 0.53 个百分点，整体性能与原实验基本保持一致。 diff --git a/docs/zh/memory_search.md b/docs/zh/memory_search.md index 1c73257f..520990ba 100644 --- a/docs/zh/memory_search.md +++ b/docs/zh/memory_search.md @@ -253,6 +253,23 @@ Memory Search 的“渐进式”不是一次把全库内容塞进结果，而是 - `response.answer`：给人看的文本，每个命中块包含路径、行号、分数和 chunk 内容，后面跟 outlinks / inlinks。 - `response.metadata`：给程序看的结构化结果，包括 `results`、`link_expansion`、`counts`。 +默认 `search` Job 没有结果可返回时，`response.answer` 保持 `""`，`success` 保持 `true`。 +此时 `results` 为空列表，`counts.returned` 为 `0`，可选字段 `response.metadata.empty_result` 说明原因： + +| `reason` | 含义 | +|----------------|-----------------------------------------------------------------| +| `no_results` | 没有匹配结果，或所有候选都被 `min_score` 过滤。 | +| `all_returned` | 符合分数要求的内容在同一 `tool_context_id` 中都已返回过。 | + +该字段还包含便于阅读的 `message`。程序调用方应通过 `reason` 区分两种情况。 +正常命中不包含 `empty_result`，正文格式保持原样。 + +通过 CLI 客户端调用已启动的服务时，可以开启已有的元数据显示选项查看原因： + +```bash +reme search query="最近关于索引的讨论" show_metadata=true +``` + 典型文本结构： ```text diff --git a/docs/zh/reme-blog.md b/docs/zh/reme-blog.md index 0c4c3826..54ff919f 100644 --- a/docs/zh/reme-blog.md +++ b/docs/zh/reme-blog.md @@ -292,6 +292,8 @@ cases / 400 题上取得 66.1%，在 1M 设置下的 35 cases / 700 题上取得 Agent 在 5 种用户角色（User Persona）上平均取得 0.580 的 PROC 分数，超出相同测试模型配置的 NanoBot 2.4%。关于该基准的详细介绍见 [$\pi$-Bench 论文](https://arxiv.org/abs/2605.14678)。 +在 **MemLens-256K** 上，ReMe Image 的 LLM-Judge 在 Qwen3.6-35B-A3B / Qwen3.8-Max 下达到 **53.44% / 72.49%**，比 Full Context Image 高 **6.35 / 6.88** 个百分点。统一过滤 6 个触发模型服务商内容安全审核的 case 后，汇报 **189/195** 个 case；五种方法的对比见[多模态记忆评测](./blog_20261009.md#memlens-256k-results)。 + ## ReMe 能帮谁？ ### 直接使用 Agent 的人 diff --git a/example.env b/example.env index 37c3aa76..b8e9e3d9 100644 --- a/example.env +++ b/example.env @@ -14,4 +14,3 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 # Optional data-source mirrors. Unset variables use the official services. # HF_MIRROR_URL=https://hf-mirror.com # ARXIV_MIRROR_URL=https://export.arxiv.org -# TUSHARE_MIRROR_URL=http://112.124.63.173:4000/tushare diff --git a/github-pages/tests/memlens-evaluation.test.mjs b/github-pages/tests/memlens-evaluation.test.mjs new file mode 100644 index 00000000..a33406e9 --- /dev/null +++ b/github-pages/tests/memlens-evaluation.test.mjs @@ -0,0 +1,122 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const siteDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const generatedDir = path.join(siteDir, ".generated/site"); +const docsDir = path.resolve(siteDir, "../docs"); +const methods = ["Full Context Image", "ReMe Text", "ReMe Image", "OpenViking", "Mem0"]; +// Published binary verdict counts, not raw benchmark outputs or a second chart generator. +const models = [ + { name: "Qwen3.6-35B-A3B", figure: "qwen36", correct: [89, 78, 101, 70, 73] }, + { name: "Qwen3.8-Max", figure: "qwen38", correct: [124, 100, 137, 80, 97] }, +]; +const excluded = ["q_028cb832", "q_1d99a36c", "q_3b01c02a", "q_564a657b", "q_600a79aa", "q_c9f8e24d"]; +const publicRevision = "962d5cd8401c1d2be17f3bb95f8ac9c185672cd8"; +const reproductionLinks = [ + ...[ + "benchmark/memlens/README.md", + "plugins/memlens/src/reme_memlens/benchmark.yaml", + "plugins/memlens-judge/src/judge_memlens/plugin.yaml", + "plugins/memlens-judge/src/judge_memlens/protocol.py", + ].map((file) => `https://github.com/agentscope-ai/ReMe/blob/${publicRevision}/${file}`), + "https://github.com/agentscope-ai/ReMe/pull/640", + `https://github.com/agentscope-ai/ReMe/commit/${publicRevision}`, +]; + +test("MemLens chart labels and geometry match the published scores", async () => { + for (const model of models) { + const svg = await readFile(path.join(docsDir, "figure", `memlens-256k-${model.figure}-judge.svg`), "utf8"); + const bars = [...svg.matchAll(/\s*]*y="([\d.]+)"[^>]*height="([\d.]+)"[^>]*\/>\s*]*>([\d.]+)%<\/text>/g)]; + assert.equal(bars.length, methods.length); + for (const [index, method] of methods.entries()) { + const score = (100 * model.correct[index] / 189).toFixed(2); + assert.equal(bars[index][1], method); + assert.equal(bars[index][2], score); + assert.equal(bars[index][5], score, `${model.name}/${method}: visible score`); + assert.ok(Math.abs(Number(bars[index][4]) - Number(score) * 3) < 0.001); + assert.ok(Math.abs(Number(bars[index][3]) + Number(bars[index][4]) - 416) < 0.001); + } + } +}); + +test("MemLens bilingual notes and blogs match the published scores", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const rows = notes.split("\n").filter((line) => /^\| Qwen.*\| \d+ \/ 189 \|/.test(line)); + assert.equal(rows.length, 10); + for (const model of models) { + for (const [index, method] of methods.entries()) { + const correct = model.correct[index]; + const score = (100 * correct / 189).toFixed(2); + assert.ok(rows.includes(`| ${model.name} | ${method} | ${correct} / 189 | ${score} |`)); + assert.ok(blog.includes(`${method} ${score}%`), `${language}: alt text for ${model.name}/${method}`); + } + const gain = (100 * (model.correct[2] - model.correct[0]) / 189).toFixed(2); + assert.ok(blog.includes(`**${gain}**`)); + assert.ok(notes.includes(`**${gain}**`)); + } + } +}); + +test("MemLens notes disclose moderation filters, paired results and Judge settings", async () => { + const triggers = { + zh: [ + ["OpenViking / Max", "记忆构建"], + ["OpenViking / A3B", "记忆构建"], + ["ReMe Text、Image / A3B；OpenViking / A3B、Max；Mem0 / A3B", "记忆构建"], + ["ReMe Image / Max", "回答"], + ["ReMe Image / Max", "记忆构建"], + ["ReMe Text / Max", "回答"], + ], + en: [ + ["OpenViking / Max", "Memory construction"], + ["OpenViking / A3B", "Memory construction"], + ["ReMe Text and Image / A3B; OpenViking / A3B and Max; Mem0 / A3B", "Memory construction"], + ["ReMe Image / Max", "Answering"], + ["ReMe Image / Max", "Memory construction"], + ["ReMe Text / Max", "Answering"], + ], + }; + for (const language of ["en", "zh"]) { + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + const filter = notes.split("{#content-moderation-filter}")[1].split("{#independent-reproduction}")[0]; + assert.deepEqual([...new Set(filter.match(/q_[0-9a-f]{8}/g))].sort(), excluded); + const rows = filter.split("\n").filter((line) => line.startsWith("| `q_")); + assert.deepEqual(rows, excluded.map((id, index) => { + const [method, stage] = triggers[language][index]; + return `| \`${id}\` | ${method} | ${stage} |`; + })); + assert.match(filter, /189\/195/); + assert.ok(notes.includes("| Qwen3.6-35B-A3B | 188 | 100 | 96 | 53.19 → 51.06 |")); + assert.ok(notes.includes("| Qwen3.8-Max | 187 | 135 | 134 | 72.19 → 71.66 |")); + const judge = notes.split("\n\n").find((paragraph) => /(?:as|作为) Judge/.test(paragraph)); + assert.ok(judge, `${language}: Judge settings must be documented`); + assert.match(judge, /\*\*Qwen3\.8-Max\*\*/); + assert.match(judge, /temperature (?:为 )?\*\*0\*\*/); + assert.match(judge, /thinking \*\*(?:disabled|关闭)\*\*/); + assert.match(judge, /\*\*256(?:-token| tokens)\*\*/); + assert.match(notes, /memory-only|restricted to memory|限定为 memory/); + } +}); + +test("MemLens notes pin the public evaluation revision without private audit details", async () => { + for (const language of ["en", "zh"]) { + const blog = await readFile(path.join(generatedDir, language, "blog_20261009.md"), "utf8"); + const notes = await readFile(path.join(generatedDir, language, "memlens_256k_evaluation.md"), "utf8"); + assert.match(blog, /\]\(\.\/memlens_256k_evaluation\.md\)/); + assert.match(notes, /\]\(\.\/blog_20261009\.md#memlens-256k-results\)/); + assert.doesNotMatch(notes, /\/mnt\/|\/Users\/|docs\/_build|file:\/\//); + const reproduction = notes.split("{#independent-reproduction}")[1]; + const targets = [...reproduction.matchAll(/\]\(([^)\r\n]+)\)/g)].map(([, target]) => target); + assert.deepEqual(targets.sort(), [...reproductionLinks].sort()); + const prose = notes.replace(/\]\([^)]+\)/g, "]"); + assert.doesNotMatch(prose, /\b[a-f0-9]{40,64}\b/); + for (const anchor of ["results", "evaluation-settings", "content-moderation-filter", "independent-reproduction"]) { + assert.ok(notes.includes(`{#${anchor}}`)); + } + } +}); diff --git a/integrations/dsh/README.md b/integrations/dsh/README.md index d84c4bf6..faac78cc 100644 --- a/integrations/dsh/README.md +++ b/integrations/dsh/README.md @@ -4,7 +4,7 @@ This guide explains how to install, configure, and use `@agentscope-ai/reme-dsh-plugin` with DeepSeek Harness (DSH), including memory guidance injection, the `reme_search` tool, automatic memory, daily consolidation, and the ReMe Status page. -The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, and `0.2.1-alpha.4`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. +The screenshots record an earlier local integration test against DSH `0.1.5-rc.2`; the supported releases are `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, and `0.2.1-rc.2`. Both the interface and ReMe guidance are set to English, and the isolated DSH and ReMe workspaces contain only fictional Project Aurora data. No `.env` values, API keys, access tokens, or personal memories appear in the screenshots. ## 1. How the plugin works @@ -26,7 +26,7 @@ The DSH adapter injects **usage guidance**, not every historical memory. Relevan ## 2. Requirements - ReMe is installed and its configuration exposes the `search`, `auto_memory`, and `auto_dream` jobs. -- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, or `0.2.1-alpha.4`. +- DeepSeek Harness `0.1.7-rc.2`, `0.2.0-rc.2`, `0.2.1-alpha.2`, `0.2.1-alpha.4`, or `0.2.1-rc.2`. - Node.js `^22.19.0` or `>=24.0.0`, matching the current DSH engine range. - The browser running DSH can reach the configured ReMe HTTP endpoint. Cross-machine deployments must also allow the DSH browser origin. @@ -110,21 +110,23 @@ If an earlier upgrade already renamed `settings.yaml` to `settings.yaml.imported ![ReMe Memory plugin configuration](./figures/reme-memory-settings.png) -| UI meaning | Configuration key | Default | Description | -| ---------------------- | --------------------- | ----------------------- | --------------------------------------------------------------- | -| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute ReMe HTTP URL using `http` or `https`. | -| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | -| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | -| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | -| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | -| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | -| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | -| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | -| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | -| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | -| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | -| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | -| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| UI meaning | Configuration key | Default | Description | +| ---------------------- | ---------------------- | ----------------------- | --------------------------------------------------------------- | +| Service URL | `endpoint` | `http://127.0.0.1:2333` | Absolute HTTP(S) URL without credentials, query, or fragment. | +| Guidance language | `language` | `en` | `en` or `zh`; controls guidance injected into new sessions. | +| Default search results | `searchLimit` | `5` | Default `reme_search` result limit, from 1 to 50. | +| Search timeout | `requestTimeoutMs` | `10000` | Search timeout in milliseconds, from 1,000 to 120,000. | +| Automatic memory | `autoMemoryEnabled` | `true` | Capture completed user/assistant turns for `auto_memory`. | +| Exclude subagents | `rootAgentsOnly` | `true` | Inject guidance and capture conversations only for root agents. | +| Submission interval | `autoMemoryInterval` | `5` | Submit after this many completed turns, from 1 to 1,000. | +| Memory consolidation | `autoDreamEnabled` | `true` | Run `auto_dream` on the daily schedule. | +| Consolidation schedule | `dreamCron` | `0 23 * * *` | Five-field cron expression interpreted in `timezone`. | +| Consolidation guidance | `dreamHint` | empty | Optional guidance passed to `auto_dream`. | +| Workspace timezone | `timezone` | `Asia/Shanghai` | IANA timezone used for batching and scheduling. | +| Background timeout | `backgroundTimeoutMs` | `3600000` | Timeout for `auto_memory` and `auto_dream`. | +| Shutdown flush timeout | `shutdownTimeoutMs` | `5000` | Budget for draining background work during shutdown. | +| Retry delay | `autoMemoryRetryMs` | `1000` | Initial retry delay; doubles after each failure. | +| Maximum retry delay | `autoMemoryRetryMaxMs` | `60000` | Upper bound for retry delay. | Deployment configuration also supports `REME_URL`, or `REME_HOST` together with `REME_PORT`. The timer-only test option `dreamIntervalMs` is intentionally excluded from user settings. @@ -169,9 +171,19 @@ An empty successful response becomes `No relevant memory found.`. Service failur With `autoMemoryEnabled=true`, the plugin listens to DSH session events and collects completed user and assistant messages per session. When `autoMemoryInterval` is reached, the batch enters a background queue and calls ReMe `auto_memory`. Plugin-generated context and tool results are excluded from capture so they cannot be laundered back into long-term memory. +Each completed turn is saved in a private JSON outbox before submission. Failures use capped exponential backoff independently of new turns; recovery drains every ready batch, including ended sessions. Payload/job rejections, such as HTTP 413, pause the affected session while other sessions continue; network, authentication, rate-limit, and service failures pause the endpoint. Each endpoint drains independently and reads one batch at a time, so a stalled old destination cannot block a newly configured service. Disabling automatic memory pauses capture and further sends; queued files remain available when re-enabled. + +The outbox defaults to `$DSH_HOME/reme-memory/outbox` (`~/.dsh/reme-memory/outbox` without `DSH_HOME`). Set the deployment-only `autoMemoryQueueDir` field and restart to choose another directory; simultaneous Hosts must use separate directories. Batches retain their original service endpoint across settings changes. An exclusive owner lock prevents concurrent writes, and a restart recovers an exited owner's queue. Queue files contain conversation text: keep them private. Disk/permission failures are surfaced as errors; preserve DSH session logs for recovery. + +Owner-lock recovery is serialized. If startup reports a busy or interrupted `.lock-recovery`, retry first; if the error persists, stop all Hosts using that queue, inspect it, and remove only the empty `.lock-recovery` directory before restarting. Preserve `.lock` and all queued JSON files. + +New batches receive strictly increasing ordering timestamps, recovered from queued files after restart, so equal wall-clock times or a clock rollback cannot reorder them. Older batches that already share a timestamp retain their existing file order. + +Sealed batches keep a stable `request_id` across timeouts and restarts. The updated ReMe service stores completed request receipts under its configured metadata directory, rejects reuse with different input, and replays completed results without invoking the memory agent again. An interruption before a receipt is committed can still repeat an unfinished operation; receipts do not make arbitrary agent file edits transactional. + ![Automatic-memory activity](./figures/reme-status-auto-memory.png) -Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until running and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. +Chat completion and durable memory completion are asynchronous. To verify persistence, open **ReMe Status → Auto Memory**, wait until queued turns, running tasks, and queued tasks return to zero, and confirm that the latest submission is marked **Completed**. ## 8. ReMe Status tabs diff --git a/integrations/dsh/README_ZH.md b/integrations/dsh/README_ZH.md index 08262b95..e7d7cd77 100644 --- a/integrations/dsh/README_ZH.md +++ b/integrations/dsh/README_ZH.md @@ -26,7 +26,7 @@ DSH 启动新会话时，插件向根 Agent 注入一段“如何使用长期记 ## 2. 环境要求 - ReMe Python 服务已安装，且配置中提供 `search`、`auto_memory` 和 `auto_dream` Job。 -- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2` 或 `0.2.1-alpha.4`。 +- DeepSeek Harness `0.1.7-rc.2`、`0.2.0-rc.2`、`0.2.1-alpha.2`、`0.2.1-alpha.4` 或 `0.2.1-rc.2`。 - Node.js `^22.19.0` 或 `>=24.0.0`，与当前 DSH 的 engine 范围一致。 - DSH 页面能够访问 ReMe HTTP 地址；跨机器部署时还要允许 DSH 页面所在的浏览器 Origin。 @@ -115,21 +115,23 @@ dsh web --no-open --port 3090 截图中的测试配置使用 `http://127.0.0.1:3457`、English 指引、默认搜索数量 5、搜索超时 10 秒，并启用了自动记忆和“Exclude subagents”。完整字段如下： -| 界面含义 | 配置键 | 默认值 | 说明 | -| -------------------- | --------------------- | ----------------------- | -------------------------------------------------------------------------- | -| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的 ReMe HTTP 服务绝对地址，只支持 `http`/`https`。 | -| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | -| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | -| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | -| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | -| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | -| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | -| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | -| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | -| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | -| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | -| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | -| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 界面含义 | 配置键 | 默认值 | 说明 | +| -------------------- | ---------------------- | ----------------------- | -------------------------------------------------------------------------- | +| 服务地址 | `endpoint` | `http://127.0.0.1:2333` | DSH 访问的绝对 HTTP(S) 地址，不支持 URL 内嵌凭据、查询参数或片段。 | +| 记忆指引语言 | `language` | `en` | `en` 或 `zh`；决定新会话中注入的指引语言。 | +| 默认搜索数量 | `searchLimit` | `5` | `reme_search` 未传 `limit` 时的默认结果上限，范围 1–50。 | +| 搜索超时（毫秒） | `requestTimeoutMs` | `10000` | `search` 请求的超时，范围 1,000–120,000。 | +| 自动记录已完成的对话 | `autoMemoryEnabled` | `true` | 是否捕获完成的用户/助手回合并提交给 `auto_memory`。 | +| 排除子 Agent | `rootAgentsOnly` | `true` | 开启时，只为根 Agent 注入指引并捕获对话。 | +| 自动记忆提交间隔 | `autoMemoryInterval` | `5` | 每完成多少轮对话提交一次，范围 1–1,000。退出时插件会在时间预算内尽力排空。 | +| 自动整理记忆 | `autoDreamEnabled` | `true` | 是否按照每日计划调用 `auto_dream`。 | +| 整理计划 | `dreamCron` | `0 23 * * *` | 五段 cron 表达式，按 `timezone` 解释；默认每天 23:00。 | +| 整理指引 | `dreamHint` | 空 | 可选的整理提示，原样交给 `auto_dream`。 | +| 工作区时区 | `timezone` | `Asia/Shanghai` | 有效的 IANA 时区，用于每日批次和 cron 计划。 | +| 后台任务超时（毫秒） | `backgroundTimeoutMs` | `3600000` | `auto_memory` 和 `auto_dream` 的最大等待时间。 | +| 退出等待（毫秒） | `shutdownTimeoutMs` | `5000` | DSH 退出时等待后台任务排空的时间预算。 | +| 失败重试间隔（毫秒） | `autoMemoryRetryMs` | `1000` | 首次失败后的等待时间，连续失败时翻倍。 | +| 最大重试间隔（毫秒） | `autoMemoryRetryMaxMs` | `60000` | 重试等待时间的上限。 | 部署层还支持 `REME_URL`，或组合使用 `REME_HOST` 与 `REME_PORT`。仅供定时器测试的 `dreamIntervalMs` 不出现在用户设置中。 @@ -180,11 +182,21 @@ only on retrieved memory and cite the returned paths. 启用 `autoMemoryEnabled` 后，插件监听 DSH `session/event`，按会话收集完成的用户和助手消息。达到 `autoMemoryInterval` 后进入提交队列，后台调用 ReMe `auto_memory`。插件生成的上下文以及工具结果不会再次进入自动记忆，避免把指引或检索回显循环写回长期记忆。 +每个完成回合会先写入私有 JSON 提交队列。失败后按有上限的指数退避独立重试，不依赖下一轮对话；恢复后持续排空可发送批次，也包括已结束的会话。请求内容或 Job 被拒绝（例如 HTTP 413）时，仅暂停受影响的会话，其他会话继续处理；网络、认证、限流或服务故障会暂停对应服务地址。每个服务地址独立排空队列，每次读取一个批次，旧地址挂起不会阻塞新配置的服务。关闭自动记忆会暂停捕获及后续发送，重新启用后可以继续处理保留的队列。 + +队列默认位于 `$DSH_HOME/reme-memory/outbox`，未配置 `DSH_HOME` 时使用 `~/.dsh/reme-memory/outbox`。可在部署配置中设置 `autoMemoryQueueDir`，重启后生效；同时运行的 Host 必须分别使用不同目录。积压批次保留捕获时的服务地址，修改地址不会把旧内容发送到另一个服务。独占锁阻止并发写入，重启时可以恢复已退出进程的积压。队列包含对话正文，请保留私有权限。磁盘或权限错误会明确报告，原始 DSH 会话日志应保留用于恢复。 + +进程锁的回收会串行执行。若启动报告 `.lock-recovery` 正忙或恢复中断，先重试；错误持续时，停止所有使用该队列的 Host，检查队列后仅移除空的 `.lock-recovery` 目录，再重新启动。保留 `.lock` 和所有排队的 JSON 文件。 + +新批次使用严格递增的排序时间戳，重启时从排队文件恢复，避免系统时间相同或回拨导致批次乱序。已有相同时间戳的旧批次仍沿用原有文件顺序。 + +已封存的批次在超时、重启后沿用相同的 `request_id`。更新后的 ReMe 服务在配置的 metadata 目录保存已完成请求的回执，拒绝把同一个 ID 用于不同输入，并直接重放已完成结果，避免重复调用记忆 Agent。回执提交前发生中断，未完成的操作仍可能重试；回执不会把任意 Agent 文件修改变成事务。 + 截图测试把间隔临时设为 1；运行记录中可以看到英文测试会话形成的已完成提交： ![自动记忆运行记录](./figures/reme-status-auto-memory.png) -自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 +自动记忆是后台任务：聊天回答完成不代表磁盘写入已经在同一毫秒完成。需要确认时，打开 **ReMe 状态 → 自动记忆**，等待“待处理回合”“运行中任务”和“排队任务”归零，并检查最近提交是否为“已完成”。 ## 8. ReMe 状态页六个标签 diff --git a/integrations/dsh/package-lock.json b/integrations/dsh/package-lock.json index 94cd94bc..f6796838 100644 --- a/integrations/dsh/package-lock.json +++ b/integrations/dsh/package-lock.json @@ -9,18 +9,18 @@ "version": "0.1.3", "license": "Apache-2.0", "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -328,21 +328,21 @@ } }, "node_modules/@deepseek-ai/cordis": { - "version": "4.0.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5-alpha.2.tgz", - "integrity": "sha512-rd5Qp6PNrMk+0saWVMLUaVv08s/WBwf2TXM84ST6tppFqlHS0Yl9l/NwPrA4oVOuS/1BKCYOGlZ6zJRj+JG6IQ==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis/-/cordis-4.0.5.tgz", + "integrity": "sha512-Sktq5axy0owk9PdnivBRNOJjHEVEibCt0+TDDZcv0+UH2KS1M910fX4kn5y45YFsrpZIi1mJf/1s5VzywMUnyg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" }, "bin": { "cordis": "bin.js" }, "peerDependencies": { - "@deepseek-ai/cordis-plugin-include": "~1.0.10-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "~1.0.6-alpha.2" + "@deepseek-ai/cordis-plugin-include": "~1.0.10", + "@deepseek-ai/cordis-plugin-loader": "~1.0.6" }, "peerDependenciesMeta": { "@deepseek-ai/cordis-plugin-include": { @@ -354,17 +354,17 @@ } }, "node_modules/@deepseek-ai/cordis-plugin-loader": { - "version": "1.0.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6-alpha.2.tgz", - "integrity": "sha512-ea4WjgWsrUxVJ0UYujTbGgaSIR/mV46myY/s6isYRTQRCmlkyVPGagkTxrRR2c/MqGwwbC4KXxusT4uM7H9BXg==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cordis-plugin-loader/-/cordis-plugin-loader-1.0.6.tgz", + "integrity": "sha512-fr011A782oEjkJXCY4pePt2qaPpkGvem7Rmst6Rhwgg6icgwI4eN9QBRb6S/q9XOYGVQUdvL0wXUWyNxO8WipA==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/cosmokit": "~1.8.6", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", + "@deepseek-ai/cordis": "~4.0.5", "node-addon-require-builtin": "^0.1.9" }, "peerDependenciesMeta": { @@ -374,28 +374,28 @@ } }, "node_modules/@deepseek-ai/cosmokit": { - "version": "1.8.6-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6-alpha.2.tgz", - "integrity": "sha512-FcxdMUal80CeFURh3lPYsw9TPDMVSGpovWnA3JtTGrBG15XFli4k38z1Egso1lgvifKpIW4Hx42X6UTelLEOAA==", + "version": "1.8.6", + "resolved": "https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.6.tgz", + "integrity": "sha512-ZSD/fN52L498LBIOYwjIOlaLv5veY6ZEwI2yzvZUKQtctn0QAPqu8tKdJ+OF5+Tv6NBq1U+wJWH+QI6OGy5EvQ==", "dev": true, "license": "MIT" }, "node_modules/@deepseek-ai/dsh-agent": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-alpha.4.tgz", - "integrity": "sha512-QIL78OhM5csPWkrdGfhUJQr3zdyUd/DZyM4euwUnkMS0BA5Hp21hCWg7N69HY73bqvV20oyU10g5huPtqJHSyQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-agent/-/dsh-agent-0.2.1-rc.2.tgz", + "integrity": "sha512-e96Tnq8g/RNCDDUW7kAypY/4cho5QxjLHMhk86BGGIEu9b31XEkSFqwqQIwtYXQez3cDR5z7LZpAM8Uqx9Y8KA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/dsh-workspace": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/dsh-workspace": "0.2.1-rc.2" }, "peerDependenciesMeta": { "@deepseek-ai/dsh-workspace": { @@ -404,148 +404,148 @@ } }, "node_modules/@deepseek-ai/dsh-brand": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-alpha.4.tgz", - "integrity": "sha512-SlrYMisnHPNREhc9jsohRylcZmpQPuzcul7DxUCJNjK08fDUUuzcyrQI7uDjvbw1n34IwYHNNV6o4mrhIIForQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-brand/-/dsh-brand-0.2.1-rc.2.tgz", + "integrity": "sha512-T2kcygikRS8rv8cKjgho2x381EgGprv5yQmvHl4/iYbuHE19OW+SLGtHWbV27PGdSHz5U8CJxasYzFosKW4e2A==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-primitives": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-alpha.4.tgz", - "integrity": "sha512-nHzG2k7l8UuNG2rcIkzny1ko/fn/mQjiWrSh6wIe6X9d/qQi55Tdo87N3usmwce1akcZrvsXQ27v451tC1gzwA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-primitives/-/dsh-client-ui-primitives-0.2.1-rc.2.tgz", + "integrity": "sha512-3K6W2ID4sbDUDHhZHs8cNO15XDXBWAYFnEzJ/iD8ZASqidazcJActxlEf8GO6NHmbbeX0lEq0P20Ftwhovt+Mw==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-client-ui-settings": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-alpha.4.tgz", - "integrity": "sha512-SG6wzQyBmkNUuVt+5Da4tSnGeVVgNE8cJ6Qa6MG4zOJqWnEkJTCIRuOmsN1+7IGcWSNooFG76RGJpO+5uKkDdA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-client-ui-settings/-/dsh-client-ui-settings-0.2.1-rc.2.tgz", + "integrity": "sha512-oU8f/0+DsdbgF0XhM+Po7W4PDJmTbhifQVwIoAcKfj29JN/4tLxKqIKS0IRcMbIGD39BGFITne78KIMgrGJy3Q==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-fs": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-alpha.4.tgz", - "integrity": "sha512-K5TK7QxC3fRM8s1Uvfy1O3C4kaybSXOLGFMSb9WLwfTe1QGsq7JZN0v8jmiv3VFA0cFUkTPybz/PokyDtRWvRg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-fs/-/dsh-fs-0.2.1-rc.2.tgz", + "integrity": "sha512-Z6d2ZKsAiJc0FaiES8wC66vuUQMGohl9tCc2p3bAsPFYkJ0+L2zTViQ9Y5BArtRe9Br8qhNjNgLlN0jb/uwv1g==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-llm": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-alpha.4.tgz", - "integrity": "sha512-rjAT6x0iNUNLgCC3pCNaQxrtQxlcMAAdCh8x27C4PiGCXMaa7kNge8HnjPC3iaZoopSrr5XRBLcyIFmg46Cc+A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-llm/-/dsh-llm-0.2.1-rc.2.tgz", + "integrity": "sha512-3xfoMDHEpUJGcH2Lu+lNXUJYRBclYfBkdCG9Au02VzIjYAm9BrFmc/MNuaVTg0vwgvPpTsd6Fx1IeGtfmQFvlg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-timeout": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-crypto": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-timeout": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-crypto": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-ptc-runtime": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-alpha.4.tgz", - "integrity": "sha512-r+4GYR1UYbr0zyuu9tOO+iJuQvxYjtCJyTHr6YR9UZNJrfbwkrSeqlRIbR1cw5NaIh7dplZK1+a90FHURkrgWA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-ptc-runtime/-/dsh-ptc-runtime-0.2.1-rc.2.tgz", + "integrity": "sha512-e81rbNwp3YNIpzIbkFI4NsWQzTD3uAP8zXU16diL9wLmITUWjMS+kvrWEXtrkWAWp5cnK2Pgn4xCbqVZ6n4I8w==", "dev": true, "license": "MIT", "peer": true, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-alpha.4.tgz", - "integrity": "sha512-PRCb2xDAvX609LagauDjLvYFmzYLBO3QYAYXcc7btfiqOdgfeDb7FgstzKjdbz2o31iVdvKUdDK1iFzXjVg6eg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox/-/dsh-sandbox-0.2.1-rc.2.tgz", + "integrity": "sha512-nnxNIkgi4A4tkPrTV8sbVMse3c32OT0U+mwYhLqDz/+cIm5tUGufQD+Ax0zqVeClyzXsxZo0/pVtcqDJrsp+7w==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-sandbox-policy": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-alpha.4.tgz", - "integrity": "sha512-xEfAiquD87cg1aOzdPtFoB82e0P4hzWkVEimwRk1s85ESb8YiV+DRzWLrIGprfi9fKhsxSFFIFnCKgzOKjURuA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-sandbox-policy/-/dsh-sandbox-policy-0.2.1-rc.2.tgz", + "integrity": "sha512-fBb8zQ13CLKM60ynl3EHvFmO8F0GARofQYf7CXUkqJEuBcjxXUG14rzDKRS/PhE8oJ023lEvd5HJjE6qUjslhw==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-scope": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-alpha.4.tgz", - "integrity": "sha512-oo1AraYsu5D3oYGdKim43HGg0Hf6NbYLmV5LZO8jiDnaXiE+xfNDqI8ARwq049lODJMxP0ONX2hTo5DTG5zjuQ==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-scope/-/dsh-scope-0.2.1-rc.2.tgz", + "integrity": "sha512-CVGPI4lzmUy7O3zg8X1LavUSmD3JKFkUdykfHN7VkyWXG5oAXSvig9s5PLnIb+sxRItmSpiqPXGnT/48X1T1qg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-session": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-alpha.4.tgz", - "integrity": "sha512-YMdG+ZdvM+jXFDkfzO+ksCn6YpUFuoN4YuMEIcbSqLnwNo0Q7ba0GJWJEH7oP5v0sCwGtMKQySKSgqBjkUyw+w==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session/-/dsh-session-0.2.1-rc.2.tgz", + "integrity": "sha512-WDA4lIrXtG5nf1dn2wrA4UzEr7D6e8T3AdPpVSZ7eY4N4ICMDZ0RYuD4GkM+iSb7WkLojdO8ljr0etiIMEbqJg==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-session-projection": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-alpha.4.tgz", - "integrity": "sha512-+Tm9xJd7ETbcLlolf9wx+Nm2z1O0SFqXermWU3GZ7ffQqo+tRKQ2oEa6NhSS5Mw1wVkY0xuxye5RRnVk3gbb4g==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-session-projection/-/dsh-session-projection-0.2.1-rc.2.tgz", + "integrity": "sha512-GBGgIAaC+Cw1rV7tpyGy9aaENUOkQCosP+LJHcmzqPWg64Ufts36mDEU3f0lC/nyGb0z66Ov+RbyaLLsCnZMwA==", "dev": true, "license": "MIT", "peer": true, @@ -553,144 +553,144 @@ "zod": "4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-session": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-system-prompt": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-alpha.4.tgz", - "integrity": "sha512-oHk8b0F2ecwrcN58ZtkMkR8FIRTULNSMTkHB1KbOz1LpTYuD6M01VFhWG0FvzZw01xPeuDlbO9wNmYtYi6/RsA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-system-prompt/-/dsh-system-prompt-0.2.1-rc.2.tgz", + "integrity": "sha512-X1WG2FMyfZapAkcCYyTl0UO5vVqDcCsA9wECJY7eeDTLHJDsqrilQJg2DRgHQzM+2/7T01QyzFZhKd7hfALJxg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-timeout": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-alpha.4.tgz", - "integrity": "sha512-hXCp1YWwDZAaTrNNon3ypb/Z6ZgDNKi/3zdC2JLHb9gU2f7DdSexbvRs37xKfyXO8HhtXDKvp/FUCWvI6rpRlg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-timeout/-/dsh-timeout-0.2.1-rc.2.tgz", + "integrity": "sha512-R75r+GkYP+ZhtIsIfn8MMuRXPkugJr0OyQ1Y+rauoTae2sdVfnE8v5zvQORRCHdh7MiOcmh10dQN70KjHh7qhQ==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-tools": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-alpha.4.tgz", - "integrity": "sha512-oTurHJ91LmlirKbEMyU0K+ES8LYtfJW4MAGrLjSBSaRKrq8eERTWNJafcxlhEKNpTWeUNU7h3OHaX/rqcMmLyA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-tools/-/dsh-tools-0.2.1-rc.2.tgz", + "integrity": "sha512-K3gET3VPbtxM66pkeRsEGrZ+bQKiBqrTbNXiIreBKu0hP+qPvlSwhiSI5MTnFp2tMFK5rpuYMZosRIdbfrbJ/g==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-util-values": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-util-values": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-ptc-runtime": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox-policy": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4", - "@deepseek-ai/dsh-user-approval": "0.2.1-alpha.4", - "@deepseek-ai/dsh-working-directory": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-ptc-runtime": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox-policy": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2", + "@deepseek-ai/dsh-user-approval": "0.2.1-rc.2", + "@deepseek-ai/dsh-working-directory": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-typert-protocol": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-alpha.4.tgz", - "integrity": "sha512-LBNUgiwbRqhyRNPvaAca++eLAxJ1jmmFJnd3RJAa8pT+oULJgZvqPgr9jVnTfX/P5pQT0p7AMZeelTc7QpQ/OA==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-typert-protocol/-/dsh-typert-protocol-0.2.1-rc.2.tgz", + "integrity": "sha512-rtsHJHGE9SWX+zOgtDbPUqz3h7r+qFm3h1WASzRJxDnULqo7JaBdIiGshNTFPmetsX+r1T69CKFdTZAJmHwMug==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-user-approval": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-alpha.4.tgz", - "integrity": "sha512-J6bcZV3rFuQiRDuMej5S/DalSeztfVIFngoGA15M31khb0E74kfrgWUIxvS7pRwy2j1MKz89wR8BbbDzufHTRw==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-user-approval/-/dsh-user-approval-0.2.1-rc.2.tgz", + "integrity": "sha512-pwhw1U1IZZm5tKyPayj2APZBq5I7/I0C0ad+ohd4vKuNb6twDIltt1GEHdCNt7I19IGHLRiUY+/Q9S/8+q5K0A==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2" + "@deepseek-ai/schemastery": "~3.18.5" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-brand": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-brand": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/dsh-util-crypto": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-alpha.4.tgz", - "integrity": "sha512-BLMlmD3y+HhTZPGij4viPRFWx8DNdsPNMauUW5XxyE2Iq5KG9ohQ+6xULju/J2Ah9JldVr3XAku7ItDPDk8Mpg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-crypto/-/dsh-util-crypto-0.2.1-rc.2.tgz", + "integrity": "sha512-V4SxvXL4ZbY5RYJckVtmo3qz4YsA52pYKvDF13+8hsDCEmQIv9YfmLRMA61pjK/i05pGMe3p1pPhAB6L+NOIEA==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-util-values": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-alpha.4.tgz", - "integrity": "sha512-RM3gAmdyUsldAb4AIsU++KYbBY98UvHRCskcHX2bojZ6kefGd1a8P1Yl/Fo+bUWIAHZj/VoA1pnAl7r5e5456A==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-util-values/-/dsh-util-values-0.2.1-rc.2.tgz", + "integrity": "sha512-mgnByPadOgZtoBxbwMI184li494O6sNn66/NvdqN0RIrzAJmFGpFxpjFkQtQN/yetwWhlx7Ic17+6p7eKl8DNg==", "dev": true, "license": "MIT", "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2" + "@deepseek-ai/cordis": "~4.0.5" } }, "node_modules/@deepseek-ai/dsh-working-directory": { - "version": "0.2.1-alpha.4", - "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-alpha.4.tgz", - "integrity": "sha512-VtgMMlvYJ4iOPtpasI+ey6ioBekJy5MLSw3ky6lvLdLnKPmj0zobVOFhldr7KEvZFRizKYN5ktU8D85qKZG9Cg==", + "version": "0.2.1-rc.2", + "resolved": "https://registry.npmjs.org/@deepseek-ai/dsh-working-directory/-/dsh-working-directory-0.2.1-rc.2.tgz", + "integrity": "sha512-zhPtvAaCxa9U0WUN9OEG37zVBq7GOXXXwnDx2Jds4DFksGY5Lmpl3ZK1QMGz+F/Aye4VHt9zoISI7m4wDA8DEg==", "dev": true, "license": "MIT", "peer": true, "dependencies": { - "@deepseek-ai/schemastery": "~3.18.5-alpha.2", + "@deepseek-ai/schemastery": "~3.18.5", "zod": "^4.4.3" }, "peerDependencies": { - "@deepseek-ai/cordis": "~4.0.5-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-fs": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-session-projection": "0.2.1-alpha.4", - "@deepseek-ai/dsh-system-prompt": "0.2.1-alpha.4" + "@deepseek-ai/cordis": "~4.0.5", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-fs": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-session-projection": "0.2.1-rc.2", + "@deepseek-ai/dsh-system-prompt": "0.2.1-rc.2" } }, "node_modules/@deepseek-ai/schemastery": { - "version": "3.18.5-alpha.2", - "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5-alpha.2.tgz", - "integrity": "sha512-W+2TSjdeKOEKqaii9BNpw7ev2Yf8C+J74i/BE/VBx56O7BX0zxV2zQKwRY4ADLTXD5O69iJ+/Skm91/ES7iMgw==", + "version": "3.18.5", + "resolved": "https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.5.tgz", + "integrity": "sha512-Mnobvqf9rHsZYv4g5edj5ddxS1oF1kIdSlz0DunUgnY3LWG5F26MqvH3YOOsSaodCgyzJ16OMN9O99/s5Crtdw==", "dev": true, "license": "MIT", "dependencies": { - "@deepseek-ai/cosmokit": "~1.8.6-alpha.2", + "@deepseek-ai/cosmokit": "~1.8.6", "@standard-schema/spec": "^1.1.0" } }, @@ -1488,9 +1488,9 @@ } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.12", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.12.tgz", - "integrity": "sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==", + "version": "7.0.13", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.13.tgz", + "integrity": "sha512-2E8Q6rh+1RRSq/YdaGsNfXsEZy9FVqv6BSSrkmZJP7BnaoPnRfKL+YaofPvP+5rzdk144qSm6i4+hvPgUcbV4Q==", "dev": true, "license": "MIT", "engines": { @@ -1811,9 +1811,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.11.29", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.29.tgz", - "integrity": "sha512-YeesxhUmR4h5y5V8KF8iBCqJb8YFrBJ++XBTeGcWIaQOYXGN19JM0FoiOjAhwuDoln8owMgXcWBmgHtYSkky2w==", + "version": "2.11.30", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.30.tgz", + "integrity": "sha512-YJs6Pz17RMNv8HoyKREtCY38E3BV1nPkWVmIb+XN1uHC1Vqt3KZCH/uJH9ZKRoiKb/icZ6tvehyee6RZNbFfRA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1835,9 +1835,9 @@ } }, "node_modules/browserslist": { - "version": "4.29.3", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", - "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "version": "4.29.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.4.tgz", + "integrity": "sha512-2TcbbiwXEq+N0kD36tykpCfyyQigUXmuwNQbnPp4Xk4PX9JOqVIKjJlIDjpH7uY6tmRGoAd+apeT1p60T8wsdA==", "dev": true, "funding": [ { @@ -1855,11 +1855,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.11.26", - "caniuse-lite": "^1.0.30001813", - "electron-to-chromium": "^1.5.439", - "node-releases": "^2.0.57", - "update-browserslist-db": "^1.3.3" + "baseline-browser-mapping": "^2.11.29", + "caniuse-lite": "^1.0.30001816", + "electron-to-chromium": "^1.5.459", + "node-releases": "^2.0.59", + "update-browserslist-db": "^1.3.4" }, "bin": { "browserslist": "cli.js" @@ -1879,9 +1879,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001816", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001816.tgz", - "integrity": "sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==", + "version": "1.0.30001817", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001817.tgz", + "integrity": "sha512-uZ1JG80BwhtMalIAGPxPfmNhHyd/B/nTdr3hITWlRamSyAdUStP1Gop01e7SD5q6J90ixJvWDbvmiFVfDgn4bw==", "dev": true, "funding": [ { @@ -1998,9 +1998,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.457", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.457.tgz", - "integrity": "sha512-RNKsgErRmkzRgC8nOV1wzFhSbfGlYJ6k4LT/aE7oFrDYaQgHhTh5oTp0ZgSbRRybKR8HlZIdCWc38Y+CDTk1Jw==", + "version": "1.5.463", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.463.tgz", + "integrity": "sha512-nVWNU6ltFjopvOdD44J4zoBkgO0M+WZiFKKFV7EjebOH9z0KNphk6bCEjplXtJofL21ARIuuwWOCsFkBVklm6w==", "dev": true, "license": "ISC" }, diff --git a/integrations/dsh/package.json b/integrations/dsh/package.json index 86388a31..0ec3eaa9 100644 --- a/integrations/dsh/package.json +++ b/integrations/dsh/package.json @@ -79,14 +79,14 @@ } }, "devDependencies": { - "@deepseek-ai/cordis": "4.0.5-alpha.2", - "@deepseek-ai/cordis-plugin-loader": "1.0.6-alpha.2", - "@deepseek-ai/dsh-agent": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-alpha.4", - "@deepseek-ai/dsh-llm": "0.2.1-alpha.4", - "@deepseek-ai/dsh-tools": "0.2.1-alpha.4", - "@deepseek-ai/dsh-typert-protocol": "0.2.1-alpha.4", - "@deepseek-ai/schemastery": "3.18.5-alpha.2", + "@deepseek-ai/cordis": "4.0.5", + "@deepseek-ai/cordis-plugin-loader": "1.0.6", + "@deepseek-ai/dsh-agent": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-primitives": "0.2.1-rc.2", + "@deepseek-ai/dsh-llm": "0.2.1-rc.2", + "@deepseek-ai/dsh-tools": "0.2.1-rc.2", + "@deepseek-ai/dsh-typert-protocol": "0.2.1-rc.2", + "@deepseek-ai/schemastery": "3.18.5", "@eslint/js": "9.39.4", "@types/node": "^22.15.0", "@types/react": "~18.3.1", @@ -98,10 +98,10 @@ "react": "^18.2.0", "typescript": "^5.9.2", "typescript-eslint": "8.59.3", - "@deepseek-ai/dsh-session": "0.2.1-alpha.4", - "@deepseek-ai/dsh-client-ui-settings": "0.2.1-alpha.4", - "@deepseek-ai/dsh-scope": "0.2.1-alpha.4", - "@deepseek-ai/dsh-sandbox": "0.2.1-alpha.4", + "@deepseek-ai/dsh-session": "0.2.1-rc.2", + "@deepseek-ai/dsh-client-ui-settings": "0.2.1-rc.2", + "@deepseek-ai/dsh-scope": "0.2.1-rc.2", + "@deepseek-ai/dsh-sandbox": "0.2.1-rc.2", "react-test-renderer": "^18.3.1", "semver": "^7.8.5" }, diff --git a/integrations/dsh/scripts/test-package.mjs b/integrations/dsh/scripts/test-package.mjs index 67884661..3e8f3f46 100644 --- a/integrations/dsh/scripts/test-package.mjs +++ b/integrations/dsh/scripts/test-package.mjs @@ -37,6 +37,7 @@ const releases = [ loader: "1.0.6-alpha.2", schema: "3.18.5-alpha.2", }, + { dsh: "0.2.1-rc.2", cordis: "4.0.5", loader: "1.0.6", schema: "3.18.5" }, ]; async function run(command, args, cwd) { diff --git a/integrations/dsh/src/client/index.tsx b/integrations/dsh/src/client/index.tsx index f8674bd0..d14a0151 100644 --- a/integrations/dsh/src/client/index.tsx +++ b/integrations/dsh/src/client/index.tsx @@ -2,6 +2,7 @@ import { useState, useSyncExternalStore } from "react"; import type { ConfigForm } from "@deepseek-ai/dsh-client-ui-settings/client"; import type { ReMeSettings } from "../types.js"; +import { validEndpoint } from "../endpoint.js"; import { ReMeStatusPage, statusEn, @@ -19,6 +20,8 @@ const FIELDS = [ "requestTimeoutMs", "backgroundTimeoutMs", "shutdownTimeoutMs", + "autoMemoryRetryMs", + "autoMemoryRetryMaxMs", "autoMemoryEnabled", "autoMemoryInterval", "autoDreamEnabled", @@ -65,6 +68,8 @@ interface Draft { requestTimeoutMs: string; backgroundTimeoutMs: string; shutdownTimeoutMs: string; + autoMemoryRetryMs: string; + autoMemoryRetryMaxMs: string; autoMemoryEnabled: boolean; autoMemoryInterval: string; autoDreamEnabled: boolean; @@ -113,6 +118,8 @@ const en = { advanced: "Advanced", backgroundTimeout: "Background request timeout (ms)", shutdownTimeout: "Shutdown flush timeout (ms)", + retryDelay: "Retry delay (ms)", + retryMaxDelay: "Maximum retry delay (ms)", diagnostics: "Service diagnostics", processMemory: "Process memory", componentMemory: "Component memory", @@ -186,6 +193,8 @@ const zh: typeof en = { advanced: "高级设置", backgroundTimeout: "后台请求超时（毫秒）", shutdownTimeout: "关闭时写入等待（毫秒）", + retryDelay: "失败重试间隔（毫秒）", + retryMaxDelay: "最大重试间隔（毫秒）", diagnostics: "服务诊断", processMemory: "进程内存", componentMemory: "组件内存", @@ -568,6 +577,28 @@ function SettingsForm({ } /> + + + update("autoMemoryRetryMs", event.target.value) + } + /> + + + + update("autoMemoryRetryMaxMs", event.target.value) + } + /> + value === undefined) @@ -676,6 +721,8 @@ function parseDraft(draft: Draft): ReMeSettings | undefined { requestTimeoutMs: requestTimeoutMs!, backgroundTimeoutMs: backgroundTimeoutMs!, shutdownTimeoutMs: shutdownTimeoutMs!, + autoMemoryRetryMs: autoMemoryRetryMs!, + autoMemoryRetryMaxMs: autoMemoryRetryMaxMs!, autoMemoryEnabled: draft.autoMemoryEnabled, autoMemoryInterval: autoMemoryInterval!, autoDreamEnabled: draft.autoDreamEnabled, @@ -699,15 +746,6 @@ function boundedInteger( : undefined; } -function validEndpoint(value: string): boolean { - try { - const url = new URL(value.trim()); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - function validTimezone(value: string): boolean { try { new Intl.DateTimeFormat("en", { timeZone: value.trim() }).format(0); diff --git a/integrations/dsh/src/config.ts b/integrations/dsh/src/config.ts index ce4421bc..e517814c 100644 --- a/integrations/dsh/src/config.ts +++ b/integrations/dsh/src/config.ts @@ -1,4 +1,7 @@ import z from "@deepseek-ai/schemastery"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { validEndpoint } from "./endpoint.js"; import { validTimezone, validateDailyCron } from "./scheduling.js"; import type { ReMeConfig, ReMeConfigInput } from "./types.js"; @@ -26,6 +29,17 @@ export const Config = z.object({ shutdownTimeoutMs: z.natural().min(100).max(60000).default(5000).volatile(), autoMemoryEnabled: z.boolean().default(true).volatile(), autoMemoryInterval: z.natural().min(1).max(1000).default(5).volatile(), + autoMemoryQueueDir: z + .string() + .default("") + .description("Private persistent automatic-memory queue directory"), + autoMemoryRetryMs: z.natural().min(1000).max(60000).default(1000).volatile(), + autoMemoryRetryMaxMs: z + .natural() + .min(1000) + .max(3600000) + .default(60000) + .volatile(), autoDreamEnabled: z.boolean().default(true).volatile(), dreamCron: checkedString( DEFAULT_DREAM_CRON, @@ -77,6 +91,9 @@ const DEFAULT_CONFIG: Readonly = Object.freeze({ shutdownTimeoutMs: 5000, autoMemoryEnabled: true, autoMemoryInterval: 5, + autoMemoryQueueDir: "", + autoMemoryRetryMs: 1000, + autoMemoryRetryMaxMs: 60000, autoDreamEnabled: true, dreamCron: "0 23 * * *", dreamHint: "", @@ -121,6 +138,23 @@ export function resolveConfig( config.endpoint = stripTrailingSlashes(String(config.endpoint)); assertEndpoint(config.endpoint); + const home = env.DSH_HOME?.trim() || `${homedir()}/.dsh`; + const queueDir = config.autoMemoryQueueDir || `${home}/reme-memory/outbox`; + config.autoMemoryQueueDir = resolve( + queueDir.replace(/^~(?=\/|$)/, homedir()), + ); + config.autoMemoryRetryMs = integer( + config.autoMemoryRetryMs, + 1000, + 60000, + 1000, + ); + config.autoMemoryRetryMaxMs = integer( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs, + 3600000, + 60000, + ); config.requestTimeoutMs = integer( config.requestTimeoutMs, 1000, @@ -163,15 +197,10 @@ export function resolveConfig( } function assertEndpoint(value: string): void { - let endpoint: URL; - try { - endpoint = new URL(value); - } catch { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } - if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") { - throw new TypeError("ReMe endpoint must be an absolute http(s) URL"); - } + if (!validEndpoint(value)) + throw new TypeError( + "ReMe endpoint must be an absolute http(s) URL without credentials, query, or fragment", + ); } function stripTrailingSlashes(value: string): string { diff --git a/integrations/dsh/src/endpoint.ts b/integrations/dsh/src/endpoint.ts new file mode 100644 index 00000000..d50eb5f9 --- /dev/null +++ b/integrations/dsh/src/endpoint.ts @@ -0,0 +1,23 @@ +/** HTTP endpoints exclude credentials, query strings, and fragments. */ +export function validEndpoint(value: string): boolean { + try { + const url = new URL(value.trim()); + return ( + (url.protocol === "http:" || url.protocol === "https:") && + !url.username && + !url.password && + !url.search && + !url.hash && + !/[?#]/.test(value) + ); + } catch { + return false; + } +} + +/** Remove URL userinfo before errors cross model, browser, or log boundaries. */ +export function safeError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + // URL parsing encodes spaces and strips tabs/newlines in userinfo; do not treat them as boundaries. + return message.replace(/(https?:\/\/)[^/?#\\]+@/gi, "$1[redacted]@"); +} diff --git a/integrations/dsh/src/index.ts b/integrations/dsh/src/index.ts index fbdc3640..8a14eebd 100644 --- a/integrations/dsh/src/index.ts +++ b/integrations/dsh/src/index.ts @@ -20,6 +20,10 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { const current = () => config; const client = new ReMeClient(current); const runtime = new ReMeRuntime(client, current, ctx.logger); + ctx.effect(() => { + runtime.start(); + return () => runtime.disposeAll(); + }, "remeMemory.lifecycle()"); ctx.provide("remeMemory", runtime); void ctx.plugin(ReMeStatusGateway); ctx.effect( @@ -27,11 +31,6 @@ export function apply(ctx: Context, input: ReMeConfigInput = {}): void { "remeMemory.tools()", ); - ctx.effect(() => { - runtime.start(); - return () => runtime.disposeAll(); - }, "remeMemory.lifecycle()"); - ctx.on("loader/volatile-update", () => { config = resolveConfig(input); runtime.reconfigure(); diff --git a/integrations/dsh/src/outbox.ts b/integrations/dsh/src/outbox.ts new file mode 100644 index 00000000..1bc9e43c --- /dev/null +++ b/integrations/dsh/src/outbox.ts @@ -0,0 +1,244 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + mkdirSync, + openSync, + opendirSync, + readFileSync, + renameSync, + rmdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import type { ReMeMessage } from "./reme/types.js"; +import { validEndpoint } from "./endpoint.js"; + +export interface MemoryBatch { + version: 1; + id: string; + createdAt: number; + endpoint: string; + sessionId: string; + date: string; + turns: ReMeMessage[][]; +} + +/** Private JSON outbox: seal batches before sending and remove only after acknowledgement. */ +export class MemoryOutbox { + private readonly lock: string; + private readonly owner = `${process.pid}:${randomUUID()}`; + private latestCreatedAt = 0; + + constructor(readonly directory: string) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + this.lock = join(directory, ".lock"); + // Serialize inspection, stale-owner removal, and replacement as one operation. + // An interrupted recovery fails closed; never reclaim this guard automatically. + const guard = join(directory, ".lock-recovery"); + try { + mkdirSync(guard, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + throw new Error( + "ReMe outbox lock recovery is busy or interrupted; retry, or inspect .lock-recovery after stopping all Hosts", + ); + } + try { + if (existsSync(this.lock)) { + const pid = Number(readFileSync(this.lock, "utf8").split(":")[0]); + if (!Number.isSafeInteger(pid) || pid <= 0) + throw new Error( + "Invalid ReMe outbox lock; inspect the queue directory", + ); + try { + process.kill(pid, 0); + throw new Error( + "ReMe outbox is in use; configure a separate autoMemoryQueueDir for each Host", + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + unlinkSync(this.lock); + } + const fd = openSync(this.lock, "wx", 0o600); + try { + writeFileSync(fd, this.owner); + } finally { + closeSync(fd); + } + for (const name of this.files()) + this.latestCreatedAt = Math.max( + this.latestCreatedAt, + this.read(name).createdAt, + ); + } catch (error) { + this.close(); + throw error; + } finally { + rmdirSync(guard); + } + } + + append( + endpoint: string, + sessionId: string, + date: string, + messages: ReMeMessage[], + interval: number, + ): void { + const key = createHash("sha256") + .update(JSON.stringify([endpoint, sessionId, date])) + .digest("hex"); + const name = `partial-${key}.json`; + const path = join(this.directory, name); + const batch: MemoryBatch = existsSync(path) + ? this.read(name) + : { + version: 1, + id: randomUUID(), + // Keep FIFO ordering when the clock repeats or moves backwards, including after restart. + createdAt: (this.latestCreatedAt = Math.max( + Date.now(), + this.latestCreatedAt + 1, + )), + endpoint, + sessionId, + date, + turns: [], + }; + batch.turns.push(messages); + this.write(name, batch); + if (batch.turns.length >= interval) this.seal(name, batch); + // Finish the previous workspace day without combining dates in one request. + for (const file of this.files()) { + if (!file.startsWith("partial-") || file === name) continue; + const previous = this.read(file); + if ( + previous.endpoint === endpoint && + previous.sessionId === sessionId && + previous.date !== date + ) + this.seal(file, previous); + } + } + + flush(sessionId?: string): void { + for (const name of this.files()) { + if (!name.startsWith("partial-")) continue; + const batch = this.read(name); + if (sessionId === undefined || batch.sessionId === sessionId) + this.seal(name, batch); + } + } + + next( + accepts: (batch: MemoryBatch) => boolean = () => true, + ): { name: string; batch: MemoryBatch } | undefined { + let oldest: string | undefined; + for (const name of this.files()) { + if ( + name.startsWith("ready-") && + (oldest === undefined || name < oldest) && + accepts(this.read(name)) + ) + oldest = name; + } + return oldest === undefined + ? undefined + : { name: oldest, batch: this.read(oldest) }; + } + + acknowledge(name: string): void { + unlinkSync(join(this.directory, name)); + } + + stats(): { turns: number; batches: number } { + let turns = 0; + let batches = 0; + for (const name of this.files()) { + turns += this.read(name).turns.length; + if (name.startsWith("ready-")) batches += 1; + } + return { turns, batches }; + } + + close(): void { + if (existsSync(this.lock) && readFileSync(this.lock, "utf8") === this.owner) + unlinkSync(this.lock); + } + + private *files(): Generator { + const dir = opendirSync(this.directory); + try { + for (let entry = dir.readSync(); entry; entry = dir.readSync()) + if (entry.isFile() && /^(partial|ready)-.*\.json$/.test(entry.name)) + yield entry.name; + } finally { + dir.closeSync(); + } + } + + private read(name: string): MemoryBatch { + try { + const batch = JSON.parse( + readFileSync(join(this.directory, name), "utf8"), + ) as MemoryBatch; + if ( + batch.version !== 1 || + typeof batch.id !== "string" || + !/^[a-f0-9-]{36}$/.test(batch.id) || + !validEndpoint(batch.endpoint) || + typeof batch.sessionId !== "string" || + typeof batch.date !== "string" || + !Number.isFinite(batch.createdAt) || + !Array.isArray(batch.turns) || + !batch.turns.length || + !batch.turns.every( + (turn) => + Array.isArray(turn) && + turn.every( + (message) => + typeof message.id === "string" && + ["user", "assistant"].includes(message.role) && + Array.isArray(message.content) && + message.content.every( + (part) => + part.type === "text" && typeof part.text === "string", + ), + ), + ) + ) + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + return batch; + } catch { + // JSON parser errors can quote message content; leave the record untouched. + throw new Error( + "Invalid ReMe outbox record; preserve the file and inspect the queue directory", + ); + } + } + + private write(name: string, batch: MemoryBatch): void { + const temporary = join(this.directory, `${randomUUID()}.tmp`); + const fd = openSync(temporary, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify(batch)); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(temporary, join(this.directory, name)); + } + + private seal(name: string, batch: MemoryBatch): void { + renameSync( + join(this.directory, name), + join(this.directory, `ready-${batch.createdAt}-${batch.id}.json`), + ); + } +} diff --git a/integrations/dsh/src/reme/client.ts b/integrations/dsh/src/reme/client.ts index ff0bccbc..10d9050c 100644 --- a/integrations/dsh/src/reme/client.ts +++ b/integrations/dsh/src/reme/client.ts @@ -13,6 +13,8 @@ import type { SearchOptions, } from "./types.js"; +import { safeError, validEndpoint } from "../endpoint.js"; + interface ReMeResponseBody { success?: boolean; answer?: unknown; @@ -50,7 +52,10 @@ export class ReMeClient { sessionId: string, options: AutoMemoryOptions = {}, ): Promise { - const config = this.configSource(); + const config = { + ...this.configSource(), + ...(options.endpoint ? { endpoint: options.endpoint } : {}), + }; return this.request( "auto_memory", { @@ -58,6 +63,7 @@ export class ReMeClient { session_id: sessionId, memory_hint: options.memoryHint || "", date: options.date || "", + ...(options.requestId ? { request_id: options.requestId } : {}), }, config, config.backgroundTimeoutMs, @@ -189,6 +195,10 @@ export class ReMeClient { ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal; try { + if (!validEndpoint(config.endpoint)) + throw new TypeError( + "Invalid ReMe endpoint: credentials, query, and fragment are unsupported", + ); const response = await fetch(`${config.endpoint}/${job}`, { method: "POST", headers: { @@ -208,7 +218,7 @@ export class ReMeClient { metadata: body.metadata ?? {}, error: ok ? "" - : String(body.answer || body.detail || `HTTP ${response.status}`), + : safeError(body.answer || body.detail || `HTTP ${response.status}`), }; } catch (error) { return { @@ -216,7 +226,7 @@ export class ReMeClient { status: 0, answer: "", metadata: {}, - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }; } finally { clearTimeout(timer); diff --git a/integrations/dsh/src/reme/types.ts b/integrations/dsh/src/reme/types.ts index a6864637..cf32dcb2 100644 --- a/integrations/dsh/src/reme/types.ts +++ b/integrations/dsh/src/reme/types.ts @@ -95,6 +95,8 @@ export interface SearchOptions { /** Automatic-memory request controls supported by the plugin client. */ export interface AutoMemoryOptions { + endpoint?: string; + requestId?: string; date?: string; memoryHint?: string; signal?: AbortSignal; diff --git a/integrations/dsh/src/runtime.ts b/integrations/dsh/src/runtime.ts index 407ec3f4..986a86f7 100644 --- a/integrations/dsh/src/runtime.ts +++ b/integrations/dsh/src/runtime.ts @@ -1,35 +1,37 @@ import type { LoggerLike, ReMeClientLike, ReMeMessage } from "./reme/types.js"; - import { messagesDay, nextDailyRun } from "./scheduling.js"; import { captureMessage, remeSessionId } from "./messages.js"; +import { MemoryOutbox, type MemoryBatch } from "./outbox.js"; +import { safeError } from "./endpoint.js"; import type { DshSession, ReMeConfig, SessionEvent } from "./types.js"; import type { ReMeRuntimeSnapshot, ReMeRuntimeTask } from "./runtime-status.js"; -interface PendingTurn { - messages: ReMeMessage[]; - day: string; -} - -interface SessionState { - session: DshSession; - sessionId: string; - activeTurn: unknown; - activeMessages: ReMeMessage[]; - pendingTurns: PendingTurn[]; - unconfirmedTurns: number; - writes: Promise; - requestController: AbortController; +interface MemoryWorker { + controller: AbortController; + task: Promise; + active?: ReMeRuntimeTask; } export class ReMeRuntime { - readonly states = new Map(); + private readonly states = new Map(); private readonly configSource: () => ReMeConfig; + private readonly outbox: MemoryOutbox; + private memoryTask: Promise | null = null; + private readonly memoryWorkers = new Map(); + private readonly retryTimers = new Map< + string, + ReturnType + >(); + private readonly failures = new Map(); + private memoryError: string | undefined; + private captureError: string | undefined; private dreamTimer: ReturnType | null = null; private dreamScheduleGeneration = 0; private dreamTask: Promise | null = null; private dreamController: AbortController | null = null; private started = false; private stopping = false; + private closed = false; private taskSequence = 0; private readonly recentTasks: ReMeRuntimeTask[] = []; private nextDreamAt: string | undefined; @@ -44,148 +46,232 @@ export class ReMeRuntime { readonly logger: LoggerLike = console, ) { this.configSource = typeof config === "function" ? config : () => config; - } - - stateFor(session: DshSession): SessionState { - const existing = this.states.get(session.id); - if (existing) { - existing.session = session; - if (existing.requestController.signal.aborted) - existing.requestController = new AbortController(); - return existing; - } - const state: SessionState = { - session, - sessionId: remeSessionId(session.id), - activeTurn: null, - activeMessages: [], - pendingTurns: [], - unconfirmedTurns: 0, - writes: Promise.resolve(), - requestController: new AbortController(), - }; - this.states.set(session.id, state); - return state; + this.outbox = new MemoryOutbox(this.configSource().autoMemoryQueueDir); } capture(session: DshSession, event: SessionEvent): void { const config = this.configSource(); - if (!config.autoMemoryEnabled) return; - const state = this.stateFor(session); - const data = isRecord(event.data) ? event.data : undefined; + if (this.stopping || !config.autoMemoryEnabled) return; if (event.type === "turn/start") { - state.activeTurn = data?.turn ?? null; - state.activeMessages = []; + this.states.set(session.id, []); return; } + const messages = this.states.get(session.id); + if (!messages) return; const message = captureMessage(event, session.id); - if (message) state.activeMessages.push(message); + if (message) messages.push(message); if (event.type !== "turn/end") return; - - const reason = data?.reason; - const reasonKind = isRecord(reason) ? reason.kind : undefined; - const completed = reasonKind === "completed" || reasonKind === "max-tokens"; - const hasUser = state.activeMessages.some((item) => item.role === "user"); - const hasAssistant = state.activeMessages.some( - (item) => item.role === "assistant", - ); - if (completed && hasUser && hasAssistant) { - const day = messagesDay(state.activeMessages, config.timezone); - const previousDay = state.pendingTurns.at(-1)?.day; - if (previousDay && day && previousDay !== day) - this.scheduleAutoMemory(state, true); - state.pendingTurns.push({ messages: state.activeMessages, day }); + const data = isRecord(event.data) ? event.data : undefined; + const reason = isRecord(data?.reason) ? data.reason.kind : undefined; + if ( + (reason === "completed" || reason === "max-tokens") && + messages.some((item) => item.role === "user") && + messages.some((item) => item.role === "assistant") + ) { + try { + this.outbox.append( + config.endpoint, + remeSessionId(session.id), + messagesDay(messages, config.timezone), + messages, + config.autoMemoryInterval, + ); + } catch (error) { + this.captureError = safeError(error); + this.recordMemoryError(error); + } } - state.activeTurn = null; - state.activeMessages = []; - this.scheduleAutoMemory(state); + this.states.set(session.id, []); + void this.drain(); } - private scheduleAutoMemory(state: SessionState, force = false): void { - const interval = this.configSource().autoMemoryInterval; - const firstDay = state.pendingTurns[0]?.day; - const dayCount = state.pendingTurns.findIndex((turn) => - Boolean(firstDay && turn.day && turn.day !== firstDay), - ); - const available = dayCount === -1 ? state.pendingTurns.length : dayCount; - const crossesDayBoundary = dayCount !== -1; - if (!force && !crossesDayBoundary && available < interval) return; - const count = force || crossesDayBoundary ? available : interval; - if (count === 0) return; - const turns = state.pendingTurns.splice(0, count); - const messages = turns.flatMap((turn) => turn.messages); - const date = turns[0]?.day || ""; - const task: ReMeRuntimeTask = { - id: `auto-memory-${++this.taskSequence}`, - phase: "queued", - queuedAt: new Date().toISOString(), - turns: turns.length, - messages: messages.length, - }; - this.recentTasks.unshift(task); - this.recentTasks.length = Math.min(this.recentTasks.length, 20); - state.unconfirmedTurns += turns.length; - state.writes = state.writes.then(async () => { - task.phase = "running"; - task.startedAt = new Date().toISOString(); - try { - const result = await this.client.autoMemory(messages, state.sessionId, { - date, - signal: state.requestController.signal, - }); - if (result.ok) { - task.phase = "completed"; - task.result = resultSummary(result.answer, result.metadata); - this.log("debug", "auto_memory_complete", { - sessionId: state.sessionId, - turns: turns.length, + /** Drain each endpoint independently; failed batches stay on disk for retry. */ + drain(): Promise { + if (this.closed) return Promise.resolve(); + if (this.retryTimers.has("*") || !this.configSource().autoMemoryEnabled) + return this.memoryTask ?? Promise.resolve(); + try { + for (;;) { + const next = this.outbox.next( + (batch) => + !this.memoryWorkers.has(batch.endpoint) && !this.blocked(batch), + ); + if (!next) break; + const { endpoint } = next.batch; + const worker: MemoryWorker = { + controller: new AbortController(), + task: Promise.resolve(), + }; + this.memoryWorkers.set(endpoint, worker); + worker.task = Promise.resolve() + .then(() => this.drainEndpoint(endpoint, worker)) + .catch((error: unknown) => { + this.recordMemoryError(error); + this.scheduleRetry(); + }) + .finally(() => { + this.memoryWorkers.delete(endpoint); + // A capture may have arrived after this worker observed an empty queue. + if (!this.stopping) void this.drain(); }); + } + } catch (error) { + this.recordMemoryError(error); + this.scheduleRetry(); + } + if (this.memoryTask) return this.memoryTask; + if (!this.memoryWorkers.size) return Promise.resolve(); + // Share one completion promise instead of accumulating waiters on stalled requests. + this.memoryTask = Promise.resolve() + .then(async () => { + while (this.memoryWorkers.size) + await Promise.all( + [...this.memoryWorkers.values()].map((worker) => worker.task), + ); + }) + .finally(() => { + this.memoryTask = null; + if (!this.stopping) return this.drain(); + }); + return this.memoryTask; + } + + private blocked(batch: MemoryBatch): boolean { + return ( + this.retryTimers.has(endpointKey(batch.endpoint)) || + this.retryTimers.has(sessionKey(batch)) + ); + } + + private async drainEndpoint( + endpoint: string, + worker: MemoryWorker, + ): Promise { + const { signal } = worker.controller; + while ( + !signal.aborted && + !this.closed && + this.configSource().autoMemoryEnabled && + !this.retryTimers.has("*") + ) { + const next = this.outbox.next( + (batch) => batch.endpoint === endpoint && !this.blocked(batch), + ); + if (!next) return; + const { name, batch } = next; + const messages = batch.turns.flat(); + const task: ReMeRuntimeTask = { + id: `${batch.id}-${++this.taskSequence}`, + phase: "running", + queuedAt: new Date(batch.createdAt).toISOString(), + startedAt: new Date().toISOString(), + turns: batch.turns.length, + messages: messages.length, + }; + worker.active = task; + this.recentTasks.unshift(task); + this.recentTasks.length = Math.min(this.recentTasks.length, 20); + let retryKey = endpointKey(endpoint); + try { + const result = await this.client.autoMemory(messages, batch.sessionId, { + date: batch.date, + endpoint, + requestId: batch.id, + signal, + }); + if (signal.aborted) { + task.phase = "cancelled"; return; } - state.pendingTurns.unshift(...turns); - task.phase = "failed"; - task.error = - result.error || "ReMe rejected the automatic-memory request"; - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: result.error, - }); + if (!result.ok) { + // Payload/job errors affect this session; network, auth and service errors affect the endpoint. + if (requestFailure(result.status)) retryKey = sessionKey(batch); + throw new Error( + `ReMe automatic-memory request failed (HTTP ${result.status ?? 0})`, + ); + } + this.outbox.acknowledge(name); + task.phase = "completed"; + // Reply text and file names can contain personal information. + this.failures.delete(endpointKey(endpoint)); + this.failures.delete(sessionKey(batch)); + if (!this.retryTimers.size) this.memoryError = undefined; } catch (error) { - state.pendingTurns.unshift(...turns); - task.phase = state.requestController.signal.aborted - ? "cancelled" - : "failed"; - task.error = error instanceof Error ? error.message : String(error); - this.log("warn", "auto_memory_failed", { - sessionId: state.sessionId, - error: error instanceof Error ? error.message : String(error), - }); + task.phase = signal.aborted ? "cancelled" : "failed"; + task.error = safeError(error); + this.recordMemoryError(error); + if (this.stopping || signal.aborted) return; + this.scheduleRetry(retryKey); } finally { task.finishedAt = new Date().toISOString(); - state.unconfirmedTurns -= turns.length; + worker.active = undefined; } - }); + } + } + + private recordMemoryError(error: unknown): void { + this.memoryError = safeError(error); + this.log("warn", "auto_memory_failed", { error: this.memoryError }); + } + + private scheduleRetry(key = "*"): void { + if ( + this.stopping || + this.retryTimers.has(key) || + !this.configSource().autoMemoryEnabled + ) + return; + const config = this.configSource(); + const failures = this.failures.get(key) ?? 0; + this.failures.set(key, failures + 1); + const delay = Math.min( + config.autoMemoryRetryMaxMs, + config.autoMemoryRetryMs * 2 ** Math.min(failures, 20), + ); + const timer = setTimeout(() => { + this.retryTimers.delete(key); + void this.drain(); + }, delay); + timer.unref?.(); + this.retryTimers.set(key, timer); + } + + private clearRetries(): void { + for (const timer of this.retryTimers.values()) clearTimeout(timer); + this.retryTimers.clear(); + this.failures.clear(); } start(): void { + if (this.started || this.stopping) return; this.started = true; - if (!this.configSource().autoDreamEnabled || this.stopping) return; - this.scheduleDream(this.dreamScheduleGeneration); + // A previous Host may have stopped before the batching interval was reached. + try { + this.outbox.flush(); + } catch (error) { + this.outbox.close(); + throw error; + } + void this.drain(); + if (this.configSource().autoDreamEnabled) + this.scheduleDream(this.dreamScheduleGeneration); } - /** Apply a changed settings snapshot to pending batching and dream scheduling. */ + /** Apply settings without discarding captured work or replacing the runtime. */ reconfigure(): void { if (this.stopping) return; - const config = this.configSource(); - if (config.autoMemoryEnabled) { - for (const state of this.states.values()) this.scheduleAutoMemory(state); + this.clearRetries(); + if (!this.configSource().autoMemoryEnabled) this.states.clear(); + if (this.configSource().autoMemoryEnabled) { + this.outbox.flush(); + void this.drain(); } if (!this.started) return; this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); this.dreamTimer = null; this.nextDreamAt = undefined; - if (config.autoDreamEnabled) + if (this.configSource().autoDreamEnabled) this.scheduleDream(this.dreamScheduleGeneration); } @@ -206,7 +292,7 @@ export class ReMeRuntime { Date.now(); } catch (error) { this.log("warn", "auto_dream_schedule_invalid", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); return; } @@ -220,6 +306,7 @@ export class ReMeRuntime { } async runDream(): Promise { + if (this.stopping) return; if (this.dreamTask) return this.dreamTask; const config = this.configSource(); this.dreamController = new AbortController(); @@ -237,21 +324,20 @@ export class ReMeRuntime { result.ok ? "debug" : "warn", result.ok ? "auto_dream_complete" : "auto_dream_failed", { - error: result.ok ? undefined : result.error, + error: result.ok ? undefined : safeError(result.error), }, ); this.dreamLastResult = result.ok ? "completed" : "failed"; this.dreamLastError = result.ok ? undefined - : result.error || "ReMe rejected the Auto Dream request"; + : safeError(result.error || "ReMe rejected the Auto Dream request"); } catch (error) { this.dreamLastResult = this.dreamController?.signal.aborted ? "cancelled" : "failed"; - this.dreamLastError = - error instanceof Error ? error.message : String(error); + this.dreamLastError = safeError(error); this.log("warn", "auto_dream_failed", { - error: error instanceof Error ? error.message : String(error), + error: safeError(error), }); } })().finally(() => { @@ -263,84 +349,66 @@ export class ReMeRuntime { } async dispose(session: DshSession): Promise { - const state = this.states.get(session.id); - if (!state) return; - if (state.requestController.signal.aborted) - state.requestController = new AbortController(); - const flush = (async () => { - await state.writes; - const retryCount = state.pendingTurns.length; - let scheduled = 0; - while (state.pendingTurns.length && scheduled < retryCount) { - const before = state.pendingTurns.length; - this.scheduleAutoMemory(state, true); - scheduled += before - state.pendingTurns.length; - } - await state.writes; - })(); - const completed = await this.withinShutdownBudget(flush, () => - state.requestController.abort(), - ); - const unsentTurns = state.pendingTurns.length + state.unconfirmedTurns; - if (unsentTurns) { - this.log( - "warn", - completed ? "auto_memory_retained" : "auto_memory_shutdown_timeout", - { - sessionId: state.sessionId, - unsentTurns, - }, - ); - } else { - this.states.delete(session.id); - } + this.states.delete(session.id); + this.outbox.flush(remeSessionId(session.id)); + void this.drain(); } async disposeAll(): Promise { + if (this.stopping) return; this.stopping = true; this.started = false; + this.states.clear(); this.dreamScheduleGeneration += 1; if (this.dreamTimer) clearTimeout(this.dreamTimer); + this.clearRetries(); this.dreamTimer = null; this.nextDreamAt = undefined; this.dreamController?.abort(); - const shutdown = Promise.all([ - ...[...this.states.values()].map((state) => this.dispose(state.session)), - ...(this.dreamTask ? [this.dreamTask] : []), - ]).then(() => undefined); - await this.withinShutdownBudget(shutdown, () => { - this.dreamController?.abort(); - for (const state of this.states.values()) state.requestController.abort(); - }); + try { + this.outbox.flush(); + await this.withinShutdownBudget( + Promise.all([this.drain(), this.dreamTask]).then(() => undefined), + () => { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.dreamController?.abort(); + }, + ); + } finally { + for (const worker of this.memoryWorkers.values()) + worker.controller.abort(); + this.outbox.close(); + this.closed = true; + } } - /** Return a content-free snapshot for the local DSH status page. */ + /** Return queue counts and bounded diagnostics without reply content. */ snapshot(): ReMeRuntimeSnapshot { const config = this.configSource(); - const tasksRunning = this.recentTasks.filter( - (task) => task.phase === "running", + const queued = this.outbox.stats(); + const running = [...this.memoryWorkers.values()].filter( + (worker) => worker.active !== undefined, ).length; - const tasksQueued = this.recentTasks.filter( - (task) => task.phase === "queued", - ).length; - const lastError = this.recentTasks.find( - (task) => task.phase === "failed" || task.phase === "cancelled", - )?.error; return { - phase: this.stopping ? "stopping" : this.started ? "running" : "stopped", + phase: this.closed + ? "stopped" + : this.stopping + ? "stopping" + : this.started + ? "running" + : "stopped", autoMemory: { enabled: config.autoMemoryEnabled, interval: config.autoMemoryInterval, activeSessions: this.states.size, - queuedTurns: [...this.states.values()].reduce( - (total, state) => - total + state.pendingTurns.length + state.unconfirmedTurns, - 0, - ), - tasksRunning, - tasksQueued, + queuedTurns: queued.turns, + tasksRunning: running, + tasksQueued: Math.max(0, queued.batches - running), recentTasks: this.recentTasks.map((task) => ({ ...task })), - ...(lastError ? { lastError } : {}), + ...(this.captureError || this.memoryError + ? { lastError: this.captureError || this.memoryError } + : {}), }, autoDream: { enabled: config.autoDreamEnabled, @@ -363,16 +431,18 @@ export class ReMeRuntime { private async withinShutdownBudget( task: Promise, abort: () => void, - ): Promise { + ): Promise { let timer: ReturnType | undefined; - const timeout = new Promise((resolve) => { - timer = setTimeout(() => { - abort(); - resolve(false); - }, this.configSource().shutdownTimeoutMs); - }); try { - return await Promise.race([task.then(() => true), timeout]); + await Promise.race([ + task, + new Promise((resolve) => { + timer = setTimeout(() => { + abort(); + resolve(); + }, this.configSource().shutdownTimeoutMs); + }), + ]); } finally { if (timer) clearTimeout(timer); } @@ -388,18 +458,23 @@ export class ReMeRuntime { } } -function resultSummary( - answer: unknown, - metadata: Record | undefined, -): string | undefined { - const path = metadata?.path; - if (typeof path === "string" && path.length > 0) return path; - if (typeof answer !== "string") return undefined; - const normalized = answer.trim(); - if (!normalized) return undefined; - return normalized.length > 240 ? `${normalized.slice(0, 237)}…` : normalized; -} - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } + +function endpointKey(endpoint: string): string { + return `endpoint:${endpoint}`; +} + +function sessionKey(batch: MemoryBatch): string { + return `session:${JSON.stringify([batch.endpoint, batch.sessionId])}`; +} + +function requestFailure(status = 0): boolean { + return ( + (status >= 200 && status < 300) || + (status >= 400 && + status < 500 && + ![401, 403, 404, 405, 408, 429].includes(status)) + ); +} diff --git a/integrations/dsh/src/types.ts b/integrations/dsh/src/types.ts index ea38c35a..13e870e4 100644 --- a/integrations/dsh/src/types.ts +++ b/integrations/dsh/src/types.ts @@ -7,6 +7,9 @@ export interface ReMeConfigInput { shutdownTimeoutMs?: number; autoMemoryEnabled?: boolean; autoMemoryInterval?: number; + autoMemoryQueueDir?: string; + autoMemoryRetryMs?: number; + autoMemoryRetryMaxMs?: number; autoDreamEnabled?: boolean; dreamCron?: string; dreamHint?: string; @@ -21,6 +24,9 @@ export interface ReMeConfig extends ReMeClientConfig { shutdownTimeoutMs: number; autoMemoryEnabled: boolean; autoMemoryInterval: number; + autoMemoryQueueDir: string; + autoMemoryRetryMs: number; + autoMemoryRetryMaxMs: number; autoDreamEnabled: boolean; dreamCron: string; dreamHint: string; @@ -32,7 +38,10 @@ export interface ReMeConfig extends ReMeClientConfig { } /** ReMe integration fields owned by the DSH user-settings document. */ -export type ReMeSettings = Omit; +export type ReMeSettings = Omit< + ReMeConfig, + "dreamIntervalMs" | "autoMemoryQueueDir" +>; export interface SessionEvent { type: string; diff --git a/integrations/dsh/tests/bundle.test.mjs b/integrations/dsh/tests/bundle.test.mjs index c7b3f2a9..39bd23ec 100644 --- a/integrations/dsh/tests/bundle.test.mjs +++ b/integrations/dsh/tests/bundle.test.mjs @@ -30,6 +30,7 @@ test("declares one installable DeepSeek Harness plugin", async () => { "0.2.0-rc.2", "0.2.1-alpha.2", "0.2.1-alpha.4", + "0.2.1-rc.2", ]) { assert.ok( semver.satisfies(version, manifest.peerDependencies[dependency]), diff --git a/integrations/dsh/tests/client-settings.test.mjs b/integrations/dsh/tests/client-settings.test.mjs index 93db8c85..d8406ec5 100644 --- a/integrations/dsh/tests/client-settings.test.mjs +++ b/integrations/dsh/tests/client-settings.test.mjs @@ -168,7 +168,7 @@ test("resets overrides in one mutation only after Save", async () => { await ui.button("save").props.onClick(); }); assert.equal(ui.calls.length, 1); - assert.equal(ui.calls[0].ops.length, 13); + assert.equal(ui.calls[0].ops.length, 15); assert.ok(ui.calls[0].ops.every((op) => op.op === "unset")); assert.equal(ui.calls[0].revision, 7); } finally { @@ -191,3 +191,30 @@ test("read-only configuration cannot be saved", async () => { ui.close(); } }); + +for (const endpoint of [ + "http://alice:SECRET@localhost:2333", + "http://localhost:2333?token=SECRET", + "http://localhost:2333#search", + "http://localhost:2333?", + "http://localhost:2333#", +]) { + test( + "rejects unsupported endpoint components before saving: " + + new URL(endpoint).hash + + new URL(endpoint).search, + async () => { + const ui = await editor(async () => true); + try { + ui.change(0, endpoint); + assert.equal(ui.button("save").props.disabled, true); + await act(async () => { + await ui.button("save").props.onClick(); + }); + assert.equal(ui.calls.length, 0); + } finally { + ui.close(); + } + }, + ); +} diff --git a/integrations/dsh/tests/client.test.mjs b/integrations/dsh/tests/client.test.mjs index 40424ab0..21ed3d3c 100644 --- a/integrations/dsh/tests/client.test.mjs +++ b/integrations/dsh/tests/client.test.mjs @@ -1,6 +1,104 @@ import assert from "node:assert/strict"; import test from "node:test"; import { ReMeClient } from "../dist/reme/client.js"; +import { safeError } from "../dist/endpoint.js"; + +test("redacts URL credentials containing spaces or stripped URL whitespace", async () => { + const originalFetch = globalThis.fetch; + const client = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + try { + for (const password of [ + "SECRET CANARY", + "SECRET\tCANARY", + "SECRET\nCANARY", + ]) { + const url = `http://alice:${password}@localhost:2333/search`; + assert.ok(new URL(url).password); + const expected = "refused http://[redacted]@localhost:2333/search"; + globalThis.fetch = async () => + Response.json({ success: false, answer: `refused ${url}` }); + assert.equal((await client.search("fact")).error, expected); + globalThis.fetch = async () => { + throw new Error(`refused ${url}`); + }; + assert.equal((await client.search("fact")).error, expected); + } + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("rejects credentials before fetch and redacts URL credentials from service and transport errors", async () => { + const originalFetch = globalThis.fetch; + let calls = 0; + const secretUrl = "http://alice:SECRET-CANARY@localhost:2333/search"; + globalThis.fetch = async () => { + calls++; + return Response.json({ success: false, answer: `refused ${secretUrl}` }); + }; + try { + const client = new ReMeClient({ + endpoint: secretUrl, + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await client.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.equal(calls, 0); + const valid = new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + globalThis.fetch = async () => { + throw new Error(`failed ${secretUrl}`); + }; + assert.doesNotMatch( + (await valid.search("fact")).error, + /alice|SECRET-CANARY/, + ); + assert.doesNotMatch( + safeError(new Error(`failed ${secretUrl}`)), + /alice|SECRET-CANARY/, + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("uses the persisted destination and request ID when sending a recovered batch", async () => { + const originalFetch = globalThis.fetch; + let request; + globalThis.fetch = async (url, init) => { + request = { url, body: JSON.parse(init.body) }; + return Response.json({ success: true }); + }; + try { + const client = new ReMeClient({ + endpoint: "http://new.test", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }); + await client.autoMemory([], "session", { + endpoint: "http://original.test", + requestId: "persisted-batch", + }); + assert.equal(request.url, "http://original.test/auto_memory"); + assert.equal(request.body.request_id, "persisted-batch"); + } finally { + globalThis.fetch = originalFetch; + } +}); test("calls ReMe jobs with their native request and response envelopes", async () => { const calls = []; diff --git a/integrations/dsh/tests/config.test.mjs b/integrations/dsh/tests/config.test.mjs index 8e2613eb..ce03debd 100644 --- a/integrations/dsh/tests/config.test.mjs +++ b/integrations/dsh/tests/config.test.mjs @@ -29,6 +29,11 @@ test("exports a Cordis schema that rejects invalid configuration", async () => { for (const input of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:bad-port" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://[:::]" }, @@ -142,6 +147,11 @@ test("Loader keeps the previous live values when an update fails schema validati await entry.fiber.await(); for (const patch of [ { endpoint: "not-a-url" }, + { endpoint: "http://localhost?" }, + { endpoint: "http://localhost#" }, + { endpoint: "http://alice:SECRET@localhost" }, + { endpoint: "http://localhost?token=SECRET" }, + { endpoint: "http://localhost#search" }, { endpoint: "http://localhost:65536" }, { endpoint: "http://999.0.0.1" }, { dreamCron: "every night" }, diff --git a/integrations/dsh/tests/dsh-index.test.mjs b/integrations/dsh/tests/dsh-index.test.mjs index 7a346710..f19b3476 100644 --- a/integrations/dsh/tests/dsh-index.test.mjs +++ b/integrations/dsh/tests/dsh-index.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { apply, Config } from "../dist/index.js"; import { hasGuidance } from "../dist/guidance.js"; @@ -28,7 +31,7 @@ test("recognizes guidance already stored by the previous DSH plugin", () => { ); }); -test("composes root-agent guidance and reme_search on supported DSH releases", async () => { +test("composes root-agent guidance and reme_search on supported DSH releases", async (t) => { const handlers = new Map(); const tools = []; const cleanups = []; @@ -58,7 +61,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a handlers.set(name, handler); }, }; - apply(ctx, { + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false, language: "zh", @@ -97,7 +100,7 @@ test("composes root-agent guidance and reme_search on supported DSH releases", a await Promise.all(cleanups.map((cleanup) => cleanup())); }); -test("keeps prompt injection and capture out of subagents by default", async () => { +test("keeps prompt injection and capture out of subagents by default", async (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -118,7 +121,7 @@ test("keeps prompt injection and capture out of subagents by default", async () handlers.set(name, handler); }, }; - apply(ctx, { autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoDreamEnabled: false }); let injected = false; handlers.get("agent/created")({ agent: { @@ -138,7 +141,7 @@ test("keeps prompt injection and capture out of subagents by default", async () assert.equal(injected, false); }); -test("reads live DSH configuration for new sessions", async () => { +test("reads live DSH configuration for new sessions", async (t) => { const handlers = new Map(); const ctx = { fiber: { state: 0 }, @@ -165,7 +168,7 @@ test("reads live DSH configuration for new sessions", async () => { language: "zh", }); assert.equal(config.issues, undefined); - apply(ctx, config.value); + applyPlugin(t, ctx, config.value); const injected = []; handlers.get("agent/created")({ agent: { @@ -204,7 +207,7 @@ test("reads live DSH configuration for new sessions", async () => { assert.match(injected[1].content[0].text, /Long-term Memory/); }); -test("session events read a validated configuration snapshot", () => { +test("session events read a validated configuration snapshot", (t) => { const handlers = new Map(); const ctx = { logger: { debug() {}, warn() {}, log() {} }, @@ -224,7 +227,7 @@ test("session events read a validated configuration snapshot", () => { handlers.set(name, handler); }, }; - apply(ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); + applyPlugin(t, ctx, { autoMemoryEnabled: false, autoDreamEnabled: false }); const original = Intl.DateTimeFormat.prototype.formatToParts; let scans = 0; Intl.DateTimeFormat.prototype.formatToParts = function (...args) { @@ -243,3 +246,18 @@ test("session events read a validated configuration snapshot", () => { Intl.DateTimeFormat.prototype.formatToParts = original; } }); + +function applyPlugin(t, ctx, input) { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-plugin-")); + let runtime; + const provide = ctx.provide; + ctx.provide = function (name, value) { + if (name === "remeMemory") runtime = value; + return provide.call(this, name, value); + }; + t.after(async () => { + await runtime?.disposeAll(); + rmSync(directory, { recursive: true, force: true }); + }); + return apply(ctx, { ...input, autoMemoryQueueDir: directory }); +} diff --git a/integrations/dsh/tests/dsh-tools.test.mjs b/integrations/dsh/tests/dsh-tools.test.mjs index a21dd4a4..6c61a706 100644 --- a/integrations/dsh/tests/dsh-tools.test.mjs +++ b/integrations/dsh/tests/dsh-tools.test.mjs @@ -1,6 +1,45 @@ import assert from "node:assert/strict"; import test from "node:test"; import { registerReMeTools } from "../dist/tools.js"; +import { ReMeClient } from "../dist/reme/client.js"; + +test("reme_search does not expose whitespace-bearing URL credentials from service errors", async () => { + const originalFetch = globalThis.fetch; + const registered = []; + globalThis.fetch = async () => + Response.json({ + success: false, + answer: "refused http://alice:SECRET CANARY@localhost:2333/search", + }); + try { + registerReMeTools( + { + tools: { + register(tool) { + registered.push(tool); + return () => {}; + }, + }, + }, + new ReMeClient({ + endpoint: "http://localhost:2333", + requestTimeoutMs: 1000, + backgroundTimeoutMs: 1000, + }), + { searchLimit: 5 }, + ); + const result = await registered[0].execute( + { query: "history" }, + { signal: new AbortController().signal }, + ); + assert.equal( + result, + "ReMe search failed: refused http://[redacted]@localhost:2333/search", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); test("reme_search uses the ReMe search contract and renders model-facing text", async () => { const registered = []; diff --git a/integrations/dsh/tests/outbox.test.mjs b/integrations/dsh/tests/outbox.test.mjs new file mode 100644 index 00000000..9755c123 --- /dev/null +++ b/integrations/dsh/tests/outbox.test.mjs @@ -0,0 +1,236 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { MemoryOutbox } from "../dist/outbox.js"; + +function directory(t) { + const path = mkdtempSync(join(tmpdir(), "reme-outbox-")); + t.after(() => rmSync(path, { recursive: true, force: true })); + return path; +} + +const messages = [ + { + id: "m1", + name: "user", + role: "user", + content: [{ type: "text", text: "private fact" }], + }, +]; + +test("preserves batch order across equal timestamps, clock rollback, and restart", (t) => { + const now = Date.parse("2026-10-21T12:00:00Z"); + t.mock.timers.enable({ apis: ["Date"], now }); + const path = directory(t); + let queue = new MemoryOutbox(path); + const append = (id) => + queue.append( + "http://localhost:2333", + "session", + "2026-10-21", + [{ ...messages[0], id }], + 1, + ); + try { + append("first"); + append("second"); + const batches = readdirSync(path) + .filter((name) => name.startsWith("ready-")) + .map((name) => JSON.parse(readFileSync(join(path, name), "utf8"))); + assert.equal(new Set(batches.map((batch) => batch.createdAt)).size, 2); + queue.close(); + t.mock.timers.setTime(now - 1000); + queue = new MemoryOutbox(path); + append("third"); + const received = []; + for (let next = queue.next(); next; next = queue.next()) { + received.push(next.batch.turns[0][0].id); + queue.acknowledge(next.name); + } + assert.deepEqual(received, ["first", "second", "third"]); + } finally { + queue.close(); + } +}); + +test("recovers a partial batch after the owning process exits without cleanup", (t) => { + const path = directory(t); + const child = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { MemoryOutbox } from ${JSON.stringify( + new URL("../dist/outbox.js", import.meta.url).href, + )}; + const queue = new MemoryOutbox(${JSON.stringify(path)}); + queue.append("http://localhost:2333", "session", "2026-10-21", ${JSON.stringify( + messages, + )}, 5); + `, + ], + { encoding: "utf8" }, + ); + assert.equal(child.status, 0, child.stderr); + const recovered = new MemoryOutbox(path); + try { + recovered.flush(); + assert.deepEqual(recovered.stats(), { turns: 1, batches: 1 }); + const { name, batch } = recovered.next(); + assert.deepEqual(batch.turns, [messages]); + if (process.platform !== "win32") + assert.equal(statSync(join(path, name)).mode & 0o777, 0o600); + recovered.acknowledge(name); + assert.deepEqual(recovered.stats(), { turns: 0, batches: 0 }); + } finally { + recovered.close(); + } +}); + +test("refuses another owner without changing the queue", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + assert.throws(() => new MemoryOutbox(path), /in use/); + assert.equal(queue.stats().turns, 1); + } finally { + queue.close(); + } +}); + +test("serializes concurrent stale-lock recovery without deleting a replacement owner", async (t) => { + const path = directory(t); + const moduleUrl = new URL("../dist/outbox.js", import.meta.url).href; + const seed = spawnSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + new MemoryOutbox(${JSON.stringify(path)});`, + ], + { encoding: "utf8" }, + ); + assert.equal(seed.status, 0, seed.stderr); + const staleOwner = readFileSync(join(path, ".lock"), "utf8"); + const child = spawn( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import { existsSync, writeFileSync } from "node:fs"; + import { join } from "node:path"; + import { MemoryOutbox } from ${JSON.stringify(moduleUrl)}; + const path = ${JSON.stringify(path)}; + const kill = process.kill.bind(process); + process.kill = (pid, signal) => { + writeFileSync(join(path, "observed-stale"), "ready"); + const deadline = Date.now() + 5000; + while (!existsSync(join(path, "release"))) { + if (Date.now() > deadline) throw new Error("Recovery probe timed out"); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + return kill(pid, signal); + }; + const queue = new MemoryOutbox(path); + process.on("message", () => { queue.close(); process.exit(0); }); + writeFileSync(join(path, "acquired"), "ready"); + `, + ], + { stdio: ["ignore", "ignore", "pipe", "ipc"] }, + ); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr += data; + }); + const exited = new Promise((resolve) => child.once("exit", resolve)); + async function waitFor(name) { + const deadline = Date.now() + 5000; + while (!existsSync(join(path, name))) { + assert.equal(child.exitCode, null, stderr); + assert.ok( + Date.now() < deadline, + `Timed out waiting for ${name}: ${stderr}`, + ); + await delay(5); + } + } + try { + await waitFor("observed-stale"); + assert.throws(() => new MemoryOutbox(path), /lock recovery is busy/); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), staleOwner); + writeFileSync(join(path, "release"), "ready"); + await waitFor("acquired"); + assert.equal( + Number(readFileSync(join(path, ".lock"), "utf8").split(":")[0]), + child.pid, + ); + assert.throws(() => new MemoryOutbox(path), /in use/); + } finally { + writeFileSync(join(path, "release"), "ready"); + if (child.connected) child.send("close"); + const timer = setTimeout(() => child.kill(), 1000); + await exited; + clearTimeout(timer); + } +}); + +test("fails closed on an interrupted recovery guard without touching the owner or records", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const owner = readFileSync(join(path, ".lock"), "utf8"); + mkdirSync(join(path, ".lock-recovery")); + assert.throws( + () => new MemoryOutbox(path), + /lock recovery is busy or interrupted/, + ); + assert.equal(readFileSync(join(path, ".lock"), "utf8"), owner); + assert.equal(queue.stats().turns, 1); + assert.equal(existsSync(join(path, ".lock-recovery")), true); + } finally { + queue.close(); + } +}); + +test("preserves invalid records and keeps parser errors free of message content", (t) => { + const path = directory(t); + const queue = new MemoryOutbox(path); + try { + queue.append("http://localhost:2333", "session", "2026-10-21", messages, 1); + const name = readdirSync(path).find((file) => file.startsWith("ready-")); + const stored = JSON.parse(readFileSync(join(path, name), "utf8")); + stored.id = "../outside"; + writeFileSync(join(path, name), JSON.stringify(stored)); + assert.throws(() => queue.next(), /Invalid ReMe outbox record/); + writeFileSync(join(path, name), "PRIVATE-CONTENT"); + assert.throws( + () => queue.next(), + (error) => !error.message.includes("PRIVATE-CONTENT"), + ); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + queue.close(); + assert.throws(() => new MemoryOutbox(path), /Invalid ReMe outbox record/); + assert.equal(existsSync(join(path, ".lock")), false); + assert.equal(readFileSync(join(path, name), "utf8"), "PRIVATE-CONTENT"); + } finally { + queue.close(); + } +}); diff --git a/integrations/dsh/tests/runtime.test.mjs b/integrations/dsh/tests/runtime.test.mjs index 9de94c9b..7cb8136f 100644 --- a/integrations/dsh/tests/runtime.test.mjs +++ b/integrations/dsh/tests/runtime.test.mjs @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setImmediate as settle } from "node:timers/promises"; import { ReMeRuntime } from "../dist/runtime.js"; const CONFIG = { + endpoint: "http://127.0.0.1:2333", autoMemoryEnabled: true, autoMemoryInterval: 2, + autoMemoryRetryMs: 10, + autoMemoryRetryMaxMs: 40, autoDreamEnabled: false, shutdownTimeoutMs: 50, dreamIntervalMs: 0, @@ -13,215 +20,525 @@ const CONFIG = { timezone: "Asia/Shanghai", }; -test("submits completed turns to auto-memory in background batches", async () => { +function makeRuntime( + t, + client, + config = CONFIG, + logger = silentLogger(), + directory, +) { + const ownsDirectory = directory === undefined; + directory ??= mkdtempSync(join(tmpdir(), "reme-dsh-runtime-")); + const current = typeof config === "function" ? config : () => config; + const runtime = new ReMeRuntime( + client, + () => ({ ...current(), autoMemoryQueueDir: directory }), + logger, + ); + t.after(async () => { + await runtime.disposeAll(); + if (ownsDirectory) rmSync(directory, { recursive: true, force: true }); + }); + return runtime; +} + +test("persists partial turns and submits one batch at the interval", async (t) => { const calls = []; - const client = { + const runtime = makeRuntime(t, { async autoMemory(messages, sessionId) { calls.push({ messages, sessionId }); return { ok: true }; }, - }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + }); const session = { id: "session-one" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 0); - + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); completeTurn(runtime, session, 2, 20); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.equal(calls[0].messages.length, 4); assert.match(calls[0].sessionId, /^dsh-[a-f0-9]{24}$/); - const status = runtime.snapshot(); - assert.equal(status.autoMemory.queuedTurns, 0); - assert.equal(status.autoMemory.recentTasks[0].phase, "completed"); - assert.equal(status.autoMemory.recentTasks[0].turns, 2); - assert.equal(status.autoMemory.recentTasks[0].messages, 4); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); }); -test("requeues failed auto-memory batches and flushes them on disposal", async () => { +test("backs off across new turns and drains every batch after recovery", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; let attempts = 0; - const client = { + const runtime = makeRuntime(t, { async autoMemory() { - attempts += 1; - return { ok: attempts > 1, error: "offline" }; + attempts++; + return { ok: available, error: "offline" }; }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retry-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); - - await runtime.dispose(session); - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("retries an in-flight failed batch before disposal completes", async () => { - let attempts = 0; - let markStarted; - let releaseFirst; - const started = new Promise((resolve) => { - markStarted = resolve; }); - const firstRequest = new Promise((resolve) => { - releaseFirst = resolve; - }); - const client = { - async autoMemory() { - attempts += 1; - if (attempts === 1) { - markStarted(); - await firstRequest; - return { ok: false, error: "offline" }; - } - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "in-flight-retry-session" }; - completeTurn(runtime, session, 1, 10); - await started; - - const disposal = runtime.dispose(session); - releaseFirst(); - await disposal; - - assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), false); -}); - -test("splits auto-memory batches at workspace date boundaries", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: true }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 1); - assert.equal(calls[0].options.date, "2026-08-19"); - await runtime.dispose(session); - assert.equal(calls.length, 2); - assert.equal(calls[1].options.date, "2026-08-20"); -}); - -test("retries a failed partial prior-day batch after later activity", async () => { - const calls = []; - const client = { - async autoMemory(messages, _sessionId, options) { - calls.push({ messages, options }); - return { ok: calls.length > 1, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 5 }, - silentLogger(), - ); - const session = { id: "midnight-retry-session" }; - - completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); - completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); - await runtime.stateFor(session).writes; - assert.equal(calls.length, 1); - assert.equal(runtime.stateFor(session).pendingTurns.length, 2); - - for (let turn = 3; turn <= 7; turn += 1) { - completeTurn( - runtime, - session, - turn, - turn * 10, - Date.parse(`2026-08-20T00:0${turn}:00Z`), - ); + const session = { id: "outage" }; + for (let turn = 1; turn <= 200; turn++) { + completeTurn(runtime, session, turn, turn * 10); + await runtime.drain(); } - await runtime.stateFor(session).writes; - - assert.equal(calls.length, 3); - assert.equal(calls[1].options.date, "2026-08-19"); - assert.equal(calls[1].messages.length, 2); - assert.equal(calls[2].options.date, "2026-08-20"); - assert.equal(calls[2].messages.length, 10); - assert.equal(runtime.stateFor(session).pendingTurns.length, 1); -}); - -test("bounds session disposal and aborts an unresponsive write", async () => { - let observedSignal; - const client = { - async autoMemory(_messages, _sessionId, options) { - observedSignal = options.signal; - return new Promise(() => {}); - }, - }; - const runtime = new ReMeRuntime( - client, - { - ...CONFIG, - autoMemoryInterval: 1, - shutdownTimeoutMs: 20, - }, - silentLogger(), - ); - const session = { id: "stuck-session" }; - completeTurn(runtime, session, 1, 10); - - const started = Date.now(); - await runtime.dispose(session); - - assert.equal(observedSignal.aborted, true); - assert.ok(Date.now() - started < 500); - assert.equal(runtime.states.has(session.id), true); -}); - -test("retains a failed final batch for a later plugin-shutdown retry", async () => { - let attempts = 0; - const client = { - async autoMemory() { - attempts += 1; - return { ok: attempts > 2, error: "offline" }; - }, - }; - const runtime = new ReMeRuntime( - client, - { ...CONFIG, autoMemoryInterval: 1 }, - silentLogger(), - ); - const session = { id: "retained-session" }; - completeTurn(runtime, session, 1, 10); - await runtime.stateFor(session).writes; - - await runtime.dispose(session); + assert.equal(attempts, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 200); + assert.equal(runtime.states.get(session.id).length, 0); + t.mock.timers.tick(10); + await runtime.drain(); assert.equal(attempts, 2); - assert.equal(runtime.states.has(session.id), true); - - await runtime.disposeAll(); - assert.equal(attempts, 3); - assert.equal(runtime.states.has(session.id), false); + available = true; + t.mock.timers.tick(19); + await settle(); + assert.equal(attempts, 2); + t.mock.timers.tick(1); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + assert.equal(attempts, 102); }); -test("runs only one auto-dream task at a time", async () => { +for (const status of [413, 200]) { + test(`isolates a rejected batch (HTTP ${status}) while preserving its session's order`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout", "Date"] }); + let accepted = false; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _session, options) { + const text = messages[0].content[0].text; + calls.push({ text, requestId: options.requestId }); + return { ok: accepted || text !== "question 101", status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "rejected" }, 101, 10); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "rejected" }, 102, 20); + t.mock.timers.tick(1); + completeTurn(runtime, { id: "healthy" }, 201, 30); + await runtime.drain(); + assert.deepEqual( + calls.map((call) => call.text), + ["question 101", "question 201"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + + t.mock.timers.tick(10); + await runtime.drain(); + assert.deepEqual(calls[2], calls[0]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + accepted = true; + t.mock.timers.tick(19); + await runtime.drain(); + assert.equal(calls.length, 3); + t.mock.timers.tick(1); + await runtime.drain(); + assert.deepEqual( + calls.slice(3).map((call) => call.text), + ["question 101", "question 102"], + ); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +for (const status of [429, 503]) { + test(`backs off the whole endpoint for HTTP ${status}`, async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: available, status }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + completeTurn(runtime, { id: "second" }, 1, 10); + await runtime.drain(); + assert.equal(calls, 1); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls, 3); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); + }); +} + +test("retries ended sessions without another turn and releases session state", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let available = false; + const runtime = makeRuntime(t, { + async autoMemory() { + return { ok: available, error: "offline" }; + }, + }); + const session = { id: "ended" }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(runtime.states.has(session.id), false); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + available = true; + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("recovers queued and partial batches after restarting the Host", async (t) => { + const directory = mkdtempSync(join(tmpdir(), "reme-dsh-restart-")); + const original = makeRuntime( + t, + { + async autoMemory() { + return { ok: false, error: "offline" }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + completeTurn(original, { id: "restart" }, 1, 10); + await original.disposeAll(); + const calls = []; + const recovered = makeRuntime( + t, + { + async autoMemory(messages) { + calls.push(messages); + return { ok: true }; + }, + }, + CONFIG, + silentLogger(), + directory, + ); + recovered.start(); + await recovered.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].length, 2); + assert.equal(recovered.snapshot().autoMemory.queuedTurns, 0); + t.after(() => rmSync(directory, { recursive: true, force: true })); +}); + +test("keeps the same request ID and batch when retrying after a timeout", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + const calls = []; + const runtime = makeRuntime(t, { + async autoMemory(messages, sessionId, options) { + calls.push({ messages, sessionId, requestId: options.requestId }); + return { ok: calls.length > 1, error: "timeout" }; + }, + }); + const session = { id: "timeout" }; + completeTurn(runtime, session, 1, 10); + completeTurn(runtime, session, 2, 20); + await runtime.drain(); + await runtime.dispose(session); + t.mock.timers.tick(10); + await runtime.drain(); + assert.equal(calls.length, 2); + assert.deepEqual(calls[0], calls[1]); + const tasks = runtime.snapshot().autoMemory.recentTasks; + assert.equal(new Set(tasks.map((task) => task.id)).size, tasks.length); +}); + +test("keeps counts accurate beyond the twenty-entry diagnostic history", async (t) => { + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const runtime = makeRuntime( + t, + { + async autoMemory() { + await gate; + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + const session = { id: "many-batches" }; + for (let n = 1; n <= 25; n++) completeTurn(runtime, session, n, n * 10); + await settle(); + const status = runtime.snapshot().autoMemory; + assert.equal(status.tasksRunning, 1); + assert.equal(status.tasksQueued, 24); + assert.equal(status.queuedTurns, 25); + release(); + await runtime.drain(); + assert.equal(runtime.snapshot().autoMemory.recentTasks.length, 20); +}); + +test("does not put service reply text or file names into diagnostic snapshots", async (t) => { + const runtime = makeRuntime( + t, + { + async autoMemory() { + return { + ok: true, + answer: "PRIVATE-ANSWER", + metadata: { path: "PRIVATE-PATH" }, + }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "privacy" }, 1, 10); + await runtime.drain(); + assert.doesNotMatch(JSON.stringify(runtime.snapshot()), /PRIVATE/); +}); + +test("keeps whitespace-bearing URL credentials out of snapshots and logs", async (t) => { + const logs = []; + const error = "refused http://alice:SECRET CANARY\n@localhost:2333"; + const runtime = makeRuntime( + t, + { + async autoMemory() { + throw new Error(error); + }, + async autoDream() { + return { ok: false, error }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + { + debug() {}, + warn(...args) { + logs.push(args); + }, + }, + ); + completeTurn(runtime, { id: "credentials" }, 1, 10); + await runtime.drain(); + await runtime.runDream(); + const snapshot = runtime.snapshot(); + assert.equal(snapshot.autoMemory.recentTasks[0].phase, "failed"); + assert.equal(snapshot.autoDream.lastResult, "failed"); + assert.equal(logs.length, 2); + assert.doesNotMatch( + JSON.stringify({ snapshot, logs }), + /alice|SECRET|CANARY/, + ); +}); + +test("splits dates and flushes a partial previous day without a new interval", async (t) => { + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(messages, _id, options) { + calls.push({ messages, date: options.date }); + return { ok: true }; + }, + }, + { ...CONFIG, autoMemoryInterval: 5 }, + ); + const session = { id: "midnight" }; + completeTurn(runtime, session, 1, 10, Date.parse("2026-08-19T15:59:00Z")); + completeTurn(runtime, session, 2, 20, Date.parse("2026-08-19T16:01:00Z")); + await runtime.drain(); + assert.equal(calls.length, 1); + assert.equal(calls[0].date, "2026-08-19"); + await runtime.dispose(session); + await runtime.drain(); + assert.equal(calls[1].date, "2026-08-20"); +}); + +test("aborts shutdown within its budget and leaves an unacknowledged batch on disk", async (t) => { + let signal; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _id, options) { + signal = options.signal; + return new Promise(() => {}); + }, + }, + { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }, + ); + completeTurn(runtime, { id: "stuck" }, 1, 10); + await settle(); + const started = Date.now(); + await runtime.disposeAll(); + assert.equal(signal.aborted, true); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.ok(Date.now() - started < 500); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("does not retry per ended session while its service is in backoff", async (t) => { + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: false }; + }, + }, + { ...CONFIG, autoMemoryInterval: 1 }, + ); + completeTurn(runtime, { id: "first" }, 1, 10); + await runtime.drain(); + for (let n = 0; n < 5; n++) { + const session = { id: "ended-" + n }; + completeTurn(runtime, session, 1, 10); + await runtime.dispose(session); + } + assert.equal(calls, 1); +}); + +test("pauses further sends while disabled and resumes persisted work when enabled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let calls = 0; + const runtime = makeRuntime( + t, + { + async autoMemory() { + calls++; + return { ok: true }; + }, + }, + () => config, + ); + runtime.capture({ id: "partial" }, { type: "turn/start" }); + runtime.capture( + { id: "partial" }, + { + type: "user/message", + seq: 1, + data: { content: "unfinished", source: { kind: "user" } }, + }, + ); + config = { ...config, autoMemoryEnabled: false }; + runtime.reconfigure(); + assert.equal(runtime.states.size, 0); + runtime.outbox.append( + config.endpoint, + "session", + "2026-10-21", + [{ id: "m", name: "user", role: "user", content: [] }], + 1, + ); + await runtime.drain(); + assert.equal(calls, 0); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + config = { ...config, autoMemoryEnabled: true }; + runtime.reconfigure(); + await runtime.drain(); + assert.equal(calls, 1); +}); + +test("a failed old destination does not block a newly configured service", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + const calls = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + calls.push(options.endpoint); + return { ok: options.endpoint === "http://new.test" }; + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await runtime.drain(); + config = { ...config, endpoint: "http://new.test" }; + completeTurn(runtime, { id: "new" }, 1, 10); + await runtime.drain(); + assert.deepEqual(calls, [CONFIG.endpoint, "http://new.test"]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); +}); + +test("drains a new destination while the old request remains stalled", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1 }; + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const calls = []; + const active = new Map(); + const maximum = new Map(); + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + const endpoint = options.endpoint; + calls.push(endpoint); + active.set(endpoint, (active.get(endpoint) ?? 0) + 1); + maximum.set( + endpoint, + Math.max(maximum.get(endpoint) ?? 0, active.get(endpoint)), + ); + await (endpoint === CONFIG.endpoint ? gate : settle()); + active.set(endpoint, active.get(endpoint) - 1); + return { ok: true }; + }, + }, + () => config, + ); + try { + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + completeTurn(runtime, { id: "new" }, 2, 20); + await settle(); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 2); + await settle(); + await settle(); + assert.deepEqual(calls, [ + CONFIG.endpoint, + "http://new.test", + "http://new.test", + ]); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 1); + assert.equal(runtime.snapshot().autoMemory.tasksRunning, 1); + assert.equal(runtime.snapshot().autoMemory.tasksQueued, 0); + assert.deepEqual([...maximum.values()], [1, 1]); + } finally { + release(); + await runtime.drain(); + } + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 0); +}); + +test("aborts all destination workers at the shutdown deadline without discarding batches", async (t) => { + let config = { ...CONFIG, autoMemoryInterval: 1, shutdownTimeoutMs: 20 }; + const signals = []; + const runtime = makeRuntime( + t, + { + async autoMemory(_messages, _session, options) { + signals.push(options.signal); + return new Promise(() => {}); + }, + }, + () => config, + ); + completeTurn(runtime, { id: "old" }, 1, 10); + await settle(); + config = { ...config, endpoint: "http://new.test" }; + runtime.reconfigure(); + completeTurn(runtime, { id: "new" }, 1, 10); + await settle(); + assert.equal(signals.length, 2); + await runtime.disposeAll(); + assert.ok(signals.every((signal) => signal.aborted)); + assert.equal(runtime.snapshot().phase, "stopped"); + assert.equal(runtime.snapshot().autoMemory.queuedTurns, 2); +}); + +test("runs only one auto-dream task at a time", async (t) => { let calls = 0; let release; const client = { @@ -233,7 +550,7 @@ test("runs only one auto-dream task at a time", async () => { return { ok: true }; }, }; - const runtime = new ReMeRuntime(client, CONFIG, silentLogger()); + const runtime = makeRuntime(t, client, CONFIG, silentLogger()); const first = runtime.runDream(); const second = runtime.runDream(); assert.equal(calls, 1); @@ -242,9 +559,10 @@ test("runs only one auto-dream task at a time", async () => { assert.equal(calls, 1); }); -test("contains unexpected auto-dream client failures", async () => { +test("contains unexpected auto-dream client failures", async (t) => { const warnings = []; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { throw new Error("broken transport"); @@ -266,10 +584,11 @@ test("contains unexpected auto-dream client failures", async () => { assert.match(runtime.snapshot().autoDream.lastError, /broken transport/); }); -test("applies changed batching and dream settings without replacing the runtime", async () => { +test("applies changed batching and dream settings without replacing the runtime", async (t) => { const calls = []; let config = { ...CONFIG, autoMemoryInterval: 5, autoDreamEnabled: false }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoMemory(messages) { calls.push(messages); @@ -294,7 +613,7 @@ test("applies changed batching and dream settings without replacing the runtime" dreamIntervalMs: 100000, }; runtime.reconfigure(); - await runtime.stateFor(session).writes; + await runtime.drain(); assert.equal(calls.length, 1); assert.notEqual(runtime.dreamTimer, null); assert.equal(runtime.snapshot().autoDream.enabled, true); @@ -302,7 +621,7 @@ test("applies changed batching and dream settings without replacing the runtime" await runtime.disposeAll(); }); -test("keeps one auto-dream schedule when reconfigured during a run", async () => { +test("keeps one auto-dream schedule when reconfigured during a run", async (t) => { let calls = 0; let releaseFirst; const firstRequest = new Promise((resolve) => { @@ -314,7 +633,8 @@ test("keeps one auto-dream schedule when reconfigured during a run", async () => dreamIntervalMs: 20, shutdownTimeoutMs: 100, }; - const runtime = new ReMeRuntime( + const runtime = makeRuntime( + t, { async autoDream() { calls += 1; diff --git a/plugins/memlens-judge/LICENSE-CODE b/plugins/memlens-judge/LICENSE-CODE new file mode 100644 index 00000000..a6bed67f --- /dev/null +++ b/plugins/memlens-judge/LICENSE-CODE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Xiyu Ren, Zhaowei Wang, and the MemLens Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/memlens-judge/README.md b/plugins/memlens-judge/README.md new file mode 100644 index 00000000..da0b4e0b --- /dev/null +++ b/plugins/memlens-judge/README.md @@ -0,0 +1,30 @@ +# ReMe MemLens Judge + +Independent `answer_judge` Job for saved MemLens predictions. Install with +`pip install -e plugins/memlens-judge` and enable the `memlens-judge` plugin +in a separate evaluation application/workspace, not the candidate memory application. + +Inputs are `question`, `reference`, `prediction`, and `category`, with optional +`subtype` and `old_answer`. The result is available as +`response.metadata.llm_judge_score` and `acc`. Metadata includes the raw response, +task rubric, native answer-preprocessing diagnostics and judge token usage. +Judge usage is evaluation cost, not method cost. + +The official category/subtype rubrics, outdated-answer handling, verdict parser, +long-answer preprocessing and over-500-word zero rule are preserved. A successful +but unparseable judge output receives the official zero score; API errors instead +remain failed jobs for resumable judging of the same saved prediction. + +The default judge is `qwen3.8-max`, with temperature 0 and thinking disabled. +Set `JUDGE_MODEL_NAME` to override the judge model independently of the candidate. +The benchmark's `settings.json` records the effective judge model, generation +parameters and protocol from the validated application configuration, after plugin +defaults and environment expansion. The snapshot excludes credentials and endpoints. +The output +budget is 256 tokens from the released OpenAI-compatible API judge branch +(`max_completion_tokens`), not the separate vLLM branch's 2,048-token setting. +These judge settings are independent of the candidate reader's thinking mode. + +Source: [official MemLens judge](https://github.com/xrenaf/MEMLENS/blob/77f3ab9a52fa2d6a17978e2dffe80438a4ecced2/llm_judge.py). +Verbatim prompt strings and their MIT notice are packaged in `_memlens_judge_prompts.json`; +see also `LICENSE-CODE`. The plugin is independent of candidate code and other benchmarks. diff --git a/plugins/memlens-judge/pyproject.toml b/plugins/memlens-judge/pyproject.toml new file mode 100644 index 00000000..8782eb5b --- /dev/null +++ b/plugins/memlens-judge/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "reme-memlens-judge" +version = "0.1.0" +description = "Official memlens judge protocol as a ReMe plugin." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11"] + +[project.entry-points."reme.plugins"] +memlens-judge = "judge_memlens" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["judge_memlens*"] + +[tool.setuptools.package-data] +judge_memlens = ["plugin.yaml", "*.json"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/memlens-judge/src/judge_memlens/__init__.py b/plugins/memlens-judge/src/judge_memlens/__init__.py new file mode 100644 index 00000000..39c8fd4a --- /dev/null +++ b/plugins/memlens-judge/src/judge_memlens/__init__.py @@ -0,0 +1,5 @@ +"""Independent memlens evaluation-only plugin.""" + +from .llm_judge import MemLensJudgeStep + +__all__ = ["MemLensJudgeStep"] diff --git a/plugins/memlens-judge/src/judge_memlens/_memlens_judge_prompts.json b/plugins/memlens-judge/src/judge_memlens/_memlens_judge_prompts.json new file mode 100644 index 00000000..bf34840b --- /dev/null +++ b/plugins/memlens-judge/src/judge_memlens/_memlens_judge_prompts.json @@ -0,0 +1,29 @@ +{ + "source_revision": "77f3ab9a52fa2d6a17978e2dffe80438a4ecced2", + "license": "MIT License\n\nCopyright (c) 2026 Xiyu Ren, Zhaowei Wang, and the MemLens Authors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n", + "template": "Now your role is a grading teacher. Your task is to review and score student answers based on reference standard answers for a question-answering benchmark. You need to notice the following key points:\n- First, extract the final answer from the student's solution, then analyze and judge whether the answer is correct.\n- Scoring should only refer to the final answer obtained by the student; there is no need to examine whether the intermediate problem-solving steps are correct.\n- If the response contains both hesitation and a clear answer, judge the answer itself.\n- If the response contains both a refusal and a guessed answer, judge the final committed answer.\n- If the response gives multiple inconsistent answers, assign 0 points.\n- If the student's response shows circular reasoning — repeatedly revisiting the same evidence, flip-flopping between answers (e.g., \"yes... wait no... actually yes... let me reconsider... no\"), with no clear final commitment — assign 0 points. A correct answer mentioned during reasoning does NOT count unless it is the clearly stated final conclusion.\n- Only the student's LAST clearly stated answer counts. If the student initially says \"yes\" but later concludes \"no\" (or vice versa), score based on the LAST position only. Intermediate answers during reasoning are not final answers.\n- If the student's output is entirely a reasoning trace (e.g., \"let me check... scanning evidence... re-reading session 5...\") with no clear concluding answer statement, assign 0 points — even if the correct answer appears somewhere within the reasoning.\n- If the student's output begins with \"[...truncated earlier reasoning...]\", this means only the final portion of a longer response is shown. Focus on extracting the answer from this final portion.\n\nBelow are examples of problematic student outputs that apply to ALL task types. Study these before grading.\n\n[Universal Example A — Circular Reasoning -> 0 points]\n: Is the cat in Mark and Jenny's 5-year anniversary post named Mittens?\n: Yes\n: let me check session 5 mark has cat named mittens session 7 mark and jenny anniversary post there is no link wait let me re-read session 5 says mittens session 7 says anniversary actually looking at this again wait is it possible let me reconsider mark has mittens in session 5 the post is about couple therefore answer is likely no wait let me double check session 5 mark has cat mittens actually maybe yes but text doesnt say\n[Scoring Rationale]: The output flip-flops between \"yes\" and \"no\" multiple times with no clear final commitment. The last position is ambiguous (\"maybe yes but text doesnt say\"). This is circular reasoning.\nIn summary, the student's answer deserves 0 points.\n[JSON]: {{\"answer_score\": 0}}\n\n[Universal Example B — Redundant But Committed -> Score Normally]\n: Is the trailing plant in the cafe safe for my kitten?\n: No\n: the plant looks like pothos pothos is toxic to cats therefore answer is no let me double check is pothos safe for cats no is answer no yes wait let me verify image 3 is pothos image 9 is pothos pothos is toxic therefore answer must be no\n[Scoring Rationale]: Despite excessive self-verification, the student consistently commits to \"no\" throughout and never wavers to a different answer. The final answer is clearly \"No\", which matches the standard answer.\nIn summary, the student's answer deserves 1 point.\n[JSON]: {{\"answer_score\": 1}}\n\n[Universal Example C — Reasoning Trace With No Answer -> 0 points]\n: How many sessions mentioned the coffee shop?\n: 3\n: scanning session 1 yes coffee mentioned session 2 no session 3 yes coffee again session 4 no session 5 maybe let me re-read session 5 it mentions cafe is that same as coffee shop need to check session 6 no session 7 unclear let me look at session 5 again\n[Scoring Rationale]: The student walks through sessions but never states a final count. The output is entirely a reasoning trace with no concluding answer.\nIn summary, the student's answer deserves 0 points.\n[JSON]: {{\"answer_score\": 0}}\n\nNow proceed with grading. Remember: the universal examples above apply regardless of task type.\n\n- When analyzing and judging whether the answer is correct, you need to write down the scoring rationale, organize it into clear statements that follow the logical flow. The summary of the scoring rationale should be placed at the end, using the following format: \"In summary, the student's answer deserves x points\" (where x represents the student's specific score).\n- Keep the whole process concise, within 150 words.\n- Provide the score based on your analysis and display it in a code block in \"JSON\" format.\n- An item is covered if it is strictly mentioned or unambiguously implied by a semantic equivalence. This includes numerical equivalence (e.g., 10% and 0.1), synonyms (e.g., UK and United Kingdom), plural/singular forms (e.g., \"apple\" and \"apples\"), and equivalent date formats (e.g., 2024-01-15 and January 15, 2024). However, do not accept loosely related concepts.\n- Ignore minor formatting differences, capitalization, punctuation, and equivalent wording when meaning is unchanged.\nYour output format is:\n[Scoring Rationale]:\n[Score]: x points\n[JSON]:\n{{\"answer_score\": }}\n\nBelow is the grading rubric:\n[Scores]:\nThe scoring scale consists of 2 levels in total, from highest to lowest: 1 point, 0 points (the minimum is 0 points).\n[Tier Details]:\n1 point: Assign 1 point if the student's final answer matches the standard answer under the task-specific criteria below.\n0 points: Assign 0 points if the student's final answer does not match the standard answer, the student refuses to answer, claims insufficient information (except for AR tasks), or does not clearly answer.\n\n[Task-Specific Criteria]:\n{task_criteria}\n\n{examples}\n", + "criteria": { + "IE_Entity": "[IE — Entity / Attribute Extraction]\nThis is an information extraction question about a concrete entity, count, attribute, text, object, location, or visual detail from the provided memory context.\n- Assign 1 point if the student's response contains the core information from the standard answer. Minor wording differences are acceptable, but the essential information must be present and correct.\n- Assign 0 points if the core information is missing, contradicted, too vague, refused, or incorrect.\n- For numeric or count answers, require the exact value unless the standard answer itself is approximate.", + "IE_PreviousInfo": "[IE — Previous Information Extraction]\nThis is an information extraction question about previously mentioned information, spatial relations, counts, or attributes from earlier conversation/image context.\n- Assign 1 point if the student's response recovers the same previous information as the standard answer, including the correct relation, location, count, or attribute.\n- Minor wording differences are acceptable if they preserve the same meaning.\n- Assign 0 points if the answer refers to the wrong previous item, gives the wrong relation/count/location, is too vague, refuses, or is unsupported.", + "KU_KnowledgeUpdate": "[KU — Knowledge Update]\nThis is a knowledge update question testing whether the model tracks the latest state.\nThe correct current answer and the outdated old answer are both provided.\n- Assign 1 point if the student gives the current (correct) answer.\n- Assign 0 points if the student gives the outdated old answer, any other incorrect answer, refuses, or does not clearly answer.", + "TR_DurationComparison": "[TR — Duration Comparison (A/B)]\nThis is a binary choice question. The student must select one of two options.\n- Assign 1 point if the student clearly selects the same option as the standard answer, regardless of phrasing.\n- Assign 0 points if the student selects the other option, refuses, or does not make a clear selection.", + "TR_OrderRanking": "[TR — Order Ranking]\nThis is an ordering question. The standard answer is an exact sequence of items.\n- Assign 1 point ONLY if the student gives the exact same sequence in the same order.\n- Assign 0 points if any item is missing, added, duplicated, misplaced, or if the order is unclear.", + "TR_DateExtraction": "[TR — Date Extraction]\nThis is a date extraction question. The standard answer is a specific date.\n- Assign 1 point if the student gives the same date in any unambiguous equivalent format (e.g., \"Jan 15, 2024\" and \"2024-01-15\" and \"15/01/2024\").\n- Assign 0 points if the date is different, incomplete when a full date is required, ambiguous, or missing.", + "MSR_YesNo": "[MSR — Yes/No (Entity Resolution)]\nThis is a yes/no question.\n- Assign 1 point if the student's final answer is semantically equivalent to the standard answer (e.g., \"Yes\" matches \"Yes\", \"No\" matches \"No\"), even if phrased differently (e.g., \"That's correct\" for \"Yes\").\n- Assign 0 points if the answer means the opposite, refuses, or does not clearly answer.", + "MSR_Counting": "[MSR — Counting]\nThis is a counting question. The standard answer is a specific number.\n- Assign 1 point if the student clearly states the exact number in any equivalent form (e.g., \"3\", \"three\", \"3.0\").\n- Assign 0 points if the student gives a different number, a range, an approximation, refuses, or does not clearly answer.", + "MSR_Arithmetic": "[MSR — Arithmetic]\nThis is an arithmetic question. The standard answer is a specific value.\n- Assign 1 point if the student gives the same value in an equivalent format (e.g., \"0.5\" and \"1/2\" and \"50%\").\n- Assign 0 points if the student gives a different value, an unsupported approximation, refuses, or does not clearly answer.", + "AR_AnswerRefusal": "[AR — Answer Refusal]\n- Assign 1 point if the student explicitly refuses, says there is not enough information, or indicates the question is unanswerable from the provided context.\n- Assign 0 points if the student gives a substantive answer, even if hedged." + }, + "examples": { + "IE_Entity": "\n[Example 1]\n: What specific word is painted in large white letters on the pavement in the middle lanes?\n: SCHOOL\n: SCHOOL\n\n[Scoring Rationale]: The student's answer is \"SCHOOL\", which exactly matches the standard answer.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: Where on the sign is the photo of the mountains located?\n: In the bottom right corner\n: The photo of the mountains is located in the upper right corner of the sign.\n\n[Scoring Rationale]: The student says \"upper right corner\" but the standard answer is \"bottom right corner\". The location is incorrect — \"upper\" vs \"bottom\" is a substantive difference.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "IE_PreviousInfo": "\n[Example 1]\n: Where on the sign is the photo of the mountains located?\n: In the bottom right corner\n: The photo is in the bottom-right corner of the sign.\n\n[Scoring Rationale]: The student's answer preserves the same location relation as the standard answer.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: Relative to the baking sheet, where is the salmon fillet placed?\n: The right side\n: It is on the left side of the baking sheet.\n\n[Scoring Rationale]: The student's answer gives the opposite spatial relation from the standard answer.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "KU_KnowledgeUpdate": "\n[Example 1]\n: What do I treat as my must-have now that I plan my meals around it?\n: brioche\n: sourdough\n: You now consider brioche your must-have bread that you plan meals around.\n\n[Scoring Rationale]: The student's answer is \"brioche\", which matches the current standard answer, not the outdated answer \"sourdough\".\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: What do I treat as my must-have now that I plan my meals around it?\n: brioche\n: sourdough\n: A versatile, high-quality protein source like chicken, beans, or tofu is a great starting point for building a week's worth of meals.\n\n[Scoring Rationale]: The student's answer mentions generic protein sources, which is neither the current answer \"brioche\" nor the outdated answer \"sourdough\". The answer is simply incorrect.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "TR_DurationComparison": "\n[Example 1]\n: Which of the following two durations is longer? Duration 1: The time I spent on Miller Hall. Duration 2: The time I spent on Music Festival Job. A. Duration 1 is longer. B. Duration 2 is longer.\n: B\n: B. Duration 2 is longer.\n\n[Scoring Rationale]: The student clearly selects \"B\", which matches the standard answer.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: Which of the following two durations is longer? Duration 1: The time I spent with the NutriPeak Nutrition. Duration 2: The time I spent with the BlendIt Pro Blender. A. Duration 1 is longer. B. Duration 2 is longer.\n: A\n: B. Duration 2 is longer.\n\n[Scoring Rationale]: The student selects \"B\", but the standard answer is \"A\". The selection is opposite.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "TR_OrderRanking": "\n[Example 1]\n: Rank the following cities by population from largest to smallest: Tokyo, Delhi, Shanghai.\n: Tokyo, Delhi, Shanghai\n: 1. Tokyo 2. Delhi 3. Shanghai\n\n[Scoring Rationale]: The student's sequence is Tokyo, Delhi, Shanghai, which exactly matches the standard answer.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: Rank events A, B, C in chronological order.\n: B, A, C\n: A, B, C\n\n[Scoring Rationale]: The student's sequence is A, B, C, but the standard answer is B, A, C. The order does not match.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "TR_DateExtraction": "\n[Example 1]\n: When did I complete the Scandinavian design course at Copenhagen Design Academy?\n: 2024/08/31\n: You completed the Scandinavian design course on August 31st, 2024.\n\n[Scoring Rationale]: The student's answer \"August 31st, 2024\" is an unambiguous equivalent of the standard answer \"2024/08/31\".\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: When did I visit Marutama Ra-men to taste their chicken broth?\n: 2023/01/20\n: January 20, 2021\n\n[Scoring Rationale]: The student's answer is \"January 20, 2021\", but the standard answer is \"2023/01/20\" — the year is wrong (2021 vs 2023).\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "MSR_YesNo": "\n[Example 1]\n: Is the cat in Mark and Jenny's 5-year anniversary post named Mittens?\n: Yes\n: Yes, the cat in Mark and Jenny's 5-year anniversary post is named Mittens.\n\n[Scoring Rationale]: The student's final answer is \"Yes\", which matches the standard answer \"Yes\". The additional explanation does not contradict the answer.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: Did the cat named after a crypto creator knock over the phone displaying the Binance alert?\n: Yes\n: The text does not explicitly state whether the cat knocked over the phone. It only mentions the cat was sitting next to the phone when it crashed.\n\n[Scoring Rationale]: The student does not give a clear \"Yes\". Instead, the student hedges and effectively refuses to commit to an answer. The standard answer is \"Yes\".\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "MSR_Counting": "", + "MSR_Arithmetic": "\n[Example 1]\n: How much total have I spent on coffee makers?\n: $260.00\n: Based on the conversation, you spent $110 on the first coffee maker and $150 on the second one, totaling $260.\n\n[Scoring Rationale]: The student's final answer is $260, which is numerically equivalent to the standard answer $260.00.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: How much total have I spent on coffee makers?\n: $260.00\n: I do not have access to your personal financial data, including your coffee maker purchases. I cannot provide you with a total amount spent on coffee makers.\n\n[Scoring Rationale]: The student refuses to answer and claims insufficient information, but this is not an AR task. The information was available in the conversation context.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n", + "AR_AnswerRefusal": "\n[Example 1]\n: What is the model number printed on the power bank?\n: Insufficient information\n: I am sorry, but I cannot fulfill your request. The provided conversation history does not contain any information about a power bank or its model number.\n\n[Scoring Rationale]: The student correctly refuses to answer, stating the information is not available in the conversation. This is the expected behavior for an unanswerable question.\nIn summary, the student's answer deserves 1 point.\n[Score]: 1 point\n[JSON]:\n{\n \"answer_score\": 1\n}\n\n[Example 2]\n: How many vertical cables are visible on one side of the bridge?\n: Insufficient information\n: Three\n\n[Scoring Rationale]: The student provides a substantive answer (\"Three\") rather than refusing. For an unanswerable question, the correct response is to refuse or indicate insufficient information.\nIn summary, the student's answer deserves 0 points.\n[Score]: 0 points\n[JSON]:\n{\n \"answer_score\": 0\n}\n" + } +} diff --git a/plugins/memlens-judge/src/judge_memlens/llm_judge.py b/plugins/memlens-judge/src/judge_memlens/llm_judge.py new file mode 100644 index 00000000..998e1685 --- /dev/null +++ b/plugins/memlens-judge/src/judge_memlens/llm_judge.py @@ -0,0 +1,72 @@ +"""Official MemLens task-specific judge as an isolated ReMe Job.""" + +from dataclasses import asdict + +from agentscope.message import UserMsg + +from reme.steps.base_step import BaseStep +from reme.steps.evolve.compressor import CompressorStep + +from .protocol import ( + SOURCE_REVISION, + judge_prompt, + normalize_for_judge, + parse_judge_response, + task_key, +) + + +class MemLensJudgeStep(BaseStep): + """Score saved answers without accessing the candidate's workspace or tools.""" + + async def execute(self): + """Apply the official judge prompt/parser to one saved prediction.""" + question = self.context["question"] + reference = self.context["reference"] + prediction = self.context["prediction"] + category = self.context["category"] + subtype = self.context.get("subtype", "") + normalized = normalize_for_judge(prediction) + prompt = judge_prompt( + question, + reference, + normalized["text"], + category, + subtype, + self.context.get("old_answer", ""), + ) + words = len(normalized["text"].split()) + result = { + "source_revision": SOURCE_REVISION, + "task_key": task_key(category, subtype, reference), + "original_len": normalized["original_len"], + "was_truncated": normalized["was_truncated"], + "is_degenerate": normalized["is_degenerate"], + **normalized["diagnostics"], + "parsed_words": words, + "auto_zero": words > 500, + } + raw, usage = "", None + if words > 500: + parsed = {"acc": 0.0, "parse_default_zero": False} + else: + response = await self.as_llm([UserMsg(name="user", content=prompt)]) + if hasattr(type(response), "__aiter__"): + last = None + async for chunk in response: + last = chunk + response = last + raw = await CompressorStep._response_text(response) # pylint: disable=protected-access + usage = asdict(response.usage) if response is not None and response.usage is not None else None + parsed = parse_judge_response(raw) + score = parsed["acc"] + self.context.response.success = True + self.context.response.answer = str(score) + self.context.response.metadata.update( + **result, + **parsed, + llm_judge_score=score, + raw_response=raw, + usage=usage, + ) + return self.context.response diff --git a/plugins/memlens-judge/src/judge_memlens/plugin.yaml b/plugins/memlens-judge/src/judge_memlens/plugin.yaml new file mode 100644 index 00000000..330566eb --- /dev/null +++ b/plugins/memlens-judge/src/judge_memlens/plugin.yaml @@ -0,0 +1,38 @@ +backends: + memlens_judge_step: judge_memlens.llm_judge:MemLensJudgeStep + +application_defaults: + jobs: + answer_judge: + backend: base + description: "memlens official judge protocol for a saved prediction" + parameters: + type: object + properties: + question: {type: string} + reference: {type: string} + prediction: {type: string} + category: {type: string, default: ""} + subtype: {type: string, default: ""} + old_answer: {type: string, default: ""} + required: [question, reference, prediction, category] + steps: + - backend: memlens_judge_step + as_llm: judge + components: + as_llm: + judge: + backend: openai + model: ${JUDGE_MODEL_NAME:-qwen3.8-max} + stream: false + max_retries: 3 + retry_delay: 60 + client_kwargs: {timeout: 600, max_retries: 0} + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + extra_body: {enable_thinking: false} + parameters: + temperature: 0 + thinking_enable: false + max_tokens: 256 diff --git a/plugins/memlens-judge/src/judge_memlens/protocol.py b/plugins/memlens-judge/src/judge_memlens/protocol.py new file mode 100644 index 00000000..19ef3039 --- /dev/null +++ b/plugins/memlens-judge/src/judge_memlens/protocol.py @@ -0,0 +1,139 @@ +"""MemLens's task-specific binary judge for fixed saved predictions. + +Prompts, answer preprocessing and verdict parsing follow SOURCE_REVISION. +Copyright (c) 2026 Xiyu Ren, Zhaowei Wang, and the MemLens Authors. +The complete MIT notice accompanies the verbatim _memlens_judge_prompts.json. +Only exhausted API errors differ: they remain coverage failures, not wrong answers. +""" + +from functools import lru_cache +import json +from pathlib import Path +import re + +SOURCE_REVISION = "77f3ab9a52fa2d6a17978e2dffe80438a4ecced2" + +PROTOCOL_ID = "memlens-task-binary-api-qwen-v1" +_HEDGING_MARKERS = ( + "wait", + "let me re-read", + "let me re read", + "let me double check", + "let me double-check", + "let me reconsider", + "let me look again", + "let's reconsider", + "let s reconsider", + "re-evaluating", + "re evaluating", + "actually looking", + "actually let me", + "hold on", + "let me look at", + "let me check", + "let's re-read", + "let s re-read", + "i need to reconsider", + "wait let me", +) + + +@lru_cache(maxsize=1) +def _prompts() -> dict: + """Load the checked-in, verbatim official prompt strings, never a network resource.""" + return json.loads(Path(__file__).with_name("_memlens_judge_prompts.json").read_text(encoding="utf-8")) + + +def task_key(category: str, subtype: str, reference: str) -> str: + """Official question-type/subtype routing, including reference-based MSR choices.""" + if category == "multi_session_reasoning": + if subtype == "arithmetic": + return "MSR_Arithmetic" + if subtype == "entity_resolution" and reference.strip() in {"Yes", "No"}: + return "MSR_YesNo" + return "MSR_Counting" + if category == "temporal_reasoning": + return { + "duration_comparison": "TR_DurationComparison", + "order_ranking": "TR_OrderRanking", + }.get(subtype, "TR_DateExtraction") + fixed = {"knowledge_update": "KU_KnowledgeUpdate", "answer_refusal": "AR_AnswerRefusal"} + if category in fixed: + return fixed[category] + if category == "information_extraction": + return "IE_PreviousInfo" if subtype == "previnfo" else "IE_Entity" + raise ValueError(f"Unsupported MemLens judge category: {category}") + + +def judge_prompt( + question: str, + reference: str, + prediction: str, + category: str, + subtype: str = "", + old_answer: str = "", +) -> str: + """Render the exact official rubric/examples and current-case fields.""" + if any(not isinstance(value, str) for value in (question, reference, prediction, category, subtype, old_answer)): + raise ValueError("MemLens judge inputs must be strings") + key = task_key(category, subtype, reference) + prompts = _prompts() + prompt = prompts["template"].format(task_criteria=prompts["criteria"][key], examples=prompts["examples"][key]) + if key == "AR_AnswerRefusal": + reference = "Insufficient information" + old = f": {old_answer}\n" if key == "KU_KnowledgeUpdate" and old_answer else "" + return ( + f"{prompt}[Current Case]\n: {question}\n: {reference}\n" + f"{old}: {prediction}\n\n[Scoring Rationale]:" + ) + + +def normalize_for_judge(text: str) -> dict: + """Preserve native parsing, circularity diagnostics and final-30% tail selection.""" + if not text: + return {"text": "", "original_len": 0, "was_truncated": False, "is_degenerate": False, "diagnostics": {}} + original_len = len(text) + text = parse_model_output(text) + hedging, ratio = 0, 1.0 + if len(text) >= 500: + lower = text.lower() + hedging = sum(lower.count(marker) for marker in _HEDGING_MARKERS) + words = lower.split() + ngrams = [tuple(words[index : index + 5]) for index in range(len(words) - 4)] + ratio = len(set(ngrams)) / len(ngrams) if ngrams else 1.0 + truncated = len(text) > 3000 and len(text) > 6000 + if truncated: + text = "[...truncated earlier reasoning...]\n" + text[int(len(text) * (1 - 0.3)) :] + return { + "text": text, + "original_len": original_len, + "was_truncated": truncated, + "is_degenerate": hedging >= 6 or ratio < 0.40, + "diagnostics": {"hedging_count": hedging, "unique_5gram_ratio": round(ratio, 3)}, + } + + +def parse_judge_response(text: str) -> dict: + """Official first-match priority/clamping; even unparseable outputs natively score zero.""" + patterns = ( + (r'"answer_score"\s*:\s*(\d+)', 0), + (r"deserves\s+(\d+)\s+point", re.IGNORECASE), + (r"\[Score\]\s*:\s*(\d+)", 0), + ) + for pattern, flags in patterns: + match = re.search(pattern, text, flags) if text else None + if match: + return {"acc": float(min(int(match.group(1)), 1)), "parse_default_zero": False} + return {"acc": 0.0, "parse_default_zero": True} + + +def parse_model_output(text: str) -> str: + """Native special-token/think-trace parsing; retained raw outputs remain separate.""" + if not text: + return "" + if "" in text: + text = text.split("", 1)[1] + text = re.sub(r"<\|[^>]*\|>", "", text) + text = re.sub(r"<\|begin_of_box\|>(.*?)<\|end_of_box\|>", r"\1", text, flags=re.DOTALL) + text = text.replace("<|begin_of_box|>", "").replace("<|end_of_box|>", "") + return text.replace("[CONTENT_FILTER_REJECTED]", "").strip() diff --git a/plugins/memlens/README.md b/plugins/memlens/README.md new file mode 100644 index 00000000..81e982c4 --- /dev/null +++ b/plugins/memlens/README.md @@ -0,0 +1,48 @@ +# MemLens plugin + +Install with `pip install -e plugins/memlens` and load the `memlens-benchmark` configuration +with the `memlens` plugin. The executable ReMe Image evaluation lives +in [`benchmark/memlens`](../../benchmark/memlens). + +`auto_memory` extends native Auto Memory with a MemLens-specific system prompt. +It retains native source-session persistence and interleaved image inputs; `include_images` +selects whether the memory model sees historical images. No question or reference answer +is given during ingestion. + +`agentic_answer` accepts the official answer prompt as `query`, optional `query_time`, +and native image DataBlocks as `query_images`. It creates a fresh AgentScope Agent for +each question with BM25 `search` and scoped `read`. Question images remain visible in +both the text-memory and image-memory settings. +Its `max_iters` input defaults to 30 ReAct iterations and applies only +to the current answer Agent. The effective value is returned in answer metadata; +Auto Memory and context compression keep their existing settings. + +`include_source_sessions` defaults to `true`: memory prompts include exact physical +JSONL line references, and the answering Agent can search/read those source sessions. +When false, only generated daily/digest markdown is searchable and readable, regardless +of whether an old session index exists. Link expansion is disabled. JSONL persistence +is unchanged. The runner must also omit session directories from index updates for +the source-disabled setting. + +With `include_images=true`, native Auto Memory saves input images beneath +`session/images/` and supplies their paths to the memory Agent. The answer tool +`read_image(path)` calls ReMe's existing `read_image` Job and converts its successful +image response to an AgentScope DataBlock. The plugin supplies the native Job with +`max_bytes=52428800` (50 MiB). Image inputs retain their original bytes and resolution. +Native size errors and non-image results +stay textual tool errors; the generic ReMe Job is unchanged. + +Image access is independent of `include_source_sessions`: a source-disabled Agent +can still reopen images cited by its memory notes. ReMe Text has no historical-image +tool. All image paths are confined to the current scene's session image directory. +The runner retains official Image IDs as text beside the corresponding image blocks; +the IDs are answer evidence labels, not a separate registry or tool argument. +Do not put answer keys, judge results, another scene's data, or agent state under the +daily/digest/source evidence directories. Every scene starts from a separate workspace; +sessions within one scene accumulate in order. Query state and search deduplication +are independent even when questions execute concurrently. + +The plugin preserves the ReMe Image experiment's thinking, output budgets and context +settings. The runner selects the candidate model; scoring belongs to `memlens-judge`. +Answer metadata includes portable usage and the raw final-message usage when available; +missing provider cache/reasoning counters are not reported as zero. diff --git a/plugins/memlens/pyproject.toml b/plugins/memlens/pyproject.toml new file mode 100644 index 00000000..2bc97530 --- /dev/null +++ b/plugins/memlens/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "reme-memlens" +version = "0.1.0" +description = "MemLens multimodal memory benchmark plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11", "pillow>=10.0.0"] + +[project.entry-points."reme.plugins"] +memlens = "reme_memlens" + +[project.entry-points."reme.configs"] +memlens-benchmark = "reme_memlens:BENCHMARK_CONFIG" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["reme_memlens*"] + +[tool.setuptools.package-data] +reme_memlens = ["plugin.yaml", "*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/memlens/src/reme_memlens/__init__.py b/plugins/memlens/src/reme_memlens/__init__.py new file mode 100644 index 00000000..a81a2c73 --- /dev/null +++ b/plugins/memlens/src/reme_memlens/__init__.py @@ -0,0 +1,5 @@ +"""MemLens benchmark Jobs for ReMe.""" + +from pathlib import Path + +BENCHMARK_CONFIG = Path(__file__).with_name("benchmark.yaml") diff --git a/plugins/memlens/src/reme_memlens/agentic_answer.py b/plugins/memlens/src/reme_memlens/agentic_answer.py new file mode 100644 index 00000000..2115d95d --- /dev/null +++ b/plugins/memlens/src/reme_memlens/agentic_answer.py @@ -0,0 +1,136 @@ +"""Agentic BM25 answer with scene-local, optional multimodal source access.""" + +from pathlib import Path +from uuid import uuid4 + +from agentscope.message import Base64Source, DataBlock, TextBlock, ToolResultState, UserMsg +from agentscope.tool import FunctionTool, ToolChunk, Toolkit + +from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper +from reme.steps.base_step import BaseStep +from reme.steps.file_io._path import _check_path_permission, resolve_path + + +def evidence_paths(workspace: Path, daily_dir: str, digest_dir: str, session_dir: str, sources: bool) -> list[str]: + """List exact allowed evidence files, never agent state, settings, or another workspace.""" + roots = [(daily_dir, "*.md"), (digest_dir, "*.md")] + if sources: + roots.append((f"{session_dir}/dialog", "*.jsonl")) + workspace = workspace.resolve() + paths = [] + for directory, pattern in roots: + root = (workspace / directory).resolve() + if not root.is_relative_to(workspace): + raise ValueError("Evidence directory must stay inside this scene workspace") + for path in sorted(root.rglob(pattern)): + if path.is_file() and path.resolve().is_relative_to(root): + paths.append(path.relative_to(workspace).as_posix()) + return paths + + +def image_reader(step: BaseStep) -> FunctionTool: + """Bridge the native read_image Job response to an AgentScope image result.""" + workspace = step.workspace_path.resolve() + image_directory = f"{step.config_value('session_dir')}/images" + + async def read_image(path: str) -> ToolChunk: + """View an image using its session/images path cited in a memory note.""" + target, error = resolve_path(workspace, path) + if error or target is None or not _check_path_permission(workspace, target, [image_directory]): + return ToolChunk( + content=[TextBlock(text=error or "Image path must stay inside this scene's session image directory.")], + state=ToolResultState.ERROR, + ) + relative = target.relative_to(workspace).as_posix() + response = await step.run_job("read_image", path=relative) + metadata = response.metadata + media_type = metadata.get("mime", "") + if ( + not response.success + or metadata.get("oversized") + or metadata.get("non_image_warning") + or not media_type.startswith("image/") + ): + notice = str(response.answer) if not response.success or metadata.get("oversized") else "Not an image file." + return ToolChunk(content=[TextBlock(text=notice)], state=ToolResultState.ERROR) + image = DataBlock(source=Base64Source(media_type=media_type, data=response.answer)) + return ToolChunk(content=[TextBlock(text=f"Image source: {relative}"), image], state=ToolResultState.SUCCESS) + + return FunctionTool(func=read_image, name="read_image", is_concurrency_safe=True, is_read_only=True) + + +class MemLensAgenticAnswerStep(BaseStep): + """Run a fresh Agent for each question, with the official answer prompt unchanged.""" + + async def execute(self): + """Answer through a fresh, scene-scoped Agent and its allowed tools.""" + wrapper = self.agent_wrapper + if not isinstance(wrapper, AsAgentWrapper): + raise NotImplementedError("MemLens agentic_answer requires the AgentScope wrapper") + query = self.context.get("query", "") + if not query: + raise ValueError("query must contain the benchmark answer prompt") + max_iters = self.context.get("max_iters", 30) + if isinstance(max_iters, bool) or not isinstance(max_iters, int) or max_iters < 1: + raise ValueError("max_iters must be a positive integer") + sources = bool(self.context.get("include_source_sessions", True)) + images = bool(self.context.get("include_images", False)) + paths = evidence_paths( + self.workspace_path, + self.config_value("daily_dir"), + self.config_value("digest_dir"), + self.config_value("session_dir"), + sources, + ) + tool_context_id = f"memlens_answer_{uuid4().hex}" + tools = [] + if images: + tools.append(image_reader(self)) + injected = { + "_allowed_paths": paths, + "read_step_format_session": True, + "search_filter": {"paths": paths}, + "vector_weight": 0.0, + } + # Standard ReMe job tools keep their normal file handling and lifecycle. + tools.extend( + wrapper._make_tool(job, tool_context_id, injected) # pylint: disable=protected-access + for job in wrapper._resolve_job_tools(["search", "read"]) # pylint: disable=protected-access + ) + blocks = [TextBlock(text=query)] + blocks.extend(DataBlock.model_validate(block) for block in self.context.get("query_images", [])) + if any(not block.source.media_type.startswith("image/") for block in blocks[1:]): + raise ValueError("query_images accepts image DataBlocks only") + prompt = self.get_prompt("system_prompt") + if query_time := self.context.get("query_time"): + prompt += f"\nQuestion time: {query_time}" + if images: + prompt += ( + "\nUse read_image(path=...) for the session/images paths cited in memory, including source_images." + ) + else: + prompt += "\nHistory images cannot be reopened in this setting." + try: + result = await wrapper.reply( + UserMsg(name="user", content=blocks), + system_prompt=prompt, + toolkit=Toolkit(tools=tools), + job_tools=[], + builtin_tools=False, + session_id=str(uuid4()), + resume="", + react_config={"max_iters": max_iters}, + ) + self.context.response.answer = (result.get("result") or "").strip() + self.context.response.metadata.update( + usage=result.get("usage"), + agentscope_usage=(result.get("last_message") or {}).get("usage"), + agent_session_id=result.get("session_id"), + include_source_sessions=sources, + include_images=images, + max_iters=max_iters, + ) + finally: + if self.app_context is not None: + self.app_context.metadata.get("tool_contexts", {}).pop(tool_context_id, None) + return self.context.response diff --git a/plugins/memlens/src/reme_memlens/agentic_answer.yaml b/plugins/memlens/src/reme_memlens/agentic_answer.yaml new file mode 100644 index 00000000..42f862e3 --- /dev/null +++ b/plugins/memlens/src/reme_memlens/agentic_answer.yaml @@ -0,0 +1,10 @@ +system_prompt: | + Answer the supplied benchmark question by retrieving evidence from this scene's memory. + Use search to find relevant facts, then read exact files or line ranges when more context is needed. + Search again with different names, dates or distinctive visual attributes if the evidence is incomplete. + Keep different people, times and visually similar objects distinct. Source line references point to + the original conversation, when source access is enabled. If read_image is available, use the image + path cited in the memory note or its source_images field to verify the actual historical image. + Preserve Image IDs exactly when the question asks which image. Do not invent IDs or unseen details. + Follow the question's supplied answer-format instructions and answer only from supported evidence. + Do not include the retrieval process or tool transcript in the final answer. diff --git a/plugins/memlens/src/reme_memlens/auto_memory.py b/plugins/memlens/src/reme_memlens/auto_memory.py new file mode 100644 index 00000000..3f473536 --- /dev/null +++ b/plugins/memlens/src/reme_memlens/auto_memory.py @@ -0,0 +1,34 @@ +"""MemLens memory prompts with optional, exact source-line references.""" + +from agentscope.message import Msg + +from reme.steps.evolve.auto_memory import AutoMemoryStep + + +class MemLensAutoMemoryStep(AutoMemoryStep): + """Keep core persistence and multimodal input, adding benchmark instructions only.""" + + def _format_history(self, messages: list[Msg]) -> str: + if not self.context.get("include_source_sessions", True): + return "Source-session access is disabled; record self-contained facts.\n\n" + super()._format_history( + messages, + ) + + # Core execute has saved the session already. Read physical line numbers + # rather than assuming input order equals timestamp-sorted JSONL order. + session_id = self.context.get("session_id", "") + source = self._session_source_path(session_id) + # File iteration keeps Unicode line separators inside JSON strings intact. + with self._session_path(session_id).open(encoding="utf-8") as stream: + lines = {Msg.model_validate_json(line).id: number for number, line in enumerate(stream, 1) if line.strip()} + turns = [ + f"[L{lines[msg.id]} | {msg.name or msg.role} @ {msg.created_at}]\n{msg.get_text_content() or ''}" + for msg in messages + ] + return ( + f"Source file: {source}\n" + f"Cite useful evidence with the full source path, for example [[{source}#L1-L2]], " + "replacing the line range with the actual physical lines below. Never use bare links such as [[L5]]. " + "Preserve original Image IDs alongside their facts, and retain the supplied Image source paths.\n\n" + + "\n\n".join(turns) + ) diff --git a/plugins/memlens/src/reme_memlens/auto_memory.yaml b/plugins/memlens/src/reme_memlens/auto_memory.yaml new file mode 100644 index 00000000..f318b669 --- /dev/null +++ b/plugins/memlens/src/reme_memlens/auto_memory.yaml @@ -0,0 +1,25 @@ +system_prompt: | + You maintain long-horizon personal multimodal memory across dated conversations. Use tools to save useful evidence, + not a response to any future question. Retain concrete people, entities, events, dates, preferences, decisions, + quantities, places and relationships. Preserve changes over time and distinguish speakers and similar objects. + Track longitudinal updates to preferences, plans, possessions, health, work and relationships without erasing + earlier dated facts. Keep which person experienced each event explicit and resolve relative dates using the + session timestamp only when justified. Separate repeated similar events instead of conflating them. + + When images are present, jointly understand them and the surrounding conversation. Record distinctive visual + details (objects, attributes, layout, visible text, activities) that make an image identifiable. Keep each original + Image ID verbatim next to the corresponding visual facts; an ID is a reference, not evidence of image contents. + Do not invent visual details when no image is provided. Do not substitute generic summaries for specific facts. + + When physical source-line labels are supplied, cite the useful source ranges with the exact session path and + line numbers. Without source access, make the memory self-contained rather than adding source references. + Organize related facts under descriptive headings, use absolute dates when supported, preserve exact names + and numbers, and distinguish facts from uncertainty. Resolve pronouns only when the conversation supports it. + + Follow the normal daily-note contract: a stable filename stem in name, an informative description, and the + factual body. Never set status. Invoke the provided file tools as structured tool calls to actually save memory. + + Complete the task, then stop: create at most one note for a new session. After a successful daily_write, + return the requested one-sentence summary without calling daily_write again or creating a renamed duplicate. + Further writes are needed only to correct an explicit tool error or an identified omission, not to repeat a + successful save. For an existing note, finish after the necessary edits and frontmatter update succeed. diff --git a/plugins/memlens/src/reme_memlens/benchmark.yaml b/plugins/memlens/src/reme_memlens/benchmark.yaml new file mode 100644 index 00000000..1421f4fc --- /dev/null +++ b/plugins/memlens/src/reme_memlens/benchmark.yaml @@ -0,0 +1,30 @@ +extends: benchmark +plugins: [memlens] +components: + as_llm: + default: &memory_model + backend: openai + model: ${LLM_MODEL_NAME:-qwen3.8-max} + stream: false + context_size: 200000 + max_retries: 3 + retry_delay: 60 + client_kwargs: {timeout: 600, max_retries: 0} + extra_body: {enable_thinking: true} + parameters: {temperature: 0, thinking_enable: true, max_tokens: 65536} + bench: + <<: *memory_model + context_size: 400000 + parameters: {temperature: 0, thinking_enable: true, max_tokens: 16384} + compressor: + <<: *memory_model + agent_wrapper: + default: &memory_agent + react_config: {max_iters: 30} + context_config: {trigger_ratio: 0.8, reserve_ratio: 0.1, tool_result_limit: 50000, max_image_num: 4096} + model_config: {max_retries: 0} + bench: + <<: *memory_agent + file_store: + default: + embedding_store: null diff --git a/plugins/memlens/src/reme_memlens/plugin.yaml b/plugins/memlens/src/reme_memlens/plugin.yaml new file mode 100644 index 00000000..56a6f64e --- /dev/null +++ b/plugins/memlens/src/reme_memlens/plugin.yaml @@ -0,0 +1,75 @@ +backends: + memlens_auto_memory_step: reme_memlens.auto_memory:MemLensAutoMemoryStep + memlens_agentic_answer_step: reme_memlens.agentic_answer:MemLensAgenticAnswerStep + +application_defaults: + jobs: + read_image: + backend: base + description: "Read a saved session image through the native ReMe image Job." + parameters: + type: object + properties: + path: {type: string, description: "Workspace-relative image attachment path."} + required: [path] + steps: + - backend: read_image_step + max_bytes: 52428800 + auto_memory: + backend: base + description: "Record one MemLens session using native Auto Memory." + include_source_sessions: true + parameters: + type: object + properties: + messages: + type: array + items: {type: object} + session_id: {type: string} + date: {type: string, default: ""} + include_images: {type: boolean, default: false} + include_source_sessions: {type: boolean, default: true} + memory_hint: {type: string, default: ""} + required: [messages, session_id] + steps: + - backend: memlens_auto_memory_step + search: + backend: base + description: "Search this scene's allowed memory and source text using BM25." + parameters: + type: object + properties: + query: {type: string} + limit: {type: integer, description: "Number of search results."} + start_date: {type: string} + end_date: {type: string} + required: [query] + steps: + - backend: search_step + vector_weight: 0.0 + expand_links: false + agentic_answer: + backend: base + description: "Answer a MemLens query through a fresh multimodal retrieval Agent." + include_source_sessions: true + parameters: + type: object + properties: + query: {type: string, description: "Official benchmark answer prompt including the query."} + query_time: {type: string, default: ""} + query_images: + type: array + items: {type: object} + default: [] + scene_id: {type: string} + max_iters: {type: integer, minimum: 1, default: 30, description: "Maximum answer Agent ReAct iterations."} + include_images: {type: boolean, default: false} + include_source_sessions: {type: boolean, default: true} + required: [query, scene_id] + steps: + - backend: memlens_agentic_answer_step + agent_wrapper: bench + components: + file_store: + default: + embedding_store: null diff --git a/plugins/mobilemem-judge/LICENSE-CODE b/plugins/mobilemem-judge/LICENSE-CODE new file mode 100644 index 00000000..e213f97f --- /dev/null +++ b/plugins/mobilemem-judge/LICENSE-CODE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ZJUNLP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/mobilemem-judge/README.md b/plugins/mobilemem-judge/README.md new file mode 100644 index 00000000..65faee79 --- /dev/null +++ b/plugins/mobilemem-judge/README.md @@ -0,0 +1,29 @@ +# ReMe MobileMem Judge + +Independent `answer_judge` Job for saved MobileMem-Omni predictions. Install with +`pip install -e plugins/mobilemem-judge` and enable `mobilemem-judge` in a separate +evaluation application/workspace, not the candidate memory application. + +Inputs are `question`, `reference`, `prediction`, and optional `evidence` (a list +of original evidence values). `judge_protocol` defaults to `membase_default`, the +executable MemBase rubric used by this benchmark. `native_strict` selects the +separately published strict rubric. Their protocol identifiers remain distinct. + +Both modes retain the official input interpolation, WRONG-first verdict parsing, +and default-zero handling of successful responses without a valid label. +`response.metadata` contains `llm_judge_score`, `label`, `parse_default_zero`, +`default_judge_protocol`, `prompt`, `raw_response`, and `usage`. Provider errors +remain failed Jobs. Judge usage is evaluation cost, not method cost. + +Defaults are `qwen3.8-max`, temperature 0, thinking disabled and 4,096 output tokens. +`JUDGE_MODEL_NAME` can override the Judge model independently of the candidate. +Candidate methods and their workspaces are not dependencies of this plugin. +The benchmark runner invokes this Job in a separate application and saves its +prompt, provider output and verdict alongside the candidate prediction. Its +`judge-settings.json` records the validated model, generation/retry settings and +protocol identifier through a credential-free whitelist, rather than recording +only the default model name. + +Sources: [MemBase executable judge](https://github.com/zjunlp/MemBase/blob/a9ba9495d48541d3e68588ff097294cc7a1201bc/membase/inference_utils/prompts.py) +and [MobileMem published strict rubric](https://github.com/zjunlp/MobileMem/blob/fa5cd13c9c659699df9caf650987ba2bea8ca085/omni/eval/eval/question_answering_and_judge_prompts.txt). +The upstream prompt/parser code retains its MIT notice in `LICENSE-CODE`. diff --git a/plugins/mobilemem-judge/pyproject.toml b/plugins/mobilemem-judge/pyproject.toml new file mode 100644 index 00000000..c9b03c05 --- /dev/null +++ b/plugins/mobilemem-judge/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "reme-mobilemem-judge" +version = "0.1.0" +description = "Official MobileMem-Omni judge protocols as an independent ReMe plugin." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11"] + +[project.entry-points."reme.plugins"] +mobilemem-judge = "judge_mobilemem" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["judge_mobilemem*"] + +[tool.setuptools.package-data] +judge_mobilemem = ["plugin.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/mobilemem-judge/src/judge_mobilemem/__init__.py b/plugins/mobilemem-judge/src/judge_mobilemem/__init__.py new file mode 100644 index 00000000..f243bdb5 --- /dev/null +++ b/plugins/mobilemem-judge/src/judge_mobilemem/__init__.py @@ -0,0 +1,5 @@ +"""Independent MobileMem evaluation-only plugin.""" + +from .llm_judge import MobileMemJudgeStep + +__all__ = ["MobileMemJudgeStep"] diff --git a/plugins/mobilemem-judge/src/judge_mobilemem/llm_judge.py b/plugins/mobilemem-judge/src/judge_mobilemem/llm_judge.py new file mode 100644 index 00000000..8b11ca7d --- /dev/null +++ b/plugins/mobilemem-judge/src/judge_mobilemem/llm_judge.py @@ -0,0 +1,43 @@ +"""Official MobileMem binary judge as an isolated ReMe Job.""" + +from dataclasses import asdict + +from agentscope.message import UserMsg + +from reme.steps.base_step import BaseStep +from reme.steps.evolve.compressor import CompressorStep + +from .protocol import judge_prompt, parse_judge, protocol_id + + +class MobileMemJudgeStep(BaseStep): + """Score saved predictions without accessing the candidate memory or tools.""" + + async def execute(self): + """Apply the unchanged benchmark prompt and verdict parser.""" + mode = self.context.get("judge_protocol", "membase_default") + prompt = judge_prompt( + self.context["question"], + self.context["reference"], + self.context["prediction"], + self.context.get("evidence", ()), + judge_protocol=mode, + ) + response = await self.as_llm([UserMsg(name="user", content=prompt)]) + if hasattr(type(response), "__aiter__"): + last = None + async for chunk in response: + last = chunk + response = last + raw = await CompressorStep._response_text(response) # pylint: disable=protected-access + parsed = parse_judge(raw) + self.context.response.success = True + self.context.response.answer = str(parsed["llm_judge_score"]) + self.context.response.metadata.update( + **parsed, + default_judge_protocol=protocol_id(mode), + prompt=prompt, + raw_response=raw, + usage=asdict(response.usage) if response is not None and response.usage is not None else None, + ) + return self.context.response diff --git a/plugins/mobilemem-judge/src/judge_mobilemem/plugin.yaml b/plugins/mobilemem-judge/src/judge_mobilemem/plugin.yaml new file mode 100644 index 00000000..73b17a10 --- /dev/null +++ b/plugins/mobilemem-judge/src/judge_mobilemem/plugin.yaml @@ -0,0 +1,40 @@ +backends: + mobilemem_judge_step: judge_mobilemem.llm_judge:MobileMemJudgeStep + +application_defaults: + jobs: + answer_judge: + backend: base + description: "MobileMem-Omni official judge protocol for a saved prediction" + parameters: + type: object + properties: + question: {type: string} + reference: {type: string} + prediction: {type: string} + evidence: {type: array, default: []} + judge_protocol: + type: string + enum: [membase_default, native_strict] + default: membase_default + required: [question, reference, prediction] + steps: + - backend: mobilemem_judge_step + as_llm: judge + components: + as_llm: + judge: + backend: openai + model: ${JUDGE_MODEL_NAME:-qwen3.8-max} + stream: false + max_retries: 3 + retry_delay: 60 + client_kwargs: {timeout: 600, max_retries: 0} + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + extra_body: {enable_thinking: false} + parameters: + temperature: 0 + thinking_enable: false + max_tokens: 4096 diff --git a/plugins/mobilemem-judge/src/judge_mobilemem/protocol.py b/plugins/mobilemem-judge/src/judge_mobilemem/protocol.py new file mode 100644 index 00000000..c2ba851f --- /dev/null +++ b/plugins/mobilemem-judge/src/judge_mobilemem/protocol.py @@ -0,0 +1,118 @@ +"""Unchanged MobileMem/MemBase judge prompts and binary parser. + +Adapted from ZJUNLP's MobileMem and MemBase revisions below. +MIT License, Copyright (c) 2026 ZJUNLP; see LICENSE-CODE. +""" + +import json +import re +from string import Template + +SOURCE_REVISION = "fa5cd13c9c659699df9caf650987ba2bea8ca085" +MEMBASE_REVISION = "a9ba9495d48541d3e68588ff097294cc7a1201bc" +PROTOCOL_ID = "mobilemem-omni-membase-default-judge-v1" +STRICT_PROTOCOL_ID = "mobilemem-omni-final-memory-native-v1" +TEMPLATE_NAME = "mobilemem-omni-judge" +SOURCE_URL = f"https://github.com/zjunlp/MemBase/blob/{MEMBASE_REVISION}/membase/inference_utils/prompts.py" +JUDGE_TEMPLATE = ( + "Your task is to classify an answer to a question as CORRECT or WRONG. " + "You are given a question, a gold answer, a generated answer, and optional " + "evidence from the user's prior conversations.\n\n" + "GRADING RULES:\n" + "1. The generated answer must include every key element in the gold answer.\n" + "2. Unsupported or irrelevant content that changes the meaning makes the answer WRONG.\n" + "3. Equivalent date formats are acceptable when they refer to the same time.\n" + "4. Use the evidence only to verify the answer; do not infer unsupported facts.\n\n" + "Question: $question\n" + "Gold answer: $golden_answers\n" + "Evidence: $evidence\n" + "Generated answer: $prediction\n\n" + 'Return exactly one JSON object: {"label": "CORRECT"} or ' + '{"label": "WRONG"}.' +) +STRICT_JUDGE_TEMPLATE = ( + "Your task is to classify an answer to a question as CORRECT or WRONG. " + "You will be provided with the following information:\n" + "(1) A question (asked by a user to an AI assistant);\n" + "(2) A gold (ground truth) answer;\n" + "(3) A generated answer;\n" + "(4) A set of evidence sentences relevant to the question, which may be used to verify the generated answer.\n" + "The question focuses on information that an AI assistant should recall about the user " + "based on prior conversations. " + "The gold answer is typically concise and includes all key facts.\n\n" + "GRADING RULES (Must Be Strictly Followed)\n" + "1. COMPLETE COVERAGE REQUIRED: The generated answer must include every key element present in the gold answer. " + "Missing any required fact results in a WRONG classification.\n" + "2. NO REDUNDANT CONTENT: Including excessive or irrelevant details not supported by the evidence or gold answer " + "results in a WRONG classification.\n" + "3. TIME-RELATED QUESTIONS: Different date formats (e.g., May 7th vs. 7 May) are acceptable " + "as long as the same date or time period is referenced.\n\n" + "INSTRUCTIONS FOR THIS TASK\n" + "Use the provided evidence only as supporting context for verification when available. " + "Do not infer facts that are not present in the question, gold answer, or evidence.\n" + "First, provide a brief (one-sentence) explanation of your reasoning. Then, output the label CORRECT or WRONG.\n" + "Do not include both labels in your response, as this will break the evaluation script.\n" + "Return the label in JSON format with the key label.\n\n" + "QUESTION TO EVALUATE\n" + "Question: [Question]\n" + "Gold answer: [Gold Answer]\n" + "Evidence: [Evidences]\n" + "Generated answer: [Generated Answer]" +) + + +def protocol_id(judge_protocol: str) -> str: + """Identify the selected rubric without silently substituting another.""" + if judge_protocol == "membase_default": + return PROTOCOL_ID + if judge_protocol == "native_strict": + return STRICT_PROTOCOL_ID + raise ValueError(f"Unsupported MobileMem judge protocol: {judge_protocol!r}") + + +def judge_prompt( + question: str, + reference: str, + prediction: str, + evidence=(), + *, + judge_protocol="membase_default", +) -> str: + """Preserve native single-pass substitution and untrimmed evidence.""" + protocol_id(judge_protocol) + evidence_text = "\n".join(str(item) for item in evidence) or "Not provided." + if judge_protocol == "membase_default": + return Template(JUDGE_TEMPLATE).substitute( + question=question, + golden_answers=reference, + evidence=evidence_text, + prediction=prediction, + ) + replacements = { + "[Question]": question, + "[Gold Answer]": reference, + "[Evidences]": evidence_text, + "[Generated Answer]": prediction, + } + return re.sub( + r"\[Question\]|\[Gold Answer\]|\[Evidences\]|\[Generated Answer\]", + lambda match: replacements[match[0]], + STRICT_JUDGE_TEMPLATE, + ) + + +def parse_judge(text: str) -> dict: + """Preserve MemBase's WRONG-first binary parsing and native default zero.""" + text = text.strip() + try: + parsed = json.loads(text) + label = str(parsed.get("label", "")) + except (json.JSONDecodeError, AttributeError): + label = text + normalised = label.strip().upper() + score = 0.0 if "WRONG" in normalised else float("CORRECT" in normalised) + return { + "llm_judge_score": score, + "label": "CORRECT" if score else "WRONG", + "parse_default_zero": "WRONG" not in normalised and "CORRECT" not in normalised, + } diff --git a/plugins/mobilemem/README.md b/plugins/mobilemem/README.md new file mode 100644 index 00000000..268e4de7 --- /dev/null +++ b/plugins/mobilemem/README.md @@ -0,0 +1,17 @@ +# MobileMem plugin + +Install this plugin from the repository: + +```bash +pip install -e plugins/mobilemem +``` + +Enable the `mobilemem` ReMe plugin for the MobileMem-Omni benchmark. Its Steps use native ReMe Auto Memory, +source-line citations, BM25 retrieval, workspace containment, and an AgentScope bridge for the native image +reader. The plugin installs independently of MemLens. Evaluation uses the separate +[`mobilemem-judge`](../mobilemem-judge/README.md) plugin. + +The runner supplies only original dialogue messages and image attachments. Memory extraction retains dates, +updates, personal relationships, app details and visual evidence; each question uses a fresh answer Agent. Source +session reading and `read_image` remain scoped to that user's workspace. Supplied captions and benchmark gold +annotations must never be passed to either Job. diff --git a/plugins/mobilemem/pyproject.toml b/plugins/mobilemem/pyproject.toml new file mode 100644 index 00000000..5e5461ea --- /dev/null +++ b/plugins/mobilemem/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "reme-mobilemem" +version = "0.1.0" +description = "MobileMem-Omni native multimodal memory benchmark plugin for ReMe." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = ["reme-ai[as]>=0.4.1.11", "pillow>=10.0.0"] + +[project.entry-points."reme.plugins"] +mobilemem = "reme_mobilemem" + +[project.entry-points."reme.configs"] +mobilemem-benchmark = "reme_mobilemem:BENCHMARK_CONFIG" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["reme_mobilemem*"] + +[tool.setuptools.package-data] +reme_mobilemem = ["plugin.yaml", "*.yaml"] + +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/plugins/mobilemem/src/reme_mobilemem/__init__.py b/plugins/mobilemem/src/reme_mobilemem/__init__.py new file mode 100644 index 00000000..e9027ef5 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/__init__.py @@ -0,0 +1,5 @@ +"""MobileMem-Omni benchmark Jobs for ReMe.""" + +from pathlib import Path + +BENCHMARK_CONFIG = Path(__file__).with_name("benchmark.yaml") diff --git a/plugins/mobilemem/src/reme_mobilemem/agentic_answer.py b/plugins/mobilemem/src/reme_mobilemem/agentic_answer.py new file mode 100644 index 00000000..90f19510 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/agentic_answer.py @@ -0,0 +1,136 @@ +"""Agentic BM25 answer with scene-local, optional multimodal source access.""" + +from pathlib import Path +from uuid import uuid4 + +from agentscope.message import Base64Source, DataBlock, TextBlock, ToolResultState, UserMsg +from agentscope.tool import FunctionTool, ToolChunk, Toolkit + +from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper +from reme.steps.base_step import BaseStep +from reme.steps.file_io._path import _check_path_permission, resolve_path + + +def evidence_paths(workspace: Path, daily_dir: str, digest_dir: str, session_dir: str, sources: bool) -> list[str]: + """List exact allowed evidence files, never agent state, settings, or another workspace.""" + roots = [(daily_dir, "*.md"), (digest_dir, "*.md")] + if sources: + roots.append((f"{session_dir}/dialog", "*.jsonl")) + workspace = workspace.resolve() + paths = [] + for directory, pattern in roots: + root = (workspace / directory).resolve() + if not root.is_relative_to(workspace): + raise ValueError("Evidence directory must stay inside this scene workspace") + for path in sorted(root.rglob(pattern)): + if path.is_file() and path.resolve().is_relative_to(root): + paths.append(path.relative_to(workspace).as_posix()) + return paths + + +def image_reader(step: BaseStep) -> FunctionTool: + """Bridge the native read_image Job response to an AgentScope image result.""" + workspace = step.workspace_path.resolve() + image_directory = f"{step.config_value('session_dir')}/images" + + async def read_image(path: str) -> ToolChunk: + """View an image using its session/images path cited in a memory note.""" + target, error = resolve_path(workspace, path) + if error or target is None or not _check_path_permission(workspace, target, [image_directory]): + return ToolChunk( + content=[TextBlock(text=error or "Image path must stay inside this scene's session image directory.")], + state=ToolResultState.ERROR, + ) + relative = target.relative_to(workspace).as_posix() + response = await step.run_job("read_image", path=relative) + metadata = response.metadata + media_type = metadata.get("mime", "") + if ( + not response.success + or metadata.get("oversized") + or metadata.get("non_image_warning") + or not media_type.startswith("image/") + ): + notice = str(response.answer) if not response.success or metadata.get("oversized") else "Not an image file." + return ToolChunk(content=[TextBlock(text=notice)], state=ToolResultState.ERROR) + image = DataBlock(source=Base64Source(media_type=media_type, data=response.answer)) + return ToolChunk(content=[TextBlock(text=f"Image source: {relative}"), image], state=ToolResultState.SUCCESS) + + return FunctionTool(func=read_image, name="read_image", is_concurrency_safe=True, is_read_only=True) + + +class MobileMemAgenticAnswerStep(BaseStep): + """Run a fresh Agent for each question, with the official answer prompt unchanged.""" + + async def execute(self): + """Answer through a fresh, scene-scoped Agent and its allowed tools.""" + wrapper = self.agent_wrapper + if not isinstance(wrapper, AsAgentWrapper): + raise NotImplementedError("MobileMem agentic_answer requires the AgentScope wrapper") + query = self.context.get("query", "") + if not query: + raise ValueError("query must contain the benchmark answer prompt") + max_iters = self.context.get("max_iters", 30) + if isinstance(max_iters, bool) or not isinstance(max_iters, int) or max_iters < 1: + raise ValueError("max_iters must be a positive integer") + sources = bool(self.context.get("include_source_sessions", True)) + images = bool(self.context.get("include_images", False)) + paths = evidence_paths( + self.workspace_path, + self.config_value("daily_dir"), + self.config_value("digest_dir"), + self.config_value("session_dir"), + sources, + ) + tool_context_id = f"mobilemem_answer_{uuid4().hex}" + tools = [] + if images: + tools.append(image_reader(self)) + injected = { + "_allowed_paths": paths, + "read_step_format_session": True, + "search_filter": {"paths": paths}, + "vector_weight": 0.0, + } + # Standard ReMe job tools keep their normal file handling and lifecycle. + tools.extend( + wrapper._make_tool(job, tool_context_id, injected) # pylint: disable=protected-access + for job in wrapper._resolve_job_tools(["search", "read"]) # pylint: disable=protected-access + ) + blocks = [TextBlock(text=query)] + blocks.extend(DataBlock.model_validate(block) for block in self.context.get("query_images", [])) + if any(not block.source.media_type.startswith("image/") for block in blocks[1:]): + raise ValueError("query_images accepts image DataBlocks only") + prompt = self.get_prompt("system_prompt") + if query_time := self.context.get("query_time"): + prompt += f"\nQuestion time: {query_time}" + if images: + prompt += ( + "\nUse read_image(path=...) for the session/images paths cited in memory, including source_images." + ) + else: + prompt += "\nHistory images cannot be reopened in this setting." + try: + result = await wrapper.reply( + UserMsg(name="user", content=blocks), + system_prompt=prompt, + toolkit=Toolkit(tools=tools), + job_tools=[], + builtin_tools=False, + session_id=str(uuid4()), + resume="", + react_config={"max_iters": max_iters}, + ) + self.context.response.answer = (result.get("result") or "").strip() + self.context.response.metadata.update( + usage=result.get("usage"), + agentscope_usage=(result.get("last_message") or {}).get("usage"), + agent_session_id=result.get("session_id"), + include_source_sessions=sources, + include_images=images, + max_iters=max_iters, + ) + finally: + if self.app_context is not None: + self.app_context.metadata.get("tool_contexts", {}).pop(tool_context_id, None) + return self.context.response diff --git a/plugins/mobilemem/src/reme_mobilemem/agentic_answer.yaml b/plugins/mobilemem/src/reme_mobilemem/agentic_answer.yaml new file mode 100644 index 00000000..661c9f20 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/agentic_answer.yaml @@ -0,0 +1,8 @@ +system_prompt: | + Answer the supplied MobileMem question from this user's saved memories and original source conversations. + Search for the relevant people, events, dates and distinctive details, then read the source files or exact line + ranges when additional context is needed. Distinguish changes over time and similar events. When read_image is + available, reopen cited session/images paths to verify visual details rather than guessing them from an ID. + Use another evidence search when the first result is incomplete. Follow the supplied benchmark answer rules. + Include all facts needed to answer the question, without unsupported or irrelevant details. Do not include the + retrieval process or tool transcript in the final answer. diff --git a/plugins/mobilemem/src/reme_mobilemem/auto_memory.py b/plugins/mobilemem/src/reme_mobilemem/auto_memory.py new file mode 100644 index 00000000..3c3c5dcc --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/auto_memory.py @@ -0,0 +1,34 @@ +"""MobileMem memory prompts with optional, exact source-line references.""" + +from agentscope.message import Msg + +from reme.steps.evolve.auto_memory import AutoMemoryStep + + +class MobileMemAutoMemoryStep(AutoMemoryStep): + """Keep core persistence and multimodal input, adding benchmark instructions only.""" + + def _format_history(self, messages: list[Msg]) -> str: + if not self.context.get("include_source_sessions", True): + return "Source-session access is disabled; record self-contained facts.\n\n" + super()._format_history( + messages, + ) + + # Core execute has saved the session already. Read physical line numbers + # rather than assuming input order equals timestamp-sorted JSONL order. + session_id = self.context.get("session_id", "") + source = self._session_source_path(session_id) + # File iteration keeps Unicode line separators inside JSON strings intact. + with self._session_path(session_id).open(encoding="utf-8") as stream: + lines = {Msg.model_validate_json(line).id: number for number, line in enumerate(stream, 1) if line.strip()} + turns = [ + f"[L{lines[msg.id]} | {msg.name or msg.role} @ {msg.created_at}]\n{msg.get_text_content() or ''}" + for msg in messages + ] + return ( + f"Source file: {source}\n" + f"Cite useful evidence with the full source path, for example [[{source}#L1-L2]], " + "replacing the line range with the actual physical lines below. Never use bare links such as [[L5]]. " + "Preserve original Image IDs alongside their facts, and retain the supplied Image source paths.\n\n" + + "\n\n".join(turns) + ) diff --git a/plugins/mobilemem/src/reme_mobilemem/auto_memory.yaml b/plugins/mobilemem/src/reme_mobilemem/auto_memory.yaml new file mode 100644 index 00000000..7afcac59 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/auto_memory.yaml @@ -0,0 +1,17 @@ +system_prompt: | + Maintain the user's long-term mobile memory from this original dated conversation. Save concrete personal facts, + names, relationships, dates, quantities, locations, plans, decisions and preferences. Distinguish separate events + across the year and preserve earlier dated facts when newer information updates them. Keep whose experience + each fact describes explicit. Use the supplied session timestamp for relative dates only when justified. + + When original images are present, understand them jointly with surrounding text. Retain distinctive visual + details, visible app text, amounts, dates, people, objects and activities. Preserve each supplied Image ID and + image source path with its facts so the original can be reopened. An identifier is not evidence of its contents. + Do not infer unavailable image details or introduce outside information. + + Cite useful physical source-line ranges with the full supplied source path when source access is enabled. + Otherwise retain self-contained facts. Follow the normal daily-note contract with a stable filename stem, + informative description and factual body. Never set status. Use structured file tool calls to save the memory. + + Create at most one new note for this session. After a successful daily_write, finish with the requested + one-sentence summary. Further writes are needed only to correct a tool error or a concrete omission. diff --git a/plugins/mobilemem/src/reme_mobilemem/benchmark.yaml b/plugins/mobilemem/src/reme_mobilemem/benchmark.yaml new file mode 100644 index 00000000..0fba4066 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/benchmark.yaml @@ -0,0 +1,32 @@ +extends: benchmark +plugins: [mobilemem] +components: + file_store: + default: + embedding_store: null + as_llm: + default: &candidate_model + backend: openai + model: ${LLM_MODEL_NAME:-qwen3.8-max} + stream: false + context_size: 1000000 + max_retries: 3 + retry_delay: 60 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + extra_body: {enable_thinking: true} + parameters: + temperature: 0 + thinking_enable: true + max_tokens: 65536 + client_kwargs: {timeout: 600, max_retries: 0} + bench: *candidate_model + compressor: *candidate_model + agent_wrapper: + default: + context_config: {max_image_num: 4096} + model_config: {max_retries: 0} + bench: + context_config: {max_image_num: 4096} + model_config: {max_retries: 0} diff --git a/plugins/mobilemem/src/reme_mobilemem/plugin.yaml b/plugins/mobilemem/src/reme_mobilemem/plugin.yaml new file mode 100644 index 00000000..c4e2bdc5 --- /dev/null +++ b/plugins/mobilemem/src/reme_mobilemem/plugin.yaml @@ -0,0 +1,71 @@ +backends: + mobilemem_auto_memory_step: reme_mobilemem.auto_memory:MobileMemAutoMemoryStep + mobilemem_agentic_answer_step: reme_mobilemem.agentic_answer:MobileMemAgenticAnswerStep + +application_defaults: + jobs: + read_image: + backend: base + description: "Read an original saved session image through ReMe's native image Job." + parameters: + type: object + properties: + path: {type: string, description: "Workspace-relative image attachment path."} + required: [path] + steps: + - backend: read_image_step + max_bytes: 52428800 + auto_memory: + backend: base + description: "Record one original MobileMem-Omni session using native Auto Memory." + include_images: true + include_source_sessions: true + parameters: + type: object + properties: + messages: {type: array, items: {type: object}} + session_id: {type: string} + date: {type: string, default: ""} + include_images: {type: boolean, default: true} + include_source_sessions: {type: boolean, default: true} + memory_hint: {type: string, default: ""} + required: [messages, session_id] + steps: + - backend: mobilemem_auto_memory_step + search: + backend: base + description: "Search this user's allowed memory and source text using BM25." + parameters: + type: object + properties: + query: {type: string} + limit: {type: integer, description: "Number of search results."} + start_date: {type: string} + end_date: {type: string} + required: [query] + steps: + - backend: search_step + vector_weight: 0.0 + expand_links: false + agentic_answer: + backend: base + description: "Answer a MobileMem question through a fresh evidence retrieval Agent." + include_images: true + include_source_sessions: true + parameters: + type: object + properties: + query: {type: string, description: "Official benchmark answer prompt including the query."} + query_time: {type: string, default: ""} + query_images: {type: array, items: {type: object}, default: []} + max_iters: {type: integer, minimum: 1, default: 30} + include_images: {type: boolean, default: true} + include_source_sessions: {type: boolean, default: true} + required: [query] + steps: + - backend: mobilemem_agentic_answer_step + agent_wrapper: bench + components: + file_store: + default: + embedding_store: null diff --git a/reme/components/as_decision_model/agentscope.py b/reme/components/as_decision_model/agentscope.py index 2e8ff025..a5699acf 100644 --- a/reme/components/as_decision_model/agentscope.py +++ b/reme/components/as_decision_model/agentscope.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter from .base_as_decision_model import BaseAsDecisionModel from ..component_registry import R +from ...utils.async_utils import complete_in_thread if TYPE_CHECKING: from agentscope.classifier import ClassifierModelBase, ClassifierResponse @@ -31,8 +32,8 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): """Wrap AgentScope's TypeSafe decision model with lazy provider loading. ``questions`` accepts AgentScope question objects or JSON dictionaries with - a ``type`` discriminator (``binary``, ``choice``, or ``score``). Only this - component's configuration supplies credentials and transport options. + a ``type`` discriminator (``binary``, ``choice``, or ``score``). Hosts may inject a borrowed model + through ``model``; otherwise this component constructs and owns its client. """ def __init__( @@ -53,10 +54,31 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): retry_delay=retry_delay, ) self._model: "ClassifierModelBase | None" = None + self._owns_model = False self._active_calls = 0 self._idle = asyncio.Event() self._idle.set() + @property + def model(self) -> "ClassifierModelBase | None": + """Return the current instance without constructing a provider client.""" + return self._model + + @model.setter + def model(self, value: "ClassifierModelBase | None") -> None: + """Borrow a host-owned instance, including via Application.update_component. + + Close the component before replacing an internally owned client. An + active request must finish before any replacement or detachment. + """ + if value is self._model: + return + if value is not None and not callable(value): + raise TypeError("An injected decision model must be callable.") + if self._active_calls or self._owns_model: + raise RuntimeError("Close the decision component before replacing its active or owned model.") + self._model = value + @staticmethod def _load_decision_model(): """Load the optional AgentScope adapter on the first decision request.""" @@ -96,17 +118,19 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): raise ValueError("Decision model inference requires DECISION_MODEL_API_KEY or credential.api_key.") if not credential.get("base_url"): credential["base_url"] = None + # Reserve ownership before yielding so host injection cannot race + # initialization. The worker publishes the model to this component + # before cancellation can release the lifecycle lock. + self._owns_model = True try: - credential_cls = import_module("agentscope.credential").TypeSafeCredential - self._model = model_cls( - credential=credential_cls(**credential), - **self.options.model_dump(exclude_none=True), - ) + await complete_in_thread(self._create_model, model_cls, credential) except ImportError as exc: raise ImportError( "Decision model inference requires a compatible typesafe-sdk. " "Install it with `pip install 'reme-ai[decision]'`.", ) from exc + finally: + self._owns_model = self._model is not None model = self._model self._active_calls += 1 self._idle.clear() @@ -119,11 +143,33 @@ class AgentScopeAsDecisionModel(BaseAsDecisionModel): if self._active_calls == 0: self._idle.set() + def _create_model(self, model_cls, credential) -> None: + """Construct and retain the owned client, including synchronous TLS setup.""" + credential_cls = import_module("agentscope.credential").TypeSafeCredential + self._model = model_cls( + credential=credential_cls(**credential), + **self.options.model_dump(exclude_none=True), + ) + async def _close(self) -> None: + """Finish draining and cleanup before propagating shutdown cancellation.""" + cleanup = asyncio.create_task(self._release_model()) + cancellation = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as exc: + # Keep the lifecycle lock until cleanup finishes, even after + # repeated cancellation. No cleanup task outlives close(). + cancellation = exc + cleanup.result() + if cancellation is not None: + raise cancellation + + async def _release_model(self) -> None: """Release the owned TypeSafe client and allow a later fresh start.""" - try: - await self._idle.wait() - finally: - model, self._model = self._model, None - if model is not None: - await model.client.aclose() + await self._idle.wait() + model, owned = self._model, self._owns_model + self._model, self._owns_model = None, False + if model is not None and owned: + await model.client.aclose() diff --git a/reme/components/embedding_store/_embedding_types.py b/reme/components/embedding_store/_embedding_types.py new file mode 100644 index 00000000..69aebebe --- /dev/null +++ b/reme/components/embedding_store/_embedding_types.py @@ -0,0 +1,109 @@ +"""Internal embedding results and normalized provider errors.""" + +import re +from dataclasses import dataclass + +_PROVIDER_WIDE_STATUS_CODES = frozenset({401, 403, 404}) + + +# AgentScope's DashScope wrapper serializes the response into a RuntimeError. +_STATUS_IN_MESSAGE = re.compile(r'"status_code"\s*:\s*(\d{3})') + + +_CODE_IN_MESSAGE = re.compile(r'"code"\s*:\s*"([^"]+)"') + + +_INPUT_DEPENDENT_ERROR_TYPES = frozenset( + { + "ContextLengthError", + "ContextWindowExceededError", + "InputLengthError", + "TokenLimitError", + }, +) + + +_INPUT_DEPENDENT_MARKERS = ( + "context length", + "context_length", + "context window", + "input length", + "input is too long", + "maximum context", + "reduce the length", + "token limit", + "too many tokens", + "too long", +) + + +@dataclass(frozen=True) +class _EmbeddingResult: + """Keep recovered vectors together with the failure state that ends isolation.""" + + embeddings: list[list[float] | None] | None = None + terminal_failure: bool = False + + +@dataclass(frozen=True) +class _EmbeddingError: + """Normalize provider error shapes once before applying the retry policy.""" + + status: int | None + code: str | None + text: str + error_type: str + + @classmethod + def from_exception(cls, error: Exception) -> "_EmbeddingError": + """Read attributes, structured bodies, and serialized DashScope responses.""" + message = str(error) + body = getattr(error, "body", None) + status = None + for value in ( + getattr(error, "status_code", None), + getattr(error, "code", None), + getattr(getattr(error, "response", None), "status_code", None), + ): + # Integer codes are HTTP statuses; strings name business errors. + if isinstance(value, int) and not isinstance(value, bool): + status = value + break + if status is None and (match := _STATUS_IN_MESSAGE.search(message)): + status = int(match.group(1)) + + code = getattr(error, "code", None) + if not isinstance(code, str) or not code: + details = body.get("error", body) if isinstance(body, dict) else None + code = details.get("code") if isinstance(details, dict) else None + if not isinstance(code, str) or not code: + match = _CODE_IN_MESSAGE.search(message) + code = match.group(1) if match else None + text = f"{message} {body}" if isinstance(body, dict) else message + return cls(status, code, text.lower(), type(error).__name__) + + @property + def is_insufficient_quota(self) -> bool: + """Quota exhaustion takes precedence over a generic HTTP 429.""" + return self.code == "insufficient_quota" + + @property + def is_rate_limited(self) -> bool: + """Recognize rate limits while preserving the separate quota policy.""" + return not self.is_insufficient_quota and (self.code == "rate_limit_exceeded" or self.status == 429) + + @property + def is_provider_wide(self) -> bool: + """Credentials, missing routes, and server outages cannot be isolated per input.""" + return ( + self.status is not None and (self.status in _PROVIDER_WIDE_STATUS_CODES or 500 <= self.status < 600) + ) or self.error_type in ("AuthenticationError", "PermissionDeniedError", "NotFoundError", "InternalServerError") + + @property + def is_input_dependent(self) -> bool: + """Only input-length rejections may split; a 413 always requires a text signal.""" + if self.status not in (None, 400, 413): + return False + if self.status != 413 and self.error_type in _INPUT_DEPENDENT_ERROR_TYPES: + return True + return any(marker in self.text for marker in _INPUT_DEPENDENT_MARKERS) diff --git a/reme/components/embedding_store/base_embedding_store.py b/reme/components/embedding_store/base_embedding_store.py index bb7703e2..f499b0eb 100644 --- a/reme/components/embedding_store/base_embedding_store.py +++ b/reme/components/embedding_store/base_embedding_store.py @@ -36,7 +36,7 @@ class BaseEmbeddingStore(BaseComponent): self.health_check_timeout = health_check_timeout self.is_healthy: bool = True - def _truncate(self, text: str) -> str: + def _truncate(self, text: str, *, max_length: int | None = None) -> str: """Truncate text using a CJK-aware character budget. ASCII text keeps its historical character limit. For non-ASCII text, @@ -44,8 +44,12 @@ class BaseEmbeddingStore(BaseComponent): characters cost 1.5 units because they commonly consume more embedding tokens. The estimate reserves a 5% safety margin, and integer half-units avoid floating-point boundary errors. + + ``max_length`` overrides the configured ``max_input_length`` for this call only, so a + caller can re-prepare a rejected input under a smaller budget without mutating shared + configuration or changing the vector space. """ - limit = max(0, self.max_input_length) + limit = max(0, self.max_input_length if max_length is None else max_length) if text.isascii(): return text[:limit] diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index 4fad8361..14e5cc9c 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np +from ._embedding_types import _EmbeddingError, _EmbeddingResult from .base_embedding_store import BaseEmbeddingStore from ..component_registry import R from ..as_embedding import BaseAsEmbedding @@ -15,6 +16,26 @@ Miss = tuple[int, str, str] # (result_index, text, cache_key) _MAX_VECTOR_SPACE_ATTEMPTS = 3 +# Match SDK/httpx transport exceptions without importing optional providers. +_TRANSPORT_FAILURE_TYPES = frozenset( + { + "APIConnectionError", + "APITimeoutError", + "ConnectError", + "ConnectTimeout", + "PoolTimeout", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "WriteError", + "WriteTimeout", + }, +) + +# A client retry budget only applies to the transport exceptions it retries. +_SDK_TRANSPORT_RETRY_TYPES = frozenset({"APIConnectionError", "APITimeoutError"}) + + @R.register("local") class LocalEmbeddingStore(BaseEmbeddingStore): """Embedding store with LRU cache, disk persistence, and serial batching. @@ -162,7 +183,8 @@ class LocalEmbeddingStore(BaseEmbeddingStore): async def _compute_batch(self, batch: list[Miss], **kwargs) -> list[tuple[int, str, np.ndarray]]: texts = [text for _, text, _ in batch] - embeddings = await self._call_with_retry(texts, **kwargs) + result = await self._call_with_retry(texts, **kwargs) + embeddings = result.embeddings if not embeddings or len(embeddings) != len(texts): return [] out: list[tuple[int, str, np.ndarray]] = [] @@ -178,70 +200,146 @@ class LocalEmbeddingStore(BaseEmbeddingStore): if bad_dims: details = ", ".join(f"{count} with dim {dim}" for dim, count in sorted(bad_dims.items())) self.logger.error(f"Embedding dimension mismatch in batch: expected {self.dimensions}; rejected {details}") - if out: - self.is_healthy = True - else: - self.is_healthy = False + # Retain partial vectors without hiding the outage that ended isolation. + self.is_healthy = bool(out) and not result.terminal_failure return out - async def _call_with_retry(self, texts: list[str], **kwargs) -> list[list[float] | None] | None: + async def _call_with_retry(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Retry failures owned by the store and isolate input-length rejections.""" + shortened = False for attempt in range(self.max_retries): try: - result = await self.as_embedding(texts, **kwargs) - if result and len(result) == len(texts): - return result - except (TimeoutError, ConnectionError, OSError): - if attempt < self.max_retries - 1: - await asyncio.sleep(2**attempt) + embeddings = await self.as_embedding(texts, **kwargs) + if embeddings and len(embeddings) == len(texts): + return _EmbeddingResult(embeddings) except Exception as error: - if self._is_rate_limited(error): + if self._is_transport_failure(error): + # Preserve recovery where the provider has no matching retry policy; + # otherwise leave its budget intact instead of adding another loop. + if self._provider_owns_transport_retry(error): + self.logger.exception( + "Embedding request failed: transport error owned by the provider, not splitting", + ) + break + if attempt < self.max_retries - 1: + await asyncio.sleep(2**attempt) + continue + self.logger.exception("Embedding request failed: transport error after exhausting retries") + break + + details = _EmbeddingError.from_exception(error) + if details.is_rate_limited: if attempt < self.max_retries - 1: delay = 2**attempt self.logger.warning(f"Embedding rate limited; retrying in {delay:.1f}s") await asyncio.sleep(delay) continue self.logger.exception("Embedding request failed after exhausting rate-limit retries") + break + if details.is_insufficient_quota: + if self.quota_retry_delay is not None and attempt < self.max_retries - 1: + self.logger.warning(f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s") + await asyncio.sleep(self.quota_retry_delay) + continue + self.logger.exception("Embedding request failed: quota exhausted") + break + if details.is_provider_wide: + # Server-error retries belong to the wrapper/SDK, not to item isolation. + self.logger.exception("Embedding request failed: provider-wide error, not splitting") + break + if details.is_input_dependent: + # Retry one item with half the character budget without changing its source text. + # Require a remaining attempt so exhausted input failures stay non-terminal. + if not shortened and len(texts) == 1 and attempt < self.max_retries - 1: + shorter = self._truncate(texts[0], max_length=max(1, self.max_input_length // 2)) + if shorter != texts[0]: + self.logger.warning( + f"Embedding input rejected as too long; retrying with a smaller budget " + f"(chars={len(texts[0])} -> {len(shorter)})", + ) + texts = [shorter] + shortened = True + await asyncio.sleep(2**attempt) + continue + self.logger.exception("Embedding request failed: input-dependent error") self.is_healthy = False - return None - if ( - self.quota_retry_delay is not None - and self._is_insufficient_quota(error) - and attempt < self.max_retries - 1 - ): - self.logger.warning( - f"Embedding quota exhausted; retrying in {self.quota_retry_delay:.1f}s", - ) - await asyncio.sleep(self.quota_retry_delay) - continue - self.logger.exception("Embedding request failed") - self.is_healthy = False - return None + if len(texts) > 1: + return await self._embed_individually(texts, **kwargs) + return _EmbeddingResult() + self.logger.exception("Embedding request failed: unrecognized error, not splitting") + break self.is_healthy = False - return None + return _EmbeddingResult(terminal_failure=True) + + async def _embed_individually(self, texts: list[str], **kwargs) -> _EmbeddingResult: + """Keep healthy siblings, stopping when a failure is no longer specific to an input.""" + embeddings: list[list[float] | None] = [] + for index, text in enumerate(texts): + result = await self._call_with_retry([text], **kwargs) + if result.embeddings: + embeddings.append(result.embeddings[0]) + continue + self.logger.error(f"Embedding request failed for a single item (chars={len(text)})") + embeddings.append(None) + if result.terminal_failure: + self.logger.error( + f"Stopping per-item isolation after {index + 1}/{len(texts)} items: " + "the failure is not specific to any input", + ) + embeddings.extend([None] * (len(texts) - index - 1)) + return _EmbeddingResult(embeddings, terminal_failure=True) + return _EmbeddingResult(embeddings) @staticmethod - def _is_rate_limited(error: Exception) -> bool: - """Recognize an OpenAI-compatible 429 response without importing a provider SDK.""" - if LocalEmbeddingStore._is_insufficient_quota(error): - return False - if getattr(error, "status_code", None) == 429: + def _is_transport_failure(error: Exception) -> bool: + """Recognize transport errors that must never trigger per-item isolation.""" + if isinstance(error, (TimeoutError, ConnectionError, OSError)): return True - body = getattr(error, "body", None) - if not isinstance(body, dict): + return any(cls.__name__ in _TRANSPORT_FAILURE_TYPES for cls in type(error).__mro__) + + def _provider_owns_transport_retry(self, error: Exception) -> bool: + """Require both a matching exception policy and an enabled retry budget.""" + model = getattr(self.as_embedding, "model", None) + if model is None: return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "rate_limit_exceeded" + policy = self._declared_retry_policy(model) + if policy: + max_retries = getattr(model, "max_retries", None) + if isinstance(max_retries, int) and max_retries > 0 and self._policy_covers(policy, error): + return True + return self._sdk_owns_transport_retry(model, error) @staticmethod - def _is_insufficient_quota(error: Exception) -> bool: - """Recognize OpenAI-compatible quota errors without importing a provider SDK.""" - if getattr(error, "code", None) == "insufficient_quota": - return True - body = getattr(error, "body", None) - if not isinstance(body, dict): + def _declared_retry_policy(model) -> tuple[type[Exception], ...] | None: + """Return the model's declared retryable types, or ``None`` when it declares none.""" + retryable = getattr(model, "_get_retryable_exceptions", None) + if not callable(retryable): + return None + try: + return tuple(retryable()) # pylint: disable=not-callable + except Exception: # pragma: no cover - a broken policy owns nothing + return None + + @staticmethod + def _policy_covers(policy: tuple[type[Exception], ...], error: Exception) -> bool: + """Return whether the declared policy actually retries this exception.""" + try: + return isinstance(error, policy) + except TypeError: # pragma: no cover - a malformed policy covers nothing return False - details = body.get("error", body) - return isinstance(details, dict) and details.get("code") == "insufficient_quota" + + @staticmethod + def _sdk_owns_transport_retry(model, error: Exception) -> bool: + """Check the SDK's budget and supported transport errors independently of the model.""" + if not LocalEmbeddingStore._is_transport_failure(error): + return False + client = getattr(model, "client", None) + if client is None: + return False + max_retries = getattr(client, "max_retries", None) + if not isinstance(max_retries, int) or max_retries <= 0: + return False + return any(cls.__name__ in _SDK_TRANSPORT_RETRY_TYPES for cls in type(error).__mro__) def _validate_dim(self, emb: np.ndarray) -> bool: """Return whether an embedding exactly matches the configured dimension.""" diff --git a/reme/components/file_chunker/markdown_file_chunker.py b/reme/components/file_chunker/markdown_file_chunker.py index 06a07793..e8fe3277 100644 --- a/reme/components/file_chunker/markdown_file_chunker.py +++ b/reme/components/file_chunker/markdown_file_chunker.py @@ -126,6 +126,8 @@ class MarkdownFileChunker(DefaultFileChunker): async def chunk(self, path: str | Path) -> tuple[FileNode, list[FileChunk]]: file_path = Path(path) rel_path = self.to_workspace_relative(path) + # A write during the read must stay detectable by the next scan. + stat = file_path.stat() text = await self._read_text_for_indexing(file_path) front_matter, content, line_offset = self._parse_front_matter(text) @@ -157,7 +159,7 @@ class MarkdownFileChunker(DefaultFileChunker): node = FileNode( path=rel_path, - st_mtime=file_path.stat().st_mtime, + st_mtime=stat.st_mtime, chunk_ids=[chunk.id for chunk in chunks], links=links, front_matter=front_matter, diff --git a/reme/components/file_store/faiss_local_file_store.py b/reme/components/file_store/faiss_local_file_store.py index 40dfd654..c78ae41d 100644 --- a/reme/components/file_store/faiss_local_file_store.py +++ b/reme/components/file_store/faiss_local_file_store.py @@ -1,7 +1,6 @@ """FAISS-backed file store: chunk JSONL stays authoritative; FAISS HNSW replaces the linear vector scan.""" import asyncio -import hashlib import json from collections.abc import Callable from contextlib import suppress @@ -389,28 +388,6 @@ class FaissLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory - (possibly float32 from the provider; assignment bypasses the pydantic - validator) or has round-tripped through the JSONL. Written into the - idmap sidecar at dump time and recomputed from ``self.file_chunks`` at - load time: a mismatch means the sidecar vectors belong to a different - chunk generation than the authoritative JSONL, which the live-id set - check alone cannot detect for same-ID in-place updates. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach FAISS state (sidecar or rebuild).""" await super().load() diff --git a/reme/components/file_store/local_file_store.py b/reme/components/file_store/local_file_store.py index e5448f65..ee36674b 100644 --- a/reme/components/file_store/local_file_store.py +++ b/reme/components/file_store/local_file_store.py @@ -3,6 +3,7 @@ import asyncio import base64 import datetime +import hashlib import heapq import json import time @@ -43,6 +44,7 @@ _SEARCH_FILTER_RESERVED_KEYS = frozenset( "strict_date_filter", }, ) +_FAILED_CHUNK_PREVIEW = 5 @R.register("local") @@ -106,6 +108,23 @@ class LocalFileStore(BaseFileStore): return 0 return max(0, int(getattr(self.embedding_store, "dimensions", 0))) + def _chunks_embedding_digest(self) -> str: + """Order-independent digest of the live (chunk_id, embedding) set. + + Float16 bytes match the authoritative chunk JSONL serialization even + before a fresh provider vector has round-tripped through persistence. + Vector backends use this digest to reject stale index sidecars. + """ + digest = hashlib.sha256() + eligible = sorted( + cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) + ) + for cid in eligible: + digest.update(cid.encode("utf-8")) + digest.update(b"\x00") + digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) + return digest.hexdigest() + # -- lifecycle ------------------------------------------------------------ async def _start(self) -> None: @@ -440,7 +459,12 @@ class LocalFileStore(BaseFileStore): if chunk.text and not self._embedding_dim_matches(chunk.embedding) ] if missing: - raise RuntimeError(f"embedding reindex incomplete: {len(missing)} chunks failed") + preview = ", ".join(missing[:_FAILED_CHUNK_PREVIEW]) + if len(missing) > _FAILED_CHUNK_PREVIEW: + preview += f" (+{len(missing) - _FAILED_CHUNK_PREVIEW} more)" + raise RuntimeError( + f"embedding reindex incomplete: {len(missing)} chunks failed: {preview}", + ) await self._finalize_embedding_reindex() await self._dump_owned_state() if self.embedding_store is not None: @@ -544,6 +568,7 @@ class LocalFileStore(BaseFileStore): return processed = 0 + filled = 0 batch_count = 0 embedding_started_at = time.monotonic() next_percent = _PROGRESS_LOG_PERCENT_STEP @@ -552,10 +577,11 @@ class LocalFileStore(BaseFileStore): await self.embedding_store.get_node_embeddings(batch) self._drop_stale_embeddings(batch, "backfill") processed += len(batch) + filled += sum(1 for chunk in batch if chunk.embedding is not None) batch_count += 1 next_percent = self._log_progress("embedding backfill", processed, total, next_percent) self.logger.info( - f"{self.name}: embedding batches complete: processed={processed}/{total}, " + f"{self.name}: embedding batches complete: processed={processed}/{total}, filled={filled}, " f"batches={batch_count}, elapsed={time.monotonic() - embedding_started_at:.3f}s", ) except asyncio.CancelledError: diff --git a/reme/components/file_store/zvec_local_file_store.py b/reme/components/file_store/zvec_local_file_store.py index 84db2dbd..8d81609d 100644 --- a/reme/components/file_store/zvec_local_file_store.py +++ b/reme/components/file_store/zvec_local_file_store.py @@ -1,6 +1,5 @@ """Zvec-backed file store: chunk JSONL stays authoritative; a zvec collection replaces the linear vector scan.""" -import hashlib import json import shutil import time @@ -227,26 +226,6 @@ class ZvecLocalFileStore(LocalFileStore): # -- persistence ------------------------------------------------------ - def _chunks_embedding_digest(self) -> str: - """Order-independent digest of the live (chunk_id, embedding) set. - - Hashes float16 canonical bytes (the chunk JSONL serialization dtype) so - the value is identical whether an embedding sits fresh in memory or has - round-tripped through the JSONL. Written into the sidecar at dump time - and recomputed from ``self.file_chunks`` at load time: a mismatch means - the collection belongs to a different chunk generation than the - authoritative JSONL. - """ - digest = hashlib.sha256() - eligible = sorted( - cid for cid, chunk in self.file_chunks.items() if self._embedding_dim_matches(chunk.embedding) - ) - for cid in eligible: - digest.update(cid.encode("utf-8")) - digest.update(b"\x00") - digest.update(np.asarray(self.file_chunks[cid].embedding, dtype=np.float16).tobytes()) - return digest.hexdigest() - async def load(self) -> None: """Load chunks via the parent, then attach the zvec collection (open or rebuild).""" await super().load() diff --git a/reme/config/default.yaml b/reme/config/default.yaml index dbab8ac2..347f5a1d 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -16,6 +16,7 @@ jobs: dispatch_steps: [update_index_step] - backend: watch_changes_step watch_mode: awatch # Set to watch to isolate the blocking wait from AnyIO's shared thread limiter. + monitor_type: file_store dispatch_steps: - backend: update_index_step persist: False @@ -37,6 +38,8 @@ jobs: - auto_text_resource_step - backend: watch_changes_step watch_mode: awatch + monitor_type: file_catalog + monitor_name: resource dispatch_steps: - backend: update_catalog_step file_catalog: resource @@ -59,6 +62,8 @@ jobs: - backend: log_changes_step - backend: watch_changes_step watch_mode: awatch + monitor_type: file_catalog + monitor_name: digest dispatch_steps: - backend: update_catalog_step file_catalog: digest @@ -173,6 +178,9 @@ jobs: parameters: type: object properties: + request_id: + type: string + description: "Optional stable batch ID (1-128 characters); replay completed requests without rerunning the memory agent" messages: type: array description: "messages" diff --git a/reme/steps/compaction/__init__.py b/reme/steps/compaction/__init__.py index 57aeeab5..eaeb79b4 100644 --- a/reme/steps/compaction/__init__.py +++ b/reme/steps/compaction/__init__.py @@ -1,5 +1,6 @@ -"""Tool-history compaction Steps.""" +"""Tool-history compaction and shared host validation helpers.""" from .compact_context import CompactContextStep +from .messages import ToolPair, collect_pairs, preview_result -__all__ = ["CompactContextStep"] +__all__ = ["CompactContextStep", "ToolPair", "collect_pairs", "preview_result"] diff --git a/reme/steps/evolve/auto_memory.py b/reme/steps/evolve/auto_memory.py index ee38fcff..5e513fab 100644 --- a/reme/steps/evolve/auto_memory.py +++ b/reme/steps/evolve/auto_memory.py @@ -3,6 +3,8 @@ import base64 import binascii import datetime +import hashlib +import json from pathlib import Path import re from urllib.parse import urlsplit @@ -21,6 +23,7 @@ from ..file_io import validate_filename_component, validate_session_id from ..file_io._path import IMAGE_MIME_BY_EXT, _check_path_permission, resolve_path from ..index import normalize_posix_path from ...components import R +from ...schema import Response from ...utils.wikilink_handler import WikilinkHandler _SESSION_ID_KEY = "session_id" @@ -458,8 +461,68 @@ class AutoMemoryStep(BaseStep): content=[images[part] if part in images else TextBlock(text=part) for part in parts if part], ) - # pylint: disable=too-many-return-statements async def execute(self): + """Replay completed request IDs without repeating the memory agent.""" + assert self.context is not None + request_id = self.context.get("request_id") + if request_id is None: + return await self._execute() + if not isinstance(request_id, str) or not 1 <= len(request_id) <= 128: + self.context.response.success = False + self.context.response.answer = "Error: request_id must be a string of 1 to 128 characters" + return + + payload = { + key: self.context.get(key) for key in ("session_id", "messages", "date", "memory_hint", "include_images") + } + if payload["messages"]: + # Keep caller-provided IDs/timestamps; parsing dictionaries would generate defaults on every retry. + payload["messages"] = [ + message.model_dump(mode="json") if isinstance(message, Msg) else message + for message in payload["messages"] + ] + fingerprint = hashlib.sha256( + json.dumps(payload, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode(), + ).hexdigest() + session_key = hashlib.sha256(str(payload["session_id"]).encode()).hexdigest() + request_key = hashlib.sha256(request_id.encode()).hexdigest() + directory = f"{self.config_value('metadata_dir')}/auto_memory_requests/{session_key}" + workspace = self.file_store.workspace_path + allowed = self.context.get("_allowed_paths") + lock_path = _checked_write_path(workspace, f"{directory}/.lock", allowed) + async with await get_path_lock(lock_path): + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + if path.exists(): + record = json.loads(path.read_text(encoding="utf-8")) + if record["fingerprint"] != fingerprint: + self.context.response.success = False + self.context.response.answer = "Error: request_id was already used for different input" + return + response = Response.model_validate(record["response"]) + self.context.response.answer = response.answer + self.context.response.success = response.success + self.context.response.metadata.update(response.metadata) + self.context["changes"] = record["changes"] + return + await self._execute() + if self.context.response.success: + path = _checked_write_path(workspace, f"{directory}/{request_key}.json", allowed) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = _checked_write_path(workspace, f"{directory}/{request_key}.tmp", allowed) + await write_file_safe( + temporary, + json.dumps( + { + "fingerprint": fingerprint, + "response": self.context.response.model_dump(mode="json"), + "changes": self.context.get("changes") or [], + }, + ), + ) + temporary.replace(path) + + # pylint: disable=too-many-return-statements + async def _execute(self): assert self.context is not None self.context["changes"] = [] raw_messages = self.context.get("messages") or [] diff --git a/reme/steps/evolve/auto_tag.py b/reme/steps/evolve/auto_tag.py index 05ea0604..823d59cb 100644 --- a/reme/steps/evolve/auto_tag.py +++ b/reme/steps/evolve/auto_tag.py @@ -123,6 +123,8 @@ class AutoTagStep(BaseStep): path = self.workspace_path / target.path metadata = dict(frontmatter.loads(path.read_text(encoding="utf-8")).metadata or {}) + if tag_key not in metadata: + raise RuntimeError(f"Auto-tag agent did not write the required {tag_key!r} field") normalized = normalize_memory_tags( metadata.get(tag_key), max_tags_per_file=tag_index.max_tags_per_file, diff --git a/reme/steps/evolve/dream/extract.py b/reme/steps/evolve/dream/extract.py index 59f6b9c9..d3d4c3b7 100644 --- a/reme/steps/evolve/dream/extract.py +++ b/reme/steps/evolve/dream/extract.py @@ -7,7 +7,7 @@ from ...file_io import refresh_day_index from .._evolve import agent_reply_result_text from ....components import R from ....enumeration import DreamBucketEnum -from ....schema import DreamState +from ....schema import DreamExtractOutput, DreamState from .utils import ( clean_paths, daily_dir, @@ -113,32 +113,43 @@ class DreamExtractStep(BaseStep): return self._finish(state, False, state.errors[-1]) self.logger.info(f"[{self.name}] agent start changed={len(changed)} dates={len(dates)}") - raw_result, meta = "", {} + user_message = self.prompt_format( + "extract_user_message", + date=day, + dates_json=json.dumps(dates, ensure_ascii=False, indent=2), + hint=hint or "(none)", + max_units=max_units, + changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), + material_blob=pack_paths(workspace, changed), + ) + retry_feedback = "" + output_schema = DreamExtractOutput.model_json_schema() + output_schema["required"] = ["units"] for attempt in range(2): try: result = await self.agent_wrapper.reply( - self.prompt_format( - "extract_user_message", - date=day, - dates_json=json.dumps(dates, ensure_ascii=False, indent=2), - hint=hint or "(none)", - max_units=max_units, - changed_paths_json=json.dumps(changed, ensure_ascii=False, indent=2), - material_blob=pack_paths(workspace, changed), - ), + user_message + retry_feedback, system_prompt=self.prompt_format( "extract_system_prompt", workspace_dir=str(workspace), buckets=", ".join(bucket.value for bucket in DreamBucketEnum), max_units=max_units, + output_schema_json=json.dumps(output_schema, ensure_ascii=False, indent=2), ), job_tools=list(_TOOLS), ) self.logger.info(f"[{self.name}] agent done has_result={bool(result.get('result'))}") raw_result = agent_reply_result_text(result) - meta = parse_structured_reply(raw_result) + state.extract_summary = raw_result + meta = parse_structured_reply(raw_result, strict=True) + self.clean_output(state, meta, max_units=max_units) + break except Exception as e: # noqa: BLE001 if attempt == 0: + retry_feedback = "\n\n" + self.prompt_format( + "extract_retry_message", + error=f"{type(e).__name__}: {e}", + ) self.logger.warning( f"[{self.name}] extract attempt 1 returned no usable receipt; retrying once: " f"{type(e).__name__}: {e}", @@ -150,20 +161,7 @@ class DreamExtractStep(BaseStep): self.logger.error(f"[{self.name}] {error}") return self._finish(state, False, error) - units = meta.get("units") if "units" in meta else meta.get("memory_units") - if isinstance(units, list): - break - if attempt == 0: - self.logger.warning(f"[{self.name}] extract attempt 1 returned an unusable receipt; retrying once") - continue - # Keep the warning-only result checkpointable after one retry so a bad source cannot loop forever. - warning = "dream extract skipped unusable agent receipt after retry; expected a units list" - state.warnings.append(warning) - self.logger.warning(f"[{self.name}] {warning}") - self.logger.info(f"[{self.name}] parse done keys={','.join(sorted(meta.keys())) if meta else '(none)'}") - self.clean_output(state, meta, max_units=max_units) - state.extract_summary = raw_result answer = f"Extracted {len(state.units)} unit(s) from {len(changed)} changed file(s) across {len(dates)} day(s)" return self._finish(state, True, answer) @@ -177,25 +175,34 @@ class DreamExtractStep(BaseStep): return out def clean_output(self, state: DreamState, meta: dict, max_units: int | None = None) -> None: - """Clean up output""" + """Validate the whole receipt before accepting any units or applying the cap.""" + units = meta.get("units") if "units" in meta else meta.get("memory_units") + if not isinstance(units, list): + raise ValueError("expected a units list (use units: [] only when nothing is worth remembering)") + normalized = [] + for raw in units: + if isinstance(raw, dict) and isinstance(raw.get("bucket"), str): + raw = {**raw, "bucket": raw["bucket"].strip()} + if ( + isinstance(raw, dict) + and "bucket" in raw + and raw["bucket"] not in {bucket.value for bucket in DreamBucketEnum} + ): + self.logger.warning( + f"[{self.name}] unit {raw.get('name')!r} emitted an unknown bucket; routing to wiki", + ) + raw = {**raw, "bucket": DreamBucketEnum.WIKI.value} + normalized.append(raw) + output = DreamExtractOutput.model_validate({"units": normalized}) allowed = set(state.changed_paths) - for raw in meta.get("units") or meta.get("memory_units") or []: - if max_units is not None and len(state.units) >= max_units: - break - if not isinstance(raw, dict): - continue - name = str(raw.get("name") or "").strip() - summary = str(raw.get("summary") or "").strip() - raw_bucket = str(raw.get("bucket") or "").strip() - paths = clean_paths(raw.get("paths"), allowed) + cleaned = [] + for index, unit in enumerate(output.units): + name, summary = unit.name.strip(), unit.summary.strip() + paths = clean_paths(unit.paths, allowed) if not name or not summary or not paths: - continue - try: - bucket = DreamBucketEnum(raw_bucket).value - except ValueError: - self.logger.warning(f"[{self.name}] unit {name!r} emitted bucket {raw_bucket!r}; routing to wiki") - bucket = DreamBucketEnum.WIKI.value - state.units.append({"name": name, "bucket": bucket, "summary": summary, "paths": paths}) + raise ValueError(f"units[{index}] requires non-empty name, summary, and paths from changed_paths") + cleaned.append({"name": name, "bucket": unit.bucket.value, "summary": summary, "paths": paths}) + state.units.extend(cleaned if max_units is None else cleaned[: max(max_units - len(state.units), 0)]) def _finish(self, state: DreamState, success: bool, answer: str): assert self.context is not None diff --git a/reme/steps/evolve/dream/extract.yaml b/reme/steps/evolve/dream/extract.yaml index b35847fe..2cdcb942 100644 --- a/reme/steps/evolve/dream/extract.yaml +++ b/reme/steps/evolve/dream/extract.yaml @@ -61,12 +61,20 @@ extract_system_prompt: | ## Output Format - Return only one YAML or JSON object with this exact shape: - units: - - name: - bucket: procedure|personal|wiki - summary: - paths: [, ...] + Return only one valid JSON object, without Markdown fences or surrounding prose. + The units field is required and must be a list. Every unit must contain name, + bucket, summary, and paths. name and summary must be non-empty strings; paths + must be a non-empty list of strings from changed_paths. + Double-quote all keys and string values, and escape embedded double quotes, + backslashes, and newlines according to JSON rules. Code-like text containing + `key: value` must stay inside a quoted string; never emit YAML plain scalars. + Return {{"units": []}} only when there is no reusable memory to extract. + + JSON schema: + {output_schema_json} + + Valid example (replace paths with actual changed_paths): + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout needs `submodules: recursive` to fetch nested dependencies.", "paths": ["daily/2026-10-20/ci.md"]}}]}} extract_system_prompt_zh: | 你是 dream 全局抽取 agent。一起阅读所有发生变化的 daily 文件，输出一个由合并记忆 unit 组成的精简跨文件计划。 @@ -118,12 +126,39 @@ extract_system_prompt_zh: | ## 输出格式 - 只返回一个 YAML 或 JSON object，结构必须严格如下： - units: - - name: <短名称> - bucket: procedure|personal|wiki - summary: <基于证据的可复用抽象> - paths: [, ...] + 只返回一个合法的 JSON object，不要使用 Markdown 代码围栏或添加解释文字。 + 必须包含 units 字段，且它必须是 list。每个 unit 必须包含 name、bucket、summary 和 paths。 + name 和 summary 必须是非空字符串；paths 必须是非空字符串列表，且只能使用 changed_paths 中的路径。 + 所有键和字符串值都必须使用双引号；内部双引号、反斜杠和换行必须按 JSON 规则转义。 + 含有 `key: value` 的代码文本必须放在带引号的字符串内，禁止输出 YAML 无引号标量。 + 只有在没有可复用记忆时才返回 {{"units": []}}。 + + JSON schema： + {output_schema_json} + + 合法示例（路径须替换为实际 changed_paths）： + {{"units": [{{"name": "ci-submodules", "bucket": "procedure", "summary": "CI checkout 需要 `submodules: recursive`，以获取嵌套依赖。", "paths": ["daily/2026-10-20/ci.md"]}}]}} + +extract_retry_message: | + The previous extraction failed parsing or validation: + {error} + + Retry extraction from the changed material above and return a complete corrected + JSON object following the system prompt's schema. Include the units list and all + required unit fields. Double-quote every key and string value, including any + `key: value` text, and escape quotes, backslashes, and newlines. Do not use YAML, + Markdown fences, or explanatory prose. A format error does not mean there are no + reusable memories; use {{"units": []}} only when the material warrants it. + +extract_retry_message_zh: | + 上次抽取发生解析或校验错误： + {error} + + 请根据上面的变化内容重新抽取，按系统提示的 schema 返回完整、修正后的 JSON object。 + 必须包含 units 列表和每个 unit 的必填字段。所有键和字符串值都要加双引号， + 含 `key: value` 的文本也不例外；双引号、反斜杠和换行须转义。 + 不要使用 YAML、Markdown 代码围栏或解释文字。格式错误不代表没有可复用记忆； + 只有材料确实不值得记忆时才返回 {{"units": []}}。 extract_user_message: | date: {date} diff --git a/reme/steps/evolve/dream/finish.py b/reme/steps/evolve/dream/finish.py index 243a449a..980f93be 100644 --- a/reme/steps/evolve/dream/finish.py +++ b/reme/steps/evolve/dream/finish.py @@ -74,8 +74,11 @@ class DreamFinishStep(BaseStep): def render_summary(state: DreamState) -> str: """Render a concise user-facing summary.""" dates = ", ".join(state.dates or [state.date]) + title = "AutoDream failed" if state.errors else "AutoDream completed" + if state.warnings and not state.errors: + title += " with warnings" lines = [ - ("AutoDream completed with warnings" if state.warnings else "AutoDream completed"), + title, "", f"- Date: {state.date}", f"- Scan window: {dates}", diff --git a/reme/steps/evolve/dream/integrate.py b/reme/steps/evolve/dream/integrate.py index 12569e7e..f3946318 100644 --- a/reme/steps/evolve/dream/integrate.py +++ b/reme/steps/evolve/dream/integrate.py @@ -61,7 +61,7 @@ class DreamIntegrateStep(BaseStep): state = state_from_context(self) if not state.units: self.logger.info(f"[{self.name}] skip no units") - return self._finish(state, True, "No dream units to integrate") + return self._finish(state, not state.errors, "No dream units to integrate") if not llm_available(self): err = "no llm configured; dream integrate requires an LLM" state.errors.append(err) @@ -214,10 +214,10 @@ class DreamIntegrateStep(BaseStep): ) before = after continue - # After the bounded retry, checkpoint the source so malformed input cannot loop forever. - self._record_skipped(state, unit, bucket, paths, e) - self.logger.warning( - f"[{self.name}] unit {index}/{len(state.units)} skipped invalid receipt after retry: " + # Without a valid receipt or recoverable write, keep sources eligible for retry. + self._record_failure(state, unit, paths, e) + self.logger.error( + f"[{self.name}] unit {index}/{len(state.units)} failed invalid receipt after retry: " f"{type(e).__name__}: {e}", ) return @@ -339,18 +339,6 @@ class DreamIntegrateStep(BaseStep): note=f"Recovered from {type(error).__name__}: agent receipt unavailable", ) - @classmethod - def _record_skipped(cls, state, unit: dict, bucket: str, paths: list[str], error: Exception) -> None: - message = f"unit {unit.get('name', '')!r} skipped: unusable agent receipt ({type(error).__name__})" - key = cls._unit_key(unit, bucket, paths) - if not any( - cls._unit_key(item, str(item.get("bucket") or ""), [str(p) for p in item.get("paths") or []]) == key - for item in state.skipped_units - ): - state.skipped_units.append({**unit, "bucket": bucket, "paths": paths, "reason": message}) - if message not in state.warnings: - state.warnings.append(message) - @staticmethod def _record_failure(state, unit: dict, paths: list[str], error: Exception) -> None: message = f"{type(error).__name__}: {error}" diff --git a/reme/steps/evolve/dream/utils.py b/reme/steps/evolve/dream/utils.py index 2f41d27d..bce5c135 100644 --- a/reme/steps/evolve/dream/utils.py +++ b/reme/steps/evolve/dream/utils.py @@ -134,19 +134,25 @@ def previous_dates(day: str, n_days: int) -> list[str]: return [(base - dt.timedelta(days=i)).isoformat() for i in range(1, max(n_days, 0) + 1)] -def parse_structured_reply(text: str) -> dict: - """Parse a JSON/YAML object from an agent reply, including fenced blocks.""" +def parse_structured_reply(text: str, *, strict: bool = False) -> dict: + """Parse a JSON/YAML object; strict mode rejects lossy fallbacks with diagnostics.""" candidates = [text.strip()] candidates.extend(m.group(1).strip() for m in re.finditer(r"```(?:json|ya?ml)?\s*(.*?)```", text, re.S | re.I)) + error = "expected a JSON/YAML object" for raw in candidates: if not raw: continue try: data = yaml.safe_load(raw) - except yaml.YAMLError: + except yaml.YAMLError as exc: + error = f"{type(exc).__name__}: {exc}" + if strict: + continue data = _parse_scalar_mapping(raw) - if isinstance(data, dict) and data: + if isinstance(data, dict) and (data or strict): return data + if strict: + raise ValueError(f"Invalid structured reply: {error}") return {} diff --git a/reme/steps/file_io/_base.py b/reme/steps/file_io/_base.py new file mode 100644 index 00000000..7c42bc81 --- /dev/null +++ b/reme/steps/file_io/_base.py @@ -0,0 +1,13 @@ +"""Common response handling for file operations.""" + +from ..base_step import BaseStep + + +class _FileIOStep(BaseStep): + """Preserve the file-operation failure envelope without changing its payload.""" + + def _fail(self, message: str, **meta) -> None: + assert self.context is not None + self.context.response.success = False + self.context.response.answer = f"Error: {message}" + self.context.response.metadata.update(meta) diff --git a/reme/steps/file_io/_file_io.py b/reme/steps/file_io/_file_io.py index 82c5264d..22e945b0 100644 --- a/reme/steps/file_io/_file_io.py +++ b/reme/steps/file_io/_file_io.py @@ -1,7 +1,10 @@ """Encoding-aware file IO, output truncation, and per-path write locks.""" import asyncio +import os from pathlib import Path +from secrets import token_hex +from stat import S_IMODE from typing import Iterable import aiofiles @@ -10,6 +13,7 @@ import aiofiles.os from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES, TRUNCATION_NOTICE_MARKER from ...schema import FileChunk from ...utils import get_logger +from ...utils.async_utils import complete_in_thread logger = get_logger(log_to_file=False) @@ -152,7 +156,7 @@ async def detect_file_encoding(file_path, sniff_bytes: int = 8192) -> str: async def write_file_safe(file_path: Path, content: str | bytes, encoding: str = "utf-8") -> None: - """Write ``content`` to ``file_path`` in binary mode; creates parent dirs.""" + """Publish complete bytes atomically while retaining normal file permissions.""" file_path.parent.mkdir(parents=True, exist_ok=True) if isinstance(content, str): try: @@ -165,8 +169,32 @@ async def write_file_safe(file_path: Path, content: str | bytes, encoding: str = payload = content.encode("utf-8") else: payload = content - async with aiofiles.open(str(file_path), "wb") as f: - await f.write(payload) + + def publish(): + flags = os.O_WRONLY | getattr(os, "O_BINARY", 0) + # Check OS write access without truncating the existing bytes. + try: + existing_fd = os.open(file_path, flags) + except FileNotFoundError: + mode = None + else: + try: + mode = S_IMODE(os.fstat(existing_fd).st_mode) + finally: + os.close(existing_fd) + + temporary = file_path.parent / f".reme-{token_hex(16)}.tmp" + fd = os.open(temporary, flags | os.O_CREAT | os.O_EXCL, 0o600 if mode is not None else 0o666) + try: + with os.fdopen(fd, "wb") as handle: + handle.write(payload) + if mode is not None: + os.chmod(temporary, mode) + os.replace(temporary, file_path) + finally: + Path(temporary).unlink(missing_ok=True) + + await complete_in_thread(publish) # --------------------------------------------------------------------------- diff --git a/reme/steps/file_io/daily_write.py b/reme/steps/file_io/daily_write.py index 31ec050b..addff538 100644 --- a/reme/steps/file_io/daily_write.py +++ b/reme/steps/file_io/daily_write.py @@ -1,8 +1,8 @@ """Write a daily markdown note by dispatching the generic ``write`` step.""" +from ._base import _FileIOStep from ._daily_index import parse_daily_date, refresh_day_index, validate_session_id from ._path import validate_filename_component -from ..base_step import BaseStep from ..index import normalize_posix_path from ...components import R from ...steps.evolve import now @@ -11,16 +11,9 @@ _RESERVED_METADATA_KEYS = {"name", "description", "session_id", "source_conversa @R.register("daily_write_step") -class DailyWriteStep(BaseStep): +class DailyWriteStep(_FileIOStep): """Write ``daily//.md`` with conversation frontmatter.""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _session_dir(self) -> str: return normalize_posix_path(str(self.config_value("session_dir"))) diff --git a/reme/steps/file_io/edit.py b/reme/steps/file_io/edit.py index 2e412d54..31a0fd76 100644 --- a/reme/steps/file_io/edit.py +++ b/reme/steps/file_io/edit.py @@ -1,16 +1,56 @@ """Find-and-replace text in a markdown file body (front matter is preserved).""" +import re + import frontmatter import yaml +from ._base import _FileIOStep from ._file_io import get_path_lock, read_file_safe, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R +def _replace_normalized_newlines(body: str, old: str, new: str) -> tuple[str, int]: + """Find literal LF text, mapping its spans back without normalizing untouched text.""" + normalized = body.replace("\r\n", "\n").replace("\r", "\n") + position = normalized.find(old) + if position < 0: + return body, 0 + + # Only CRLF pairs change offsets. Consume them once as match boundaries advance; + # no per-character offset array or regex search for the user-supplied needle. + crlf_starts = (match.start() for match in re.finditer("\r\n", body)) + next_crlf = next(crlf_starts, len(body)) + removed = 0 + source_cursor = 0 + count = 0 + parts = [] + while position >= 0: + source_start = position + removed + while next_crlf < source_start: + removed += 1 + source_start += 1 + next_crlf = next(crlf_starts, len(body)) + + end = position + len(old) + source_end = end + removed + while next_crlf < source_end: + removed += 1 + source_end += 1 + next_crlf = next(crlf_starts, len(body)) + + parts.extend((body[source_cursor:source_start], new)) + source_cursor = source_end + count += 1 + position = normalized.find(old, end) + + parts.append(body[source_cursor:]) + return "".join(parts), count + + @R.register("edit_step") -class EditStep(BaseStep): +class EditStep(_FileIOStep): """Replace every occurrence of ``old`` with ``new`` inside the file body. The YAML front matter block (if any) is parsed out, kept verbatim and @@ -24,13 +64,6 @@ class EditStep(BaseStep): read-modify-write cycle against the same file (multi-worker / multi- process safety is out of scope).""" - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") @@ -89,13 +122,16 @@ class EditStep(BaseStep): body = raw_text not_found_msg = f"text to replace was not found in {target}" - if old_str not in body: + if "\r" in body and "\n" in old_str and "\r" not in old_str: + new_body, count = _replace_normalized_newlines(body, old_str, new_str) + else: + count = body.count(old_str) + new_body = body.replace(old_str, new_str) + + if count == 0: self._fail(not_found_msg, path=str(target)) return None - count = body.count(old_str) - new_body = body.replace(old_str, new_str) - if is_md and post is not None: post.content = new_body # Re-serialize: keep front matter when present, otherwise emit body alone diff --git a/reme/steps/file_io/list.py b/reme/steps/file_io/list.py index df217e81..7f62185e 100644 --- a/reme/steps/file_io/list.py +++ b/reme/steps/file_io/list.py @@ -19,8 +19,8 @@ from pathlib import Path from stat import S_ISREG from typing import Iterable +from ._base import _FileIOStep from ._path import resolve_path -from ..base_step import BaseStep from ...components import R # Default cap on returned items so huge workspaces don't blow up the response. @@ -28,17 +28,9 @@ DEFAULT_LIMIT = 100 @R.register("list_step") -class ListStep(BaseStep): +class ListStep(_FileIOStep): """Enumerate files under a directory in the workspace.""" - def _fail(self, message: str, **meta) -> None: - """Set a failed response (matches the read/edit/... fail envelope).""" - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - def _collect_params(self) -> tuple[str, bool, int, str, frozenset[str]]: """Read ``path`` / ``recursive`` / ``limit`` from context; coerce permissively.""" assert self.context is not None diff --git a/reme/steps/file_io/load.py b/reme/steps/file_io/load.py index 94bc23fc..4403c72e 100644 --- a/reme/steps/file_io/load.py +++ b/reme/steps/file_io/load.py @@ -2,24 +2,18 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import read_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R DEFAULT_EDITOR_MAX_BYTES = 5 * 1024 * 1024 @R.register("load_step") -class LoadStep(BaseStep): +class LoadStep(_FileIOStep): """Return complete text without the truncation used by agent-facing reads.""" - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/read.py b/reme/steps/file_io/read.py index 48f53aea..f8093487 100644 --- a/reme/steps/file_io/read.py +++ b/reme/steps/file_io/read.py @@ -2,9 +2,9 @@ from pathlib import Path +from ._base import _FileIOStep from ._file_io import read_file_lines_safe, read_file_safe, truncate_session_output, truncate_text_output from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ..index._source_format import is_session_path from ...components import R from ...constants import DEFAULT_MAX_BYTES, MAX_FILE_READ_BYTES @@ -12,7 +12,7 @@ from ...utils import expand_links, render_expansion_lines @R.register("read_step") -class ReadStep(BaseStep): +class ReadStep(_FileIOStep): """Read a markdown file. Optional `start_line`/`end_line` for ranged reads. Step-level attributes (``kwargs``, configured in yaml under ``steps:`` — @@ -37,12 +37,8 @@ class ReadStep(BaseStep): def _fail(self, message: str, **meta) -> None: """Mark the response failed and stash a human-readable error.""" - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" + super()._fail(message, **meta) self.logger.warning(f"[{self.name}] FAILED: {message} | meta={meta}") - if meta: - self.context.response.metadata.update(meta) def _resolve_target(self, raw: str) -> Path | None: """Resolve ``raw`` under workspace and gate the markdown suffix. @@ -156,6 +152,8 @@ class ReadStep(BaseStep): if content is None: return None + # Match text-mode streaming reads and the chunker's source line numbers. + content = content.replace("\r\n", "\n").replace("\r", "\n") all_lines = content.split("\n") if content.endswith("\n"): # split() yields a trailing empty element for the final newline; diff --git a/reme/steps/file_io/read_image.py b/reme/steps/file_io/read_image.py index 67342e0c..84666eec 100644 --- a/reme/steps/file_io/read_image.py +++ b/reme/steps/file_io/read_image.py @@ -5,14 +5,14 @@ import base64 import aiofiles import aiofiles.os +from ._base import _FileIOStep from ._path import NON_IMAGE_WARNING, gate_image, resolve_path -from ..base_step import BaseStep from ...components import R from ...constants import DEFAULT_MAX_IMAGE_BYTES @R.register("read_image_step") -class ReadImageStep(BaseStep): +class ReadImageStep(_FileIOStep): """Read an image file under ``workspace_dir`` and return base64 in ``answer``. Step-level attributes (``kwargs``, configured in yaml under ``steps:`` — @@ -32,13 +32,6 @@ class ReadImageStep(BaseStep): base64; ``metadata.non_image_warning=True`` flags it. """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/save.py b/reme/steps/file_io/save.py index 0b77cf7a..9ec9845f 100644 --- a/reme/steps/file_io/save.py +++ b/reme/steps/file_io/save.py @@ -2,14 +2,14 @@ from datetime import datetime +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("save_step") -class SaveStep(BaseStep): +class SaveStep(_FileIOStep): """Persist editor content without interpreting or rebuilding frontmatter. ``expected_mtime`` is the ISO timestamp returned by ``stat``. When supplied, @@ -17,12 +17,6 @@ class SaveStep(BaseStep): filesystem authoritative when ReMe and another editor are used together. """ - def _fail(self, message: str, **metadata) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - self.context.response.metadata.update(metadata) - async def execute(self): assert self.context is not None raw_path = str(self.context.get("path") or "") diff --git a/reme/steps/file_io/write.py b/reme/steps/file_io/write.py index 44d1425d..ff5fd081 100644 --- a/reme/steps/file_io/write.py +++ b/reme/steps/file_io/write.py @@ -2,14 +2,14 @@ import frontmatter +from ._base import _FileIOStep from ._file_io import detect_file_encoding, get_path_lock, write_file_safe from ._path import _check_path_permission, NON_MD_WARNING, gate_md, resolve_path -from ..base_step import BaseStep from ...components import R @R.register("write_step") -class WriteStep(BaseStep): +class WriteStep(_FileIOStep): """Write (create or overwrite) a markdown file. Frontmatter accepts two reserved string fields (``name`` / ``description``) @@ -26,13 +26,6 @@ class WriteStep(BaseStep): scope). """ - def _fail(self, message: str, **meta) -> None: - assert self.context is not None - self.context.response.success = False - self.context.response.answer = f"Error: {message}" - if meta: - self.context.response.metadata.update(meta) - async def execute(self): # pylint: disable=too-many-return-statements assert self.context is not None raw = str(self.context.get("path") or "") diff --git a/reme/steps/index/_dedup.py b/reme/steps/index/_dedup.py index 96929e7b..230aebfb 100644 --- a/reme/steps/index/_dedup.py +++ b/reme/steps/index/_dedup.py @@ -12,15 +12,13 @@ When ``app_context`` is absent the same structure is mirrored under """ import datetime -from typing import TYPE_CHECKING, Any, Callable, Final +from typing import Callable, Final +from ._tool_context import _ToolContextMixin from ...schema import FileChunk -if TYPE_CHECKING: - from ...components import ApplicationContext - -class _ToolContextDedupMixin: +class _ToolContextDedupMixin(_ToolContextMixin): """Mixin providing tool_context-scoped chunk dedup with TTL. Must be mixed into a ``BaseStep`` subclass (e.g. ``SearchStep``); it @@ -29,7 +27,6 @@ class _ToolContextDedupMixin: set by the host step's ``__init__`` (default 24h). """ - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_RANGES_KEY: Final[str] = "search_seen_chunk_ranges" def __init_subclass__(cls, **kwargs): @@ -53,25 +50,7 @@ class _ToolContextDedupMixin: ) return super().__new__(cls, *args, **kwargs) - if TYPE_CHECKING: - # Declared by BaseStep; repeated here so static analysis resolves - # attribute access on the mixin without inheriting BaseStep. - app_context: "ApplicationContext | None" - kwargs: dict[str, Any] - seen_ttl_hours: float - - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context. - - The bucket is created lazily on first access and lives at - ``metadata["tool_contexts"][tool_context_id]`` (or the same path under - ``kwargs`` when no ``app_context`` is available, e.g. in unit tests). - """ - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) + seen_ttl_hours: float @staticmethod def _now_ts() -> float: diff --git a/reme/steps/index/_single_search.py b/reme/steps/index/_single_search.py new file mode 100644 index 00000000..06973f02 --- /dev/null +++ b/reme/steps/index/_single_search.py @@ -0,0 +1,68 @@ +"""Shared execution for single-backend search steps.""" + +from abc import abstractmethod +from typing import Final + +from ._dedup import _ToolContextDedupMixin +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries +from ._source_format import merge_session_chunk_intervals, render_chunk_entries +from ..base_step import BaseStep +from ...schema import FileChunk + +_MAX_CANDIDATES: Final = 200 +_CANDIDATE_MULTIPLIER: Final = 10 + + +class _SingleSearchStep(_ToolContextDedupMixin, BaseStep): + """Retrieve candidates, then apply the common single-backend search contract.""" + + def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.seen_ttl_hours = seen_ttl_hours + self.include_source = include_source + + async def execute(self): + assert self.context is not None + raw_query = self.context.get("query") + query: str = "" if raw_query is None else str(raw_query).strip() + limit: int = int(self.context.get("limit") or 5) + min_score: float = float(self.context.get("min_score") or 0.0) + tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() + + if not query: + self.context.response.success = False + self.context.response.answer = "Error: query cannot be empty" + return self.context.response + assert limit > 0, f"limit must be positive, got {limit}" + + candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) + results = await self._retrieve(query, candidates) + self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") + + if min_score > 0.0: + results = [chunk for chunk in results if chunk.score >= min_score] + + pre_dedup_count = 0 + if tool_context_id: + pre_dedup_count = len(results) + results, _ = self._dedupe_tool_context(results, tool_context_id, limit) + else: + results = results[:limit] + + session_dir = self.config_value("session_dir") + entries = render_chunk_entries( + merge_session_chunk_intervals(results, session_dir), + session_dir, + include_source=self.include_source, + ) + self.context.response.answer = join_chunk_entries(entries) + if not results: + self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE + self.context.response.metadata["results"] = [ + c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results + ] + return self.context.response + + @abstractmethod + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + """Fetch ranked candidates from the concrete backend.""" diff --git a/reme/steps/index/_tool_context.py b/reme/steps/index/_tool_context.py new file mode 100644 index 00000000..4b14a6d5 --- /dev/null +++ b/reme/steps/index/_tool_context.py @@ -0,0 +1,23 @@ +"""Shared access to application-lifetime tool context state.""" + +from typing import TYPE_CHECKING, Any, Final + +if TYPE_CHECKING: + from ...components import ApplicationContext + + +class _ToolContextMixin: + """Share storage access while leaving each step's state format independent.""" + + TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" + + if TYPE_CHECKING: + app_context: "ApplicationContext | None" + kwargs: dict[str, Any] + + def _tool_context_store(self, tool_context_id: str) -> dict: + if self.app_context is not None: + contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) + else: + contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) + return contexts.setdefault(tool_context_id, {}) diff --git a/reme/steps/index/bm25_search.py b/reme/steps/index/bm25_search.py index 99ca8a4d..59bd2460 100644 --- a/reme/steps/index/bm25_search.py +++ b/reme/steps/index/bm25_search.py @@ -1,64 +1,13 @@ -"""``bm25_search_step`` — plain BM25 keyword search with tool_context dedup.""" +"""Plain BM25 keyword search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("bm25_search_step") -class Bm25SearchStep(_ToolContextDedupMixin, BaseStep): - """BM25-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class Bm25SearchStep(_SingleSearchStep): + """Search the BM25 keyword backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.keyword_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.keyword_search(query, candidates, {}) diff --git a/reme/steps/index/draft.py b/reme/steps/index/draft.py index a33d3db2..5684e329 100644 --- a/reme/steps/index/draft.py +++ b/reme/steps/index/draft.py @@ -2,24 +2,17 @@ from typing import Final +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ...components import R @R.register("add_draft_step") -class AddDraftStep(BaseStep): +class AddDraftStep(_ToolContextMixin, BaseStep): """Append one draft text item to the current tool context.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" DRAFTS_KEY: Final[str] = "drafts" - def _tool_context_store(self, tool_context_id: str) -> dict: - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - async def execute(self): assert self.context is not None text = self.context.get("text", "") diff --git a/reme/steps/index/search.py b/reme/steps/index/search.py index 928587f5..f4e20b3c 100644 --- a/reme/steps/index/search.py +++ b/reme/steps/index/search.py @@ -5,6 +5,8 @@ import datetime import os from typing import Final +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE +from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ..file_io import extract_daily_date from ...components import R @@ -35,10 +37,9 @@ def _filter_values(value: object) -> set: @R.register("search_step") -class SearchStep(BaseStep): +class SearchStep(_ToolContextMixin, BaseStep): """Hybrid search: run vector + keyword in parallel, fuse via RRF, filter, truncate.""" - TOOL_CONTEXTS_KEY: Final[str] = "tool_contexts" SEARCH_SEEN_KEY: Final[str] = "search_seen_chunk_ids" def __init__( @@ -98,14 +99,6 @@ class SearchStep(BaseStep): def _now_ts() -> float: return datetime.datetime.now().timestamp() - def _tool_context_store(self, tool_context_id: str) -> dict: - """Return the mutable state bucket for a tool context.""" - if self.app_context is not None: - contexts = self.app_context.metadata.setdefault(self.TOOL_CONTEXTS_KEY, {}) - else: - contexts = self.kwargs.setdefault(self.TOOL_CONTEXTS_KEY, {}) - return contexts.setdefault(tool_context_id, {}) - def _dedupe_tool_context( self, chunks: list[FileChunk], @@ -203,6 +196,7 @@ class SearchStep(BaseStep): async def execute(self): assert self.context is not None + self.context.response.metadata.pop("empty_result", None) raw_query = self.context.get("query") query: str = "" if raw_query is None else str(raw_query).strip() limit: int = int(self.context.get("limit") or _default_limit()) @@ -349,6 +343,7 @@ class SearchStep(BaseStep): ) fused = [c for c in fused if c.score >= min_score] + pre_dedup_count = len(fused) dedup: dict | None = None if tool_context_id: fused, dedup = self._dedupe_tool_context(fused, tool_context_id, limit) @@ -369,6 +364,11 @@ class SearchStep(BaseStep): answer_lines.extend(render_expansion_lines(link_expansion.get(c.path, {}))) self.context.response.answer = "\n".join(answer_lines) + if not fused: + self.context.response.metadata["empty_result"] = { + "reason": "all_returned" if pre_dedup_count else "no_results", + "message": ALL_RETURNED_MESSAGE if pre_dedup_count else NO_RESULTS_MESSAGE, + } self.context.response.metadata["results"] = [ c.model_dump(exclude_none=True, exclude={"embedding"}) for c in fused ] diff --git a/reme/steps/index/vector_search.py b/reme/steps/index/vector_search.py index 5de5aa94..858db2bb 100644 --- a/reme/steps/index/vector_search.py +++ b/reme/steps/index/vector_search.py @@ -1,64 +1,13 @@ -"""``vector_search_step`` — plain vector search with tool_context dedup.""" +"""Plain vector search with tool-context deduplication.""" -from typing import Final - -from ._dedup import _ToolContextDedupMixin -from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE, join_chunk_entries -from ._source_format import merge_session_chunk_intervals, render_chunk_entries -from ..base_step import BaseStep +from ._single_search import _SingleSearchStep from ...components import R - -_MAX_CANDIDATES: Final = 200 -_CANDIDATE_MULTIPLIER: Final = 10 +from ...schema import FileChunk @R.register("vector_search_step") -class VectorSearchStep(_ToolContextDedupMixin, BaseStep): - """Vector-only search: retrieve, filter by min_score, dedup by tool_context, truncate.""" +class VectorSearchStep(_SingleSearchStep): + """Search the vector backend using the shared result contract.""" - def __init__(self, *args, seen_ttl_hours: float = 24, include_source: bool = True, **kwargs): - super().__init__(*args, **kwargs) - self.seen_ttl_hours = seen_ttl_hours - self.include_source = include_source - - async def execute(self): - assert self.context is not None - raw_query = self.context.get("query") - query: str = "" if raw_query is None else str(raw_query).strip() - limit: int = int(self.context.get("limit") or 5) - min_score: float = float(self.context.get("min_score") or 0.0) - tool_context_id: str = (self.context.get("tool_context_id", "") or "").strip() - - if not query: - self.context.response.success = False - self.context.response.answer = "Error: query cannot be empty" - return self.context.response - assert limit > 0, f"limit must be positive, got {limit}" - - candidates = min(_MAX_CANDIDATES, max(1, limit * _CANDIDATE_MULTIPLIER)) - results = await self.file_store.vector_search(query, candidates, {}) - self.logger.info(f"[{self.name}] query={query!r} candidates={candidates} hits={len(results)}") - - if min_score > 0.0: - results = [chunk for chunk in results if chunk.score >= min_score] - - pre_dedup_count = 0 - if tool_context_id: - pre_dedup_count = len(results) - results, _ = self._dedupe_tool_context(results, tool_context_id, limit) - else: - results = results[:limit] - - session_dir = self.config_value("session_dir") - entries = render_chunk_entries( - merge_session_chunk_intervals(results, session_dir), - session_dir, - include_source=self.include_source, - ) - self.context.response.answer = join_chunk_entries(entries) - if not results: - self.context.response.answer = ALL_RETURNED_MESSAGE if pre_dedup_count > 0 else NO_RESULTS_MESSAGE - self.context.response.metadata["results"] = [ - c.model_dump(exclude_none=True, exclude={"embedding"}) for c in results - ] - return self.context.response + async def _retrieve(self, query: str, candidates: int) -> list[FileChunk]: + return await self.file_store.vector_search(query, candidates, {}) diff --git a/reme/steps/index/watch_changes.py b/reme/steps/index/watch_changes.py index a01d205f..a39eb467 100644 --- a/reme/steps/index/watch_changes.py +++ b/reme/steps/index/watch_changes.py @@ -60,6 +60,8 @@ class WatchChangesStep(BaseStep): step: int = DEFAULT_WATCH_STEP_MS, poll_delay_ms: int = DEFAULT_LOW_POWER_POLL_MS, watch_mode: Literal["awatch", "watch"] = "awatch", + monitor_type: Literal["file_store", "file_catalog"] | None = None, + monitor_name: str = "default", **kwargs, ): if watch_mode not in ("awatch", "watch"): @@ -71,6 +73,8 @@ class WatchChangesStep(BaseStep): self.debounce: int = debounce self.step: int = step self.poll_delay_ms: int = poll_delay_ms + self.monitor_type = monitor_type + self.monitor_name = monitor_name self._rules: list[WatchRule] = [] def _get_watch_rules(self) -> list[WatchRule]: @@ -112,7 +116,9 @@ class WatchChangesStep(BaseStep): step=self.step, poll_delay_ms=self.poll_delay_ms, stop_event=stop_event, + yield_on_timeout=self.monitor_type is not None, ) + needs_initial_scan = self.monitor_type is not None async with aclosing(watcher): async for raw_changes in watcher: if stop_event.is_set(): @@ -122,6 +128,23 @@ class WatchChangesStep(BaseStep): for c, p in raw_changes if c in (Change.added, Change.modified, Change.deleted) ] + if needs_initial_scan: + # Subscribe first, then recover writes made after the + # preceding scan but before watcher initialization. + await self.dispatch_steps( + [ + { + "backend": "init_changes_step", + "monitor_type": self.monitor_type, + "monitor_name": self.monitor_name, + "recursive": self.recursive, + }, + ], + ) + # The scan only produces changes. Merge overlapping events + # before running handlers, which may have model side effects. + changes.extend(self.context.get("changes") or []) + needs_initial_scan = False changes = coalesce_changes(changes) if changes: self.logger.info(f"Detected {len(changes)} change(s)") diff --git a/reme/utils/async_utils.py b/reme/utils/async_utils.py index 290a4456..8f54fbcb 100644 --- a/reme/utils/async_utils.py +++ b/reme/utils/async_utils.py @@ -7,11 +7,21 @@ from typing import Any async def complete_in_thread(func: Callable[..., Any], /, *args) -> Any: - """Finish a side-effecting thread call before propagating cancellation.""" + """Finish a side-effecting thread call, including through repeated cancellation.""" task = asyncio.create_task(asyncio.to_thread(func, *args)) - try: - return await asyncio.shield(task) - except asyncio.CancelledError: + cancellation: asyncio.CancelledError | None = None + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as exc: + if cancellation is None: + cancellation = exc + except Exception: + if cancellation is None: + raise + break + if cancellation is not None: with suppress(Exception): - await task - raise + task.result() + raise cancellation + return task.result() diff --git a/reme/utils/tushare.py b/reme/utils/tushare.py deleted file mode 100644 index 68218151..00000000 --- a/reme/utils/tushare.py +++ /dev/null @@ -1,17 +0,0 @@ -"""TuShare client construction with an optional mirror endpoint.""" - -import os - - -def create_tushare_api(token: str): - """Create a TuShare DataApi, using TUSHARE_MIRROR_URL when configured.""" - try: - import tushare as ts - except ImportError as exc: # pragma: no cover - optional core dependency. - raise RuntimeError("tushare is required for market-data research") from exc - - api = ts.pro_api(token) - api._DataApi__timeout = 600 # pylint: disable=protected-access - if mirror_url := os.getenv("TUSHARE_MIRROR_URL", "").strip(): - api._DataApi__http_url = mirror_url.rstrip("/") # pylint: disable=protected-access - return api diff --git a/reme_studio/README.md b/reme_studio/README.md index 8667cd91..2cbeea61 100644 --- a/reme_studio/README.md +++ b/reme_studio/README.md @@ -402,8 +402,10 @@ Run proportionate checks from `reme_studio/`: ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# Build and verify the Sites, static, and demo distributions: +npm run test:build +# Run both suites: npm test ``` diff --git a/reme_studio/README_ZH.md b/reme_studio/README_ZH.md index f3a11d59..3351d3a9 100644 --- a/reme_studio/README_ZH.md +++ b/reme_studio/README_ZH.md @@ -357,8 +357,10 @@ npm run build:demo ```bash npm run format:check npm run lint -npm run build -npm run build:static +npm run test:unit +# 构建并验证 Sites、静态和演示分发： +npm run test:build +# 运行以上全部检查： npm test ``` diff --git a/reme_studio/app/chatgpt-auth.ts b/reme_studio/app/chatgpt-auth.ts deleted file mode 100644 index a0ae2ed9..00000000 --- a/reme_studio/app/chatgpt-auth.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { headers } from "next/headers"; -import { redirect } from "next/navigation"; - -export type ChatGPTUser = { - userId: string; - displayName: string; - email: string; - fullName: string | null; -}; - -const USER_ID_HEADER = "oai-authenticated-user-id"; -const USER_EMAIL_HEADER = "oai-authenticated-user-email"; -const USER_FULL_NAME_HEADER = "oai-authenticated-user-full-name"; -const USER_FULL_NAME_ENCODING_HEADER = - "oai-authenticated-user-full-name-encoding"; -const PERCENT_ENCODED_UTF8 = "percent-encoded-utf-8"; -const SIGN_IN_PATH = "/signin-with-chatgpt"; -const SIGN_OUT_PATH = "/signout-with-chatgpt"; -const CALLBACK_PATH = "/callback"; - -export async function getChatGPTUser(): Promise { - const requestHeaders = await headers(); - const userId = requestHeaders.get(USER_ID_HEADER); - const email = requestHeaders.get(USER_EMAIL_HEADER); - if (!userId || !email) return null; - - const encodedFullName = requestHeaders.get(USER_FULL_NAME_HEADER); - const fullName = - encodedFullName && - requestHeaders.get(USER_FULL_NAME_ENCODING_HEADER) === PERCENT_ENCODED_UTF8 - ? safeDecodeURIComponent(encodedFullName) - : null; - - return { - userId, - displayName: fullName ?? email, - email, - fullName, - }; -} - -export async function requireChatGPTUser( - returnTo: string, -): Promise { - const user = await getChatGPTUser(); - if (user) return user; - - redirect(chatGPTSignInPath(returnTo)); -} - -export function chatGPTSignInPath(returnTo: string): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_IN_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -export function chatGPTSignOutPath(returnTo = "/"): string { - const safeReturnTo = safeRelativeReturnPath(returnTo); - return `${SIGN_OUT_PATH}?return_to=${encodeURIComponent(safeReturnTo)}`; -} - -function safeRelativeReturnPath(value: string): string { - if (!value.startsWith("/") || value.startsWith("//")) return "/"; - - let url: URL; - try { - url = new URL(value, "https://app.local"); - } catch { - return "/"; - } - if (url.origin !== "https://app.local") return "/"; - if (isReservedAuthPath(url.pathname)) return "/"; - - return `${url.pathname}${url.search}${url.hash}`; -} - -function isReservedAuthPath(pathname: string): boolean { - return ( - pathname === SIGN_IN_PATH || - pathname === SIGN_OUT_PATH || - pathname === CALLBACK_PATH - ); -} - -function safeDecodeURIComponent(value: string): string | null { - try { - return decodeURIComponent(value); - } catch { - return null; - } -} diff --git a/reme_studio/app/globals.css b/reme_studio/app/globals.css index 2fc41164..2621b818 100644 --- a/reme_studio/app/globals.css +++ b/reme_studio/app/globals.css @@ -1610,27 +1610,6 @@ textarea:focus-visible { margin: 8px 0 10px; font-size: 12px; } -.component-row { - min-height: 48px; - display: flex; - align-items: center; - justify-content: space-between; - gap: 16px; - border-top: 1px solid var(--line); -} -.component-row > span { - display: flex; - flex-direction: column; - font-size: 12px; -} -.component-row small { - margin-top: 2px; - color: var(--muted); - font-size: 10px; -} -.component-row strong { - font-size: 12px; -} .index-card { padding: 22px; } diff --git a/reme_studio/app/open-file.ts b/reme_studio/app/open-file.ts index 41b965c0..9fe34dab 100644 --- a/reme_studio/app/open-file.ts +++ b/reme_studio/app/open-file.ts @@ -9,13 +9,24 @@ export async function openWorkspaceFile(path: string) { ); const id = store.openMarkdown(path); if (existing) return; + await loadWorkspaceFile(id, path); +} + +/** Load both newly opened and restored tabs, respecting the caller's lifetime. */ +export async function loadWorkspaceFile( + id: string, + path: string, + shouldApply: () => boolean = () => true, +) { + const store = useWorkspaceStore.getState(); try { const file = await readWorkspaceFile(path); - store.hydrateMarkdown(id, file.content, file.stat.mtime); + if (shouldApply()) store.hydrateMarkdown(id, file.content, file.stat.mtime); } catch (error) { - store.failMarkdown( - id, - error instanceof Error ? error.message : "Failed to read file", - ); + if (shouldApply()) + store.failMarkdown( + id, + error instanceof Error ? error.message : "Failed to read file", + ); } } diff --git a/reme_studio/app/store.ts b/reme_studio/app/store.ts index 5f2fb64f..98ec968d 100644 --- a/reme_studio/app/store.ts +++ b/reme_studio/app/store.ts @@ -54,6 +54,19 @@ interface WorkspaceState { finishChat: (tabId: string, messageId: string, error?: string) => void; } +function updateChatMessage( + tab: Extract, + messageId: string, + update: (message: ChatMessage) => ChatMessage, +) { + return { + ...tab, + messages: tab.messages.map((message) => + message.id === messageId ? update(message) : message, + ), + }; +} + const fileTitle = (path: string) => path.split("/").pop() || path; // Preserve the original key so existing tabs, chats, and unsaved drafts survive the Studio rename. const WORKSPACE_STORAGE_KEY = IS_DEMO @@ -62,189 +75,153 @@ const WORKSPACE_STORAGE_KEY = IS_DEMO export const useWorkspaceStore = create()( persist( - (set, get) => ({ - tabs: [], - activeTabId: undefined, - openMarkdown: (path) => { - const existing = get().tabs.find( - (tab) => tab.type === "markdown" && tab.path === path, - ); - if (existing) { - set({ activeTabId: existing.id }); - return existing.id; - } - const id = `file:${path}`; + (set, get) => { + const updateTab = ( + id: string, + type: Type, + update: (tab: Extract) => WorkspaceTab, + ) => set((state) => ({ - tabs: [ - ...state.tabs, - { - id, - type: "markdown", - title: fileTitle(path), - path, - content: "", - savedContent: "", - loading: true, - }, - ], - activeTabId: id, + tabs: state.tabs.map((tab) => + tab.id === id && tab.type === type + ? update(tab as Extract) + : tab, + ), })); - return id; - }, - openAgent: () => { - const id = `agent:${crypto.randomUUID()}`; - set((state) => ({ - tabs: [...state.tabs, { id, type: "agent", title: "", messages: [] }], - activeTabId: id, - })); - return id; - }, - openGraph: (root) => { - const id = `graph:${root}`; - const existing = get().tabs.find( - (tab) => tab.id === id && tab.type === "graph", - ); - if (existing) { - set({ activeTabId: id }); + return { + tabs: [], + activeTabId: undefined, + openMarkdown: (path) => { + const existing = get().tabs.find( + (tab) => tab.type === "markdown" && tab.path === path, + ); + if (existing) { + set({ activeTabId: existing.id }); + return existing.id; + } + const id = `file:${path}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { + id, + type: "markdown", + title: fileTitle(path), + path, + content: "", + savedContent: "", + loading: true, + }, + ], + activeTabId: id, + })); return id; - } - set((state) => ({ - tabs: [ - ...state.tabs, - { id, type: "graph", title: `${root} graph`, root }, - ], - activeTabId: id, - })); - return id; - }, - closeTab: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, false).length - ) - return state; - const index = state.tabs.findIndex((tab) => tab.id === id); - const tabs = state.tabs.filter((tab) => tab.id !== id); - const activeTabId = - state.activeTabId === id - ? tabs[Math.max(0, index - 1)]?.id - : state.activeTabId; - return { tabs, activeTabId }; - }), - closeOtherTabs: (id, discardUnsaved = false) => - set((state) => { - if ( - !discardUnsaved && - unsavedTabsClosedBy(state.tabs, id, true).length - ) - return state; - const tab = state.tabs.find((item) => item.id === id); - return tab ? { tabs: [tab], activeTabId: id } : state; - }), - setActiveTab: (activeTabId) => set({ activeTabId }), - hydrateMarkdown: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { - ...tab, - content, - savedContent: content, - mtime, - loading: false, - error: undefined, - } - : tab, - ), - })), - failMarkdown: (id, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, loading: false, error } - : tab, - ), - })), - updateMarkdown: (id, content) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? { ...tab, content } - : tab, - ), - })), - markSaved: (id, content, mtime) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === id && tab.type === "markdown" - ? markMarkdownContentSaved(tab, content, mtime) - : tab, - ), - })), - addChatTurn: (tabId, user, assistant) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - title: tab.messages.length - ? tab.title - : user.content.slice(0, 18), - messages: [...tab.messages, user, assistant], - streaming: true, - } - : tab, - ), - })), - applyChatChunk: (tabId, messageId, chunk) => - set((state) => ({ - tabs: state.tabs.map((tab) => { - if (tab.id !== tabId || tab.type !== "agent") return tab; - const messages = tab.messages.map((message) => - message.id === messageId - ? applyStreamChunk(message, chunk) - : message, - ); - return { - ...tab, - sessionId: chunk.session_id || tab.sessionId, - messages, - }; + }, + openAgent: () => { + const id = `agent:${crypto.randomUUID()}`; + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "agent", title: "", messages: [] }, + ], + activeTabId: id, + })); + return id; + }, + openGraph: (root) => { + const id = `graph:${root}`; + const existing = get().tabs.find( + (tab) => tab.id === id && tab.type === "graph", + ); + if (existing) { + set({ activeTabId: id }); + return id; + } + set((state) => ({ + tabs: [ + ...state.tabs, + { id, type: "graph", title: `${root} graph`, root }, + ], + activeTabId: id, + })); + return id; + }, + closeTab: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, false).length + ) + return state; + const index = state.tabs.findIndex((tab) => tab.id === id); + const tabs = state.tabs.filter((tab) => tab.id !== id); + const activeTabId = + state.activeTabId === id + ? tabs[Math.max(0, index - 1)]?.id + : state.activeTabId; + return { tabs, activeTabId }; }), - })), - toggleChatBlock: (tabId, messageId, blockId, expanded) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - messages: tab.messages.map((message) => - message.id === messageId - ? toggleChatBlock(message, blockId, expanded) - : message, - ), - } - : tab, + closeOtherTabs: (id, discardUnsaved = false) => + set((state) => { + if ( + !discardUnsaved && + unsavedTabsClosedBy(state.tabs, id, true).length + ) + return state; + const tab = state.tabs.find((item) => item.id === id); + return tab ? { tabs: [tab], activeTabId: id } : state; + }), + setActiveTab: (activeTabId) => set({ activeTabId }), + hydrateMarkdown: (id, content, mtime) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + content, + savedContent: content, + mtime, + loading: false, + error: undefined, + })), + failMarkdown: (id, error) => + updateTab(id, "markdown", (tab) => ({ + ...tab, + loading: false, + error, + })), + updateMarkdown: (id, content) => + updateTab(id, "markdown", (tab) => ({ ...tab, content })), + markSaved: (id, content, mtime) => + updateTab(id, "markdown", (tab) => + markMarkdownContentSaved(tab, content, mtime), ), - })), - finishChat: (tabId, messageId, error) => - set((state) => ({ - tabs: state.tabs.map((tab) => - tab.id === tabId && tab.type === "agent" - ? { - ...tab, - streaming: false, - messages: tab.messages.map((message) => - message.id === messageId - ? finishChatMessage(message, error) - : message, - ), - } - : tab, + addChatTurn: (tabId, user, assistant) => + updateTab(tabId, "agent", (tab) => ({ + ...tab, + title: tab.messages.length ? tab.title : user.content.slice(0, 18), + messages: [...tab.messages, user, assistant], + streaming: true, + })), + applyChatChunk: (tabId, messageId, chunk) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + applyStreamChunk(message, chunk), + ), + sessionId: chunk.session_id || tab.sessionId, + })), + toggleChatBlock: (tabId, messageId, blockId, expanded) => + updateTab(tabId, "agent", (tab) => + updateChatMessage(tab, messageId, (message) => + toggleChatBlock(message, blockId, expanded), + ), ), - })), - }), + finishChat: (tabId, messageId, error) => + updateTab(tabId, "agent", (tab) => ({ + ...updateChatMessage(tab, messageId, (message) => + finishChatMessage(message, error), + ), + streaming: false, + })), + }; + }, { name: WORKSPACE_STORAGE_KEY, version: 1, diff --git a/reme_studio/app/workspace-chat.tsx b/reme_studio/app/workspace-chat.tsx new file mode 100644 index 00000000..6a6475ae --- /dev/null +++ b/reme_studio/app/workspace-chat.tsx @@ -0,0 +1,279 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { Check, CircleAlert, LoaderCircle, Send } from "lucide-react"; +import WorkspaceMarkdown from "./files-workspace/WorkspaceMarkdown"; +import { streamChat } from "./api"; +import { chatStreamError, formatStreamPayloads } from "./chat-stream"; +import { useI18n } from "./i18n"; +import { useWorkspaceStore } from "./store"; +import { IS_DEMO } from "./studio-mode"; +import type { + ChatBlock, + ChatMessage, + DetailBlock, + WorkspaceTab, +} from "./types"; +import { + appendWorkspaceFileReference, + WORKSPACE_FILE_DRAG_TYPE, +} from "./workspace-drag"; +import { scenarios } from "../demo/scenarios"; + +function MarkdownView({ content }: { content: string }) { + return ( +
+ +
+ ); +} + +function ChunkStatus({ + active, + error = false, +}: { + active: boolean; + error?: boolean; +}) { + if (error) return ; + return active ? ( + + ) : ( + + ); +} + +function detailLabel( + block: DetailBlock, + t: ReturnType["t"], +): string { + if (block.type === "think") return t("thinking"); + if (block.type === "data") + return block.mediaType + ? `${t("dataChunk")} · ${block.mediaType}` + : t("dataChunk"); + if (block.type === "approval") return t("approvalChunk"); + if (block.type === "usage") return t("usageChunk"); + return t("unknownChunk", { type: block.sourceType }); +} + +function ChatBlockView({ + block, + onToggle, +}: { + block: ChatBlock; + onToggle: (blockId: string, expanded: boolean) => void; +}) { + const { t } = useI18n(); + if (block.type === "content") return ; + if (block.type === "error") + return ( +
+ + {block.text} +
+ ); + if (block.type === "usage") return null; + + const active = + block.status === "streaming" || + block.status === "calling" || + block.status === "running"; + const failed = block.status === "error"; + const tool = block.type === "tool"; + const call = tool ? formatStreamPayloads(block.callPayloads) : ""; + const result = tool ? formatStreamPayloads(block.resultPayloads) : ""; + const detail = tool ? "" : formatStreamPayloads(block.payloads); + return ( +
{ + if (event.currentTarget.open !== block.expanded) + onToggle(block.id, event.currentTarget.open); + }} + > + + + {tool ? block.name : detailLabel(block, t)} + {active ? t("streaming") : t("completed")} + + {tool ? ( +
+ {call && ( +
+ {t("toolCall")} + 
{call}
+
+ )} + {result && ( +
+ {t("toolResult")} + 
{result}
+
+ )} +
+ ) : detail ? ( +
+ 
{detail}
+
+ ) : null} +
+ ); +} + +export default function Chat({ + tab, +}: { + tab: Extract; +}) { + const { language, t } = useI18n(); + const [input, setInput] = useState(""); + const [fileDragOver, setFileDragOver] = useState(false); + const endRef = useRef(null); + const controller = useRef(null); + const { addChatTurn, applyChatChunk, toggleChatBlock, finishChat } = + useWorkspaceStore(); + + useEffect(() => { + endRef.current?.scrollIntoView({ behavior: "smooth" }); + }, [tab.messages]); + useEffect(() => () => controller.current?.abort(), []); + + const suggestions = ( +
+ {(IS_DEMO + ? scenarios[language].map((scenario) => scenario.question) + : [t("promptRecent"), t("promptTasks"), t("promptIdeas")] + ).map((question) => ( + + ))} +
+ ); + + const send = async () => { + const query = input.trim(); + if (!query || tab.streaming) return; + setInput(""); + const user: ChatMessage = { + id: crypto.randomUUID(), + role: "user", + content: query, + }; + const assistant: ChatMessage = { + id: crypto.randomUUID(), + role: "assistant", + content: "", + blocks: [], + }; + addChatTurn(tab.id, user, assistant); + const requestController = new AbortController(); + controller.current = requestController; + const error = await chatStreamError( + () => + streamChat(query, tab.sessionId, requestController.signal, (chunk) => + applyChatChunk(tab.id, assistant.id, chunk), + ), + requestController.signal, + t("chatFailed"), + ); + finishChat(tab.id, assistant.id, error); + if (controller.current === requestController) controller.current = null; + }; + + return ( +
+
+ {!tab.messages.length && ( +
+ + )} + {tab.messages.map((message) => ( +
+
+ {message.role === "user" ? ( + t("you") + ) : ( + + )} +
+
+ {message.role === "user" &&

{message.content}

} + {message.role === "assistant" && + message.blocks?.map((block) => ( + + toggleChatBlock(tab.id, message.id, blockId, expanded) + } + /> + ))} + {message.role === "assistant" && + tab.streaming && + !message.blocks?.some((block) => block.type !== "usage") && ( + + )} +
+
+ ))} + {IS_DEMO && tab.messages.length > 0 && suggestions} +
+
+
{ + if (!event.dataTransfer.types.includes(WORKSPACE_FILE_DRAG_TYPE)) + return; + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + setFileDragOver(true); + }} + onDragLeave={(event) => { + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) + setFileDragOver(false); + }} + onDrop={(event) => { + const path = event.dataTransfer.getData(WORKSPACE_FILE_DRAG_TYPE); + if (!path) return; + event.preventDefault(); + setFileDragOver(false); + setInput((current) => appendWorkspaceFileReference(current, path)); + }} + > +
+