From 4d43236db227fab89e19f33b68826d4601f7700a Mon Sep 17 00:00:00 2001 From: "jinli.yl" Date: Thu, 15 Oct 2026 16:13:03 +0800 Subject: [PATCH] test: add live decision compaction scenarios and strategy guides --- README.md | 12 +- README_ZH.md | 11 +- docs/.vitepress/config.mts | 1 + docs/en/configuration.md | 2 + docs/en/context_compaction.md | 167 +++++++ docs/figure/decision-context-compaction.svg | 64 +++ docs/zh/configuration.md | 2 + docs/zh/context_compaction.md | 147 ++++++ example.env | 3 +- tests/integration/test_decision_compaction.py | 452 ++++++++++++++++++ 10 files changed, 854 insertions(+), 7 deletions(-) create mode 100644 docs/en/context_compaction.md create mode 100644 docs/figure/decision-context-compaction.svg create mode 100644 docs/zh/context_compaction.md create mode 100644 tests/integration/test_decision_compaction.py diff --git a/README.md b/README.md index 0bafad36..ec4a46fe 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,11 @@ users retain control of the durable files. ## 📰 Latest Updates + + +- [2026.10] - **[Decision-model context compaction](docs/en/context_compaction.md)**: score old tool calls and results + against the next action, preserve protected history, and return a recoverable compaction proposal. Includes an SVG + walkthrough and two live-model integration scenarios. - [2026.10] - **[ReMe Multimodal Memory](https://reme.agentscope.io/en/blog_20261009) update**: Auto Memory reads images with their conversation context and preserves image sources; Auto Resource turns image files into searchable daily cards linked to the originals. @@ -67,6 +72,10 @@ users retain control of the durable files. - [2026.09] - **[Hermes Agent memory provider](https://reme.agentscope.io/en/integrations/hermes) available**: choose HTTP or embedded mode for automatic recall before model calls and asynchronous `auto_memory` after completed turns. The integration supports Hermes Agent 0.21+ and includes profile-aware background work. + +
+More updates + - [2026.09] - **[OpenClaw plugin](https://reme.agentscope.io/en/integrations/openclaw) released**: install it from [ClawHub](https://clawhub.ai/agentscope-ai/plugins/reme-openclaw-plugin) or [npm](https://www.npmjs.com/package/@agentscope-ai/reme-openclaw-plugin) to add native memory recall, automatic @@ -76,9 +85,6 @@ users retain control of the durable files. [npm](https://www.npmjs.com/package/@agentscope-ai/reme-dsh-plugin) for long-term-memory guidance, `reme_search`, automatic memory, Auto Dream, and ReMe Status. -
-More updates - - [2026.08] - **ReMe blog published**: the [ReMe blog](https://reme.agentscope.io/en/reme-blog) introduces the local-first memory architecture, self-evolving workflows, hybrid search, proactive discovery, and benchmark results. - [2026.08] - **New ReMe ecosystem plugins**: [Daily Paper](https://reme.agentscope.io/en/plugins/daily-paper) diff --git a/README_ZH.md b/README_ZH.md index cacac7e9..15dc0465 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -47,6 +47,10 @@ ## 📰 最新动态 + + +- [2026.10] - **[基于决策模型的上下文压缩](docs/zh/context_compaction.md)**：根据下一步任务判断旧工具调用与完整输出的保留价值， + 保护关键历史，并返回可回溯的压缩建议；提供 SVG 流程图和两个真实模型集成场景。 - [2026.10] - **[ReMe 多模态记忆更新](https://reme.agentscope.io/zh/blog_20261009)**：Auto Memory 将会话图片和上下文一起 写入记忆并保留图片来源；Auto Resource 将图片资料整理为可检索、可回溯原图的 daily 卡片。 - [2026.10] - **[ReMe Studio Playground](https://reme.agentscope.io/studio/?lang=zh) 上线**：无需安装或启动后端， @@ -62,6 +66,10 @@ 可重建 Tag Index 与标签过滤检索。 - [2026.09] - **[Hermes Agent 记忆 Provider](https://reme.agentscope.io/zh/integrations/hermes) 已可使用**：支持 HTTP 和 Embedded 两种模式，在模型调用前自动召回、每轮对话结束后异步执行 `auto_memory`。集成支持 Hermes Agent 0.21 及以上版本，后台任务也会继承当前 profile 上下文。 + +
+更多更新 + - [2026.09] - **[OpenClaw 插件](https://reme.agentscope.io/zh/integrations/openclaw) 发布**：可通过 [ClawHub](https://clawhub.ai/agentscope-ai/plugins/reme-openclaw-plugin) 或 [npm](https://www.npmjs.com/package/@agentscope-ai/reme-openclaw-plugin) 安装，为 OpenClaw 提供原生记忆召回、自动对话捕获和定时整理能力。 @@ -69,9 +77,6 @@ [Awesome DSH Plugin](https://awesome-dsh-plugin.com/p/agentscope-ai/ReMe--integrations-dsh/) 或 [npm](https://www.npmjs.com/package/@agentscope-ai/reme-dsh-plugin) 安装，提供长期记忆指引、`reme_search`、自动记忆、Auto Dream 和 ReMe Status。 -
-更多更新 - - [2026.08] - **ReMe 博客发布**：[ReMe 博客](https://reme.agentscope.io/zh/reme-blog) 系统介绍了本地优先的记忆架构、 自进化工作流、混合检索、主动发现与评测结果。 - [2026.08] - **新增 ReMe 生态插件**：[每日论文](https://reme.agentscope.io/zh/plugins/daily-paper) 可自动发现、解析论文并生成文件化简报； diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 169e3b64..e0cd0345 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -148,6 +148,7 @@ function docsSidebar(language: "zh" | "en"): DefaultTheme.SidebarItem[] { { text: "Auto Resource", link: `/${language}/auto_resource` }, { text: "Auto Dream", link: `/${language}/auto_dream` }, { text: "Proactive", link: `/${language}/proactive` }, + { text: zh ? "决策模型上下文压缩" : "Context Compaction", link: `/${language}/context_compaction` }, ], }, { diff --git a/docs/en/configuration.md b/docs/en/configuration.md index 464dde18..1f90b858 100644 --- a/docs/en/configuration.md +++ b/docs/en/configuration.md @@ -209,6 +209,8 @@ terminal with `messages` and any protection or budget options. It uses the same a proposed snapshot; the caller owns archiving and applying it. Request token budgets are estimates with overhead headroom, independent of the host model tokenizer. +See [Context compaction with decision models](./context_compaction.md) for the strategy, SVG walkthrough, and live integration cases. + ## Embeddings Vector retrieval is disabled by default. Credentials alone do not enable it: configure `as_embedding`, `embedding_store`, and connect the store to `file_store`. diff --git a/docs/en/context_compaction.md b/docs/en/context_compaction.md new file mode 100644 index 00000000..598fb408 --- /dev/null +++ b/docs/en/context_compaction.md @@ -0,0 +1,167 @@ +# Context compaction with decision models + +Long tasks accumulate source reads, stale searches, repeated diagnostics, and completed side tasks. +`compact_context` uses the **next action** to judge whether an old tool call still matters and whether its full result +must remain verbatim. It returns a proposed message snapshot and per-call decisions. The host decides whether to apply it. + +The workflow calls the provider-independent `as_decision_model` interface. The current `agentscope` adapter uses +AgentScope's TypeSafe/Jev implementation; jobs keep the same interface when another backend is registered. + +![Decision-model context compaction: guards, bounded observation, two probabilities, and host application](../figure/decision-context-compaction.svg) + +## How one pass works + +1. **The host archives the original history.** Supply an AgentScope 2.x JSON message snapshot, current `goal`, eligible + tool names, and protection lists. ReMe neither archives history nor updates a running Agent's state. Apply proposals + only when the original record remains recoverable. +2. **Deterministic guards select candidates.** Match `tool_call` and `tool_result` by ID. Calls must be `finished`, results + must be `success`, tools must be explicitly allowlisted, and outputs must be entirely text. Duplicate IDs, missing or + reversed pairs, and invalid JSON inputs are ineligible. +3. **Build a bounded observation.** Send the goal, conversation text, tool names, shortened inputs, and result character + counts. Tool-result bodies and thinking are omitted. When necessary, shorten inputs and old text, omit old text-only + messages, or merge old call records in this view. These changes do not rewrite conversation text in the returned snapshot. +4. **Ask two binary questions per candidate.** `call_` asks whether the call and its input still matter for the next + action. `result_` asks whether the full output must remain verbatim and later retrieval would be insufficient. + Keep each question pair together; batches repeat the same observation and run under concurrency and timeout limits. +5. **Rebuild only after every batch succeeds.** Validate answer types and probabilities before applying decisions to a + deep copy. Missing answers, invalid probabilities, failed batches, or timeout fail the call without a partial proposal. + The host should continue using its saved original snapshot. + +## From probabilities to actions + +With retention threshold `t` (default `0.5`), check the result probability first: + +| Condition | Action | Proposed snapshot | +| --- | --- | --- | +| `p_result >= t` | `keep` | Keep the call and its complete result, even if `p_call` is lower | +| `p_result < t` and `p_call >= t` | `drop_result` | Keep the call; shorten long text output to a prefix plus an omission marker | +| Both probabilities `< t` | `drop_call` | Remove both tool blocks; remove a message only if this makes it empty | + +The default prefix is 300 characters. The marker points back to archived history. Replacement occurs only if it makes +output shorter, so `drop_result` can leave a short output unchanged. User text, assistant text, thinking, and untouched +block fields remain intact. + +Deterministic protection takes precedence over probabilities: + +- The first message, the most recent `preserve_recent_messages` messages, and `protected_message_ids` are protected. + Protection of either endpoint protects the complete pair. +- `protected_call_ids` and tools outside `eligible_tool_names` are protected. The default allowlist is empty, so no tools + are compacted unless the caller explicitly opts them in. +- Unfinished or failed calls, non-text outputs, and mismatched tool names remain intact. Duplicate or missing IDs are + also retained, but may have no entry in `decisions`. + +The host must choose read-only tools for the allowlist; ReMe cannot infer tool side effects from their names. +Explicitly protect indispensable audit records, change receipts, and evidence; semantic probabilities are insufficient +as the sole protection mechanism. + +## Configure and call the service + +Install dependencies and configure credentials: + +```bash +pip install "reme-ai[core,decision]" +export DECISION_MODEL_API_KEY=your-typesafe-api-key +export DECISION_MODEL_BACKEND=agentscope +# Optional: DECISION_MODEL_NAME and DECISION_MODEL_BASE_URL +``` + +The CLI also loads `.env`. Migrate old `CLASSIFIER_*` variables to `DECISION_MODEL_*` as described in +[configuration](./configuration.md#decision-model-configuration); the backend name is `agentscope`. +Omitting the model name or setting it to `null` uses the adapter's SDK default. + +Start the service in one terminal: + +```bash +reme start config=decision_compaction +``` + +Call it from a second terminal. This short example illustrates the request format; the integration scenarios below +provide longer histories: + +```bash +reme compact_context \ + goal='Fix payment retries; the completed icon search is irrelevant' \ + eligible_tool_names='["search_code"]' \ + options='{"preserve_recent_messages":1}' \ + messages='[{"id":"m0","role":"user","content":[{"type":"text","text":"Preserve payment source; clean old icon searches"}]},{"id":"m1","role":"assistant","content":[{"type":"tool_call","id":"c1","name":"search_code","input":"{\"pattern\":\"logo\",\"path\":\"web/icons\"}","state":"finished"},{"type":"tool_result","id":"c1","name":"search_code","output":"Icon paths from a completed task","state":"success"}]},{"id":"m2","role":"user","content":[{"type":"text","text":"Now focus on payment retries"}]}]' +``` + +`decision_compaction` inherits the default service configuration and other jobs, including normal background workflows. +For embedding in a host process, assemble only the compaction job and its decision component as the integration tests do. + +The standard Response's `answer` contains: + +- `schema_version` and `messages`: version and proposed snapshot; `changed` and `reason`: whether it changed, with + `compacted`, `no_change`, or `no_candidates`. +- `decisions`: internal pair ID, original `call_id`, action, reason, and `keep_call` / `keep_result` probabilities for + pairs evaluated by the model. +- `stats`: message and character counts before and after, candidate/protected counts, estimated observation tokens and + fitting stage, request count, provider usage, and elapsed time. The same statistics appear in `response.metadata.compaction`. + +Check `success`, recalculate the final input budget with the host model's tokenizer, and apply, repeat, or reject the +proposal. Compaction does not automatically summarize, write long-term memory, or execute historical tool calls. + +## Budgets, concurrency, and timeout + +| `options` field | Default | Purpose | +| --- | --- | --- | +| `keep_threshold` | `0.5` | Shared retention threshold for both questions | +| `preserve_recent_messages` | `6` | Messages protected verbatim, counted as messages rather than turns | +| `max_state_tokens` | `25000` | Estimated observation token limit | +| `max_request_tokens` | `30000` | Per-batch estimated observation/question budget, including a 256-token overhead reserve | +| `truncate_head_chars` | `300` | Text prefix retained by `drop_result` | +| `max_concurrency` | `4` | Concurrent decision requests, from 1 to 32 | +| `total_timeout_seconds` | `30` | Overall deadline for decision requests and their queueing | + +Component `timeout`, `max_retries`, and `retry_delay` control individual provider requests and retries. The compaction +preset uses 10 seconds, one retry, and 0.5 seconds of initial backoff. Local token estimates differ from provider billing +and from the host model tokenizer. If the observation cannot fit, or a single question pair cannot fit a request, the +call fails and the host retains its original snapshot. + +## Two live-model integration scenarios + +The [integration tests](https://github.com/agentscope-ai/ReMe/blob/main/tests/integration/test_decision_compaction.py) +provide two synthetic long-task histories. Historical deploys, rollbacks, and database queries are fixture data and are +never executed; only the decision requests call the real TypeSafe/Jev API. + +| Scenario | Current task and important candidates | Distractors and protection boundaries | +| --- | --- | --- | +| Payment retry patch | Produce an exact patch from previously read source and regression-test lines; keep those candidate results intact | Completed theme CSS, icon searches, and obsolete inventory include removable calls; preserve billing evidence, approval receipts, failed/pending work, duplicate IDs, and recent messages | +| Replication incident | Quote exact incident logs and WAL offsets in a root-cause report; keep the verbatim evidence intact | Mix in completed marketing reports, holiday weather, and retired-system load tests; preserve audit approval, rollback receipts, non-text charts, failed queries, and recent messages | + +Both cases check real batched inference, removal of at least one irrelevant call, full retention of critical candidates, +protected history and conversation text, unchanged original input and temporary archive, and a reduction in output +characters. Semantic assertions can expose model/configuration changes; the tests do not fabricate probabilities to pass. + +A live run on 2026-10-15 against the locally configured TypeSafe/Jev endpoint passed both cases. These are JSON +character counts, not token savings: + +| Scenario | Characters before | Characters after | Reduction | Irrelevant calls removed | Decision requests | +| --- | ---: | ---: | ---: | ---: | ---: | +| Payment retry patch | 39,438 | 12,738 | 67.7% | 3 | 2 | +| Replication incident | 51,485 | 12,828 | 75.1% | 4 | 2 | + +Explicitly opt in to live API calls: + +```bash +REME_RUN_DECISION_INTEGRATION=1 \ + pytest tests/integration/test_decision_compaction.py -v -s +``` + +Tests read the environment or repository-root `.env`, with existing environment variables taking precedence. They skip +without opt-in or without an API key. Output contains only synthetic-scenario actions, probabilities, character counts, +and usage. Workspaces and original snapshots use pytest temporary directories. Savings depend on history, goal, and +model decisions; these two scenarios do not establish a general performance guarantee. + +## Boundaries + +The decision model sees a limited observation and cannot inspect omitted tool-result contents. Incomplete goals or +insufficient tool descriptions can hide relevant evidence. The host must supply a clear next-action goal, explicitly +protect indispensable information, and retain recoverable original history after applying a proposal. +This strategy reduces old tool traces; it does not provide automatic triggering, a final token-budget guarantee, +summary generation, or archival storage. + +Contracts and implementation: [compaction schema](https://github.com/agentscope-ai/ReMe/blob/main/reme/schema/compaction.py), +[pairing and rebuilding](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/messages.py), +[observation and batching](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/state.py), and +[job step](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/compact_context.py). diff --git a/docs/figure/decision-context-compaction.svg b/docs/figure/decision-context-compaction.svg new file mode 100644 index 00000000..ae06d039 --- /dev/null +++ b/docs/figure/decision-context-compaction.svg @@ -0,0 +1,64 @@ + +Decision-model context compaction +The caller archives original history. Deterministic guards protect messages and unsafe tool pairs. A bounded observation without tool outputs goes to a decision model, which estimates two probabilities per eligible pair. The result is a proposed snapshot with unchanged protected history; the caller checks its own token budget before applying it. + + + +Decision-model context compaction +Score old tool history for the next action. Return a proposal; the caller owns the original. + + + +1 ORIGINAL SNAPSHOT +Caller archives full history +User / assistant / system messages +Tool calls + full tool results +Text, thinking and metadata retained +2 DETERMINISTIC GUARDS +Choose eligible pairs +Read-only tool allowlist +Unique IDs + finished + success +Protect first / recent / pinned history +3 BOUNDED OBSERVATION +Send a small decision view +Goal + text + tool names / inputs +Result sizes; tool outputs omitted +Thinking is omitted; inputs may shrink + + + +4 DECISION MODEL +Two binary questions per tool pair +p_call Keep call + input? +p_result Keep full output verbatim? + + + +Protected pairs bypass scoring +Their original blocks stay unchanged. +Same observation in each batch +Bounded concurrency + deadline +Any failed batch → no proposal + + + + + + +KEEP +p_result ≥ threshold +Retain the call and full result. +DROP_RESULT +p_call ≥ t and p_result < t +Keep call; shorten long text output. +DROP_CALL +Both probabilities < threshold +Remove both tool blocks together. + + + + +5 PROPOSED SNAPSHOT + DECISIONS + STATS +Original text / thinking / protected blocks remain intact. +Caller checks its own model token budget, then applies or rejects the proposal. + diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 392b5e3e..769ad0f0 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -206,6 +206,8 @@ schema 定义问题与答案。接口见 `reme/components/as_decision_model/`。 `reme compact_context`，传入 `messages` 以及需要的保护和预算选项。返回值是建议的快照，归档和应用由调用方负责。 请求 token 预算使用预留开销的估算值，与宿主模型的 tokenizer 独立。 +完整策略、SVG 图解及真实模型用例见[基于决策模型的上下文压缩](./context_compaction.md)。 + ## Embedding 配置 向量检索默认关闭。只设置 `EMBEDDING_API_KEY` 不会自动启用它；还需要同时启用 `as_embedding`、`embedding_store`，并把它连接到 `file_store`： diff --git a/docs/zh/context_compaction.md b/docs/zh/context_compaction.md new file mode 100644 index 00000000..1e26a9c6 --- /dev/null +++ b/docs/zh/context_compaction.md @@ -0,0 +1,147 @@ +# 基于决策模型的上下文压缩 + +长任务会积累大量工具调用：读过的源码、过期的搜索结果、重复查询的日志、已经结束的支线任务。 +`compact_context` 根据**下一步要做什么**，判断旧工具调用是否仍然有用，以及它的完整输出是否必须保留。 +它返回一份建议的消息快照和逐调用决策，由宿主决定是否应用。 + +这一流程通过 `as_decision_model` 接口调用概率决策模型。当前 `agentscope` 适配器使用 AgentScope 的 +TypeSafe/Jev 实现；替换供应商时，Job 和压缩策略仍然使用同一个接口。 + +![决策模型上下文压缩流程：保护规则、观测视图、双问题决策和宿主应用](../figure/decision-context-compaction.svg) + +## 一次压缩如何工作 + +1. **宿主先归档原始历史。** 宿主提供 AgentScope 2.x 的 JSON 消息快照、当前 `goal`、允许处理的工具名和保护列表。 + ReMe 不负责归档，也不会替宿主更新运行中的 Agent 状态。只有原始记录可恢复时，宿主才应应用删减建议。 +2. **用确定性规则挑选候选。** 按调用 ID 配对 `tool_call` 与 `tool_result`。调用必须为 `finished`，结果必须为 + `success`，工具必须在显式允许列表中，输出必须全部是文本。重复 ID、缺失配对、反向配对和非法 JSON 输入不会参与决策。 +3. **建立有限的观测视图。** 发送当前目标、对话文本、工具名、截短后的输入和结果字符数。工具输出正文与 thinking + 不进入决策请求。观测超预算时，逐步缩短输入和旧文本、移除旧的纯文本消息或合并旧调用记录；返回的会话原文不因此改变。 +4. **每个候选问两个二分类问题。** `call_` 判断“调用及其输入对下一步是否仍然重要”，`result_` 判断 + “完整输出是否必须原样保留，之后再检索是否不够”。问题成对分批，每批携带相同观测视图，在并发和总超时约束内执行。 +5. **全部批次成功后重建快照。** 校验每个答案的类型和概率范围，再在消息的深拷贝上应用决策。缺少答案、非法概率、 + 任意批次失败或超时都会使调用失败，不返回部分压缩结果。宿主应继续使用已保存的原始快照。 + +## 两个概率如何变成动作 + +设保留阈值为 `t`，默认 `0.5`。先判断结果，再判断调用： + +| 条件 | 动作 | 对返回快照的影响 | +| --- | --- | --- | +| `p_result >= t` | `keep` | 保留调用及完整结果，即使 `p_call` 较低 | +| `p_result < t` 且 `p_call >= t` | `drop_result` | 保留调用，将长文本结果替换为前缀和省略标记 | +| 两个概率都 `< t` | `drop_call` | 同时移除调用和结果；仅移除因此变空的消息 | + +`drop_result` 的默认前缀为 300 个字符，标记提示从归档历史恢复全文。只有新文本更短时才替换，所以该动作对短输出可能不产生变化。 +用户文本、assistant 文本、thinking 和未被修改的块字段保留原样。 + +以下内容优先受规则保护，不受概率阈值影响： + +- 第一条消息、最近 `preserve_recent_messages` 条消息，以及 `protected_message_ids` 指定的消息。配对的任一端受保护，整对保留。 +- `protected_call_ids` 指定的调用，以及不在 `eligible_tool_names` 中的工具。默认允许列表为空，因此默认不会压缩任何工具。 +- 未完成或失败的调用、非纯文本输出、工具名不一致的配对。重复或缺失 ID 的块也不会删除，但不一定出现在 `decisions` 中。 + +允许列表应由宿主限定为安全的只读工具，ReMe 不会根据工具名自动识别副作用。 +对于必须保留的审计记录、变更回执或关键证据，宿主应显式保护，不能只依赖语义概率。 + +## 配置与服务调用 + +安装并配置凭证： + +```bash +pip install "reme-ai[core,decision]" +export DECISION_MODEL_API_KEY=your-typesafe-api-key +export DECISION_MODEL_BACKEND=agentscope +# 可选：DECISION_MODEL_NAME、DECISION_MODEL_BASE_URL +``` + +`.env` 也可以配置这些变量。旧的 `CLASSIFIER_*` 名称需按[配置迁移说明](./configuration.md#决策模型配置)改为 +`DECISION_MODEL_*`；backend 应为 `agentscope`。省略模型名或配置为 `null` 时使用适配器的 SDK 默认模型。 + +在一个终端启动服务： + +```bash +reme start config=decision_compaction +``` + +在另一个终端调用。以下短示例展示请求格式；复杂样例见本文后面的集成用例： + +```bash +reme compact_context \ + goal='修复支付重试逻辑，之前完成的图标搜索已无关' \ + eligible_tool_names='["search_code"]' \ + options='{"preserve_recent_messages":1}' \ + messages='[{"id":"m0","role":"user","content":[{"type":"text","text":"保留支付代码，清理旧图标搜索"}]},{"id":"m1","role":"assistant","content":[{"type":"tool_call","id":"c1","name":"search_code","input":"{\"pattern\":\"logo\",\"path\":\"web/icons\"}","state":"finished"},{"type":"tool_result","id":"c1","name":"search_code","output":"旧任务的图标路径列表","state":"success"}]},{"id":"m2","role":"user","content":[{"type":"text","text":"现在只处理支付重试"}]}]' +``` + +`decision_compaction` 继承默认服务配置和其他 Job，因此常规后台任务也会启动。 +需要嵌入宿主进程时，可像集成测试一样只装配 `compact_context` Job 和它的决策组件，不启动其他工作流。 + +标准 Response 的 `answer` 包含： + +- `schema_version`、`messages`：版本与建议快照；`changed`、`reason`：是否实际改变，以及 `compacted` / `no_change` / `no_candidates`。 +- `decisions`：内部配对 ID、原始 `call_id`、动作、原因，以及参与推理时的 `keep_call` / `keep_result` 概率。 +- `stats`：前后消息数和字符数、候选与保护配对数量、观测视图的估算 token 数及缩减阶段、请求数、供应商 usage 和耗时。 + 同一份统计也位于 `response.metadata.compaction`。 + +宿主应检查 `success`，使用自己的 tokenizer 重新计算最终上下文预算，再选择应用建议、继续压缩或恢复原始记录。 +压缩不会自动生成摘要、写入长期记忆或执行历史中的工具。 + +## 预算、并发和超时 + +| `options` 字段 | 默认值 | 作用 | +| --- | --- | --- | +| `keep_threshold` | `0.5` | 两个问题共用的保留阈值 | +| `preserve_recent_messages` | `6` | 原样保护最近的消息，按消息而非轮次数计数 | +| `max_state_tokens` | `25000` | 观测视图的估算 token 上限 | +| `max_request_tokens` | `30000` | 每批观测与问题的估算 token 预算，包含 256 token 的开销预留 | +| `truncate_head_chars` | `300` | `drop_result` 保留的文本前缀长度 | +| `max_concurrency` | `4` | 同时运行的决策请求上限，范围 1–32 | +| `total_timeout_seconds` | `30` | 所有决策请求及其排队等待的总超时 | + +组件层的 `timeout`、`max_retries`、`retry_delay` 控制单次供应商请求及重试。 +内置压缩配置分别为 10 秒、1 次重试、0.5 秒初始退避。预算使用本地估算器，不等于供应商计费 token，也不等于宿主模型的上下文预算。 +观测无法装入预算，或一对问题也放不进单批请求时，调用失败，由宿主保留原始快照。 + +## 两个真实模型集成场景 + +[集成测试源码](https://github.com/agentscope-ai/ReMe/blob/main/tests/integration/test_decision_compaction.py) +提供两个合成的长任务场景。工具调用及其返回值均为测试数据，历史中的部署、回滚、数据库查询不会实际执行； +只有决策请求使用真实的 TypeSafe/Jev API。 + +| 场景 | 当前任务与关键候选 | 干扰与保护边界 | +| --- | --- | --- | +| 支付重试缺陷修复 | 下一步根据已读源码和测试原文生成精确补丁，验证相关源码与测试结果被保留 | 已完成的主题 CSS、图标搜索、过期库存查询应有可淘汰项；显式保护账单证据与审批回执，保留失败、运行中、重复 ID 和最近消息 | +| 数据库复制延迟排障 | 下一步引用事件日志及 WAL 位点原文编写根因报告，验证关键日志与数据行被保留 | 混入历史营销报表、旅行天气和退役系统压测；保护审计审批、回滚回执、非文本图表、失败查询和最近消息 | + +两个用例都检查：真实推理按批执行、至少移除一个无关调用、关键候选完整保留、受保护记录和会话文本不变、 +原始输入与临时归档不变，以及输出字符数实际下降。模型具有不确定性，语义断言可能揭示模型或配置的变化；测试不会伪造概率来保证通过。 + +2026-10-15 使用本地配置的 TypeSafe/Jev 端点实测，两例均通过，结果如下。数字按 JSON 字符数统计，不是 token 节省率： + +| 场景 | 压缩前字符数 | 压缩后字符数 | 减少比例 | 被移除的无关调用 | 决策请求数 | +| --- | ---: | ---: | ---: | ---: | ---: | +| 支付重试缺陷修复 | 39,438 | 12,738 | 67.7% | 3 | 2 | +| 数据库复制延迟排障 | 51,485 | 12,828 | 75.1% | 4 | 2 | + +显式授权真实 API 调用后运行： + +```bash +REME_RUN_DECISION_INTEGRATION=1 \ + pytest tests/integration/test_decision_compaction.py -v -s +``` + +测试读取环境变量或仓库根目录 `.env`，环境变量优先；未显式启用时直接跳过，未配置 API key 时也跳过。 +输出仅包含合成场景的动作、概率、字符数和 usage；工作区及原始快照放在 pytest 的临时目录中。 +实际节省比例依赖任务历史、目标和模型判断，不能由这两个用例推导通用收益。 + +## 适用边界 + +决策模型只能看到有限观测视图，无法核查被省略的工具输出内容。目标写得不完整或工具描述不足时，判断可能遗漏重要信息。 +宿主需要给出明确的下一步目标，为不可丢失的信息设置保护，并在应用后继续保留可恢复的原始历史。 +该策略适合减少旧工具轨迹的体积；它不提供自动触发、最终 token 预算保证、摘要生成或归档存储。 + +接口与实现分别见 [Compaction schema](https://github.com/agentscope-ai/ReMe/blob/main/reme/schema/compaction.py)、 +[候选配对与重建](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/messages.py)、 +[观测与分批](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/state.py) 和 +[Job Step](https://github.com/agentscope-ai/ReMe/blob/main/reme/steps/compaction/compact_context.py)。 diff --git a/example.env b/example.env index 8fec65f8..37c3aa76 100644 --- a/example.env +++ b/example.env @@ -7,7 +7,8 @@ LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 # Setting these variables alone does not enable an as_decision_model component. # DECISION_MODEL_API_KEY=your-typesafe-api-key # DECISION_MODEL_BACKEND=agentscope -# DECISION_MODEL_NAME=null # Use the adapter default model. +# DECISION_MODEL_NAME=null +# Omit the model name or use null for the adapter default. # DECISION_MODEL_BASE_URL=null # Optional data-source mirrors. Unset variables use the official services. diff --git a/tests/integration/test_decision_compaction.py b/tests/integration/test_decision_compaction.py new file mode 100644 index 00000000..1567ac15 --- /dev/null +++ b/tests/integration/test_decision_compaction.py @@ -0,0 +1,452 @@ +"""Two live decision-model scenarios; explicitly opt in to paid API calls. + +REME_RUN_DECISION_INTEGRATION=1 pytest tests/integration/test_decision_compaction.py -v -s +Credentials come from DECISION_MODEL_* in the environment or the repository .env. +Only synthetic history is sent. No tools in these histories are actually executed. +""" + +# pylint: disable=redefined-outer-name + +import asyncio +from collections import Counter +from copy import deepcopy +import json +import os +from pathlib import Path + +import pytest + +from reme import Application +from reme.config import resolve_app_config +from reme.schema.compaction import CompactionRequest +from reme.steps.compaction.messages import collect_pairs +from reme.steps.compaction.state import estimate_decision_tokens, fit_state +from reme.utils.env_utils import load_env + +pytestmark = pytest.mark.skipif( + os.environ.get("REME_RUN_DECISION_INTEGRATION") != "1", + reason="Set REME_RUN_DECISION_INTEGRATION=1 to authorize live decision API calls.", +) + + +@pytest.fixture +def live_config(): + """Load credentials only after opt-in; never print or persist them.""" + load_env(Path(__file__).resolve().parents[2] / ".env", override=False) + if not os.environ.get("DECISION_MODEL_API_KEY", "").strip(): + pytest.skip("DECISION_MODEL_API_KEY is required for live decisions.") + pytest.importorskip("agentscope.classifier") + config = resolve_app_config(config="decision_compaction", log_config=False) + component = config["components"]["as_decision_model"]["compaction"] + component["timeout"] = 30 + return { + "service": {"backend": "cli"}, + "jobs": {"compact_context": config["jobs"]["compact_context"]}, + "components": {"as_decision_model": {"compaction": component}}, + "enable_logo": False, + "log_to_console": False, + "log_to_file": False, + } + + +def _rows(prefix, count=100): + """Synthetic, reproducible tool output; no production or personal data.""" + return "\n".join(f"{prefix} row={i:04d} status=ok sample=synthetic trace=sample-{i:04d}" for i in range(count)) + + +def _pair(history, call_id, tool, arguments, output, note, *, result_state="success", split=False): + """Include realistic metadata and optionally span two messages.""" + call = { + "type": "tool_call", + "id": call_id, + "name": tool, + "input": json.dumps(arguments), + "state": "finished", + } + result = { + "type": "tool_result", + "id": call_id, + "name": tool, + "output": output, + "state": result_state, + "metadata": {"source": "synthetic-fixture"}, + } + history.append( + { + "id": call_id + "-call", + "role": "assistant", + "metadata": {"fixture": True}, + "content": [{"type": "text", "text": note}, call] + ([] if split else [result]), + }, + ) + if split: + history.append({"id": call_id + "-result", "role": "assistant", "content": [result]}) + + +def _blocks(messages, block_type): + return [block for message in messages for block in message["content"] if block["type"] == block_type] + + +async def _run_case( + config, + tmp_path, + *, + name, + history, + goal, + eligible, + protected_calls, + protected_messages, + must_keep, + irrelevant, +): + original = deepcopy(history) + archive = tmp_path / "original-history.json" + archive.write_text(json.dumps(history, ensure_ascii=False), encoding="utf-8") + archived_bytes = archive.read_bytes() + request = CompactionRequest( + messages=history, + goal=goal, + eligible_tool_names=eligible, + protected_call_ids=protected_calls, + protected_message_ids=protected_messages, + options={"preserve_recent_messages": 2, "max_state_tokens": 6000}, + ) + pairs = collect_pairs(request) + state, _ = fit_state(request, pairs) + # Fit a few question pairs per request, exercising repeated state and concurrency. + options = request.options.model_dump() + options.update( + max_request_tokens=estimate_decision_tokens({"state": state, "questions": {}}) + 750, + max_concurrency=2, + total_timeout_seconds=90, + ) + async with Application(**config, workspace_dir=str(tmp_path / "workspace")) as app: + response = await app.run_job("compact_context", **request.model_dump(exclude={"options"}), options=options) + assert response.success, "Live compact_context failed; inspect the configured endpoint and SDK." + result = response.answer + assert history == original + assert archive.read_bytes() == archived_bytes + assert _blocks(result["messages"], "text") == _blocks(original, "text") + decisions = {item["call_id"]: item for item in result["decisions"]} + for call_id in must_keep: + assert decisions[call_id]["action"] == "keep", f"Current verbatim evidence was lost: {call_id}" + assert any(decisions[call_id]["action"] == "drop_call" for call_id in irrelevant) + remaining = {message["id"]: message for message in result["messages"]} + protected_ids = {original[0]["id"], *(message["id"] for message in original[-2:]), *protected_messages} + for message in original: + if message["id"] in protected_ids: + assert remaining[message["id"]] == message + for pair in pairs: + if pair.protected: + for mi, bi in (pair.call_location, pair.result_location): + message = original[mi] + assert message["content"][bi] in remaining[message["id"]]["content"] + original_tools = [ + block for message in original for block in message["content"] if block["type"] in {"tool_call", "tool_result"} + ] + proposed_tools = [ + block + for message in result["messages"] + for block in message["content"] + if block["type"] in {"tool_call", "tool_result"} + ] + untouched = { + block["id"] + for block in original_tools + if block["id"] not in decisions or decisions[block["id"]]["reason"] == "protected" + } + for call_id in untouched: + assert [block for block in proposed_tools if block["id"] == call_id] == [ + block for block in original_tools if block["id"] == call_id + ] + for item in decisions.values(): + if item["reason"] != "protected": + assert 0 <= item["keep_call"] <= 1 and 0 <= item["keep_result"] <= 1 + stats = result["stats"] + assert result["changed"] and stats["chars_after"] < stats["chars_before"] + assert stats["requests"] >= 2 + assert stats["usage"], "The live provider should report token usage." + print( + json.dumps( + { + "case": name, + "actions": dict(Counter(item["action"] for item in decisions.values())), + "chars_before": stats["chars_before"], + "chars_after": stats["chars_after"], + "saved_percent": round(100 * (1 - stats["chars_after"] / stats["chars_before"]), 1), + "requests": stats["requests"], + "usage": stats["usage"], + "decisions": { + key: {field: value[field] for field in ("action", "keep_call", "keep_result")} + for key, value in decisions.items() + }, + }, + ensure_ascii=False, + ), + ) + + +def test_payment_retry_patch_keeps_source_and_removes_completed_ui_task(live_config, tmp_path): + """Exact next-edit evidence competes with large, completed unrelated work.""" + history = [ + { + "id": "first", + "role": "user", + "content": [ + { + "type": "text", + "text": "Earlier we finished a storefront theme task. " + "Now fix duplicate charges after payment retries. " + "Keep approval receipts intact; do not deploy anything.", + }, + ], + }, + ] + _pair( + history, + "old_css", + "read_file", + {"path": "web/theme/archive.css"}, + _rows("unused-theme"), + "The storefront theme task was completed and accepted. This CSS has no payment logic.", + ) + _pair( + history, + "old_icons", + "search_code", + {"pattern": "logo", "path": "web/icons"}, + _rows("icon-index"), + "Historical icon search from the finished UI task; unrelated to the current retry defect.", + split=True, + ) + _pair( + history, + "retry_source", + "read_file", + {"path": "src/payments/retry.py"}, + "def retry_charge(payment, gateway):\n for attempt in range(3):\n" + " try:\n return gateway.charge(payment)\n" + " except TimeoutError:\n continue\n" + _rows("source-context", 25), + "Current retry source. The next edit must match its exact original lines; no further reads before patching.", + split=True, + ) + _pair( + history, + "retry_tests", + "read_file", + {"path": "tests/payments/test_retry.py"}, + 'def test_timeout_reuses_idempotency_key():\n assert keys == ["payment-42"] * 3\n' + _rows("tests", 30), + "Current regression-test file. Keep its original lines to place the next patch correctly.", + ) + _pair( + history, + "obsolete_inventory", + "query_database", + {"table": "catalog.old_inventory", "month": "2024-01"}, + _rows("old-inventory", 180), + "Completed inventory task; these obsolete SKU rows cannot explain payment retries.", + ) + _pair( + history, + "billing_receipt", + "query_database", + {"table": "billing.events", "payment": "synthetic-42"}, + _rows("billing-evidence", 35), + "Explicitly pinned billing evidence for the incident report.", + ) + _pair( + history, + "approval", + "deploy_release", + {"release": "synthetic-old-release"}, + "approved by fixture operator", + "Historical approval receipt: a side-effecting tool, never eligible for removal.", + ) + _pair( + history, + "failed_probe", + "query_database", + {"table": "unavailable_replica"}, + "replica unreachable", + "Failed diagnostic attempt; preserve the failure record.", + result_state="error", + ) + _pair( + history, + "pending_probe", + "query_database", + {"table": "billing.pending"}, + "still running", + "Unfinished work is protected.", + result_state="running", + ) + for _ in range(2): + _pair( + history, + "ambiguous", + "read_file", + {"path": "unknown-snapshot"}, + "ambiguous duplicate identifier", + "Duplicate call identifiers are intentionally ineligible.", + ) + _pair( + history, + "recent_manifest", + "read_file", + {"path": "pyproject.toml"}, + _rows("manifest", 15), + "Most recent tool exchange; retain it by the recency rule.", + ) + goal = ( + "Next action: produce an exact minimal patch to src/payments/retry.py and its regression test, fixing " + "idempotency-key reuse after timeouts. You must use the verbatim source and test lines already read, " + "without another file read before the patch. The completed storefront theme and obsolete inventory " + "tasks are irrelevant. Do not deploy. Preserve pinned billing evidence and all approval receipts." + ) + history.append({"id": "last", "role": "user", "content": [{"type": "text", "text": goal}]}) + asyncio.run( + _run_case( + live_config, + tmp_path, + name="payment_retry_patch", + history=history, + goal=goal, + eligible=["read_file", "search_code", "query_database"], + protected_calls=["billing_receipt"], + protected_messages=["approval-call"], + must_keep=["retry_source", "retry_tests"], + irrelevant=["old_css", "old_icons", "obsolete_inventory"], + ), + ) + + +def test_replication_incident_keeps_verbatim_evidence_and_removes_unrelated_reports(live_config, tmp_path): + """A forensic report needs exact event records amid mixed operational history.""" + history = [ + { + "id": "first", + "role": "user", + "content": [ + { + "type": "text", + "text": "Investigate replica lag after the synthetic region-A failover. " + "Preserve audit evidence and do not roll back.", + }, + ], + }, + ] + _pair( + history, + "old_campaign", + "query_database", + {"table": "marketing.completed_campaign", "year": 2023}, + _rows("completed-ad-campaign", 180), + "Finished marketing report; no relation to this database incident.", + ) + _pair( + history, + "old_weather", + "read_file", + {"path": "personal/completed-trip-weather.txt"}, + _rows("weather", 100), + "Historical holiday planning from a completed task; entirely unrelated to database replication.", + ) + _pair( + history, + "old_loadtest", + "query_metrics", + {"region": "retired-region-Z", "period": "2021-load-test"}, + _rows("retired-system", 140), + "Old load test of a decommissioned system, not the production replica.", + ) + _pair( + history, + "incident_logs", + "query_logs", + {"region": "region-A", "window": "10:00-10:05", "service": "wal"}, + "10:00:01 event=F001 leader_switch term=17\n10:00:04 event=W002 apply_paused slot=replica-a\n" + "10:04:59 event=L003 lag_bytes=67108864\n" + _rows("wal-log-context", 50), + "Current incident log records. The next report must quote the exact event IDs, timestamps and lines; " + "summaries or later retrieval cannot replace these records.", + split=True, + ) + _pair( + history, + "wal_offsets", + "query_database", + {"table": "replication_slots", "region": "region-A"}, + "slot=replica-a restart_lsn=0/7000000 confirmed_lsn=0/3000000 active=false\n" + _rows("slot-evidence", 40), + "Current WAL-offset evidence. Keep the exact row for a verbatim evidence table in the next answer.", + ) + _pair( + history, + "old_costs", + "query_metrics", + {"dashboard": "finished-2022-advertising-budget"}, + _rows("ad-cost", 120), + "Completed advertising cost dashboard; unrelated to region-A failover or WAL offsets.", + ) + _pair( + history, + "audit_approval", + "read_file", + {"path": "audit/synthetic-change-approval.txt"}, + "ticket=SYN-008 approved_region=region-A action=failover", + "Pinned audit record; retain the entire message.", + ) + _pair( + history, + "rollback_receipt", + "rollback_release", + {"release": "old-synthetic-release"}, + "old rollback receipt", + "Historical side effect, not an eligible read-only query.", + ) + _pair( + history, + "chart", + "query_metrics", + {"chart": "replication"}, + [{"type": "data", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}], + "Non-text chart output must remain unchanged.", + ) + _pair( + history, + "failed_metrics", + "query_metrics", + {"region": "region-A", "metric": "disk"}, + "query failed", + "Failure record must remain intact.", + result_state="error", + ) + _pair( + history, + "recent_alert", + "query_logs", + {"alert": "synthetic-replica-lag"}, + _rows("recent-alert", 20), + "Newest exchange remains protected by recency.", + ) + goal = ( + "Next action: write the final region-A failover root-cause report using the exact incident_logs events " + "and wal_offsets rows already obtained. Quote every incident event line and the slot row verbatim; " + "do not perform additional log, database or metric queries before the report. Correlate the leader switch " + "with paused WAL apply and the offset gap. Ignore completed marketing, holiday, retired-system load-test " + "and advertising-budget tasks. Preserve the audit approval, chart and failed probes. Do not roll back." + ) + history.append({"id": "last", "role": "user", "content": [{"type": "text", "text": goal}]}) + asyncio.run( + _run_case( + live_config, + tmp_path, + name="replication_incident", + history=history, + goal=goal, + eligible=["read_file", "query_database", "query_logs", "query_metrics"], + protected_calls=[], + protected_messages=["audit_approval-call"], + must_keep=["incident_logs", "wal_offsets"], + irrelevant=["old_campaign", "old_weather", "old_loadtest", "old_costs"], + ), + )