diff --git a/benchmark/README.md b/benchmark/README.md index 930a5ec1..0522b09e 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -52,6 +52,7 @@ python download.py # saves dataset/longmemeval_s_reme_cleaned.json; s **BEAM** (public repository, cloned into `benchmark/beam/dataset/`): ```bash +mkdir -p benchmark/beam/dataset cd benchmark/beam/dataset git clone https://github.com/mohammadtavakoli78/BEAM.git ``` diff --git a/benchmark/README_ZH.md b/benchmark/README_ZH.md index cf4c4bbb..0ac8fd3e 100644 --- a/benchmark/README_ZH.md +++ b/benchmark/README_ZH.md @@ -48,6 +48,7 @@ python download.py # 保存为 dataset/longmemeval_s_reme_cleaned.jso **BEAM**(公开仓库,clone 到 `benchmark/beam/dataset/` 下): ```bash +mkdir -p benchmark/beam/dataset cd benchmark/beam/dataset git clone https://github.com/mohammadtavakoli78/BEAM.git ``` diff --git a/reme/config/beam.yaml b/reme/config/beam.yaml index cf7981fc..430bb743 100644 --- a/reme/config/beam.yaml +++ b/reme/config/beam.yaml @@ -170,6 +170,28 @@ jobs: # chunking in the index.) <= 0 disables splitting. max_segment_words: 10000 + # ── Text compression (direct LLM call, no agent) ── + compressor: + backend: base + description: "Compress text via a direct LLM call, optionally guided by queries as relevance filter" + parameters: + type: object + properties: + text: + type: string + description: "the text to compress" + queries: + type: array + description: "optional list of queries; content potentially relevant to any query is kept, content certainly irrelevant to all queries may be dropped" + items: + type: string + default: [] + required: + - text + steps: + - backend: compressor_step + as_llm: compressor + # ── Reindex (full rebuild) ── reindex: backend: base @@ -516,6 +538,19 @@ components: parameters: max_tokens: 65536 thinking_enable: true + compressor: + backend: ${LLM_BACKEND:-openai} + model: ${LLM_MODEL_NAME:-qwen3.6-flash} + stream: false + context_size: 200000 + max_retries: 5 + retry_delay: 5.0 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false agent_wrapper: default: diff --git a/reme/steps/index/search_v2.py b/reme/steps/index/search_v2.py index b9e7581a..02462a76 100644 --- a/reme/steps/index/search_v2.py +++ b/reme/steps/index/search_v2.py @@ -258,8 +258,26 @@ class SearchV2Step(_ToolContextDedupMixin, BaseStep): succeeded and the result is not longer than the input; adopted bodies get a leading ``compressed session chunk:`` marker so downstream consumers can tell them from verbatim transcripts. + + Degrades gracefully when the ``compressor`` job is missing from the + active config: the whole method becomes a no-op and a warning is + logged, so search behaves as if compression were disabled. This + avoids a hard ``Job compressor not found`` failure when a benchmark + config forgets to define the compressor job/component. """ assert self.context is not None + # Guard: when the compressor job is missing from the active config + # (e.g. a benchmark config that forgot to define it), degrade + # gracefully to the no-compression behavior instead of raising + # "Job compressor not found" from run_job below. + # Skipped when there is no app_context (e.g. unit tests that mock + # run_job directly), so the mock can still drive compression. + if self.app_context is not None and self.get_job("compressor") is None: + self.logger.warning( + f"[{self.name}] compressor job not found in config; " + "skipping session chunk compression (degrading to uncompressed behavior)", + ) + return search_cfg: dict = self.context.get("_search") or {} query_type = str(search_cfg.get("type") or "query-aware").strip().lower() if query_type == "query-independent":