From c773fa7b771d6b72657b5e6e2cc3dc56ed412089 Mon Sep 17 00:00:00 2001 From: fly liu Date: Fri, 23 Oct 2026 15:53:44 +0800 Subject: [PATCH] fix(search): explain empty results in metadata (#647) * fix(search): explain empty search results * fix(codex): ignore empty search notices * fix(integrations): skip empty search results in auto recall * fix(search): explain empty results in metadata --- docs/en/memory_search.md | 17 ++ docs/zh/memory_search.md | 17 ++ reme/steps/index/search.py | 8 + tests/unit/test_search_empty_response.py | 267 +++++++++++++++++++++++ 4 files changed, 309 insertions(+) create mode 100644 tests/unit/test_search_empty_response.py diff --git a/docs/en/memory_search.md b/docs/en/memory_search.md index 9c63a725..c2a72d6b 100644 --- a/docs/en/memory_search.md +++ b/docs/en/memory_search.md @@ -276,6 +276,23 @@ connects to. If a result is worth pursuing, use `read path=...` to open the sour followed by outlinks and inlinks. - `response.metadata`: structured programmatic results containing `results`, `link_expansion`, and `counts`. +When the default `search` Job returns no results, `response.answer` remains `""` and `success` remains `true`. +The `results` list is empty and `counts.returned` is `0`. An optional `response.metadata.empty_result` explains why: + +| `reason` | Meaning | +|----------------|-----------------------------------------------------------------------------| +| `no_results` | No candidates matched or survived `min_score`. | +| `all_returned` | All eligible chunks were already returned within the same `tool_context_id`. | + +The field also contains a human-readable `message`. Programmatic consumers should use `reason` to distinguish the +outcomes. Normal hits omit `empty_result` and keep their existing answer format. + +To show the explanation through the running service's CLI client, enable the existing metadata display option: + +```bash +reme search query="recent discussions about indexing" show_metadata=true +``` + Typical text structure: ```text diff --git a/docs/zh/memory_search.md b/docs/zh/memory_search.md index 1c73257f..520990ba 100644 --- a/docs/zh/memory_search.md +++ b/docs/zh/memory_search.md @@ -253,6 +253,23 @@ Memory Search 的“渐进式”不是一次把全库内容塞进结果，而是 - `response.answer`：给人看的文本，每个命中块包含路径、行号、分数和 chunk 内容，后面跟 outlinks / inlinks。 - `response.metadata`：给程序看的结构化结果，包括 `results`、`link_expansion`、`counts`。 +默认 `search` Job 没有结果可返回时，`response.answer` 保持 `""`，`success` 保持 `true`。 +此时 `results` 为空列表，`counts.returned` 为 `0`，可选字段 `response.metadata.empty_result` 说明原因： + +| `reason` | 含义 | +|----------------|-----------------------------------------------------------------| +| `no_results` | 没有匹配结果，或所有候选都被 `min_score` 过滤。 | +| `all_returned` | 符合分数要求的内容在同一 `tool_context_id` 中都已返回过。 | + +该字段还包含便于阅读的 `message`。程序调用方应通过 `reason` 区分两种情况。 +正常命中不包含 `empty_result`，正文格式保持原样。 + +通过 CLI 客户端调用已启动的服务时，可以开启已有的元数据显示选项查看原因： + +```bash +reme search query="最近关于索引的讨论" show_metadata=true +``` + 典型文本结构： ```text diff --git a/reme/steps/index/search.py b/reme/steps/index/search.py index 77ff4a4c..f4e20b3c 100644 --- a/reme/steps/index/search.py +++ b/reme/steps/index/search.py @@ -5,6 +5,7 @@ import datetime import os from typing import Final +from ._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE from ._tool_context import _ToolContextMixin from ..base_step import BaseStep from ..file_io import extract_daily_date @@ -195,6 +196,7 @@ class SearchStep(_ToolContextMixin, BaseStep): async def execute(self): assert self.context is not None + self.context.response.metadata.pop("empty_result", None) raw_query = self.context.get("query") query: str = "" if raw_query is None else str(raw_query).strip() limit: int = int(self.context.get("limit") or _default_limit()) @@ -341,6 +343,7 @@ class SearchStep(_ToolContextMixin, BaseStep): ) fused = [c for c in fused if c.score >= min_score] + pre_dedup_count = len(fused) dedup: dict | None = None if tool_context_id: fused, dedup = self._dedupe_tool_context(fused, tool_context_id, limit) @@ -361,6 +364,11 @@ class SearchStep(_ToolContextMixin, BaseStep): answer_lines.extend(render_expansion_lines(link_expansion.get(c.path, {}))) self.context.response.answer = "\n".join(answer_lines) + if not fused: + self.context.response.metadata["empty_result"] = { + "reason": "all_returned" if pre_dedup_count else "no_results", + "message": ALL_RETURNED_MESSAGE if pre_dedup_count else NO_RESULTS_MESSAGE, + } self.context.response.metadata["results"] = [ c.model_dump(exclude_none=True, exclude={"embedding"}) for c in fused ] diff --git a/tests/unit/test_search_empty_response.py b/tests/unit/test_search_empty_response.py new file mode 100644 index 00000000..179ccca8 --- /dev/null +++ b/tests/unit/test_search_empty_response.py @@ -0,0 +1,267 @@ +"""Empty search metadata explains missing results without changing answer compatibility.""" + +# pylint: disable=protected-access + +import asyncio +import json +from unittest.mock import AsyncMock, Mock + +import pytest + +from reme.components import ApplicationContext +from reme.components.client.http_client import HttpClient +from reme.components.file_store import BaseFileStore +from reme.components.runtime_context import RuntimeContext +from reme.components.service.cli_service import CliService +from reme.schema import FileChunk +from reme.steps.index import SearchStep +from reme.steps.index._source_format import ALL_RETURNED_MESSAGE, NO_RESULTS_MESSAGE + + +def _chunk(chunk_id="match", score=2.0): + """Create a scored hit with an embedding that must not appear in results.""" + return FileChunk( + id=chunk_id, + path=f"daily/{chunk_id}.md", + text=f"memory {chunk_id}", + start_line=1, + end_line=2, + embedding=[0.1, 0.2], + scores={"score": score}, + ) + + +def _store(chunks=(), mode="keyword"): + """Mock retrieval boundaries without starting indexes or model providers.""" + store = Mock(spec=BaseFileStore) + store.vector_search = AsyncMock(return_value=list(chunks) if mode in {"vector", "hybrid"} else []) + store.keyword_search = AsyncMock(return_value=list(chunks) if mode in {"keyword", "hybrid"} else []) + return store + + +def _step(tmp_path, store, *, app_context=None, vector_weight=0.0): + """Use an isolated application, sharing it explicitly for repeated calls.""" + return SearchStep( + app_context=app_context if app_context is not None else ApplicationContext(workspace_dir=str(tmp_path)), + file_store=store, + vector_weight=vector_weight, + expand_links=False, + ) + + +@pytest.mark.parametrize( + "mode,weight,score,min_score", + [ + ("keyword", 0.7, None, 0.0), + ("keyword", 0.0, 0.1, 1.0), + ("vector", 1.0, 0.1, 1.0), + ("hybrid", 0.7, 0.1, 1.0), + ], + ids=["no-matches", "keyword-below-threshold", "vector-below-threshold", "hybrid-below-threshold"], +) +@pytest.mark.parametrize("tool_context_id", ["", "ctx-1"], ids=["without-context", "with-context"]) +def test_search_without_eligible_results_explains_empty_metadata( + tmp_path, + mode, + weight, + score, + min_score, + tool_context_id, +): + """No matches and score-filtered hits share the same empty-result behavior.""" + chunks = [] if score is None else [_chunk(score=score)] + response = asyncio.run( + _step(tmp_path, _store(chunks, mode), vector_weight=weight)( + RuntimeContext( + query="missing" if score is None else "memory", + min_score=min_score, + tool_context_id=tool_context_id, + ), + ), + ) + + assert response.success + assert response.answer == "" + assert response.metadata["empty_result"] == {"reason": "no_results", "message": NO_RESULTS_MESSAGE} + assert response.metadata["results"] == [] + assert response.metadata["link_expansion"] == {} + assert response.metadata["counts"] == { + "vector": int(score is not None and mode in {"vector", "hybrid"}), + "keyword": int(score is not None and mode in {"keyword", "hybrid"}), + "returned": 0, + "hybrid": score is not None and mode == "hybrid", + } + if tool_context_id: + assert response.metadata["dedup"]["skipped_seen"] == 0 + else: + assert "dedup" not in response.metadata + + +def test_search_all_seen_explains_empty_metadata_across_step_instances(tmp_path): + """Fresh Step instances share previously returned content through the application.""" + + async def run(): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + store = _store([_chunk()]) + first = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-1"), + ) + second = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-1"), + ) + + assert first.metadata["counts"]["returned"] == 1 + assert "empty_result" not in first.metadata + assert second.success + assert second.answer == "" + assert second.metadata["empty_result"] == {"reason": "all_returned", "message": ALL_RETURNED_MESSAGE} + assert second.metadata["results"] == [] + assert second.metadata["link_expansion"] == {} + assert second.metadata["counts"] == {"vector": 0, "keyword": 1, "returned": 0, "hybrid": False} + assert second.metadata["dedup"]["seen_before"] == 1 + assert second.metadata["dedup"]["skipped_seen"] == 1 + + asyncio.run(run()) + + +@pytest.mark.parametrize("min_score", [0.0, 3.0], ids=["no-matches", "below-threshold"]) +def test_search_history_does_not_turn_missing_results_into_all_returned(tmp_path, min_score): + """Past hits do not explain a new query with no eligible candidates.""" + + async def run(): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + store = _store([_chunk()]) + first = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-1"), + ) + assert first.metadata["counts"]["returned"] == 1 + if not min_score: + store.keyword_search.return_value = [] + response = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="another query", min_score=min_score, tool_context_id="ctx-1"), + ) + + assert response.success + assert response.answer == "" + assert response.metadata["empty_result"] == {"reason": "no_results", "message": NO_RESULTS_MESSAGE} + assert response.metadata["results"] == [] + assert response.metadata["counts"]["returned"] == 0 + assert response.metadata["dedup"]["seen_before"] == 1 + assert response.metadata["dedup"]["skipped_seen"] == 0 + + asyncio.run(run()) + + +@pytest.mark.parametrize("tool_context_id", ["", "ctx-1"], ids=["without-context", "with-context"]) +def test_search_hit_keeps_answer_and_structured_results(tmp_path, tool_context_id): + """A normal hit keeps its exact rendering and excludes the embedding from metadata.""" + chunk = _chunk() + response = asyncio.run( + _step(tmp_path, _store([chunk]))(RuntimeContext(query="memory", tool_context_id=tool_context_id)), + ) + + assert response.success + assert "empty_result" not in response.metadata + assert response.answer == "========== daily/match.md:1-2 [score=2.0000] ==========\nmemory match" + assert response.metadata["results"] == [chunk.model_dump(exclude_none=True, exclude={"embedding"})] + assert response.metadata["link_expansion"] == {} + assert response.metadata["counts"] == {"vector": 0, "keyword": 1, "returned": 1, "hybrid": False} + + +def test_search_partial_dedup_and_context_isolation_keep_hit_answers(tmp_path): + """Unseen content still renders normally and another context can retrieve seen content.""" + + async def run(): + app_context = ApplicationContext(workspace_dir=str(tmp_path)) + store = _store([_chunk()]) + await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-1"), + ) + store.keyword_search.return_value = [_chunk(), _chunk("new")] + response = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-1"), + ) + + assert response.success + assert "empty_result" not in response.metadata + assert response.answer == "========== daily/new.md:1-2 [score=2.0000] ==========\nmemory new" + assert [result["id"] for result in response.metadata["results"]] == ["new"] + assert response.metadata["dedup"]["skipped_seen"] == 1 + + other = await _step(tmp_path, store, app_context=app_context)( + RuntimeContext(query="memory", tool_context_id="ctx-2"), + ) + assert [result["id"] for result in other.metadata["results"]] == ["match", "new"] + assert "memory match" in other.answer + assert "memory new" in other.answer + assert other.metadata["dedup"]["skipped_seen"] == 0 + assert "empty_result" not in other.metadata + + asyncio.run(run()) + + +@pytest.mark.parametrize("next_query", ["memory", ""], ids=["normal-hit", "invalid-query"]) +def test_search_reused_response_does_not_keep_empty_reason(tmp_path, next_query): + """A later hit or error must not inherit an earlier empty search explanation.""" + + async def run(): + store = _store() + step = _step(tmp_path, store) + context = RuntimeContext(query="missing") + context.response.metadata["other_step"] = "preserved" + first = await step(context) + assert first.answer == "" + assert first.metadata["empty_result"]["reason"] == "no_results" + + store.keyword_search.return_value = [_chunk()] + context.update({"query": next_query}) + response = await step(context) + + assert "empty_result" not in response.metadata + assert response.metadata["other_step"] == "preserved" + if next_query: + assert response.success + assert "memory match" in response.answer + assert response.metadata["counts"]["returned"] == 1 + else: + assert not response.success + assert response.answer == "Error: query cannot be empty" + + asyncio.run(run()) + + +@pytest.mark.parametrize("all_returned", [False, True], ids=["no-results", "all-returned"]) +@pytest.mark.parametrize("show_metadata", [False, True], ids=["default-display", "show-metadata"]) +def test_empty_search_explanation_uses_existing_cli_metadata_display(tmp_path, capsys, all_returned, show_metadata): + """Both CLI paths expose reasons only when metadata display is requested.""" + + async def search(): + store = _store([_chunk()] if all_returned else []) + step = _step(tmp_path, store) + if all_returned: + await step(RuntimeContext(query="memory", tool_context_id="ctx-1")) + return await step(RuntimeContext(query="memory", tool_context_id="ctx-1")) + + response = asyncio.run(search()) + expected = { + "reason": "all_returned" if all_returned else "no_results", + "message": ALL_RETURNED_MESSAGE if all_returned else NO_RESULTS_MESSAGE, + } + assert response.answer == "" + assert response.success + assert response.metadata["empty_result"] == expected + + http_text = HttpClient(show_metadata=show_metadata)._format_for_display(response.model_dump_json()) + app = Mock(start=AsyncMock(), run_job=AsyncMock(return_value=response), close=AsyncMock()) + CliService(job="search", job_args={"query": "memory"}, show_metadata=show_metadata).start_service(app) + output = capsys.readouterr() + + assert output.err == "" + app.run_job.assert_awaited_once_with("search", query="memory") + app.close.assert_awaited_once() + if show_metadata: + assert json.loads(http_text.removeprefix("\n✅ "))["empty_result"] == expected + assert json.loads(output.out)["empty_result"] == expected + else: + assert http_text == "\n✅" + assert output.out == "\n"