From 1a8ff12df1671900f974748cb7c02a9d432842eb Mon Sep 17 00:00:00 2001 From: jinliyl <6469360+jinliyl@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:54:56 +0800 Subject: [PATCH] fix(search): clarify score thresholds and warn above hybrid bound (#642) --- docs/en/memory_search.md | 20 ++++++++ docs/zh/memory_search.md | 17 +++++++ reme/config/default.yaml | 6 ++- reme/steps/index/search.py | 8 ++++ tests/unit/test_search_score_threshold.py | 58 +++++++++++++++++++++++ 5 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_search_score_threshold.py diff --git a/docs/en/memory_search.md b/docs/en/memory_search.md index 43446c6e..9c63a725 100644 --- a/docs/en/memory_search.md +++ b/docs/en/memory_search.md @@ -208,6 +208,26 @@ fused_score = vector_weight / (60 + vector_rank) The default `vector_weight=0.7` gives semantic recall more weight when embeddings are enabled, while keyword search can still promote chunks with exact term matches. +### Score thresholds + +Keep `min_score=0` (the default) for normal hybrid search and use `limit` to select the top results. +Positive `min_score` values filter the returned score before truncation, using the scale of the branches that +actually returned hits for that query: + +| Branches with hits | Score compared with `min_score` | +| --- | --- | +| Vector only | Native vector-backend score; its range depends on the backend | +| BM25 only | Native BM25 score, without a fixed upper bound | +| Both | Weighted RRF rank score, at most `1/61` (approximately `0.01639`) | + +RRF scores express rank contributions, not normalized relevance or probabilities. The maximum assumes the same chunk +ranks first in both lists; a vector-only contribution with the default weight is at most `0.7/61`. +Changing `vector_weight` changes individual contributions, while the two weights still sum to 1. +The score scale can change between queries when one branch has no hits, so a single positive threshold is not +comparable across all modes. For example, `min_score=0.05` removes every hybrid result even if its native vector or +BM25 score is high. Search logs a warning when both branches have hits and the threshold exceeds `1/61`. +The threshold is still applied: the warning does not lower it or restore filtered results. + ## How BM25 Works `keyword_search()` calls `keyword_index.retrieve(query, limit)`. Each chunk is a document in the BM25 index: diff --git a/docs/zh/memory_search.md b/docs/zh/memory_search.md index 632a579c..1c73257f 100644 --- a/docs/zh/memory_search.md +++ b/docs/zh/memory_search.md @@ -193,6 +193,23 @@ fused_score = vector_weight / (60 + vector_rank) 默认 `vector_weight=0.7`，所以启用 embedding 后语义召回权重更高；关键词仍能把精确词命中的 chunk 拉上来。 +### 分数阈值 + +常规混合搜索建议保持 `min_score=0`（默认值），用 `limit` 选取排名靠前的结果。 +正数 `min_score` 会在截断之前过滤返回分数，其尺度取决于本次查询实际有命中的分支： + +| 有命中的分支 | 与 `min_score` 比较的分数 | +| --- | --- | +| 只有向量 | 向量后端的原始分数，范围取决于后端 | +| 只有 BM25 | BM25 原始分数，没有固定上限 | +| 两边都有 | 加权 RRF 排名分数，最大为 `1/61`（约 `0.01639`） | + +RRF 分数表示排名贡献，不是归一化相关性或概率。最大值要求同一 chunk 在两个列表中都排第一；默认权重下， +仅来自向量分支的贡献最大为 `0.7/61`。调整 `vector_weight` 会改变各分支的贡献，但两个权重之和仍为 1。 +当某个分支没有命中时，分数尺度会随查询切换，因此同一个正数阈值无法在所有模式下进行等价比较。 +例如，`min_score=0.05` 会过滤掉所有混合结果，即使它们的原始向量或 BM25 分数很高。 +当两边都有命中且阈值超过 `1/61` 时，搜索会记录警告。阈值仍会照常应用，不会因警告而降低或恢复已过滤结果。 + ## BM25 怎么工作 `keyword_search()` 调用 `keyword_index.retrieve(query, limit)`。BM25 索引里每个 chunk 是一篇文档： diff --git a/reme/config/default.yaml b/reme/config/default.yaml index 9715fa50..f01fdf66 100644 --- a/reme/config/default.yaml +++ b/reme/config/default.yaml @@ -461,7 +461,11 @@ jobs: default: 5 min_score: type: number - description: "min fused score" + description: >- + Minimum returned score; 0 disables filtering (recommended). + Uses native vector or BM25 scores when only one branch has hits, + and rank-based RRF scores (at most 1/61, about 0.01639) when both have hits. + The scale can change per query; this is not a normalized relevance threshold. default: 0.0 tags: type: array diff --git a/reme/steps/index/search.py b/reme/steps/index/search.py index 8d644aa1..928587f5 100644 --- a/reme/steps/index/search.py +++ b/reme/steps/index/search.py @@ -339,6 +339,14 @@ class SearchStep(BaseStep): fused = self._rrf_merge(vector_results, keyword_results, vector_weight) if min_score > 0.0: + max_rrf_score = 1.0 / (_RRF_K + 1) + if hybrid and min_score > max_rrf_score: + self.logger.warning( + f"[{self.name}] min_score={min_score:g} exceeds the hybrid RRF maximum " + f"{max_rrf_score:.8g}; all hybrid results will be filtered out. " + "RRF scores are rank-based, not vector similarities or BM25 scores. " + "Keep min_score=0 to disable filtering; single-branch queries use native scores.", + ) fused = [c for c in fused if c.score >= min_score] dedup: dict | None = None diff --git a/tests/unit/test_search_score_threshold.py b/tests/unit/test_search_score_threshold.py new file mode 100644 index 00000000..82cd1a9a --- /dev/null +++ b/tests/unit/test_search_score_threshold.py @@ -0,0 +1,58 @@ +"""Score-scale diagnostics preserve existing search threshold behavior.""" + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from reme.components.file_store import BaseFileStore +from reme.components.runtime_context import RuntimeContext +from reme.schema import FileChunk +from reme.steps.index import SearchStep + + +@pytest.mark.parametrize( + "mode,weight,min_score,expected_ids,warn", + [ + ("hybrid", 0.7, 0.05, [], True), + ("hybrid", 0.2, 0.05, [], True), + ("hybrid", 0.7, 1 / 61, ["match"], False), + ("hybrid", 0.7, 0.01, ["match"], False), + ("hybrid", 0.7, 0, ["match"], False), + ("vector", 0.7, 0.05, ["match"], False), + ("keyword", 0.7, 1, ["match"], False), + ("vector", 1, 0.05, ["match"], False), + ("keyword", 0, 1, ["match"], False), + ("empty", 0.7, 0.05, [], False), + ], +) +def test_search_score_threshold_warning(mode, weight, min_score, expected_ids, warn): + """Warn only above the hybrid bound, preserving native scores and exact-bound hits.""" + vector = FileChunk( + id="match", + path="daily/match.md", + text="matching memory", + start_line=1, + end_line=1, + scores={"vector": 0.9, "score": 0.9}, + ) + keyword = vector.model_copy(update={"scores": {"keyword": 8.0, "score": 8.0}}) + store = Mock(spec=BaseFileStore) + store.vector_search = AsyncMock(return_value=[vector] if mode in {"hybrid", "vector"} else []) + store.keyword_search = AsyncMock(return_value=[keyword] if mode in {"hybrid", "keyword"} else []) + step = SearchStep(file_store=store, vector_weight=weight, expand_links=False) + step.logger = Mock() + + response = asyncio.run(step(RuntimeContext(query="memory", min_score=min_score))) + + assert response.success + assert [result["id"] for result in response.metadata["results"]] == expected_ids + assert step.logger.warning.call_count == int(warn) + if warn: + message = step.logger.warning.call_args.args[0] + assert "min_score=0.05 exceeds the hybrid RRF maximum" in message + assert "all hybrid results will be filtered out" in message + assert "min_score=0" in message + elif expected_ids: + expected_score = {"hybrid": 1 / 61, "vector": 0.9, "keyword": 8.0}[mode] + assert response.metadata["results"][0]["scores"]["score"] == pytest.approx(expected_score)