From 1d592792a6f3aac2618b07dacfa07d643d1318eb Mon Sep 17 00:00:00 2001 From: BeiMu-new <2690322907@qq.com> Date: Wed, 28 Oct 2026 23:15:22 +0800 Subject: [PATCH] test(embedding-store): cover a repeated rebuild of an over-limit chunk `reindex` is what #683 was about, and while the request-level recovery had a repeatability test, the rebuild path did not. Add the rebuild that runs twice over the same over-limit chunk, with the embedding cache both off and on, so a regression on the second pass fails here. The shrinking helper keeps the reasoning behind halving the rejected text, because that is what makes each round a real reduction. --- .../embedding_store/local_embedding_store.py | 20 +++---- tests/unit/test_file_store_consistency.py | 55 +++++++++++++++++++ tests/unit/test_local_embedding_store.py | 9 ++- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/reme/components/embedding_store/local_embedding_store.py b/reme/components/embedding_store/local_embedding_store.py index 57bc79b2..e49bae9f 100644 --- a/reme/components/embedding_store/local_embedding_store.py +++ b/reme/components/embedding_store/local_embedding_store.py @@ -281,22 +281,18 @@ class LocalEmbeddingStore(BaseEmbeddingStore): @staticmethod def _shrink_rejected_input(text: str) -> str | None: - """Return a strictly smaller request built from the text the provider rejected. + """Return a strictly smaller request, or ``None`` when there is none. - The rejected text is the only reliable starting point: a budget derived from the - configured ``max_input_length`` can still sit above the provider's real per-item - limit, or leave an already short input untouched so nothing is re-sent. Halving the - *current* request keeps every round an actual reduction, and the caller bounds the - number of rounds so an impossible limit cannot spin. - - Returns ``None`` when no smaller non-empty request can be produced. + The rejected text is the only reliable starting point: a budget derived + from the configured ``max_input_length`` can still sit above the + provider's real per-item limit, or leave an already short input + untouched so nothing is re-sent. Halving the *current* request keeps + every round an actual reduction, and the caller bounds the number of + rounds so an impossible limit cannot spin. """ if len(text) <= 1: return None - target = max(1, len(text) // 2) - if target >= len(text): - return None - return text[:target] + return text[: len(text) // 2] async def _embed_individually(self, texts: list[str], **kwargs) -> _EmbeddingResult: """Keep healthy siblings, stopping when a failure is no longer specific to an input.""" diff --git a/tests/unit/test_file_store_consistency.py b/tests/unit/test_file_store_consistency.py index 346a507e..6e1ebe28 100644 --- a/tests/unit/test_file_store_consistency.py +++ b/tests/unit/test_file_store_consistency.py @@ -2235,3 +2235,58 @@ async def test_embedding_reindex_completes_when_a_chunk_exceeds_the_provider_lim assert long_text not in provider.embedded_texts await store.close() + + +@pytest.mark.parametrize("store_factory", [_new_local_store, _new_faiss_store, _new_zvec_store]) +@pytest.mark.parametrize("long_chars", [5000, 2400]) +@pytest.mark.parametrize("enable_cache", [False, True]) +@pytest.mark.asyncio +async def test_second_reindex_recovers_the_same_over_limit_chunk(store_factory, long_chars, enable_cache): + """The repeat rebuild must clear the gate again, not only the first one. + + #683 is an availability bug: reindex never finished. The test above covers the first + rebuild; this one raises the gate a second time and requires the same over-limit chunk + to be recovered again, with the embedding cache both off and on. + """ + with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp): + store = store_factory(name=f"t_reindex_twice_{long_chars}_{enable_cache}") + await store.start() + long_text = "x" * long_chars + await set_chunks_with_graph( + store, + { + "good": chunk("good", "a.md", "alpha text"), + "long": chunk("long", "b.md", long_text), + }, + ) + provider = OverLimitAsEmbedding(max_chars=2048) + embedding = LocalEmbeddingStore( + name=f"t_reindex_twice_embedding_{long_chars}_{enable_cache}", + enable_cache=enable_cache, + max_input_length=8192, + ) + embedding.as_embedding = provider + store.embedding_store = embedding + _ensure_zvec_collection(store) + if isinstance(store, FaissLocalFileStore): + store._rebuild_index() + elif isinstance(store, ZvecLocalFileStore): + store._rebuild_collection() + + shortened = long_text + while len(shortened) > 2048: + shortened = shortened[: len(shortened) // 2] + + for round_index in (1, 2): + await store.require_embedding_rebuild() + result = await store.reindex("embedding") + + assert result == {"indexed": 2, "scope": "embedding"} + assert store._embedding_rebuild_pending is False + assert provider.embedded_texts.count(shortened) >= 1, f"rebuild {round_index}" + + # Without a cache each rebuild has to reach the provider again; with one, the + # second rebuild is served from it and the shortened request is sent only once. + assert provider.embedded_texts.count(shortened) == (1 if enable_cache else 2) + + await store.close() diff --git a/tests/unit/test_local_embedding_store.py b/tests/unit/test_local_embedding_store.py index c98e8acc..d21cd6d0 100644 --- a/tests/unit/test_local_embedding_store.py +++ b/tests/unit/test_local_embedding_store.py @@ -2069,8 +2069,13 @@ async def test_shrinking_stops_when_no_smaller_request_can_fit(): @pytest.mark.asyncio -async def test_repeated_recovery_reuses_the_same_shrinking_path(): - """A second rebuild must recover the same item without any new configuration.""" +async def test_repeated_request_recovery_reuses_the_same_shrinking_path(): + """Two consecutive requests on one store both recover the same item. + + This is request-level repeatability only: no rebuild is performed and the gate is not + involved. The rebuild-side regression lives in + ``test_file_store_consistency.py::test_second_reindex_recovers_the_same_over_limit_chunk``. + """ provider = ShrinkablePerItemLimitAsEmbedding(max_chars=2048) store = LocalEmbeddingStore( name="t_local_embedding_shrink_repeat",