test(embedding-store): cover a repeated rebuild of an over-limit chunk

`reindex` is what #683 was about, and while the request-level recovery had
a repeatability test, the rebuild path did not. Add the rebuild that runs
twice over the same over-limit chunk, with the embedding cache both off and
on, so a regression on the second pass fails here.

The shrinking helper keeps the reasoning behind halving the rejected text,
because that is what makes each round a real reduction.
This commit is contained in:
BeiMu-new 2026-10-28 23:15:22 +08:00
parent d17022fc32
commit 1d592792a6
3 changed files with 70 additions and 14 deletions

View file

@ -281,22 +281,18 @@ class LocalEmbeddingStore(BaseEmbeddingStore):
@staticmethod
def _shrink_rejected_input(text: str) -> str | None:
"""Return a strictly smaller request built from the text the provider rejected.
"""Return a strictly smaller request, or ``None`` when there is none.
The rejected text is the only reliable starting point: a budget derived from the
configured ``max_input_length`` can still sit above the provider's real per-item
limit, or leave an already short input untouched so nothing is re-sent. Halving the
*current* request keeps every round an actual reduction, and the caller bounds the
number of rounds so an impossible limit cannot spin.
Returns ``None`` when no smaller non-empty request can be produced.
The rejected text is the only reliable starting point: a budget derived
from the configured ``max_input_length`` can still sit above the
provider's real per-item limit, or leave an already short input
untouched so nothing is re-sent. Halving the *current* request keeps
every round an actual reduction, and the caller bounds the number of
rounds so an impossible limit cannot spin.
"""
if len(text) <= 1:
return None
target = max(1, len(text) // 2)
if target >= len(text):
return None
return text[:target]
return text[: len(text) // 2]
async def _embed_individually(self, texts: list[str], **kwargs) -> _EmbeddingResult:
"""Keep healthy siblings, stopping when a failure is no longer specific to an input."""

View file

@ -2235,3 +2235,58 @@ async def test_embedding_reindex_completes_when_a_chunk_exceeds_the_provider_lim
assert long_text not in provider.embedded_texts
await store.close()
@pytest.mark.parametrize("store_factory", [_new_local_store, _new_faiss_store, _new_zvec_store])
@pytest.mark.parametrize("long_chars", [5000, 2400])
@pytest.mark.parametrize("enable_cache", [False, True])
@pytest.mark.asyncio
async def test_second_reindex_recovers_the_same_over_limit_chunk(store_factory, long_chars, enable_cache):
"""The repeat rebuild must clear the gate again, not only the first one.
#683 is an availability bug: reindex never finished. The test above covers the first
rebuild; this one raises the gate a second time and requires the same over-limit chunk
to be recovered again, with the embedding cache both off and on.
"""
with tempfile.TemporaryDirectory() as tmp, temp_chdir(tmp):
store = store_factory(name=f"t_reindex_twice_{long_chars}_{enable_cache}")
await store.start()
long_text = "x" * long_chars
await set_chunks_with_graph(
store,
{
"good": chunk("good", "a.md", "alpha text"),
"long": chunk("long", "b.md", long_text),
},
)
provider = OverLimitAsEmbedding(max_chars=2048)
embedding = LocalEmbeddingStore(
name=f"t_reindex_twice_embedding_{long_chars}_{enable_cache}",
enable_cache=enable_cache,
max_input_length=8192,
)
embedding.as_embedding = provider
store.embedding_store = embedding
_ensure_zvec_collection(store)
if isinstance(store, FaissLocalFileStore):
store._rebuild_index()
elif isinstance(store, ZvecLocalFileStore):
store._rebuild_collection()
shortened = long_text
while len(shortened) > 2048:
shortened = shortened[: len(shortened) // 2]
for round_index in (1, 2):
await store.require_embedding_rebuild()
result = await store.reindex("embedding")
assert result == {"indexed": 2, "scope": "embedding"}
assert store._embedding_rebuild_pending is False
assert provider.embedded_texts.count(shortened) >= 1, f"rebuild {round_index}"
# Without a cache each rebuild has to reach the provider again; with one, the
# second rebuild is served from it and the shortened request is sent only once.
assert provider.embedded_texts.count(shortened) == (1 if enable_cache else 2)
await store.close()

View file

@ -2069,8 +2069,13 @@ async def test_shrinking_stops_when_no_smaller_request_can_fit():
@pytest.mark.asyncio
async def test_repeated_recovery_reuses_the_same_shrinking_path():
"""A second rebuild must recover the same item without any new configuration."""
async def test_repeated_request_recovery_reuses_the_same_shrinking_path():
"""Two consecutive requests on one store both recover the same item.
This is request-level repeatability only: no rebuild is performed and the gate is not
involved. The rebuild-side regression lives in
``test_file_store_consistency.py::test_second_reindex_recovers_the_same_over_limit_chunk``.
"""
provider = ShrinkablePerItemLimitAsEmbedding(max_chars=2048)
store = LocalEmbeddingStore(
name="t_local_embedding_shrink_repeat",