mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(caching): re-key cached-miss embeddings to their input index
When an embedding batch mixes cached and uncached inputs, only the cache misses are forwarded upstream, so the provider indexes them 0..k-1 against that shorter batch. `_combine_cached_embedding_response_with_api_result` placed each fresh embedding back in the correct position but appended the provider's object unchanged, so it kept the subset-relative index while cached entries carried their original one. The result was duplicate indices in a 200 response: `[0, 1, 0]` for two cached inputs plus one fresh, `[0, 0, 2, 1]` when interleaved. Any client mapping data[i].embedding back to input[data[i].index] then pairs embeddings with the wrong text. All-fresh and all-cached batches were unaffected, which is why this went unnoticed. Build the corrected Embedding rather than assigning onto the item owned by `embedding_response`, which LIT011 forbids. Fixes #41002 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
27797923a3
commit
e9078eb399
2 changed files with 76 additions and 3 deletions
|
|
@ -659,9 +659,16 @@ class LLMCachingHandler:
|
|||
|
||||
idx = 0
|
||||
final_data_list: Final = []
|
||||
for item in _caching_handler_response.final_embedding_cached_response.data:
|
||||
for position, item in enumerate(_caching_handler_response.final_embedding_cached_response.data):
|
||||
if item is None and embedding_response.data is not None:
|
||||
final_data_list.append(embedding_response.data[idx])
|
||||
api_item = embedding_response.data[idx]
|
||||
final_data_list.append(
|
||||
Embedding(
|
||||
embedding=api_item["embedding"],
|
||||
index=position,
|
||||
object="embedding",
|
||||
)
|
||||
)
|
||||
idx += 1
|
||||
else:
|
||||
final_data_list.append(item)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ from fastapi.testclient import TestClient
|
|||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from litellm.caching.caching_handler import LLMCachingHandler
|
||||
from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler
|
||||
from litellm.types.utils import Embedding, EmbeddingResponse
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -780,3 +781,68 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl
|
|||
assert hit.cached_result.choices[0].message.content == "done"
|
||||
logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once()
|
||||
assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cache_pattern",
|
||||
[
|
||||
pytest.param([True, True, False], id="cached_first"),
|
||||
pytest.param([False, True, True], id="fresh_first"),
|
||||
pytest.param([True, False, True, False], id="interleaved"),
|
||||
pytest.param([False, False, False], id="all_fresh_control"),
|
||||
],
|
||||
)
|
||||
def test_embedding_cache_merge_preserves_input_index(cache_pattern):
|
||||
"""
|
||||
A batch mixing cached and uncached inputs must return data[i].index == i (#41002).
|
||||
|
||||
Cache misses are forwarded upstream as a shorter batch, so the provider indexes
|
||||
them relative to that subset. Those indices have to be re-keyed to the position
|
||||
of the input in the original request, or clients that map data[i].embedding back
|
||||
to input[data[i].index] silently pair embeddings with the wrong text.
|
||||
"""
|
||||
handler = LLMCachingHandler(
|
||||
original_function=MagicMock(),
|
||||
request_kwargs={},
|
||||
start_time=datetime.now(),
|
||||
)
|
||||
|
||||
size = len(cache_pattern)
|
||||
cached_result = [
|
||||
{"embedding": [float(i)], "index": i, "object": "embedding"} if hit else None
|
||||
for i, hit in enumerate(cache_pattern)
|
||||
]
|
||||
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.async_success_handler = AsyncMock()
|
||||
|
||||
kwargs = {"model": "text-embedding-ada-002", "input": [f"text-{i}" for i in range(size)]}
|
||||
final_cached, _ = handler._process_async_embedding_cached_response(
|
||||
final_embedding_cached_response=None,
|
||||
cached_result=cached_result,
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
start_time=datetime.now(),
|
||||
model="text-embedding-ada-002",
|
||||
)
|
||||
|
||||
missed_positions = [i for i, hit in enumerate(cache_pattern) if not hit]
|
||||
api_response = EmbeddingResponse(
|
||||
model="text-embedding-ada-002",
|
||||
data=[
|
||||
Embedding(embedding=[float(position)], index=subset_index, object="embedding")
|
||||
for subset_index, position in enumerate(missed_positions)
|
||||
],
|
||||
)
|
||||
|
||||
combined = handler._combine_cached_embedding_response_with_api_result(
|
||||
_caching_handler_response=CachingHandlerResponse(
|
||||
final_embedding_cached_response=final_cached
|
||||
),
|
||||
embedding_response=api_response,
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
|
||||
assert [d["index"] for d in combined.data] == list(range(size))
|
||||
assert [d["embedding"] for d in combined.data] == [[float(i)] for i in range(size)]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue