fix(caching): re-key cached-miss embeddings to their input index

When an embedding batch mixes cached and uncached inputs, only the cache
misses are forwarded upstream, so the provider indexes them 0..k-1
against that shorter batch. `_combine_cached_embedding_response_with_api_result`
placed each fresh embedding back in the correct position but appended the
provider's object unchanged, so it kept the subset-relative index while
cached entries carried their original one.

The result was duplicate indices in a 200 response: `[0, 1, 0]` for two
cached inputs plus one fresh, `[0, 0, 2, 1]` when interleaved. Any client
mapping data[i].embedding back to input[data[i].index] then pairs
embeddings with the wrong text. All-fresh and all-cached batches were
unaffected, which is why this went unnoticed.

Build the corrected Embedding rather than assigning onto the item owned
by `embedding_response`, which LIT011 forbids.

Fixes #41002

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
deepak7lal 2026-09-21 22:13:19 +05:30
parent 27797923a3
commit e9078eb399
2 changed files with 76 additions and 3 deletions

View file

@ -659,9 +659,16 @@ class LLMCachingHandler:
idx = 0
final_data_list: Final = []
for item in _caching_handler_response.final_embedding_cached_response.data:
for position, item in enumerate(_caching_handler_response.final_embedding_cached_response.data):
if item is None and embedding_response.data is not None:
final_data_list.append(embedding_response.data[idx])
api_item = embedding_response.data[idx]
final_data_list.append(
Embedding(
embedding=api_item["embedding"],
index=position,
object="embedding",
)
)
idx += 1
else:
final_data_list.append(item)

View file

@ -11,7 +11,8 @@ from fastapi.testclient import TestClient
from datetime import datetime
from unittest.mock import AsyncMock
from litellm.caching.caching_handler import LLMCachingHandler
from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler
from litellm.types.utils import Embedding, EmbeddingResponse
@pytest.mark.asyncio
@ -780,3 +781,68 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl
assert hit.cached_result.choices[0].message.content == "done"
logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once()
assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True
@pytest.mark.parametrize(
"cache_pattern",
[
pytest.param([True, True, False], id="cached_first"),
pytest.param([False, True, True], id="fresh_first"),
pytest.param([True, False, True, False], id="interleaved"),
pytest.param([False, False, False], id="all_fresh_control"),
],
)
def test_embedding_cache_merge_preserves_input_index(cache_pattern):
"""
A batch mixing cached and uncached inputs must return data[i].index == i (#41002).
Cache misses are forwarded upstream as a shorter batch, so the provider indexes
them relative to that subset. Those indices have to be re-keyed to the position
of the input in the original request, or clients that map data[i].embedding back
to input[data[i].index] silently pair embeddings with the wrong text.
"""
handler = LLMCachingHandler(
original_function=MagicMock(),
request_kwargs={},
start_time=datetime.now(),
)
size = len(cache_pattern)
cached_result = [
{"embedding": [float(i)], "index": i, "object": "embedding"} if hit else None
for i, hit in enumerate(cache_pattern)
]
logging_obj = MagicMock()
logging_obj.async_success_handler = AsyncMock()
kwargs = {"model": "text-embedding-ada-002", "input": [f"text-{i}" for i in range(size)]}
final_cached, _ = handler._process_async_embedding_cached_response(
final_embedding_cached_response=None,
cached_result=cached_result,
kwargs=kwargs,
logging_obj=logging_obj,
start_time=datetime.now(),
model="text-embedding-ada-002",
)
missed_positions = [i for i, hit in enumerate(cache_pattern) if not hit]
api_response = EmbeddingResponse(
model="text-embedding-ada-002",
data=[
Embedding(embedding=[float(position)], index=subset_index, object="embedding")
for subset_index, position in enumerate(missed_positions)
],
)
combined = handler._combine_cached_embedding_response_with_api_result(
_caching_handler_response=CachingHandlerResponse(
final_embedding_cached_response=final_cached
),
embedding_response=api_response,
start_time=datetime.now(),
end_time=datetime.now(),
)
assert [d["index"] for d in combined.data] == list(range(size))
assert [d["embedding"] for d in combined.data] == [[float(i)] for i in range(size)]