diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 50426ea89ea..f2af7258322 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -659,9 +659,16 @@ class LLMCachingHandler: idx = 0 final_data_list: Final = [] - for item in _caching_handler_response.final_embedding_cached_response.data: + for position, item in enumerate(_caching_handler_response.final_embedding_cached_response.data): if item is None and embedding_response.data is not None: - final_data_list.append(embedding_response.data[idx]) + api_item = embedding_response.data[idx] + final_data_list.append( + Embedding( + embedding=api_item["embedding"], + index=position, + object="embedding", + ) + ) idx += 1 else: final_data_list.append(item) diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 39018dca41d..b63c129ea54 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -11,7 +11,8 @@ from fastapi.testclient import TestClient from datetime import datetime from unittest.mock import AsyncMock -from litellm.caching.caching_handler import LLMCachingHandler +from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler +from litellm.types.utils import Embedding, EmbeddingResponse @pytest.mark.asyncio @@ -780,3 +781,68 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl assert hit.cached_result.choices[0].message.content == "done" logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once() assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True + + +@pytest.mark.parametrize( + "cache_pattern", + [ + pytest.param([True, True, False], id="cached_first"), + pytest.param([False, True, True], id="fresh_first"), + pytest.param([True, False, True, False], id="interleaved"), + pytest.param([False, False, False], id="all_fresh_control"), + ], +) +def test_embedding_cache_merge_preserves_input_index(cache_pattern): + """ + A batch mixing cached and uncached inputs must return data[i].index == i (#41002). + + Cache misses are forwarded upstream as a shorter batch, so the provider indexes + them relative to that subset. Those indices have to be re-keyed to the position + of the input in the original request, or clients that map data[i].embedding back + to input[data[i].index] silently pair embeddings with the wrong text. + """ + handler = LLMCachingHandler( + original_function=MagicMock(), + request_kwargs={}, + start_time=datetime.now(), + ) + + size = len(cache_pattern) + cached_result = [ + {"embedding": [float(i)], "index": i, "object": "embedding"} if hit else None + for i, hit in enumerate(cache_pattern) + ] + + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + + kwargs = {"model": "text-embedding-ada-002", "input": [f"text-{i}" for i in range(size)]} + final_cached, _ = handler._process_async_embedding_cached_response( + final_embedding_cached_response=None, + cached_result=cached_result, + kwargs=kwargs, + logging_obj=logging_obj, + start_time=datetime.now(), + model="text-embedding-ada-002", + ) + + missed_positions = [i for i, hit in enumerate(cache_pattern) if not hit] + api_response = EmbeddingResponse( + model="text-embedding-ada-002", + data=[ + Embedding(embedding=[float(position)], index=subset_index, object="embedding") + for subset_index, position in enumerate(missed_positions) + ], + ) + + combined = handler._combine_cached_embedding_response_with_api_result( + _caching_handler_response=CachingHandlerResponse( + final_embedding_cached_response=final_cached + ), + embedding_response=api_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert [d["index"] for d in combined.data] == list(range(size)) + assert [d["embedding"] for d in combined.data] == [[float(i)] for i in range(size)]