diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 3cf1d911d7f..3f4e54382c9 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -87,6 +87,16 @@ class CachingHandlerResponse(BaseModel): in_memory_cache_obj = InMemoryCache() +def _is_chat_completion_cached_dict(cached_result: dict) -> bool: + cached_id = cached_result.get("id") + if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"): + return True + obj = cached_result.get("object") + if isinstance(obj, str): + return obj.startswith("chat.completion") + return "choices" in cached_result + + def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool: """ When stream=True, do not run success callbacks at cache-hit time. @@ -861,27 +871,47 @@ class LLMCachingHandler: elif (call_type == "aresponses" or call_type == "responses") and isinstance( cached_result, dict ): - from litellm.responses.streaming_iterator import ( - CachedResponsesAPIStreamingIterator, - ) - - response_obj = ResponsesAPIResponse(**cached_result) - if ( - hasattr(response_obj, "_hidden_params") - and response_obj._hidden_params is not None - and isinstance(response_obj._hidden_params, dict) - ): - response_obj._hidden_params["cache_hit"] = True - - if kwargs.get("stream", False) is True: - cached_result = CachedResponsesAPIStreamingIterator( - response=response_obj, - logging_obj=logging_obj, - request_data=kwargs, - call_type=call_type, - ) + use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result) + if use_chat_completion_cache: + if kwargs.get("stream", False) is True: + bridge_call_type = ( + CallTypes.acompletion.value + if call_type == "aresponses" + else CallTypes.completion.value + ) + cached_result = self._convert_cached_stream_response( + cached_result=cached_result, + call_type=bridge_call_type, + logging_obj=logging_obj, + model=model, + ) + else: + cached_result = convert_to_model_response_object( + response_object=cached_result, + model_response_object=ModelResponse(), + ) else: - cached_result = response_obj + from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + ) + + response_obj = ResponsesAPIResponse(**cached_result) + if ( + hasattr(response_obj, "_hidden_params") + and response_obj._hidden_params is not None + and isinstance(response_obj._hidden_params, dict) + ): + response_obj._hidden_params["cache_hit"] = True + + if kwargs.get("stream", False) is True: + cached_result = CachedResponsesAPIStreamingIterator( + response=response_obj, + logging_obj=logging_obj, + request_data=kwargs, + call_type=call_type, + ) + else: + cached_result = response_obj if ( hasattr(cached_result, "_hidden_params") diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index ce398ee8288..2de7bda6467 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -37,6 +37,15 @@ class ResponsesToCompletionBridgeHandler: stream = litellm_params.get("stream", False) return bool(stream) + @staticmethod + def _is_preformatted_cached_chat_stream(result: Any) -> bool: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + return ( + isinstance(result, CustomStreamWrapper) + and result.custom_llm_provider == "cached_response" + ) + @staticmethod def _coerce_response_object( response_obj: Any, @@ -177,6 +186,8 @@ class ResponsesToCompletionBridgeHandler: **request_data, ) + from litellm.types.utils import ModelResponse + stream = self._resolve_stream_flag(optional_params, litellm_params) if isinstance(result, ResponsesAPIResponse): return self.transformation_handler.transform_response( @@ -192,6 +203,8 @@ class ResponsesToCompletionBridgeHandler: api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), ) + elif isinstance(result, ModelResponse): + return result elif not stream: responses_api_response = self._collect_response_from_stream(result) return self.transformation_handler.transform_response( @@ -208,6 +221,10 @@ class ResponsesToCompletionBridgeHandler: json_mode=kwargs.get("json_mode"), ) else: + if self._is_preformatted_cached_chat_stream(result): + return self._apply_post_stream_processing( + result, model, custom_llm_provider + ) completion_stream = self.transformation_handler.get_model_response_iterator( streaming_response=result, # type: ignore sync_stream=True, @@ -256,6 +273,8 @@ class ResponsesToCompletionBridgeHandler: aresponses=True, ) + from litellm.types.utils import ModelResponse + stream = self._resolve_stream_flag(optional_params, litellm_params) if isinstance(result, ResponsesAPIResponse): return self.transformation_handler.transform_response( @@ -271,6 +290,8 @@ class ResponsesToCompletionBridgeHandler: api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), ) + elif isinstance(result, ModelResponse): + return result elif not stream: responses_api_response = await self._collect_response_from_stream_async( result @@ -289,6 +310,10 @@ class ResponsesToCompletionBridgeHandler: json_mode=kwargs.get("json_mode"), ) else: + if self._is_preformatted_cached_chat_stream(result): + return self._apply_post_stream_processing( + result, model, custom_llm_provider + ) completion_stream = self.transformation_handler.get_model_response_iterator( streaming_response=result, # type: ignore sync_stream=False, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 32423f23314..e3cbf422e5d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1141,6 +1141,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): event_type = parsed_chunk.get("type") if isinstance(event_type, ResponsesAPIStreamEvents): event_type = event_type.value + + if parsed_chunk.get("object") == "chat.completion.chunk" or ( + event_type is None + and isinstance(parsed_chunk.get("choices"), list) + and parsed_chunk.get("choices") + ): + return ModelResponseStream(**parsed_chunk) + verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") if event_type == "response.created": diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 7fdb7267a38..373cb66696a 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -159,9 +159,20 @@ def make_redis_persister( raise CassetteNotFoundError() from exc if data is None: raise CassetteNotFoundError() - if isinstance(data, bytes): - data = data.decode("utf-8") - return deserialize(data, serializer) + try: + if isinstance(data, bytes): + data = data.decode("utf-8") + return deserialize(data, serializer) + except Exception as exc: + _record_cache_failure("load", exc) + msg = ( + f"VCR redis load failed for {cassette_path}; cached " + f"payload is corrupt, treating as cache miss: " + f"{type(exc).__name__}: {exc}" + ) + _log.warning(msg) + warnings.warn(msg, VCRCassetteCacheWarning, stacklevel=2) + raise CassetteNotFoundError() from exc @staticmethod def save_cassette(cassette_path, cassette_dict, serializer): diff --git a/tests/local_testing/test_caching_handler.py b/tests/local_testing/test_caching_handler.py index 2b6712cbaa3..0f4539162a2 100644 --- a/tests/local_testing/test_caching_handler.py +++ b/tests/local_testing/test_caching_handler.py @@ -25,6 +25,7 @@ from unittest.mock import AsyncMock, patch, MagicMock from litellm.caching.caching_handler import ( LLMCachingHandler, CachingHandlerResponse, + _is_chat_completion_cached_dict, _should_defer_streaming_cache_hit_callbacks, ) from litellm.caching.caching import LiteLLMCacheType @@ -40,6 +41,7 @@ from litellm.types.utils import ( from litellm.types.llms.openai import ResponsesAPIResponse from datetime import timedelta, datetime from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm._logging import verbose_logger import logging @@ -1072,6 +1074,70 @@ def test_convert_cached_streaming_responses_result_to_iterator(): ) +def test_is_chat_completion_cached_dict(): + assert _is_chat_completion_cached_dict( + {"id": "chatcmpl-abc", "object": "chat.completion", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "other", "object": "chat.completion.chunk", "choices": []} + ) + assert not _is_chat_completion_cached_dict( + {"id": "resp_abc", "object": "response", "output": []} + ) + + +def test_convert_cached_aresponses_bridge_chat_completion_stream(): + """ + openai/responses chat-completions bridge caches ModelResponse JSON on aresponses + cache keys; replay must not call ResponsesAPIResponse(**chatcmpl_dict). + """ + caching_handler = LLMCachingHandler( + original_function=aresponses, request_kwargs={}, start_time=datetime.now() + ) + logging_obj = LiteLLMLogging( + litellm_call_id=str(datetime.now()), + call_type=CallTypes.aresponses.value, + model="gpt-5.4", + messages=[], + function_id=str(uuid.uuid4()), + stream=True, + start_time=datetime.now(), + ) + cached_result = { + "id": "chatcmpl-bridge-cache-test", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 7, + "completion_tokens": 11, + "total_tokens": 18, + }, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.aresponses.value, + kwargs={ + "model": "gpt-5.4", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=logging_obj, + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, CustomStreamWrapper) + + def test_convert_cached_streaming_reasoning_result_to_iterator(): caching_handler = LLMCachingHandler( original_function=responses, request_kwargs={}, start_time=datetime.now() diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 742a4f410d4..3eb949d7f29 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -232,3 +232,207 @@ def test_combine_usage_handles_none_details(): combined = llm_caching_handler.combine_usage(usage_a, usage_c) assert combined.prompt_tokens_details is not None assert combined.prompt_tokens_details.image_count == 1 + + +def test_is_chat_completion_cached_dict(): + from litellm.caching.caching_handler import _is_chat_completion_cached_dict + + assert _is_chat_completion_cached_dict( + {"id": "chatcmpl-abc", "object": "chat.completion", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "other", "object": "chat.completion.chunk", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "no-object", "choices": [{"index": 0}]} + ) + assert not _is_chat_completion_cached_dict( + {"id": "resp_abc", "object": "response", "output": []} + ) + + +def _build_logging_obj(call_type: str, stream: bool): + import uuid as _uuid + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + return LiteLLMLogging( + litellm_call_id=str(datetime.now()), + call_type=call_type, + model="gpt-5.4", + messages=[], + function_id=str(_uuid.uuid4()), + stream=stream, + start_time=datetime.now(), + ) + + +def test_convert_cached_aresponses_bridge_chat_completion_stream(): + """openai/responses chat-completions bridge: streaming cache hit replays as chat stream.""" + from litellm import aresponses + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=aresponses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "chatcmpl-bridge-cache-test", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18}, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.aresponses.value, + kwargs={ + "model": "gpt-5.4", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=_build_logging_obj(CallTypes.aresponses.value, stream=True), + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, CustomStreamWrapper) + + +def test_convert_cached_responses_bridge_chat_completion_nonstream(): + """openai/responses chat-completions bridge: non-streaming cache hit replays as ModelResponse.""" + from litellm import responses + from litellm.types.utils import CallTypes, ModelResponse + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "chatcmpl-bridge-nonstream", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18}, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={ + "model": "gpt-5.4", + "stream": False, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=False), + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hi!" + + +def test_convert_cached_responses_legacy_nonstream_path(): + """Genuine ResponsesAPIResponse dict (no chatcmpl/choices) falls through legacy path.""" + from litellm import responses + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "resp_legacy_nonstream", + "created_at": int(time.time()), + "status": "completed", + "model": "gpt-4o", + "object": "response", + "output": [ + { + "type": "message", + "id": "msg_legacy", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "legacy response", + "annotations": [], + } + ], + } + ], + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={"model": "gpt-4o", "input": "hi", "stream": False}, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=False), + model="gpt-4o", + args=(), + ) + + assert isinstance(result, ResponsesAPIResponse) + assert result.id == "resp_legacy_nonstream" + + +def test_convert_cached_responses_legacy_stream_path(): + """Genuine ResponsesAPIResponse dict (no chatcmpl/choices) on stream falls through legacy path.""" + from litellm import responses + from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + ) + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "resp_legacy_stream", + "created_at": int(time.time()), + "status": "completed", + "model": "gpt-4o", + "object": "response", + "output": [ + { + "type": "message", + "id": "msg_legacy_stream", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "legacy stream", + "annotations": [], + } + ], + } + ], + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={"model": "gpt-4o", "input": "hi", "stream": True}, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=True), + model="gpt-4o", + args=(), + ) + + assert isinstance(result, CachedResponsesAPIStreamingIterator) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py new file mode 100644 index 00000000000..734033ed6be --- /dev/null +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -0,0 +1,150 @@ +import os +import sys +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.completion_extras.litellm_responses_transformation.handler import ( + ResponsesToCompletionBridgeHandler, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.utils import ModelResponse + + +def test_is_preformatted_cached_chat_stream_true(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream(stream) + is True + ) + + +def test_is_preformatted_cached_chat_stream_false_wrong_provider(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "openai" + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream(stream) + is False + ) + + +def test_is_preformatted_cached_chat_stream_false_wrong_type(): + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream( + {"object": "chat.completion.chunk"} + ) + is False + ) + + +def _bridge_kwargs(stream: bool): + logging_obj = LiteLLMLogging( + litellm_call_id="test-call", + call_type="completion", + model="gpt-5.4", + messages=[{"role": "user", "content": "hi"}], + function_id="fn-id", + stream=stream, + start_time=datetime.now(), + ) + return { + "model": "gpt-5.4", + "custom_llm_provider": "openai", + "messages": [{"role": "user", "content": "hi"}], + "optional_params": {"stream": stream}, + "litellm_params": {}, + "headers": {}, + "model_response": ModelResponse(), + "logging_obj": logging_obj, + } + + +def test_completion_returns_cached_model_response_directly(): + """Non-streaming bridge cache hit: responses() returns a ModelResponse -> bridge returns it as-is.""" + cached = ModelResponse(id="chatcmpl-cached-nonstream", model="gpt-5.4") + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.responses", return_value=cached), + ): + result = bridge.completion(**_bridge_kwargs(stream=False)) + + assert result is cached + + +@pytest.mark.asyncio +async def test_acompletion_returns_cached_model_response_directly(): + cached = ModelResponse(id="chatcmpl-cached-nonstream-async", model="gpt-5.4") + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=cached)), + ): + result = await bridge.acompletion(**_bridge_kwargs(stream=False)) + + assert result is cached + + +def test_completion_skips_rewrapping_preformatted_cached_chat_stream(): + """Streaming bridge cache hit returning CustomStreamWrapper(cached_response) -> bridge skips re-wrapping.""" + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.responses", return_value=stream), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ) as post, + ): + result = bridge.completion(**_bridge_kwargs(stream=True)) + + post.assert_called_once() + assert result is stream + + +@pytest.mark.asyncio +async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=stream)), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ) as post, + ): + result = await bridge.acompletion(**_bridge_kwargs(stream=True)) + + post.assert_called_once() + assert result is stream diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 009f432fca1..05bdc40112c 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -230,3 +230,30 @@ def test_transform_request_drops_user_metadata_with_additional_drop_params(): assert "metadata" not in result assert result["litellm_metadata"]["internal_key"] == "secret" + + +def test_translate_responses_chunk_passthrough_chat_completion_chunk(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + chat_chunk = { + "id": "chatcmpl-cache-passthrough", + "object": "chat.completion.chunk", + "created": 1779104834, + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hi! How can I help?"}, + "finish_reason": None, + } + ], + } + + result = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + chat_chunk + ) + + assert result.choices[0].delta.content == "Hi! How can I help?" + assert result.choices[0].finish_reason is None