From 477b63c5ead39bb38b804552ccd0aa04b5fde118 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 19 May 2026 04:57:06 +0530 Subject: [PATCH] fix(caching): replay openai/responses bridge cache hits as chat streams (#28158) * fix(caching): replay openai/responses bridge cache hits as chat streams When chat completions route through openai/responses, cached ModelResponse payloads under aresponses keys were deserialized as ResponsesAPIResponse (500) or re-translated as responses events (empty streaming deltas). Deserialize chat-shaped cache entries as acompletion and bypass the responses stream iterator for cached CustomStreamWrapper replay. Co-authored-by: Cursor * fix(caching): map responses bridge call_type for sync vs async stream replay Co-authored-by: Yassin Kortam * fix: handle ModelResponse cache return in responses bridge and drop dead acompletion check Co-authored-by: Yassin Kortam * fix(caching): detect chat cache hits via object field before choices fallback Prefer chat.completion object type over the broad choices-key heuristic so Responses API cached payloads are not misclassified if their schema changes. Co-authored-by: Cursor * test(caching): cover responses bridge cache-hit paths in CI-tracked test suite The new bridge cache replay logic in caching_handler.py and the preformatted-stream guard in litellm_responses_transformation/handler.py were exercised only by tests under tests/local_testing/, which the responses-caching-types and misc shards do not run. Codecov flagged the patch as 29.72% covered. Add equivalent unit tests under tests/test_litellm/ so the responses, caching, types, and misc shards execute them and ship their coverage data to Codecov: - _is_chat_completion_cached_dict happy/sad paths - aresponses streaming bridge cache hit -> CustomStreamWrapper - responses non-streaming bridge cache hit -> ModelResponse - legacy ResponsesAPIResponse stream + non-stream replay - _is_preformatted_cached_chat_stream true/false - completion/acompletion early return on cached ModelResponse - completion/acompletion skip rewrap on preformatted cached stream * fix: add negative guard on object field in _is_chat_completion_cached_dict Co-authored-by: Yassin Kortam * fix(vcr): treat corrupt cassette payloads as cache miss * test: bump EOL'd NVIDIA rerank and OpenAI realtime models in CI The NVIDIA hosted rerank endpoint for nvidia/llama-3_2-nv-rerankqa-1b-v2 reached end-of-life on 2026-05-18 and now returns HTTP 410 Gone, breaking TestNvidiaNim::test_basic_rerank. Switch to nvidia/nv-rerankqa-mistral-4b-v3, which is still hosted on the NVIDIA API catalog and is already listed in model_prices_and_context_window.json. OpenAI also retired the gpt-4o-realtime-preview-2024-12-17 model used by test_realtime_guardrails_openai (now returns model_not_found). Switch the realtime test URL to the GA gpt-realtime alias. Unrelated to the responses-bridge cache fix in this PR, but committing here to unblock CI per maintainer guidance. Co-authored-by: Mateo Wang * test(realtime): switch retired gpt-4o-realtime-preview to gpt-realtime OpenAI removed gpt-4o-realtime-preview and all its date snapshots on 2026-05-18 (every variant now returns model_not_found), breaking the live-WebSocket OpenAI realtime tests in CI: - test_openai_realtime_direct_call_no_intent - test_openai_realtime_direct_call_with_intent - TestOpenAIRealtime.test_realtime_connection - TestOpenAIRealtime.test_realtime_with_query_params Point each of those to the current GA alias gpt-realtime (verified live). Pure unit/mock tests that just assert the string value (e.g. in test_realtime_query_params_construction and the test_realtime_query_params_use_normalized_model_name mock) are left alone since they do not depend on model availability. Also relax the AI-response assertion in test_text_message_blocked_by_guardrail_no_ai_response: gpt-realtime occasionally produces a polite refusal ("I'm sorry, but I can't say that") when the cancel arrives after the model has already started generating, which is the expected outcome (no real AI content) but does not contain the words 'blocked' or 'guardrail'. The primary guardrail behaviour (guardrail_violation error event + transcript_delta block message) is still asserted unchanged. Co-authored-by: Mateo Wang * test(nvidia_nim): mock rerank live API instead of hitting EOL'd endpoint NVIDIA reached end-of-life for the hosted nvidia/llama-3.2-nv-rerankqa-1b-v2 rerank API on 2026-05-18 (returns HTTP 410 Gone), and the proposed replacement nv-rerankqa-mistral-4b-v3 returns HTTP 404 for the CI account, breaking TestNvidiaNim::test_basic_rerank. Override test_basic_rerank to mock the HTTP transport (same pattern as test_nvidia_nim_rerank_ranking_endpoint above) so the request/response transformation and cost calculation stay covered without depending on NVIDIA's hosted catalog rotation. The model identifier reverts to the original llama-3.2-nv-rerankqa-1b-v2 since the request never leaves the test process. --------- Co-authored-by: Cursor Co-authored-by: Yassin Kortam Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Mateo Wang --- litellm/caching/caching_handler.py | 70 ++++-- .../handler.py | 25 +++ .../transformation.py | 8 + tests/_vcr_redis_persister.py | 17 +- tests/local_testing/test_caching_handler.py | 66 ++++++ .../caching/test_caching_handler.py | 204 ++++++++++++++++++ ...itellm_responses_transformation_handler.py | 150 +++++++++++++ ...responses_transformation_transformation.py | 27 +++ 8 files changed, 544 insertions(+), 23 deletions(-) create mode 100644 tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 3cf1d911d7f..3f4e54382c9 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -87,6 +87,16 @@ class CachingHandlerResponse(BaseModel): in_memory_cache_obj = InMemoryCache() +def _is_chat_completion_cached_dict(cached_result: dict) -> bool: + cached_id = cached_result.get("id") + if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"): + return True + obj = cached_result.get("object") + if isinstance(obj, str): + return obj.startswith("chat.completion") + return "choices" in cached_result + + def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool: """ When stream=True, do not run success callbacks at cache-hit time. @@ -861,27 +871,47 @@ class LLMCachingHandler: elif (call_type == "aresponses" or call_type == "responses") and isinstance( cached_result, dict ): - from litellm.responses.streaming_iterator import ( - CachedResponsesAPIStreamingIterator, - ) - - response_obj = ResponsesAPIResponse(**cached_result) - if ( - hasattr(response_obj, "_hidden_params") - and response_obj._hidden_params is not None - and isinstance(response_obj._hidden_params, dict) - ): - response_obj._hidden_params["cache_hit"] = True - - if kwargs.get("stream", False) is True: - cached_result = CachedResponsesAPIStreamingIterator( - response=response_obj, - logging_obj=logging_obj, - request_data=kwargs, - call_type=call_type, - ) + use_chat_completion_cache = _is_chat_completion_cached_dict(cached_result) + if use_chat_completion_cache: + if kwargs.get("stream", False) is True: + bridge_call_type = ( + CallTypes.acompletion.value + if call_type == "aresponses" + else CallTypes.completion.value + ) + cached_result = self._convert_cached_stream_response( + cached_result=cached_result, + call_type=bridge_call_type, + logging_obj=logging_obj, + model=model, + ) + else: + cached_result = convert_to_model_response_object( + response_object=cached_result, + model_response_object=ModelResponse(), + ) else: - cached_result = response_obj + from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + ) + + response_obj = ResponsesAPIResponse(**cached_result) + if ( + hasattr(response_obj, "_hidden_params") + and response_obj._hidden_params is not None + and isinstance(response_obj._hidden_params, dict) + ): + response_obj._hidden_params["cache_hit"] = True + + if kwargs.get("stream", False) is True: + cached_result = CachedResponsesAPIStreamingIterator( + response=response_obj, + logging_obj=logging_obj, + request_data=kwargs, + call_type=call_type, + ) + else: + cached_result = response_obj if ( hasattr(cached_result, "_hidden_params") diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index ce398ee8288..2de7bda6467 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -37,6 +37,15 @@ class ResponsesToCompletionBridgeHandler: stream = litellm_params.get("stream", False) return bool(stream) + @staticmethod + def _is_preformatted_cached_chat_stream(result: Any) -> bool: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + return ( + isinstance(result, CustomStreamWrapper) + and result.custom_llm_provider == "cached_response" + ) + @staticmethod def _coerce_response_object( response_obj: Any, @@ -177,6 +186,8 @@ class ResponsesToCompletionBridgeHandler: **request_data, ) + from litellm.types.utils import ModelResponse + stream = self._resolve_stream_flag(optional_params, litellm_params) if isinstance(result, ResponsesAPIResponse): return self.transformation_handler.transform_response( @@ -192,6 +203,8 @@ class ResponsesToCompletionBridgeHandler: api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), ) + elif isinstance(result, ModelResponse): + return result elif not stream: responses_api_response = self._collect_response_from_stream(result) return self.transformation_handler.transform_response( @@ -208,6 +221,10 @@ class ResponsesToCompletionBridgeHandler: json_mode=kwargs.get("json_mode"), ) else: + if self._is_preformatted_cached_chat_stream(result): + return self._apply_post_stream_processing( + result, model, custom_llm_provider + ) completion_stream = self.transformation_handler.get_model_response_iterator( streaming_response=result, # type: ignore sync_stream=True, @@ -256,6 +273,8 @@ class ResponsesToCompletionBridgeHandler: aresponses=True, ) + from litellm.types.utils import ModelResponse + stream = self._resolve_stream_flag(optional_params, litellm_params) if isinstance(result, ResponsesAPIResponse): return self.transformation_handler.transform_response( @@ -271,6 +290,8 @@ class ResponsesToCompletionBridgeHandler: api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), ) + elif isinstance(result, ModelResponse): + return result elif not stream: responses_api_response = await self._collect_response_from_stream_async( result @@ -289,6 +310,10 @@ class ResponsesToCompletionBridgeHandler: json_mode=kwargs.get("json_mode"), ) else: + if self._is_preformatted_cached_chat_stream(result): + return self._apply_post_stream_processing( + result, model, custom_llm_provider + ) completion_stream = self.transformation_handler.get_model_response_iterator( streaming_response=result, # type: ignore sync_stream=False, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 32423f23314..e3cbf422e5d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1141,6 +1141,14 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): event_type = parsed_chunk.get("type") if isinstance(event_type, ResponsesAPIStreamEvents): event_type = event_type.value + + if parsed_chunk.get("object") == "chat.completion.chunk" or ( + event_type is None + and isinstance(parsed_chunk.get("choices"), list) + and parsed_chunk.get("choices") + ): + return ModelResponseStream(**parsed_chunk) + verbose_logger.debug(f"Chat provider: Processing event type: {event_type}") if event_type == "response.created": diff --git a/tests/_vcr_redis_persister.py b/tests/_vcr_redis_persister.py index 7fdb7267a38..373cb66696a 100644 --- a/tests/_vcr_redis_persister.py +++ b/tests/_vcr_redis_persister.py @@ -159,9 +159,20 @@ def make_redis_persister( raise CassetteNotFoundError() from exc if data is None: raise CassetteNotFoundError() - if isinstance(data, bytes): - data = data.decode("utf-8") - return deserialize(data, serializer) + try: + if isinstance(data, bytes): + data = data.decode("utf-8") + return deserialize(data, serializer) + except Exception as exc: + _record_cache_failure("load", exc) + msg = ( + f"VCR redis load failed for {cassette_path}; cached " + f"payload is corrupt, treating as cache miss: " + f"{type(exc).__name__}: {exc}" + ) + _log.warning(msg) + warnings.warn(msg, VCRCassetteCacheWarning, stacklevel=2) + raise CassetteNotFoundError() from exc @staticmethod def save_cassette(cassette_path, cassette_dict, serializer): diff --git a/tests/local_testing/test_caching_handler.py b/tests/local_testing/test_caching_handler.py index 2b6712cbaa3..0f4539162a2 100644 --- a/tests/local_testing/test_caching_handler.py +++ b/tests/local_testing/test_caching_handler.py @@ -25,6 +25,7 @@ from unittest.mock import AsyncMock, patch, MagicMock from litellm.caching.caching_handler import ( LLMCachingHandler, CachingHandlerResponse, + _is_chat_completion_cached_dict, _should_defer_streaming_cache_hit_callbacks, ) from litellm.caching.caching import LiteLLMCacheType @@ -40,6 +41,7 @@ from litellm.types.utils import ( from litellm.types.llms.openai import ResponsesAPIResponse from datetime import timedelta, datetime from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm._logging import verbose_logger import logging @@ -1072,6 +1074,70 @@ def test_convert_cached_streaming_responses_result_to_iterator(): ) +def test_is_chat_completion_cached_dict(): + assert _is_chat_completion_cached_dict( + {"id": "chatcmpl-abc", "object": "chat.completion", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "other", "object": "chat.completion.chunk", "choices": []} + ) + assert not _is_chat_completion_cached_dict( + {"id": "resp_abc", "object": "response", "output": []} + ) + + +def test_convert_cached_aresponses_bridge_chat_completion_stream(): + """ + openai/responses chat-completions bridge caches ModelResponse JSON on aresponses + cache keys; replay must not call ResponsesAPIResponse(**chatcmpl_dict). + """ + caching_handler = LLMCachingHandler( + original_function=aresponses, request_kwargs={}, start_time=datetime.now() + ) + logging_obj = LiteLLMLogging( + litellm_call_id=str(datetime.now()), + call_type=CallTypes.aresponses.value, + model="gpt-5.4", + messages=[], + function_id=str(uuid.uuid4()), + stream=True, + start_time=datetime.now(), + ) + cached_result = { + "id": "chatcmpl-bridge-cache-test", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 7, + "completion_tokens": 11, + "total_tokens": 18, + }, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.aresponses.value, + kwargs={ + "model": "gpt-5.4", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=logging_obj, + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, CustomStreamWrapper) + + def test_convert_cached_streaming_reasoning_result_to_iterator(): caching_handler = LLMCachingHandler( original_function=responses, request_kwargs={}, start_time=datetime.now() diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 742a4f410d4..3eb949d7f29 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -232,3 +232,207 @@ def test_combine_usage_handles_none_details(): combined = llm_caching_handler.combine_usage(usage_a, usage_c) assert combined.prompt_tokens_details is not None assert combined.prompt_tokens_details.image_count == 1 + + +def test_is_chat_completion_cached_dict(): + from litellm.caching.caching_handler import _is_chat_completion_cached_dict + + assert _is_chat_completion_cached_dict( + {"id": "chatcmpl-abc", "object": "chat.completion", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "other", "object": "chat.completion.chunk", "choices": []} + ) + assert _is_chat_completion_cached_dict( + {"id": "no-object", "choices": [{"index": 0}]} + ) + assert not _is_chat_completion_cached_dict( + {"id": "resp_abc", "object": "response", "output": []} + ) + + +def _build_logging_obj(call_type: str, stream: bool): + import uuid as _uuid + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + + return LiteLLMLogging( + litellm_call_id=str(datetime.now()), + call_type=call_type, + model="gpt-5.4", + messages=[], + function_id=str(_uuid.uuid4()), + stream=stream, + start_time=datetime.now(), + ) + + +def test_convert_cached_aresponses_bridge_chat_completion_stream(): + """openai/responses chat-completions bridge: streaming cache hit replays as chat stream.""" + from litellm import aresponses + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=aresponses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "chatcmpl-bridge-cache-test", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18}, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.aresponses.value, + kwargs={ + "model": "gpt-5.4", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=_build_logging_obj(CallTypes.aresponses.value, stream=True), + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, CustomStreamWrapper) + + +def test_convert_cached_responses_bridge_chat_completion_nonstream(): + """openai/responses chat-completions bridge: non-streaming cache hit replays as ModelResponse.""" + from litellm import responses + from litellm.types.utils import CallTypes, ModelResponse + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "chatcmpl-bridge-nonstream", + "object": "chat.completion", + "created": int(time.time()), + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hi!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18}, + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={ + "model": "gpt-5.4", + "stream": False, + "messages": [{"role": "user", "content": "hi"}], + }, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=False), + model="gpt-5.4", + args=(), + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hi!" + + +def test_convert_cached_responses_legacy_nonstream_path(): + """Genuine ResponsesAPIResponse dict (no chatcmpl/choices) falls through legacy path.""" + from litellm import responses + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "resp_legacy_nonstream", + "created_at": int(time.time()), + "status": "completed", + "model": "gpt-4o", + "object": "response", + "output": [ + { + "type": "message", + "id": "msg_legacy", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "legacy response", + "annotations": [], + } + ], + } + ], + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={"model": "gpt-4o", "input": "hi", "stream": False}, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=False), + model="gpt-4o", + args=(), + ) + + assert isinstance(result, ResponsesAPIResponse) + assert result.id == "resp_legacy_nonstream" + + +def test_convert_cached_responses_legacy_stream_path(): + """Genuine ResponsesAPIResponse dict (no chatcmpl/choices) on stream falls through legacy path.""" + from litellm import responses + from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + ) + from litellm.types.utils import CallTypes + + caching_handler = LLMCachingHandler( + original_function=responses, request_kwargs={}, start_time=datetime.now() + ) + cached_result = { + "id": "resp_legacy_stream", + "created_at": int(time.time()), + "status": "completed", + "model": "gpt-4o", + "object": "response", + "output": [ + { + "type": "message", + "id": "msg_legacy_stream", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "legacy stream", + "annotations": [], + } + ], + } + ], + } + + result = caching_handler._convert_cached_result_to_model_response( + cached_result=cached_result, + call_type=CallTypes.responses.value, + kwargs={"model": "gpt-4o", "input": "hi", "stream": True}, + logging_obj=_build_logging_obj(CallTypes.responses.value, stream=True), + model="gpt-4o", + args=(), + ) + + assert isinstance(result, CachedResponsesAPIStreamingIterator) diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py new file mode 100644 index 00000000000..734033ed6be --- /dev/null +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -0,0 +1,150 @@ +import os +import sys +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) + +from litellm.completion_extras.litellm_responses_transformation.handler import ( + ResponsesToCompletionBridgeHandler, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.utils import ModelResponse + + +def test_is_preformatted_cached_chat_stream_true(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream(stream) + is True + ) + + +def test_is_preformatted_cached_chat_stream_false_wrong_provider(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "openai" + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream(stream) + is False + ) + + +def test_is_preformatted_cached_chat_stream_false_wrong_type(): + assert ( + ResponsesToCompletionBridgeHandler._is_preformatted_cached_chat_stream( + {"object": "chat.completion.chunk"} + ) + is False + ) + + +def _bridge_kwargs(stream: bool): + logging_obj = LiteLLMLogging( + litellm_call_id="test-call", + call_type="completion", + model="gpt-5.4", + messages=[{"role": "user", "content": "hi"}], + function_id="fn-id", + stream=stream, + start_time=datetime.now(), + ) + return { + "model": "gpt-5.4", + "custom_llm_provider": "openai", + "messages": [{"role": "user", "content": "hi"}], + "optional_params": {"stream": stream}, + "litellm_params": {}, + "headers": {}, + "model_response": ModelResponse(), + "logging_obj": logging_obj, + } + + +def test_completion_returns_cached_model_response_directly(): + """Non-streaming bridge cache hit: responses() returns a ModelResponse -> bridge returns it as-is.""" + cached = ModelResponse(id="chatcmpl-cached-nonstream", model="gpt-5.4") + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.responses", return_value=cached), + ): + result = bridge.completion(**_bridge_kwargs(stream=False)) + + assert result is cached + + +@pytest.mark.asyncio +async def test_acompletion_returns_cached_model_response_directly(): + cached = ModelResponse(id="chatcmpl-cached-nonstream-async", model="gpt-5.4") + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=cached)), + ): + result = await bridge.acompletion(**_bridge_kwargs(stream=False)) + + assert result is cached + + +def test_completion_skips_rewrapping_preformatted_cached_chat_stream(): + """Streaming bridge cache hit returning CustomStreamWrapper(cached_response) -> bridge skips re-wrapping.""" + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.responses", return_value=stream), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ) as post, + ): + result = bridge.completion(**_bridge_kwargs(stream=True)) + + post.assert_called_once() + assert result is stream + + +@pytest.mark.asyncio +async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=stream)), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ) as post, + ): + result = await bridge.acompletion(**_bridge_kwargs(stream=True)) + + post.assert_called_once() + assert result is stream diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py index 009f432fca1..05bdc40112c 100644 --- a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -230,3 +230,30 @@ def test_transform_request_drops_user_metadata_with_additional_drop_params(): assert "metadata" not in result assert result["litellm_metadata"]["internal_key"] == "secret" + + +def test_translate_responses_chunk_passthrough_chat_completion_chunk(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + chat_chunk = { + "id": "chatcmpl-cache-passthrough", + "object": "chat.completion.chunk", + "created": 1779104834, + "model": "gpt-5.4", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hi! How can I help?"}, + "finish_reason": None, + } + ], + } + + result = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + chat_chunk + ) + + assert result.choices[0].delta.content == "Hi! How can I help?" + assert result.choices[0].finish_reason is None