diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 1d80d6952a4..5738a9c2813 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -182,14 +182,16 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo spend and callback records. A plain (non-stream) replay logs here, since nothing else will. """ - from litellm.llms.anthropic.messages.native.response_cache import ( - CachedAnthropicMessagesStreamIterator, - ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator - return isinstance( - cached_result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator, CachedAnthropicMessagesStreamIterator) - ) + streaming_types: Final[tuple[type, ...]] = (CustomStreamWrapper, BaseResponsesAPIStreamingIterator) + try: + from litellm.llms.anthropic.messages.native.response_cache import ( + CachedAnthropicMessagesStreamIterator, + ) + except ImportError: + return isinstance(cached_result, streaming_types) + return isinstance(cached_result, (*streaming_types, CachedAnthropicMessagesStreamIterator)) def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]: diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 51754c80dd6..145a6e78879 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -1,4 +1,5 @@ import time +import sys import traceback from litellm._uuid import uuid @@ -1281,6 +1282,17 @@ def test_should_defer_streaming_cache_hit_callbacks_for_any_streaming_request(): assert _should_defer_streaming_cache_hit_callbacks(cached_result={"id": "msg_1"}) is False +def test_should_defer_streaming_cache_hit_callbacks_without_proxy_extra(monkeypatch): + monkeypatch.setitem(sys.modules, "litellm.llms.anthropic.messages.native.response_cache", None) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + stream_replay = CustomStreamWrapper( + completion_stream=iter(()), model="gpt-4o", logging_obj=logging_obj + ) + assert _should_defer_streaming_cache_hit_callbacks(cached_result=stream_replay) is True + assert _should_defer_streaming_cache_hit_callbacks(cached_result=ModelResponse()) is False + + @pytest.mark.asyncio async def test_async_get_cache_defers_streaming_completion_hit_callbacks(): litellm.set_verbose = True