From dafadf3705617eb36ba6cc5fc30f5d3e456da106 Mon Sep 17 00:00:00 2001 From: Lakshya Dhariwal Date: Sat, 17 Oct 2026 14:54:45 +0530 Subject: [PATCH 1/3] fix(caching): skip proxy-only Anthropic import in cache-hit defer check Base SDK installs lack the proxy-only extras (fastapi), so the lazy import of the Anthropic Messages replay class crashes sync cache hits. Guard it; a base install can never hold that iterator. --- litellm/caching/caching_handler.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 063faa70f2f..ab823176abe 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -182,14 +182,22 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo spend and callback records. A plain (non-stream) replay logs here, since nothing else will. """ - from litellm.llms.anthropic.pass_through.messages.response_cache import ( - CachedAnthropicMessagesStreamIterator, - ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator - return isinstance( - cached_result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator, CachedAnthropicMessagesStreamIterator) - ) + stream_replay_classes: tuple[type, ...] = (CustomStreamWrapper, BaseResponsesAPIStreamingIterator) + try: + from litellm.llms.anthropic.pass_through.messages.response_cache import ( + CachedAnthropicMessagesStreamIterator, + ) + except ModuleNotFoundError: + # The Anthropic Messages replay is only reachable through proxy-only + # extras (fastapi). A base SDK install can never hold one, so the + # isinstance check can safely go ahead without it. + pass + else: + stream_replay_classes = (*stream_replay_classes, CachedAnthropicMessagesStreamIterator) + + return isinstance(cached_result, stream_replay_classes) def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]: From f00bc628a61237b9265e2ffca6359e1e5a84a43a Mon Sep 17 00:00:00 2001 From: Lakshya Dhariwal Date: Sat, 17 Oct 2026 14:55:42 +0530 Subject: [PATCH 2/3] test(caching): cover cache-hit defer check without proxy extras Simulates a base SDK install (fastapi absent) and asserts the defer check still classifies stream replays instead of raising. --- tests/unit/caching/test_caching_handler.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 7d6b1e15fc6..6fe9f547747 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -3142,3 +3142,21 @@ async def test_a_cache_hit_inside_an_internal_sub_call_is_free_and_fires_no_call assert async_logging.async_success_handler.called is not internal assert async_logging.handle_sync_success_callbacks_for_async_calls.called is not internal assert sync_logging.handle_sync_cache_hit_success_callbacks.called is not internal + +def test_should_defer_streaming_cache_hit_callbacks_without_proxy_extras(monkeypatch): + """Base SDK installs lack the proxy-only extras (fastapi); the defer check must not require them.""" + import sys + + for mod in list(sys.modules): + if mod.startswith(("fastapi", "litellm.llms.anthropic.pass_through", "litellm.proxy")): + monkeypatch.delitem(sys.modules, mod, raising=False) + monkeypatch.setitem(sys.modules, "fastapi", None) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + stream_replay = CustomStreamWrapper( + completion_stream=iter(()), model="gpt-4o", logging_obj=logging_obj + ) + assert _should_defer_streaming_cache_hit_callbacks(cached_result=stream_replay) is True + assert _should_defer_streaming_cache_hit_callbacks(cached_result=ModelResponse()) is False + assert _should_defer_streaming_cache_hit_callbacks(cached_result={"id": "msg_1"}) is False From 24fd51b58231f22b9e89b661b20444ade8288795 Mon Sep 17 00:00:00 2001 From: Lakshya Dhariwal Date: Sun, 18 Oct 2026 02:55:10 +0530 Subject: [PATCH 3/3] fix(caching): keep stream-replay class tuple free of rebinding for the type-discipline gate --- litellm/caching/caching_handler.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index ab823176abe..cc209c9f2e8 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -184,7 +184,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo """ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator - stream_replay_classes: tuple[type, ...] = (CustomStreamWrapper, BaseResponsesAPIStreamingIterator) + base_replay_classes: Final = (CustomStreamWrapper, BaseResponsesAPIStreamingIterator) try: from litellm.llms.anthropic.pass_through.messages.response_cache import ( CachedAnthropicMessagesStreamIterator, @@ -193,11 +193,8 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo # The Anthropic Messages replay is only reachable through proxy-only # extras (fastapi). A base SDK install can never hold one, so the # isinstance check can safely go ahead without it. - pass - else: - stream_replay_classes = (*stream_replay_classes, CachedAnthropicMessagesStreamIterator) - - return isinstance(cached_result, stream_replay_classes) + return isinstance(cached_result, base_replay_classes) + return isinstance(cached_result, (*base_replay_classes, CachedAnthropicMessagesStreamIterator)) def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]: