scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 175

fix(caching): skip proxy-only Anthropic import in cache-hit defer check

Browse source 
Base SDK installs lack the proxy-only extras (fastapi), so the lazy import of the Anthropic Messages replay class crashes sync cache hits. Guard it; a base install can never hold that iterator.
This commit is contained in:
Lakshya Dhariwal 2026-10-17 14:54:45 +05:30 • committed by GitHub
parent caad4d6204
commit dafadf3705
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 14 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/caching/caching_handler.py
View file

@ -182,14 +182,22 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo
spend and callback records. A plain (non-stream) replay logs here, since nothing
else will.
"""
from litellm.llms.anthropic.pass_through.messages.response_cache import (
CachedAnthropicMessagesStreamIterator,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
return isinstance(
cached_result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator, CachedAnthropicMessagesStreamIterator)
)
stream_replay_classes: tuple[type, ...] = (CustomStreamWrapper, BaseResponsesAPIStreamingIterator)
try:
from litellm.llms.anthropic.pass_through.messages.response_cache import (
CachedAnthropicMessagesStreamIterator,
)
except ModuleNotFoundError:
# The Anthropic Messages replay is only reachable through proxy-only
# extras (fastapi). A base SDK install can never hold one, so the
# isinstance check can safely go ahead without it.
pass
else:
stream_replay_classes = (*stream_replay_classes, CachedAnthropicMessagesStreamIterator)
return isinstance(cached_result, stream_replay_classes)
def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]: