From 04afc962b10d905e2ceabdfe121c65367941d513 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 12:29:43 -0700 Subject: [PATCH] feat(anthropic): add enable_anthropic_prompt_caching for automatic cache_control injection Anthropic only caches a prompt when the request carries explicit cache_control breakpoints, unlike OpenAI where prompt caching is automatic and needs no configuration. Today litellm can inject those breakpoints server-side, but only when an admin hand-writes cache_control_injection_points into a model's litellm_params (or router_settings.default_litellm_params). Clients such as Claude Code and Claude Desktop never set cache_control themselves, and the admin recipe is easy to miss, so Anthropic traffic through the proxy silently pays full price on every repeated prefix. This adds an opt-in litellm_settings flag, enable_anthropic_prompt_caching. When it is on and the request has no injection points configured and no client-supplied cache_control, litellm synthesizes a default pair of breakpoints (the system prompt and the trailing turn) so the stable prefix is cached while the breakpoint advances with the conversation. It is wired into both surfaces: /chat/completions seeds the points before the existing prompt-management gate, and /v1/messages resolves them in maybe_inject_cache_control, so the existing AnthropicCacheControlHook applies them unchanged and keeps its four-block cap and its refusal to overwrite client breakpoints. The default is off, so no existing deployment changes behavior. Injection is gated to providers that actually consume cache_control markers (anthropic and bedrock) and to models the cost map flags as supporting prompt caching; note that supports_prompt_caching alone is not a sufficient gate, since OpenAI, Azure and Gemini models report it as well but never take cache_control markers. The default ttl is Anthropic's 5 minute ephemeral cache, with an optional anthropic_prompt_caching_ttl of "5m" or "1h"; ttl is also added to ChatCompletionCachedContent, which the bedrock and anthropic transforms already read at runtime but the type never declared Resolves LIT-4478 --- litellm/__init__.py | 2 + .../anthropic_cache_control_hook.py | 117 ++++++++++++++- .../messages/handler.py | 8 +- litellm/main.py | 25 ++++ litellm/types/llms/openai.py | 1 + .../test_anthropic_cache_control_hook.py | 136 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 7 files changed, 291 insertions(+), 3 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e2a03b7c7c..c2a98497d62 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -315,6 +315,8 @@ disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False +enable_anthropic_prompt_caching: bool = False +anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = None disable_vertex_batch_output_transformation: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 608fdebc1d9..026d8b8e82e 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -296,18 +296,133 @@ class AnthropicCacheControlHook(CustomPromptManagement): return processed_messages, processed_system, remaining_points + @staticmethod + def _default_control() -> ChatCompletionCachedContent: + """Build the cache_control block for auto-injected breakpoints. + + Defaults to Anthropic's 5-minute ephemeral cache; honors the optional + ``litellm.anthropic_prompt_caching_ttl`` override ("5m" or "1h"). + """ + import litellm + + ttl = litellm.anthropic_prompt_caching_ttl + if ttl == "5m" or ttl == "1h": + return ChatCompletionCachedContent(type="ephemeral", ttl=ttl) + return ChatCompletionCachedContent(type="ephemeral") + + @staticmethod + def _request_has_cache_control(messages: list[AllMessageValues], system: Optional[Union[str, list]]) -> bool: + """Return True if the request already carries any client-supplied cache_control. + + When the client (e.g. Claude Code) already marks its own breakpoints we + stand down entirely rather than add more, per the auto-caching contract. + """ + if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages): + return True + if isinstance(system, list): + return any(isinstance(block, dict) and block.get("cache_control") is not None for block in system) + return False + + @staticmethod + def get_default_injection_points( + messages: list[AllMessageValues], + system: Optional[Union[str, list]], + model: str, + custom_llm_provider: Optional[str], + ) -> list[CacheControlInjectionPoint]: + """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. + + Caches the system prompt and the trailing turn, so the stable prefix + (system + tools + history) is reused while the breakpoint advances with + the conversation. Returns [] (stand down) when the flag is off, the + provider does not consume cache_control breakpoints (only anthropic / + bedrock do), the model lacks prompt-caching support, or the request + already carries client-supplied cache_control. + """ + import litellm + + if litellm.enable_anthropic_prompt_caching is not True: + return [] + + provider = custom_llm_provider + if provider is None: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + ) + + try: + _, provider, _, _ = get_llm_provider(model=model) + except Exception: # noqa: BLE001 # unroutable model must never block the call, just skip auto-caching + return [] + + if provider not in ("anthropic", "bedrock"): + return [] + + from litellm.utils import supports_prompt_caching + + if not supports_prompt_caching(model=model, custom_llm_provider=provider): + return [] + + if AnthropicCacheControlHook._request_has_cache_control(messages, system): + return [] + + control = AnthropicCacheControlHook._default_control() + points: list[CacheControlInjectionPoint] = [ + CacheControlMessageInjectionPoint(location="message", role="system", index=None, control=control), + CacheControlMessageInjectionPoint(location="message", role=None, index=-1, control=control), + ] + return points + + @staticmethod + def maybe_seed_default_injection_points( + non_default_params: dict[str, Any], + messages: list[AllMessageValues], + model: str, + custom_llm_provider: Optional[str], + ) -> None: + """For /chat/completions: add default injection points to the request params. + + No-op when injection points are already configured (explicit config wins). + Seeding the param lets the existing prompt-management gate and the + AnthropicCacheControlHook run unchanged. + """ + if non_default_params.get("cache_control_injection_points"): + return + points = AnthropicCacheControlHook.get_default_injection_points( + messages=messages, + system=None, + model=model, + custom_llm_provider=custom_llm_provider, + ) + if points: + non_default_params["cache_control_injection_points"] = points + @staticmethod def maybe_inject_cache_control( messages: List[Dict], system: str | list | None, kwargs: Dict[str, Any], + model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> Tuple[List[Dict], str | list | None]: """Extract cache_control_injection_points from kwargs and apply if present. + When none are configured but ``litellm.enable_anthropic_prompt_caching`` + is on, synthesize default breakpoints for the native /v1/messages path. Pops the key from kwargs; if remaining (non-message) points exist they are written back so downstream transforms can handle them. """ - injection_points = kwargs.pop("cache_control_injection_points", None) + configured = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list + Optional[list[CacheControlInjectionPoint]], kwargs.pop("cache_control_injection_points", None) + ) + injection_points: list[CacheControlInjectionPoint] = configured or [] + if not injection_points and model is not None: + injection_points = AnthropicCacheControlHook.get_default_injection_points( + messages=cast(list[AllMessageValues], messages), # cast-ok: Anthropic-shaped dicts from v1/messages + system=system, + model=model, + custom_llm_provider=custom_llm_provider, + ) if not injection_points: return messages, system diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index dd983f0c344..c205d7516e6 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -236,7 +236,9 @@ async def anthropic_messages( AnthropicCacheControlHook, ) - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + ) original_stream = stream or kwargs.get("_websearch_interception_converted_stream", False) @@ -425,7 +427,9 @@ def anthropic_messages_handler( AnthropicCacheControlHook, ) - messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(messages, system, kwargs) + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider + ) metadata = validate_anthropic_api_metadata(metadata) diff --git a/litellm/main.py b/litellm/main.py index 6fd68921fb0..4a9b5bdc76f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -510,6 +510,19 @@ async def acompletion( ######################################################### ######################################################### litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + from litellm.types.llms.openai import AllMessageValues + + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=kwargs, + messages=cast(list[AllMessageValues], messages), # cast-ok: acompletion types messages as a bare List + model=model, + custom_llm_provider=cast(Optional[str], custom_llm_provider), # cast-ok: read from untyped kwargs + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=kwargs.get("prompt_id", None), @@ -5055,6 +5068,18 @@ def completion( # type: ignore litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## + from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + ) + from litellm.types.llms.openai import AllMessageValues + + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=non_default_params, + messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List + model=model, + custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs + ) + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( prompt_id=prompt_id, non_default_params=non_default_params diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index daac1e4506f..9f689a2dd31 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -529,6 +529,7 @@ class ChatCompletionDeltaToolCallChunk(TypedDict, total=False): class ChatCompletionCachedContent(TypedDict): type: Literal["ephemeral"] + ttl: NotRequired[Literal["5m", "1h"]] class ChatCompletionThinkingBlock(TypedDict, total=False): diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 4664cc86303..ef63555bdac 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1533,3 +1533,139 @@ class TestApplyToAnthropicMessagesRequest: sys_blocks = sum(1 for b in (result_sys or []) if isinstance(b, dict) and b.get("cache_control") is not None) total_blocks = sys_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(m) for m in result_msgs) assert total_blocks <= 4 + + +class TestEnableAnthropicPromptCaching: + """Auto-injected default breakpoints via litellm.enable_anthropic_prompt_caching.""" + + MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "a long system prompt"}, + {"role": "user", "content": "first turn"}, + {"role": "assistant", "content": "a reply"}, + {"role": "user", "content": "latest turn"}, + ] + + def _points(self, model="claude-sonnet-4-5", provider="anthropic", messages=None, system=None): + return AnthropicCacheControlHook.get_default_injection_points( + messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, + system=system, + model=model, + custom_llm_provider=provider, + ) + + def test_disabled_by_default(self): + assert litellm.enable_anthropic_prompt_caching is False + assert self._points() == [] + + def test_injects_system_and_trailing_turn(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points() == [ + {"location": "message", "role": "system", "index": None, "control": {"type": "ephemeral"}}, + {"location": "message", "role": None, "index": -1, "control": {"type": "ephemeral"}}, + ] + + def test_bedrock_claude_is_injected(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") + assert [p["index"] for p in points] == [None, -1] + + @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai"), ("gemini-2.0-flash", "gemini")]) + def test_non_anthropic_providers_never_injected(self, monkeypatch, model, provider): + """These report supports_prompt_caching=True but never consume cache_control markers.""" + from litellm.utils import supports_prompt_caching + + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert supports_prompt_caching(model=model, custom_llm_provider=provider) is True + assert self._points(model=model, provider=provider) == [] + + def test_model_without_caching_support_not_injected(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert self._points(model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] + + def test_stands_down_when_client_sent_cache_control(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [ + {"role": "system", "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "latest turn"}, + ] + assert self._points(messages=messages) == [] + + def test_stands_down_when_system_block_has_cache_control(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}] + assert self._points(messages=[{"role": "user", "content": "hi"}], system=system) == [] + + def test_default_ttl_is_anthropics_five_minute_cache(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert all(p["control"] == {"type": "ephemeral"} for p in self._points()) + + @pytest.mark.parametrize("ttl", ["5m", "1h"]) + def test_ttl_override_applied(self, monkeypatch, ttl): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", ttl) + assert all(p["control"] == {"type": "ephemeral", "ttl": ttl} for p in self._points()) + + def test_seed_does_not_override_configured_points(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + configured = [{"location": "message", "role": "user", "index": 0}] + params = {"cache_control_injection_points": configured} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params["cache_control_injection_points"] is configured + + def test_seed_adds_defaults_when_enabled(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert [p["index"] for p in params["cache_control_injection_points"]] == [None, -1] + + def test_seed_is_noop_when_disabled(self): + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert params == {} + + def test_v1_messages_applies_defaults_end_to_end(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + messages = [ + {"role": "user", "content": [{"type": "text", "text": "first"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "reply"}]}, + {"role": "user", "content": [{"type": "text", "text": "latest"}]}, + ] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + "a system prompt", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + + assert result_sys == [{"type": "text", "text": "a system prompt", "cache_control": {"type": "ephemeral"}}] + assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result_msgs[0]["content"][-1] + + def test_v1_messages_is_noop_when_disabled(self): + messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, + "sys", + {}, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + + assert result_sys == "sys" + assert result_msgs == messages diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 87c760257d1..501a30110c0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21870,6 +21870,11 @@ export interface components { }; /** ChatCompletionCachedContent */ ChatCompletionCachedContent: { + /** + * Ttl + * @enum {string} + */ + ttl?: "5m" | "1h"; /** * Type * @constant