diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index f2ef8d63a07..4df6fce74c0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -382,19 +382,23 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + enable_prompt_caching: bool | None = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. - Caches the system prompt and the trailing turn, so the stable prefix - (system + tools + history) is reused while the breakpoint advances with - the conversation. Returns [] (stand down) when the flag is off, the - provider does not consume cache_control breakpoints (only anthropic / - bedrock do), the model lacks prompt-caching support, or the request - already carries client-supplied cache_control. + ``enable_prompt_caching`` is the per-request override (stamped from key + metadata by the proxy); True turns auto-injection on for this request + even when the global flag is off. Caches the system prompt and the + trailing turn, so the stable prefix (system + tools + history) is + reused while the breakpoint advances with the conversation. Returns [] + (stand down) when neither flag is on, the provider does not consume + cache_control breakpoints (only anthropic / bedrock do), the model + lacks prompt-caching support, or the request already carries + client-supplied cache_control. """ import litellm - if litellm.enable_anthropic_prompt_caching is not True: + if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True: return [] provider = custom_llm_provider @@ -433,6 +437,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model: str, custom_llm_provider: str | None, tools: list | None = None, + enable_prompt_caching: bool | None = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -458,6 +463,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): model=model, custom_llm_provider=custom_llm_provider, tools=tools, + enable_prompt_caching=enable_prompt_caching, ) if points: non_default_params["cache_control_injection_points"] = points @@ -478,12 +484,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): judgment happens once per request; points a prior pass wrote back carry the judged stamp and are never re-judged (see ``_should_stand_down``). When none are configured but - ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default - breakpoints for the native /v1/messages path. Pops the key from kwargs; + ``litellm.enable_anthropic_prompt_caching`` or the per-request + ``enable_prompt_caching`` kwarg (stamped from key metadata) is on, + synthesize default breakpoints for the native /v1/messages path. Pops + both keys from kwargs; if remaining (non-message) points exist they are written back so downstream transforms can handle them. """ typed_messages = cast(list[AllMessageValues], messages) # cast-ok: Anthropic-shaped dicts from v1/messages + enable_prompt_caching: Final = cast( # cast-ok: kwargs is untyped; key stamped as bool by the proxy + bool | None, kwargs.pop("enable_prompt_caching", None) + ) configured: Final = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) ) @@ -497,6 +508,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools=tools, model=model, custom_llm_provider=custom_llm_provider, + enable_prompt_caching=enable_prompt_caching, ) if not injection_points: return messages, system diff --git a/litellm/main.py b/litellm/main.py index c70a41c891a..bf6ec8004ce 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -504,6 +504,7 @@ async def acompletion( model=model, custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs tools=tools, + enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( @@ -5105,6 +5106,7 @@ def completion( model=model, custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs tools=tools, + enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dc4f17c7b31..08348187645 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1108,6 +1108,7 @@ class KeyRequestBase(GenerateRequestBase): budget_id: str | None = None tags: list[str] | None = None disable_global_guardrails: bool | None = None + enable_prompt_caching: bool | None = None throttle_on_budget_exceeded: bool | None = None enforced_params: list[str] | None = None allowed_routes: list | None = [] @@ -4124,6 +4125,7 @@ LiteLLM_ManagementEndpoint_MetadataFields: Final = [ "enforced_batch_output_expires_after", "enforced_file_expires_after", "throttle_on_budget_exceeded", + "enable_prompt_caching", ] LiteLLM_ManagementEndpoint_MetadataFields_Premium: Final = [ diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0924b6aebea..f83061a15ce 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -201,6 +201,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "mock_tool_calls", "disable_global_guardrails", "disable_global_guardrail", + "enable_prompt_caching", "opted_out_global_guardrails", "applied_guardrails", "applied_policies", @@ -1333,6 +1334,9 @@ class LiteLLMProxyRequestSetup: if "disable_fallbacks" in key_metadata and isinstance(key_metadata["disable_fallbacks"], bool): data["disable_fallbacks"] = key_metadata["disable_fallbacks"] + if isinstance(key_metadata.get("enable_prompt_caching"), bool): + data["enable_prompt_caching"] = key_metadata["enable_prompt_caching"] # rebind-ok: data is an out-param + ## KEY-LEVEL METADATA data = LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata( data=data, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 836b223c6ad..ebf44497658 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1593,6 +1593,7 @@ async def generate_key_fn( - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules. - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely. + - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Anthropic and Bedrock Claude models only. - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false} - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget. - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. @@ -2693,6 +2694,7 @@ async def update_key_fn( - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules. - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key. - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely. + - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Anthropic and Bedrock Claude models only. - prompts: Optional[List[str]] - List of prompts that the key is allowed to use. - blocked: Optional[bool] - Whether the key is blocked - aliases: Optional[dict] - Model aliases for the key - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 74311d59d8e..d4c26df54d8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3467,6 +3467,7 @@ all_litellm_params = ( "caching_groups", "ttl", "cache", + "enable_prompt_caching", "no-log", "base_model", "stream_timeout", diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 47baacd61d7..cc43a424419 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1728,6 +1728,87 @@ class TestEnableAnthropicPromptCaching: assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} assert "cache_control" not in result_msgs[0]["content"][-1] + +class TestPerKeyEnablePromptCaching: + """Per-request enable_prompt_caching override (stamped from key metadata) with the global flag off.""" + + MESSAGES: List[AllMessageValues] = [ + {"role": "system", "content": "a long system prompt"}, + {"role": "user", "content": "latest turn"}, + ] + + def _points(self, enable_prompt_caching, model="claude-sonnet-4-5", provider="anthropic", messages=None): + return AnthropicCacheControlHook.get_default_injection_points( + messages=copy.deepcopy(self.MESSAGES) if messages is None else messages, + system=None, + model=model, + custom_llm_provider=provider, + enable_prompt_caching=enable_prompt_caching, + ) + + def test_true_injects_with_global_flag_off(self): + assert litellm.enable_anthropic_prompt_caching is False + assert self._points(True) == [ + {"location": "message", "role": "system", "index": None, "control": {"type": "ephemeral"}}, + {"location": "message", "role": None, "index": -1, "control": {"type": "ephemeral"}}, + ] + + @pytest.mark.parametrize("enable_prompt_caching", [False, None]) + def test_false_and_none_fall_back_to_global_flag(self, enable_prompt_caching): + assert self._points(enable_prompt_caching) == [] + + def test_false_does_not_suppress_global_flag(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + assert [p["index"] for p in self._points(False)] == [None, -1] + + def test_provider_gate_still_applies(self): + assert self._points(True, model="gpt-4o", provider="openai") == [] + + def test_unsupported_model_gate_still_applies(self): + assert self._points(True, model="anthropic.claude-3-5-sonnet-20240620-v1:0", provider="bedrock") == [] + + def test_client_markers_still_win(self): + messages = [ + {"role": "system", "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": "latest turn"}, + ] + assert self._points(True, messages=messages) == [] + + def test_seed_injects_with_global_flag_off(self): + params: dict = {} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + enable_prompt_caching=True, + ) + assert [p["index"] for p in params["cache_control_injection_points"]] == [None, -1] + + def test_v1_messages_injects_and_pops_flag_from_kwargs(self): + kwargs: dict = {"enable_prompt_caching": True} + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": [{"type": "text", "text": "latest"}]}], + "a system prompt", + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert result_sys == [{"type": "text", "text": "a system prompt", "cache_control": {"type": "ephemeral"}}] + assert result_msgs[-1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert "enable_prompt_caching" not in kwargs + + def test_v1_messages_pops_flag_even_when_noop(self): + kwargs: dict = {"enable_prompt_caching": True} + AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + None, + kwargs, + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "enable_prompt_caching" not in kwargs + def test_v1_messages_is_noop_when_disabled(self): messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 4c13a3367d9..0a88f59f677 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1733,6 +1733,21 @@ async def test_update_service_account_works_with_team_id(): await prepare_key_update_data(data=data, existing_key_row=existing_key) +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_value", [True, False]) +async def test_update_key_enable_prompt_caching_folds_into_metadata(flag_value): + """Top-level enable_prompt_caching on /key/update lands in key metadata, including flipping back to False.""" + data = UpdateKeyRequest(key="sk-1", enable_prompt_caching=flag_value) + existing_key = LiteLLM_VerificationToken( + token="hashed", metadata={"enable_prompt_caching": not flag_value} + ) + + updated = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + assert updated["metadata"]["enable_prompt_caching"] is flag_value + assert "enable_prompt_caching" not in {k for k in updated if k != "metadata"} + + @pytest.mark.asyncio async def test_update_preserves_service_account_id_when_metadata_replaced(): """ diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 293cce5fa5d..e31058f402e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -688,6 +688,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "mock_response": "free response", "mock_tool_calls": [{"id": "call_1"}], "disable_global_guardrails": True, + "enable_prompt_caching": True, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, "metadata": copy.deepcopy(malicious_metadata), "litellm_metadata": copy.deepcopy(malicious_metadata), @@ -705,6 +706,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "mock_response" not in updated assert "mock_tool_calls" not in updated assert "disable_global_guardrails" not in updated + assert "enable_prompt_caching" not in updated assert "routing_decision" not in updated stripped_keys = { @@ -741,6 +743,42 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "pillar_response_headers" not in snapshot_body["metadata"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_value, expected", + [(True, True), (False, False), ("yes", None), (None, None)], +) +async def test_key_metadata_enable_prompt_caching_promoted_to_request_root(key_value, expected): + """Key metadata enable_prompt_caching is stamped onto the request root (bools only), even when the client spoofs it.""" + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hello"}], + "enable_prompt_caching": "spoofed-by-client", + } + key_metadata = {} if key_value is None else {"enable_prompt_caching": key_value} + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", metadata=key_metadata), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated.get("enable_prompt_caching") == expected + + @pytest.mark.asyncio @pytest.mark.parametrize( "control_field", diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 19abae73549..8951a471f84 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1191,6 +1191,21 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp > + + Enable Prompt Caching{" "} + + + + + } + name="enable_prompt_caching" + valuePropName="checked" + > + + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index 29b32c1f7fc..fb8ab87e45f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -410,6 +410,36 @@ describe("KeyEditView", () => { }); }); + it("should initialize and submit enable_prompt_caching from key metadata", async () => { + const onSubmitMock = vi.fn().mockResolvedValue(undefined); + const keyDataWithPromptCaching = { + ...MOCK_KEY_DATA, + metadata: { ...MOCK_KEY_DATA.metadata, enable_prompt_caching: true }, + }; + + renderWithProviders( + {}} + onSubmit={onSubmitMock} + accessToken={"test-token"} + userID={"test-user"} + userRole={"admin"} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("Enable Prompt Caching")).toBeInTheDocument(); + }); + + await userEvent.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(onSubmitMock).toHaveBeenCalledWith(expect.objectContaining({ enable_prompt_caching: true })); + }); + }); + it("should disable models field when management routes are selected", async () => { const keyDataWithManagementRoutes = { ...MOCK_KEY_DATA, diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index e407527f562..36dc02f7bd0 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -150,6 +150,7 @@ export function KeyEditView({ guardrails: keyData.metadata?.guardrails, disable_global_guardrails: keyData.metadata?.disable_global_guardrails || false, throttle_on_budget_exceeded: keyData.metadata?.throttle_on_budget_exceeded || false, + enable_prompt_caching: keyData.metadata?.enable_prompt_caching || false, ...estimateFields(keyData.metadata), prompts: keyData.metadata?.prompts, tags: keyData.metadata?.tags, @@ -178,36 +179,8 @@ export function KeyEditView({ }; useEffect(() => { - form.setFieldsValue({ - ...keyData, - token: keyData.token || keyData.token_id, - budget_duration: canonicalBudgetDuration(keyData.budget_duration), - metadata: formatMetadataForDisplay(stripTagsFromMetadata(keyData.metadata)), - guardrails: keyData.metadata?.guardrails, - disable_global_guardrails: keyData.metadata?.disable_global_guardrails || false, - prompts: keyData.metadata?.prompts, - tags: keyData.metadata?.tags, - vector_stores: keyData.object_permission?.vector_stores || [], - mcp_servers_and_groups: { - servers: keyData.object_permission?.mcp_servers || [], - accessGroups: keyData.object_permission?.mcp_access_groups || [], - toolsets: keyData.object_permission?.mcp_toolsets || [], - }, - mcp_tool_permissions: keyData.object_permission?.mcp_tool_permissions || {}, - throttle_on_budget_exceeded: keyData.metadata?.throttle_on_budget_exceeded || false, - ...estimateFields(keyData.metadata), - logging_settings: extractLoggingSettings(keyData.metadata), - disabled_callbacks: Array.isArray(keyData.metadata?.litellm_disabled_callbacks) - ? mapInternalToDisplayNames(keyData.metadata.litellm_disabled_callbacks) - : [], - access_group_ids: keyData.access_group_ids || [], - auto_rotate: keyData.auto_rotate || false, - ...(keyData.rotation_interval && { rotation_interval: keyData.rotation_interval }), - allowed_routes: - Array.isArray(keyData.allowed_routes) && keyData.allowed_routes.length > 0 - ? keyData.allowed_routes.join(", ") - : "", - }); + form.setFieldsValue(initialValues); + // eslint-disable-next-line react-hooks/exhaustive-deps -- initialValues is rebuilt from keyData every render; depending on it would re-run each render }, [keyData, form]); // Sync auto-rotation state with form values @@ -532,6 +505,21 @@ export function KeyEditView({ + + Enable Prompt Caching{" "} + + + + + } + name="enable_prompt_caching" + valuePropName="checked" + > + + + diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 6a2a4a88425..6fd6547a995 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -782,6 +782,13 @@ export default function KeyInfoView({ {currentKeyData.expires ? formatTimestamp(currentKeyData.expires) : "Never"} + {Boolean(currentKeyData.metadata?.enable_prompt_caching) && ( +
+ Prompt Caching + Enabled (auto-injects cache_control markers on Anthropic and Bedrock Claude requests) +
+ )} +