diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 3b7ea66ad6b..1e7c2f5492b 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -102,18 +102,29 @@ def _is_conversation_item(item: object) -> bool: return block.get("type") != "file" -def _request_message_count(kwargs: Mapping[str, object]) -> int: +REQUEST_MESSAGE_COUNT_KWARG: Final = "_litellm_request_message_count" + + +def request_message_count(kwargs: Mapping[str, object]) -> int: """Chat and Messages API `messages`, else Responses API `input` items; embedding strings and file blocks count as none""" + caller_count: Final = kwargs.get(REQUEST_MESSAGE_COUNT_KWARG) + if isinstance(caller_count, int) and not isinstance(caller_count, bool): + return caller_count messages: Final = kwargs.get("messages") if isinstance(messages, list): return len(messages) input_items: Final = kwargs.get("input") + if isinstance(input_items, str): + return 0 if kwargs.get("call_type") in ("embedding", "aembedding") else 1 if not isinstance(input_items, list): return 0 items: Final = cast(list[object], input_items) # cast-ok: isinstance leaves the element type unknown return sum(1 for item in items if _is_conversation_item(item)) +_request_message_count = request_message_count + + class Cache: def __init__( self, diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index f6879c313de..00fdf5ca2c7 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -13,6 +13,7 @@ from typing import ( from typing_extensions import ReadOnly, TypedDict +from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: @@ -203,6 +204,8 @@ class ResponsesToCompletionBridgeHandler: request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) if kwargs.get("cache") is not None: request_data["cache"] = kwargs["cache"] + if REQUEST_MESSAGE_COUNT_KWARG not in request_data: + request_data[REQUEST_MESSAGE_COUNT_KWARG] = len(messages) result: Final = responses( **request_data, ) @@ -307,6 +310,8 @@ class ResponsesToCompletionBridgeHandler: request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) if kwargs.get("cache") is not None: request_data["cache"] = kwargs["cache"] + if REQUEST_MESSAGE_COUNT_KWARG not in request_data: + request_data[REQUEST_MESSAGE_COUNT_KWARG] = len(messages) result: Final = await aresponses( **request_data, aresponses=True, diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index da120cad0e1..c1e01b3bcd4 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger +from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -744,6 +745,8 @@ class LiteLLMMessagesToCompletionTransformationHandler: if chat_reasoning is not None and chat_reasoning.summary is not None: completion_kwargs["reasoning_summary"] = chat_reasoning.summary + if REQUEST_MESSAGE_COUNT_KWARG not in completion_kwargs: + completion_kwargs[REQUEST_MESSAGE_COUNT_KWARG] = len(messages) return completion_kwargs, tool_name_mapping @staticmethod @@ -821,7 +824,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: top_k=top_k, top_p=top_p, output_format=output_format, - extra_kwargs=kwargs, + extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs}, ) completion_response: Final = await litellm.acompletion(**completion_kwargs) @@ -968,7 +971,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: top_k=top_k, top_p=top_p, output_format=output_format, - extra_kwargs=kwargs, + extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs}, ) completion_response: Final = litellm.completion(**completion_kwargs) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 85f042f621c..c0d6f6704d6 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger +from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, routed_model_info, @@ -388,6 +389,8 @@ def _build_responses_kwargs( *(item for item in deployment_include if item not in bridge_include), ] + if REQUEST_MESSAGE_COUNT_KWARG not in responses_kwargs: + responses_kwargs[REQUEST_MESSAGE_COUNT_KWARG] = len(messages) return responses_kwargs @@ -439,7 +442,7 @@ class LiteLLMMessagesToResponsesAPIHandler: top_k=top_k, top_p=top_p, output_format=output_format, - extra_kwargs=kwargs, + extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs}, ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) @@ -543,7 +546,7 @@ class LiteLLMMessagesToResponsesAPIHandler: top_k=top_k, top_p=top_p, output_format=output_format, - extra_kwargs=kwargs, + extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs}, ) result: Final = litellm.responses(**responses_kwargs) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4444d6d25f9..35c49e6f1b4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -27,6 +27,7 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm._uuid import uuid +from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG from litellm.constants import ( BAGGAGE_MAPPED_TAGS_METADATA_KEY, BILLED_RETRY_ATTEMPT_METADATA_KEY, @@ -422,6 +423,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "guardrail_scan_metadata", "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, + REQUEST_MESSAGE_COUNT_KWARG, "pillar_response_headers", "_guardrail_pipelines", "_pipeline_managed_guardrails", diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 802ed6bf9c5..752192a549c 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -10,6 +10,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG, request_message_count from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools @@ -91,10 +92,14 @@ class LiteLLMCompletionTransformationHandler: | BaseResponsesAPIStreamingIterator | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] ): + request_kwargs: Final = { + REQUEST_MESSAGE_COUNT_KWARG: request_message_count({"input": input}), + **_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + } hoisted: Final = hoist_additional_tools( input, responses_api_request.get("tools"), - drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")), + drop_params=drops_unsupported_params(request_kwargs.get("drop_params")), ) previous_response_id: Final = responses_api_request.get("previous_response_id") if _is_async and previous_response_id and has_client_tool_search(hoisted.tools): @@ -108,7 +113,7 @@ class LiteLLMCompletionTransformationHandler: custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + request_kwargs=request_kwargs, ) bridged: Final = _bridged_request(hoisted, responses_api_request, ()) bridged_input: Final = bridged.input @@ -123,7 +128,7 @@ class LiteLLMCompletionTransformationHandler: custom_llm_provider=custom_llm_provider, stream=stream, extra_headers=extra_headers, - **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), + **_without_bridge_transform_keywords(request_kwargs), ), **_chat_verbosity(bridged_request), } @@ -134,11 +139,11 @@ class LiteLLMCompletionTransformationHandler: litellm_completion_request=litellm_completion_request, request_input=bridged_input, responses_api_request=bridged_request, - **kwargs, + **request_kwargs, ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **{**request_kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, ) if isinstance(litellm_completion_response, ModelResponse): @@ -237,7 +242,12 @@ class LiteLLMCompletionTransformationHandler: ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **{ + REQUEST_MESSAGE_COUNT_KWARG: request_message_count({"input": request_input}), + **kwargs, + **litellm_completion_request, + "_skip_responses_api_bridge": True, + }, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index e9a075af507..e12768524fb 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -1091,6 +1091,154 @@ def test_completion_past_max_messages_is_neither_served_from_nor_written_to_the_ assert answer(five, "five second") == "five second", "a 5-message repeat was served from the cache" +@respx.mock +def test_bridged_responses_counts_caller_input_items_against_max_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4)) + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + five_items: Final = [ + {"role": "user", "content": "list a and b"}, + {"type": "function_call", "call_id": "c1", "name": "ls", "arguments": '{"p":"a"}'}, + {"type": "function_call", "call_id": "c2", "name": "ls", "arguments": '{"p":"b"}'}, + {"type": "function_call_output", "call_id": "c1", "output": "a"}, + {"type": "function_call_output", "call_id": "c2", "output": "b"}, + ] + + litellm.responses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k") + second: Final = litellm.responses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k") + + assert route.call_count == 2 + assert second._hidden_params.get("cache_hit") is None + + +@respx.mock +def test_bridged_responses_counts_string_input_as_one_message_against_max_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG + from litellm.proxy.litellm_pre_call_utils import _UNTRUSTED_ROOT_CONTROL_FIELDS + + assert REQUEST_MESSAGE_COUNT_KWARG in _UNTRUSTED_ROOT_CONTROL_FIELDS + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=0)) + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + + litellm.responses(model="anthropic/claude-sonnet-4-5", input="hello", api_key="k") + second: Final = litellm.responses(model="anthropic/claude-sonnet-4-5", input="hello", api_key="k") + + assert route.call_count == 2 + assert second._hidden_params.get("cache_hit") is None + + +@pytest.mark.asyncio +@respx.mock +async def test_bridged_aresponses_counts_caller_input_items_against_max_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "done"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + five_items: Final = [ + {"role": "user", "content": "list a and b"}, + {"type": "function_call", "call_id": "c1", "name": "ls", "arguments": '{"p":"a"}'}, + {"type": "function_call", "call_id": "c2", "name": "ls", "arguments": '{"p":"b"}'}, + {"type": "function_call_output", "call_id": "c1", "output": "a"}, + {"type": "function_call_output", "call_id": "c2", "output": "b"}, + ] + + await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k") + second: Final = await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k") + + assert route.call_count == 2 + assert second._hidden_params.get("cache_hit") is None + + +@respx.mock +def test_bridged_anthropic_messages_counts_caller_messages_against_max_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4)) + route: Final = respx.post( + url__regex=r"https://generativelanguage\.googleapis\.com/.*/models/gemini-2\.5-flash:generateContent.*" + ).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6}, + }, + ) + ) + three_messages: Final = [ + {"role": "user", "content": "check a, b, c"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "ls", "input": {"p": "a"}}, + {"type": "tool_use", "id": "t2", "name": "ls", "input": {"p": "b"}}, + {"type": "tool_use", "id": "t3", "name": "ls", "input": {"p": "c"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "a"}, + {"type": "tool_result", "tool_use_id": "t2", "content": "b"}, + {"type": "tool_result", "tool_use_id": "t3", "content": "c"}, + ], + }, + ] + + litellm.anthropic.messages.create( + model="gemini/gemini-2.5-flash", max_tokens=16, messages=three_messages, api_key="k" + ) + litellm.anthropic.messages.create( + model="gemini/gemini-2.5-flash", max_tokens=16, messages=three_messages, api_key="k" + ) + + assert route.call_count == 1 + + @respx.mock def test_enable_cache_serves_a_repeated_bedrock_titan_embedding_from_the_local_cache( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], fake_provider_credentials: None