scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge b878e1b599 into 08f2a98250

Browse source
This commit is contained in:
izahai 2026-11-06 16:36:18 +09:00 • committed by GitHub
commit c5e84dd28e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 193 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/caching/caching.py
View file

@ -102,18 +102,29 @@ def _is_conversation_item(item: object) -> bool:
return block.get("type") != "file"
def _request_message_count(kwargs: Mapping[str, object]) -> int:
REQUEST_MESSAGE_COUNT_KWARG: Final = "_litellm_request_message_count"
def request_message_count(kwargs: Mapping[str, object]) -> int:
"""Chat and Messages API `messages`, else Responses API `input` items; embedding strings and file blocks count as none"""
caller_count: Final = kwargs.get(REQUEST_MESSAGE_COUNT_KWARG)
if isinstance(caller_count, int) and not isinstance(caller_count, bool):
return caller_count
messages: Final = kwargs.get("messages")
if isinstance(messages, list):
return len(messages)
input_items: Final = kwargs.get("input")
if isinstance(input_items, str):
return 0 if kwargs.get("call_type") in ("embedding", "aembedding") else 1
if not isinstance(input_items, list):
return 0
items: Final = cast(list[object], input_items) # cast-ok: isinstance leaves the element type unknown
return sum(1 for item in items if _is_conversation_item(item))
_request_message_count = request_message_count
class Cache:
def __init__(
self,

5
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -13,6 +13,7 @@ from typing import (
from typing_extensions import ReadOnly, TypedDict
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG
from litellm.types.llms.openai import ResponsesAPIResponse
if TYPE_CHECKING:
@ -203,6 +204,8 @@ class ResponsesToCompletionBridgeHandler:
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
if kwargs.get("cache") is not None:
request_data["cache"] = kwargs["cache"]
if REQUEST_MESSAGE_COUNT_KWARG not in request_data:
request_data[REQUEST_MESSAGE_COUNT_KWARG] = len(messages)
result: Final = responses(
**request_data,
)
@ -307,6 +310,8 @@ class ResponsesToCompletionBridgeHandler:
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
if kwargs.get("cache") is not None:
request_data["cache"] = kwargs["cache"]
if REQUEST_MESSAGE_COUNT_KWARG not in request_data:
request_data[REQUEST_MESSAGE_COUNT_KWARG] = len(messages)
result: Final = await aresponses(
**request_data,
aresponses=True,

7
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -12,6 +12,7 @@ from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
@ -744,6 +745,8 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if chat_reasoning is not None and chat_reasoning.summary is not None:
completion_kwargs["reasoning_summary"] = chat_reasoning.summary
if REQUEST_MESSAGE_COUNT_KWARG not in completion_kwargs:
completion_kwargs[REQUEST_MESSAGE_COUNT_KWARG] = len(messages)
return completion_kwargs, tool_name_mapping
@staticmethod
@ -821,7 +824,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs},
)
completion_response: Final = await litellm.acompletion(**completion_kwargs)
@ -968,7 +971,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs},
)
completion_response: Final = litellm.completion(**completion_kwargs)

7
litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
routed_model_info,
@ -388,6 +389,8 @@ def _build_responses_kwargs(
*(item for item in deployment_include if item not in bridge_include),
]
if REQUEST_MESSAGE_COUNT_KWARG not in responses_kwargs:
responses_kwargs[REQUEST_MESSAGE_COUNT_KWARG] = len(messages)
return responses_kwargs
@ -439,7 +442,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs},
)
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
@ -543,7 +546,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
extra_kwargs={REQUEST_MESSAGE_COUNT_KWARG: len(messages), **kwargs},
)
result: Final = litellm.responses(**responses_kwargs)

2
litellm/proxy/litellm_pre_call_utils.py
View file

@ -27,6 +27,7 @@ import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm._uuid import uuid
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG
from litellm.constants import (
BAGGAGE_MAPPED_TAGS_METADATA_KEY,
BILLED_RETRY_ATTEMPT_METADATA_KEY,
@ -422,6 +423,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
"guardrail_scan_metadata",
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
REQUEST_MESSAGE_COUNT_KWARG,
"pillar_response_headers",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",

22
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -10,6 +10,7 @@ from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG, request_message_count
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
@ -91,10 +92,14 @@ class LiteLLMCompletionTransformationHandler:
| BaseResponsesAPIStreamingIterator
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
):
request_kwargs: Final = {
REQUEST_MESSAGE_COUNT_KWARG: request_message_count({"input": input}),
**_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
}
hoisted: Final = hoist_additional_tools(
input,
responses_api_request.get("tools"),
drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")),
drop_params=drops_unsupported_params(request_kwargs.get("drop_params")),
)
previous_response_id: Final = responses_api_request.get("previous_response_id")
if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
@ -108,7 +113,7 @@ class LiteLLMCompletionTransformationHandler:
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
request_kwargs=request_kwargs,
)
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
bridged_input: Final = bridged.input
@ -123,7 +128,7 @@ class LiteLLMCompletionTransformationHandler:
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
**_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
**_without_bridge_transform_keywords(request_kwargs),
),
**_chat_verbosity(bridged_request),
}
@ -134,11 +139,11 @@ class LiteLLMCompletionTransformationHandler:
litellm_completion_request=litellm_completion_request,
request_input=bridged_input,
responses_api_request=bridged_request,
**kwargs,
**request_kwargs,
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**{**request_kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
)
if isinstance(litellm_completion_response, ModelResponse):
@ -237,7 +242,12 @@ class LiteLLMCompletionTransformationHandler:
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**{
REQUEST_MESSAGE_COUNT_KWARG: request_message_count({"input": request_input}),
**kwargs,
**litellm_completion_request,
"_skip_responses_api_bridge": True,
},
)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (

148
tests/unit/caching/test_caching.py
View file

@ -1091,6 +1091,154 @@ def test_completion_past_max_messages_is_neither_served_from_nor_written_to_the_
assert answer(five, "five second") == "five second", "a 5-message repeat was served from the cache"
@respx.mock
def test_bridged_responses_counts_caller_input_items_against_max_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4))
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "done"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
)
five_items: Final = [
{"role": "user", "content": "list a and b"},
{"type": "function_call", "call_id": "c1", "name": "ls", "arguments": '{"p":"a"}'},
{"type": "function_call", "call_id": "c2", "name": "ls", "arguments": '{"p":"b"}'},
{"type": "function_call_output", "call_id": "c1", "output": "a"},
{"type": "function_call_output", "call_id": "c2", "output": "b"},
]
litellm.responses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k")
second: Final = litellm.responses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k")
assert route.call_count == 2
assert second._hidden_params.get("cache_hit") is None
@respx.mock
def test_bridged_responses_counts_string_input_as_one_message_against_max_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.caching.caching import REQUEST_MESSAGE_COUNT_KWARG
from litellm.proxy.litellm_pre_call_utils import _UNTRUSTED_ROOT_CONTROL_FIELDS
assert REQUEST_MESSAGE_COUNT_KWARG in _UNTRUSTED_ROOT_CONTROL_FIELDS
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=0))
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "done"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
)
litellm.responses(model="anthropic/claude-sonnet-4-5", input="hello", api_key="k")
second: Final = litellm.responses(model="anthropic/claude-sonnet-4-5", input="hello", api_key="k")
assert route.call_count == 2
assert second._hidden_params.get("cache_hit") is None
@pytest.mark.asyncio
@respx.mock
async def test_bridged_aresponses_counts_caller_input_items_against_max_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4))
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
route: Final = respx.post("https://api.anthropic.com/v1/messages").mock(
return_value=httpx.Response(
200,
json={
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "done"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
)
five_items: Final = [
{"role": "user", "content": "list a and b"},
{"type": "function_call", "call_id": "c1", "name": "ls", "arguments": '{"p":"a"}'},
{"type": "function_call", "call_id": "c2", "name": "ls", "arguments": '{"p":"b"}'},
{"type": "function_call_output", "call_id": "c1", "output": "a"},
{"type": "function_call_output", "call_id": "c2", "output": "b"},
]
await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k")
second: Final = await litellm.aresponses(model="anthropic/claude-sonnet-4-5", input=five_items, api_key="k")
assert route.call_count == 2
assert second._hidden_params.get("cache_hit") is None
@respx.mock
def test_bridged_anthropic_messages_counts_caller_messages_against_max_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, max_messages=4))
route: Final = respx.post(
url__regex=r"https://generativelanguage\.googleapis\.com/.*/models/gemini-2\.5-flash:generateContent.*"
).mock(
return_value=httpx.Response(
200,
json={
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6},
},
)
)
three_messages: Final = [
{"role": "user", "content": "check a, b, c"},
{
"role": "assistant",
"content": [
{"type": "tool_use", "id": "t1", "name": "ls", "input": {"p": "a"}},
{"type": "tool_use", "id": "t2", "name": "ls", "input": {"p": "b"}},
{"type": "tool_use", "id": "t3", "name": "ls", "input": {"p": "c"}},
],
},
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": "t1", "content": "a"},
{"type": "tool_result", "tool_use_id": "t2", "content": "b"},
{"type": "tool_result", "tool_use_id": "t3", "content": "c"},
],
},
]
litellm.anthropic.messages.create(
model="gemini/gemini-2.5-flash", max_tokens=16, messages=three_messages, api_key="k"
)
litellm.anthropic.messages.create(
model="gemini/gemini-2.5-flash", max_tokens=16, messages=three_messages, api_key="k"
)
assert route.call_count == 1
@respx.mock
def test_enable_cache_serves_a_repeated_bedrock_titan_embedding_from_the_local_cache(
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], fake_provider_credentials: None