scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

feat(openai_like): let a passthrough deployment keep cache_control ttl via model_info.cache_control_ttl

Browse source 
The supported_endpoints passthrough had no way to keep ttl for an upstream
that honors it, so the deployment now opts in with
model_info.cache_control_ttl: true, injected into the config the same way
the providers.json constraint is for JSON providers
This commit is contained in:
mateo-berri 2026-08-29 14:17:13 -07:00
parent 0baf376efd
commit c32eb41aad
4 changed files with 84 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -99,6 +99,10 @@ def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
return isinstance(supported_endpoints, (list, tuple)) and "/v1/messages" in supported_endpoints
def _deployment_supports_cache_control_ttl(model_info: object) -> bool:
return isinstance(model_info, dict) and model_info.get("cache_control_ttl") is True
####### ENVIRONMENT VARIABLES ###################
# Initialize any necessary instances or variables here
base_llm_http_handler = BaseLLMHTTPHandler()
@ -568,7 +572,9 @@ def anthropic_messages_handler(
OpenAILikeAnthropicMessagesConfig,
)
anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig()
anthropic_messages_provider_config = OpenAILikeAnthropicMessagesConfig(
cache_control_ttl=_deployment_supports_cache_control_ttl(kwargs.get("model_info")),
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if _should_route_to_responses_api(custom_llm_provider, original_model, model):

16
litellm/llms/openai_like/messages/transformation.py
View file

@ -23,10 +23,15 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
``{api_base}/v1/messages``, so Anthropic-only features that the
Anthropic->OpenAI translation would otherwise drop are preserved. The one
exception is ``cache_control``, whose Anthropic-only extensions (``ttl``)
are stripped unless ``supports_cache_control_ttl`` says otherwise. Response
parsing and streaming are inherited from the native Anthropic config.
are stripped unless the deployment opts in with
``model_info.cache_control_ttl: true``. Response parsing and streaming are
inherited from the native Anthropic config.
"""
def __init__(self, cache_control_ttl: bool = False) -> None:
super().__init__()
self._cache_control_ttl: Final = cache_control_ttl
def validate_anthropic_messages_environment(
self,
headers: dict[str, str],
@ -58,7 +63,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
return False
def supports_cache_control_ttl(self) -> bool:
return False
return self._cache_control_ttl
def transform_anthropic_messages_request(
self,
@ -114,7 +119,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig):
"""
def __init__(self, provider: SimpleProviderConfig):
super().__init__()
super().__init__(cache_control_ttl=bool(provider.constraints.get("cache_control_ttl")))
self._provider = provider
@property
@ -124,9 +129,6 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig):
def should_strip_billing_metadata(self) -> bool:
return True
def supports_cache_control_ttl(self) -> bool:
return bool(self._provider.constraints.get("cache_control_ttl"))
def _resolve_api_key(self, api_key: str | None) -> str | None:
return api_key or get_secret_str(self._provider.api_key_env) or litellm.api_key

97
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -296,21 +296,15 @@ async def test_bedrock_converse_budget_tokens_preserved():
mock_acompletion.assert_called_once()
call_kwargs = mock_acompletion.call_args.kwargs
print(
"acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str)
)
print("acompletion call kwargs: ", json.dumps(call_kwargs, indent=4, default=str))
# Verify thinking parameter is passed through with budget_tokens preserved
thinking_param = call_kwargs.get("thinking")
assert (
thinking_param is not None
), "thinking parameter should be passed to acompletion"
assert (
thinking_param.get("type") == "enabled"
), "thinking.type should be 'enabled'"
assert (
thinking_param.get("budget_tokens") == 1024
), f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}"
assert thinking_param is not None, "thinking parameter should be passed to acompletion"
assert thinking_param.get("type") == "enabled", "thinking.type should be 'enabled'"
assert thinking_param.get("budget_tokens") == 1024, (
f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}"
)
def test_openai_model_with_thinking_converts_to_reasoning():
@ -342,23 +336,18 @@ def test_openai_model_with_thinking_converts_to_reasoning():
call_kwargs = mock_responses.call_args.kwargs
# Verify reasoning is set (converted from thinking)
assert (
"reasoning" in call_kwargs
), "reasoning should be passed to litellm.responses"
assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses"
# budget_tokens=1024 -> effort="low" (at the LOW budget threshold)
# reasoning_auto_summary is False by default, so no summary key
expected_reasoning = {"effort": "low"}
assert call_kwargs["reasoning"] == expected_reasoning, (
f"reasoning should be {expected_reasoning} for budget_tokens=1024, "
f"got {call_kwargs.get('reasoning')}"
f"reasoning should be {expected_reasoning} for budget_tokens=1024, got {call_kwargs.get('reasoning')}"
)
assert "summary" not in call_kwargs["reasoning"]
# Verify thinking is NOT passed directly to the Responses API
assert (
"thinking" not in call_kwargs
), "thinking should NOT be passed directly to litellm.responses"
assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses"
class TestThinkingParameterTransformation:
@ -411,9 +400,7 @@ class TestThinkingParameterTransformation:
thinking=thinking,
model="openai/gpt-5.2",
)
assert result == {
"reasoning_effort": {"effort": "high", "summary": "detailed"}
}
assert result == {"reasoning_effort": {"effort": "high", "summary": "detailed"}}
finally:
litellm.reasoning_auto_summary = original
@ -611,9 +598,9 @@ class TestThinkingSummaryPreservation:
mock_responses.assert_called_once()
call_kwargs = mock_responses.call_args.kwargs
reasoning = call_kwargs["reasoning"]
assert (
reasoning["summary"] == "concise"
), f"Expected summary='concise', got summary='{reasoning.get('summary')}'"
assert reasoning["summary"] == "concise", (
f"Expected summary='concise', got summary='{reasoning.get('summary')}'"
)
def test_responses_adapter_preserves_summary(self):
"""translate_thinking_to_reasoning should include summary when user provides it."""
@ -622,9 +609,7 @@ class TestThinkingSummaryPreservation:
)
thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"}
result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(
thinking
)
result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking)
assert result == {"effort": "high", "summary": "concise"}
def test_responses_adapter_no_summary_by_default(self):
@ -638,11 +623,7 @@ class TestThinkingSummaryPreservation:
try:
litellm.reasoning_auto_summary = False
thinking = {"type": "enabled", "budget_tokens": 5000}
result = (
LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(
thinking
)
)
result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking)
assert result == {"effort": "high"}
assert result is not None and "summary" not in result
finally:
@ -659,9 +640,7 @@ class TestThinkingSummaryPreservation:
thinking=thinking,
model="openai/gpt-5.2",
)
assert result == {
"reasoning_effort": {"effort": "high", "summary": "concise"}
}
assert result == {"reasoning_effort": {"effort": "high", "summary": "concise"}}
def test_translate_thinking_for_model_disabled_stays_plain_string_when_auto_summary_enabled(self):
"""Disabled thinking must stay a plain string even when reasoning_auto_summary is on."""
@ -807,9 +786,7 @@ def test_presanitized_flag_not_leaked_to_provider_params():
def fake_base_handler(*args, **kwargs):
captured.update(kwargs)
captured["optional"] = kwargs.get(
"anthropic_messages_optional_request_params", {}
)
captured["optional"] = kwargs.get("anthropic_messages_optional_request_params", {})
return "stub"
with patch.object(
@ -974,6 +951,38 @@ def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypat
assert "config" not in captured
@pytest.mark.parametrize(
"model_info, expected_ttl_support",
[
({"supported_endpoints": ["/v1/messages"]}, False),
({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": True}, True),
({"supported_endpoints": ["/v1/messages"], "cache_control_ttl": "yes"}, False),
],
)
def test_gate_passthrough_forwards_cache_control_ttl_only_when_deployment_opts_in(
monkeypatch, model_info, expected_ttl_support
):
"""The passthrough config strips cache_control.ttl unless the deployment sets
model_info.cache_control_ttl to exactly true."""
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages_handler,
)
captured, _ = _gate_stubs(monkeypatch)
result = anthropic_messages_handler(
max_tokens=100,
messages=[{"role": "user", "content": "Hello"}],
model="openai/some-model",
api_key="sk-test",
api_base="https://host/v1",
model_info=model_info,
)
assert result == "native-passthrough"
assert captured["config"].supports_cache_control_ttl() is expected_ttl_support
def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Regional and provider-prefixed Claude 4.8+/5 entries carry
``supports_mid_conversation_system``, but the bare first-party keys
@ -987,9 +996,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys
import litellm
cost_map_path = os.path.join(
os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json"
)
cost_map_path = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json")
with open(cost_map_path) as f:
cost_map = json.load(f)
rules = cost_map["fallback_generalizations"]["rules"]
@ -1028,9 +1035,7 @@ def test_first_party_claude_4_8_plus_cost_map_entries_carry_mid_conversation_sys
("perplexity/sonar", "sonar", "https://api.perplexity.ai/chat/completions"),
],
)
async def test_messages_strips_provider_prefix_exactly_once(
requested_model, expected_wire_model, expected_url
):
async def test_messages_strips_provider_prefix_exactly_once(requested_model, expected_wire_model, expected_url):
"""
BerriAI/litellm#37716: only the leading provider segment may be stripped on the way upstream.

17
tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py
View file

@ -414,6 +414,23 @@ def test_native_anthropic_config_keeps_cache_control_ttl():
assert payload["system"][0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"}
def test_deployment_opt_in_keeps_cache_control_ttl():
config = OpenAILikeAnthropicMessagesConfig(cache_control_ttl=True)
payload = config.transform_anthropic_messages_request(
model="some-model",
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
}
],
anthropic_messages_optional_request_params={"max_tokens": 16},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert payload["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
def test_json_provider_constraint_opts_into_cache_control_ttl():
from litellm.llms.openai_like.json_loader import SimpleProviderConfig
from litellm.llms.openai_like.messages.transformation import (