scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(cache): keep /v1/messages marks only where the chat route turns them into breakpoints

Browse source
This commit is contained in:
mateo-berri 2026-11-05 21:46:55 -08:00
parent 756320e843
commit 8d90603ba8
2 changed files with 64 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -397,9 +397,20 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter()
########################################################
_ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED: Final = frozenset(("azure",))
_ROUTES_READING_ONLY_THE_SERVED_MODEL: Final = frozenset(("openai",))
def _target_reads_prompt_cache_breakpoint(model: str, call_kwargs: Mapping[str, object]) -> bool:
provider: Final = call_kwargs.get("custom_llm_provider")
if provider in _ROUTES_FORWARDING_CACHE_CONTROL_UNMAPPED:
return False
base_model: Final = call_kwargs.get("base_model")
named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
named_models: Final = (
(model, base_model)
if isinstance(base_model, str) and base_model and provider not in _ROUTES_READING_ONLY_THE_SERVED_MODEL
else (model,)
)
return any(call_targets_openai_prompt_cache_breakpoint(name, call_kwargs) for name in named_models)

52
tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py
View file

@ -224,3 +224,55 @@ def test_anthropic_messages_to_a_deployment_that_opts_out_of_breakpoints_carry_n
assert list(_texts_with("cache_control", completion_kwargs)) == []
assert list(_texts_with("prompt_cache_breakpoint", wire)) == []
_OPENAI_GPT_ROW: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_cache_breakpoint": True}
@pytest.mark.parametrize(
("model", "extra_kwargs", "keeps_marks"),
[
pytest.param(
"azure/gpt-5.6-sol",
{"custom_llm_provider": "azure", "model_info": {"supports_prompt_cache_breakpoint": True}},
False,
id="azure-chat-forwards-cache-control-unmapped",
),
pytest.param(
"openai/team-alias",
{
"custom_llm_provider": "openai",
"base_model": "gpt-5.6-sol",
"api_base": "https://llm.example.com/v1",
"prompt_cache_options": {"mode": "explicit"},
},
False,
id="openai-chat-reads-only-the-served-model",
),
pytest.param(
"bedrock_mantle/team-alias",
{
"custom_llm_provider": "bedrock_mantle",
"base_model": "openai.gpt-5.6-sol",
"prompt_cache_options": {"mode": "explicit"},
},
True,
id="bridge-reads-base-model",
),
],
)
def test_anthropic_messages_keep_marks_only_for_a_route_that_turns_them_into_breakpoints(
monkeypatch: pytest.MonkeyPatch, model: str, extra_kwargs: dict[str, object], keeps_marks: bool
) -> None:
monkeypatch.setitem(litellm.model_cost, "gpt-5.6-sol", _OPENAI_GPT_ROW)
monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
max_tokens=64,
messages=json.loads(json.dumps(_CLAUDE_CODE_MESSAGES)),
model=model,
system=json.loads(json.dumps(_SYSTEM)),
extra_kwargs=extra_kwargs,
)
assert bool(list(_texts_with("cache_control", completion_kwargs["messages"]))) is keeps_marks