mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
fix(responses): keep only the fallback-branch marker carry, main already routes the hook
#42281 landed the text, image_url and file branch carry and the implicit default on main, and the Responses routing lookup this branch changed has no observable effect there, so the PR shrinks to the unknown-block fallback branch and its regression test
This commit is contained in:
mateo-berri
parent
5b47c9dcb2
commit
d89bd9ee29
3 changed files with 1 additions and 125 deletions
|
|
@ -521,15 +521,6 @@ def _resolve_responses_api_provider_config(
|
|||
return OpenAILikeResponsesConfig()
|
||||
|
||||
|
||||
def _responses_config_lookup_model(model: str, custom_llm_provider: str) -> str:
|
||||
"""The model id the dispatch hands the config lookup, with the routing prefixes off.
|
||||
|
||||
A config keyed on the model, such as bedrock_mantle reading its cost-map entry, misses on
|
||||
the prefixed id, so every one of those models reads as having no native Responses support.
|
||||
"""
|
||||
return _strip_responses_routing_prefix(model.removeprefix(f"{custom_llm_provider}/"))
|
||||
|
||||
|
||||
def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None:
|
||||
api_base: Final = kwargs.get("api_base")
|
||||
return api_base if isinstance(api_base, str) else None
|
||||
|
|
@ -554,12 +545,7 @@ def _will_bridge_to_chat_completions(
|
|||
if custom_llm_provider is None:
|
||||
return True
|
||||
return _bridges_to_chat_completions(
|
||||
_resolve_responses_api_provider_config(
|
||||
_responses_config_lookup_model(normalized_model[0], custom_llm_provider),
|
||||
custom_llm_provider,
|
||||
model_info,
|
||||
api_base,
|
||||
),
|
||||
_resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info, api_base),
|
||||
use_chat_completions_api or normalized_model[1],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4609,29 +4609,6 @@ def test_every_bridged_chunk_after_response_created_carries_the_served_service_t
|
|||
assert relayed == ["default"] * len(events), relayed
|
||||
|
||||
|
||||
def test_convert_chat_completion_messages_to_responses_api_omits_absent_prompt_cache_breakpoint():
|
||||
"""Unmarked blocks must not grow the key, or every request would look explicitly cached."""
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
|
||||
handler = LiteLLMResponsesTransformationHandler()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "no caching here"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
response, _ = handler.convert_chat_completion_messages_to_responses_api(messages)
|
||||
|
||||
assert all("prompt_cache_breakpoint" not in block for block in response[0]["content"])
|
||||
|
||||
|
||||
def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint_on_unknown_block():
|
||||
"""A block type the bridge cannot map still has to keep the marker.
|
||||
|
||||
|
|
|
|||
|
|
@ -85,24 +85,6 @@ def _patch_responses_dispatch():
|
|||
]
|
||||
|
||||
|
||||
def _make_hook_backed_logging_obj() -> MagicMock:
|
||||
"""A logging object that runs the real cache-control hook, the way a configured proxy's does."""
|
||||
hook = AnthropicCacheControlHook()
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.__class__ = LiteLLMLoggingObj
|
||||
logging_obj.should_run_prompt_management_hooks.return_value = True
|
||||
logging_obj.get_chat_completion_prompt.side_effect = lambda **call: hook.get_chat_completion_prompt(
|
||||
model=call["model"],
|
||||
messages=call["messages"],
|
||||
non_default_params=call["non_default_params"],
|
||||
prompt_id=call["prompt_id"],
|
||||
prompt_variables=call["prompt_variables"],
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
logging_obj.model_call_details = {}
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _make_cache_control_case() -> tuple[
|
||||
ResponseInputParam,
|
||||
list[AllMessageValues],
|
||||
|
|
@ -367,75 +349,6 @@ class TestResponsesAPIPromptManagement:
|
|||
assert sent_input[1] == reasoning_item
|
||||
assert sent_input[2]["id"] == "msg_1"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["bedrock_mantle/openai.gpt-5.6-sol", "bedrock_mantle/responses/openai.gpt-5.6-sol"],
|
||||
)
|
||||
def test_native_responses_provider_places_role_targeted_cache_point(self, model: str):
|
||||
"""A provider serving Responses natively gets no second pass, so the point lands on this one.
|
||||
|
||||
Handing a role-targeted point forward here sends it to the chat-completions bridge, which
|
||||
never runs for this request, and the call goes upstream with nothing marked for caching.
|
||||
The ``responses/`` routing prefix reaches the same deployment, so it has to read the same way.
|
||||
"""
|
||||
system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"})
|
||||
user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"})
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler,
|
||||
"_should_use_litellm_mcp_gateway",
|
||||
return_value=False,
|
||||
),
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"response_api_handler",
|
||||
return_value=MagicMock(),
|
||||
) as mock_handler,
|
||||
):
|
||||
import litellm
|
||||
|
||||
litellm.responses(
|
||||
input=cast(ResponseInputParam, [system_message, user_message]),
|
||||
model=model,
|
||||
litellm_logging_obj=_make_hook_backed_logging_obj(),
|
||||
cache_control_injection_points=[{"location": "message", "role": "system"}],
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
sent_input = mock_handler.call_args.kwargs["input"]
|
||||
assert sent_input[0]["cache_control"] == {"type": "ephemeral"}
|
||||
|
||||
def test_bridged_responses_request_still_defers_role_targeted_cache_point(self):
|
||||
"""The bridge builds its own system message, so its own pass is the one that marks it."""
|
||||
system_message = cast(AllMessageValues, {"role": "system", "content": "Analyze the request"})
|
||||
user_message = cast(AllMessageValues, {"role": "user", "content": "Check for security issues"})
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler,
|
||||
"_should_use_litellm_mcp_gateway",
|
||||
return_value=False,
|
||||
),
|
||||
patch.object(
|
||||
import_module("litellm.responses.main").litellm_completion_transformation_handler,
|
||||
"response_api_handler",
|
||||
return_value=MagicMock(),
|
||||
) as mock_handler,
|
||||
):
|
||||
import litellm
|
||||
|
||||
litellm.responses(
|
||||
input=cast(ResponseInputParam, [system_message, user_message]),
|
||||
model="openai/chat_completions/gpt-5.6",
|
||||
litellm_logging_obj=_make_hook_backed_logging_obj(),
|
||||
cache_control_injection_points=[{"location": "message", "role": "system"}],
|
||||
api_key="fake-key",
|
||||
)
|
||||
|
||||
sent_input = mock_handler.call_args.kwargs["input"]
|
||||
assert "cache_control" not in sent_input[0]
|
||||
|
||||
def test_all_non_message_input_items_remain_unchanged(self):
|
||||
reasoning_item = {
|
||||
"type": "reasoning",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue