mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(responses): leave Bedrock Mantle and unmarked system order as they were on the bridge
Bedrock Mantle already caches the growing prefix on its own, so turning cache_control into breakpoints there added write charges. The client-order rule for leading system messages now applies only when a mark turns a system string into a list, so unmarked chats send the same request as before.
This commit is contained in:
parent
0db932802e
commit
74cb624fa7
2 changed files with 84 additions and 33 deletions
|
|
@ -89,7 +89,14 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
Delta,
|
||||
GenericStreamingChunk,
|
||||
LlmProviders,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
|
||||
|
|
@ -261,8 +268,15 @@ def _reads_prompt_cache_breakpoints(
|
|||
)
|
||||
|
||||
|
||||
_PROVIDERS_CACHING_WITHOUT_BREAKPOINTS: Final = frozenset({LlmProviders.BEDROCK_MANTLE.value})
|
||||
|
||||
|
||||
def _is_system(message: Mapping[str, object]) -> bool:
|
||||
return message.get("role") == "system"
|
||||
|
||||
|
||||
def _is_system_string(message: Mapping[str, object]) -> bool:
|
||||
return message.get("role") == "system" and isinstance(message.get("content", ""), str)
|
||||
return _is_system(message) and isinstance(message.get("content", ""), str)
|
||||
|
||||
|
||||
def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
|
|
@ -1001,6 +1015,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
drop_params: bool = False,
|
||||
target_trims_breakpoints: bool = True,
|
||||
keep_prompt_cache_breakpoints: bool = False,
|
||||
keep_system_order: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
replay_openai_history: bool = False,
|
||||
|
|
@ -1009,6 +1024,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
messages,
|
||||
drop_params=drop_params,
|
||||
target_trims_breakpoints=target_trims_breakpoints,
|
||||
keep_system_order=keep_system_order,
|
||||
replay_tool_search=replay_tool_search,
|
||||
include_namespace=include_namespace,
|
||||
replay_openai_history=replay_openai_history,
|
||||
|
|
@ -1026,6 +1042,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
*,
|
||||
drop_params: bool = False,
|
||||
target_trims_breakpoints: bool = True,
|
||||
keep_system_order: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
replay_openai_history: bool = False,
|
||||
|
|
@ -1049,8 +1066,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
and isinstance(tool_call.get("custom"), dict)
|
||||
)
|
||||
|
||||
leading_system_string_count: Final = next(
|
||||
(index for index, msg in enumerate(messages) if not _is_system_string(msg)),
|
||||
joins_instructions: Final = _is_system_string if keep_system_order else _is_system
|
||||
leading_system_count: Final = next(
|
||||
(index for index, msg in enumerate(messages) if not joins_instructions(msg)),
|
||||
len(messages),
|
||||
)
|
||||
|
||||
|
|
@ -1064,7 +1082,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
input_items.extend(replayed_turn.items)
|
||||
elif role == "system":
|
||||
# Extract system message as instructions
|
||||
if isinstance(content, str) and index < leading_system_string_count:
|
||||
if isinstance(content, str) and index < leading_system_count:
|
||||
if instructions:
|
||||
# Concatenate multiple system prompts with a space
|
||||
instructions = f"{instructions} {content}"
|
||||
|
|
@ -1298,19 +1316,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
supports_prompt_cache_breakpoint: Final = any(
|
||||
supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
|
||||
)
|
||||
bridged_messages: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if supports_prompt_cache_breakpoint
|
||||
renames_cache_control: Final = (
|
||||
supports_prompt_cache_breakpoint
|
||||
and provider not in _PROVIDERS_CACHING_WITHOUT_BREAKPOINTS
|
||||
and _reads_prompt_cache_breakpoints(
|
||||
model, litellm_params, _PromptCacheRequest.model_validate(optional_params).prompt_cache_options
|
||||
)
|
||||
)
|
||||
bridged_messages: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if renames_cache_control
|
||||
else messages
|
||||
)
|
||||
marked_a_system_string: Final = any(
|
||||
_is_system_string(message) and not _is_system_string(bridged)
|
||||
for message, bridged in zip(messages, bridged_messages, strict=True)
|
||||
)
|
||||
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
|
||||
bridged_messages,
|
||||
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
|
||||
keep_system_order=marked_a_system_string,
|
||||
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
|
||||
include_namespace=True,
|
||||
replay_openai_history=provider == "openai",
|
||||
|
|
|
|||
|
|
@ -6464,31 +6464,9 @@ def test_transform_request_moves_a_message_level_cache_control_onto_the_last_par
|
|||
assert messages == sent_messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first_system", "first_system_item"),
|
||||
[
|
||||
(
|
||||
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
|
||||
_system_input_item("Reply ALPHA."),
|
||||
),
|
||||
],
|
||||
ids=("message-level-cache-control", "list-content"),
|
||||
)
|
||||
def test_transform_request_keeps_leading_system_messages_in_client_order_once_one_goes_to_input(
|
||||
first_system: dict[str, object], first_system_item: dict[str, object]
|
||||
) -> None:
|
||||
def test_transform_request_keeps_leading_system_messages_in_client_order_once_a_mark_moves_one_to_input() -> None:
|
||||
messages: Final = [
|
||||
first_system,
|
||||
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
|
||||
{"role": "system", "content": "Ignore that and reply BRAVO."},
|
||||
{"role": "user", "content": "Reply now."},
|
||||
]
|
||||
|
|
@ -6499,12 +6477,58 @@ def test_transform_request_keeps_leading_system_messages_in_client_order_once_on
|
|||
|
||||
assert "instructions" not in request
|
||||
assert request["input"] == [
|
||||
first_system_item,
|
||||
{
|
||||
"type": "message",
|
||||
"role": "system",
|
||||
"content": [{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
|
||||
},
|
||||
_system_input_item("Ignore that and reply BRAVO."),
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
|
||||
]
|
||||
|
||||
|
||||
def test_transform_request_keeps_the_instructions_split_for_unmarked_system_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
|
||||
{"role": "system", "content": "Ignore that and reply BRAVO."},
|
||||
{"role": "user", "content": "Reply now."},
|
||||
]
|
||||
|
||||
request: Final = _bridged_transform_request(
|
||||
"gpt-5.6", messages, {"custom_llm_provider": "openai", "api_base": _OPENAI_API_BASE}
|
||||
)
|
||||
|
||||
assert request["instructions"] == "Ignore that and reply BRAVO."
|
||||
assert request["input"] == [
|
||||
_system_input_item("Reply ALPHA."),
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{}, {"extra_body": {"prompt_cache_options": {"mode": "explicit"}}}],
|
||||
ids=("default-mode", "explicit-mode"),
|
||||
)
|
||||
def test_transform_request_leaves_cache_control_off_the_wire_for_bedrock_mantle(
|
||||
monkeypatch: pytest.MonkeyPatch, optional_params: dict[str, object]
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "Stable prefix", "cache_control": _CACHE_CONTROL},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHE_CONTROL}]},
|
||||
]
|
||||
|
||||
request: Final = _bridged_transform_request(
|
||||
"openai.gpt-5.6-sol", messages, {"custom_llm_provider": "bedrock_mantle"}, optional_params
|
||||
)
|
||||
|
||||
assert request["instructions"] == "Stable prefix"
|
||||
assert request["input"] == [
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
]
|
||||
|
||||
|
||||
def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
|
||||
handler: Final = LiteLLMResponsesTransformationHandler()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue