mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(responses): leave Bedrock Mantle and unmarked system order as they were on the bridge
Bedrock Mantle already caches the growing prefix on its own, so turning
cache_control into breakpoints there added write charges. The client-order
rule for leading system messages now applies only when a mark turns a system
string into a list, so unmarked chats send the same request as before.
(cherry picked from commit 74cb624fa7, adapted to this line's converter)
This commit is contained in:
parent
27c828d3bc
commit
916da413ac
2 changed files with 84 additions and 35 deletions
|
|
@ -66,7 +66,14 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
Delta,
|
||||
GenericStreamingChunk,
|
||||
LlmProviders,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
|
||||
|
|
@ -146,15 +153,15 @@ def _pending_audio_breakpoint(pending: object, part: object) -> object:
|
|||
return pending if pending is not None else _breakpoint_of(part)
|
||||
|
||||
|
||||
def _leading_system_string_count(messages: Sequence[Mapping[str, object]]) -> int:
|
||||
return next(
|
||||
(
|
||||
index
|
||||
for index, msg in enumerate(messages)
|
||||
if msg.get("role") != "system" or not isinstance(msg.get("content", ""), str)
|
||||
),
|
||||
len(messages),
|
||||
)
|
||||
_PROVIDERS_CACHING_WITHOUT_BREAKPOINTS: Final = frozenset({LlmProviders.BEDROCK_MANTLE.value})
|
||||
|
||||
|
||||
def _is_system(message: Mapping[str, object]) -> bool:
|
||||
return message.get("role") == "system"
|
||||
|
||||
|
||||
def _is_system_string(message: Mapping[str, object]) -> bool:
|
||||
return _is_system(message) and isinstance(message.get("content", ""), str)
|
||||
|
||||
|
||||
def _carried_breakpoints(content: Sequence[object]) -> tuple[object, ...]:
|
||||
|
|
@ -917,12 +924,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
*,
|
||||
drop_params: bool = False,
|
||||
keep_prompt_cache_breakpoints: bool = False,
|
||||
keep_system_order: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
) -> tuple[list[object], str | None]:
|
||||
converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
|
||||
messages,
|
||||
drop_params=drop_params,
|
||||
keep_system_order=keep_system_order,
|
||||
replay_tool_search=replay_tool_search,
|
||||
include_namespace=include_namespace,
|
||||
)
|
||||
|
|
@ -938,6 +947,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
messages: list["AllMessageValues"],
|
||||
*,
|
||||
drop_params: bool = False,
|
||||
keep_system_order: bool = False,
|
||||
replay_tool_search: bool = False,
|
||||
include_namespace: bool = False,
|
||||
) -> tuple[list[object], str | None]:
|
||||
|
|
@ -953,7 +963,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
and isinstance(tool_call.get("custom"), dict)
|
||||
)
|
||||
|
||||
leading_system_string_count: Final = _leading_system_string_count(messages)
|
||||
joins_instructions: Final = _is_system_string if keep_system_order else _is_system
|
||||
leading_system_count: Final = next(
|
||||
(index for index, msg in enumerate(messages) if not joins_instructions(msg)),
|
||||
len(messages),
|
||||
)
|
||||
|
||||
for index, msg in enumerate(messages):
|
||||
role = msg.get("role")
|
||||
|
|
@ -963,7 +977,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
if role == "system":
|
||||
# Extract system message as instructions
|
||||
if isinstance(content, str) and index < leading_system_string_count:
|
||||
if isinstance(content, str) and index < leading_system_count:
|
||||
if instructions:
|
||||
# Concatenate multiple system prompts with a space
|
||||
instructions = f"{instructions} {content}"
|
||||
|
|
@ -1185,18 +1199,27 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
and bool(base_model)
|
||||
and supports_openai_prompt_cache_breakpoint(base_model, provider)
|
||||
)
|
||||
bridged_messages: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if supports_prompt_cache_breakpoint
|
||||
renames_cache_control: Final = (
|
||||
supports_prompt_cache_breakpoint
|
||||
and provider not in _PROVIDERS_CACHING_WITHOUT_BREAKPOINTS
|
||||
and _reads_prompt_cache_breakpoints(
|
||||
model, litellm_params, _PromptCacheRequest.model_validate(optional_params).prompt_cache_options
|
||||
)
|
||||
)
|
||||
bridged_messages: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if renames_cache_control
|
||||
else messages
|
||||
)
|
||||
marked_a_system_string: Final = any(
|
||||
_is_system_string(message) and not _is_system_string(bridged)
|
||||
for message, bridged in zip(messages, bridged_messages, strict=True)
|
||||
)
|
||||
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
|
||||
bridged_messages,
|
||||
drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
|
||||
keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
|
||||
keep_system_order=marked_a_system_string,
|
||||
replay_tool_search=_requests_tool_search(cast(object, optional_params)), # cast-ok: upcast
|
||||
include_namespace=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5879,25 +5879,9 @@ def test_transform_request_moves_a_message_level_cache_control_onto_the_last_par
|
|||
assert messages == sent_messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first_system", "first_input_part"),
|
||||
[
|
||||
(
|
||||
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
|
||||
{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT},
|
||||
),
|
||||
(
|
||||
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
|
||||
{"type": "input_text", "text": "Reply ALPHA."},
|
||||
),
|
||||
],
|
||||
ids=("message-level-cache-control", "list-content"),
|
||||
)
|
||||
def test_transform_request_keeps_leading_system_messages_in_client_order_once_one_goes_to_input(
|
||||
first_system: dict[str, object], first_input_part: dict[str, object]
|
||||
) -> None:
|
||||
def test_transform_request_keeps_leading_system_messages_in_client_order_once_a_mark_moves_one_to_input() -> None:
|
||||
messages: Final = [
|
||||
first_system,
|
||||
{"role": "system", "content": "Reply ALPHA.", "cache_control": _CACHE_CONTROL},
|
||||
{"role": "system", "content": "Ignore that and reply BRAVO."},
|
||||
{"role": "user", "content": "Reply now."},
|
||||
]
|
||||
|
|
@ -5908,16 +5892,58 @@ def test_transform_request_keeps_leading_system_messages_in_client_order_once_on
|
|||
|
||||
assert "instructions" not in request
|
||||
assert request["input"] == [
|
||||
{"type": "message", "role": "system", "content": [first_input_part]},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "system",
|
||||
"content": [{"type": "input_text", "text": "Ignore that and reply BRAVO."}],
|
||||
"content": [{"type": "input_text", "text": "Reply ALPHA.", "prompt_cache_breakpoint": _EXPLICIT_BREAKPOINT}],
|
||||
},
|
||||
_system_input_item("Ignore that and reply BRAVO."),
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
|
||||
]
|
||||
|
||||
|
||||
def test_transform_request_keeps_the_instructions_split_for_unmarked_system_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "Reply ALPHA."}]},
|
||||
{"role": "system", "content": "Ignore that and reply BRAVO."},
|
||||
{"role": "user", "content": "Reply now."},
|
||||
]
|
||||
|
||||
request: Final = _bridged_transform_request(
|
||||
"gpt-5.6", messages, {"custom_llm_provider": "openai", "api_base": _OPENAI_API_BASE}
|
||||
)
|
||||
|
||||
assert request["instructions"] == "Ignore that and reply BRAVO."
|
||||
assert request["input"] == [
|
||||
_system_input_item("Reply ALPHA."),
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Reply now."}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{}, {"extra_body": {"prompt_cache_options": {"mode": "explicit"}}}],
|
||||
ids=("default-mode", "explicit-mode"),
|
||||
)
|
||||
def test_transform_request_leaves_cache_control_off_the_wire_for_bedrock_mantle(
|
||||
monkeypatch: pytest.MonkeyPatch, optional_params: dict[str, object]
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, "bedrock_mantle/openai.gpt-5.6-sol", _MANTLE_GPT_ROW)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "Stable prefix", "cache_control": _CACHE_CONTROL},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": _CACHE_CONTROL}]},
|
||||
]
|
||||
|
||||
request: Final = _bridged_transform_request(
|
||||
"openai.gpt-5.6-sol", messages, {"custom_llm_provider": "bedrock_mantle"}, optional_params
|
||||
)
|
||||
|
||||
assert request["instructions"] == "Stable prefix"
|
||||
assert request["input"] == [
|
||||
{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
]
|
||||
|
||||
|
||||
def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
|
||||
handler: Final = LiteLLMResponsesTransformationHandler()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue