mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(bedrock): drop another model's signed thinking when replaying to a non-Claude Converse target (#50073)
* fix(bedrock/converse): omit reasoning signature for non-Anthropic models (#49969) * test(bedrock): cover signed reasoning replay in mapped tests * fix(bedrock): drop another model's signed thinking when replaying to a non-Claude Converse target A signed thinking block from Claude replayed to a Converse model known not to be Claude, such as GPT-6 Luna or Nova 2, now leaves the request instead of going out as reasoningContent. Bedrock rejects that field on those models, so the whole request answered 400 A target that may still be Claude, such as an application inference profile ARN with no base_model, keeps the signed block, since Claude needs it on a tool-use continuation. Unsigned blocks keep the existing text path The thinking-block error matcher added earlier on this branch is removed, because Converse and non-Claude Bedrock requests never reach that retry * fix(bedrock): keep signed thinking for Claude behind a region prefix litellm does not list ca. and in. inference profiles name Claude after a region prefix get_bedrock_cross_region_inference_regions does not have yet, so they counted as non-Claude and lost their signed thinking. The thinking-only turn test now pins the exact merged and continue-message shapes. * fix(bedrock): limit the region-prefix Claude match to signed thinking and drop a dropped block's cache point * fix(bedrock): drop Claude's bridged reasoning items before a Bedrock Responses API replay * fix(bedrock_mantle): drop litellm-minted reasoning items before a Mantle Responses call * test(integration): expect Bedrock and Bedrock Mantle Responses to drop minted reasoning Bedrock and Bedrock Mantle answer 400 on a reasoning item litellm minted, so the wire test now expects both to drop it, like OpenAI and Azure --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
b02bb99b08
commit
2a4f8b1d19
8 changed files with 377 additions and 39 deletions
|
|
@ -5034,11 +5034,15 @@ class BedrockConverseMessagesProcessor:
|
|||
drop_params: bool = False,
|
||||
model_info: Mapping[str, object] | None = None,
|
||||
) -> list[BedrockMessageBlock]:
|
||||
from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_converse_accepts_redacted_thinking,
|
||||
bedrock_converse_rejects_signed_thinking,
|
||||
)
|
||||
|
||||
contents: list[BedrockMessageBlock] = []
|
||||
msg_i = 0
|
||||
to_claude: Final = bedrock_converse_accepts_redacted_thinking(model, base_model)
|
||||
drop_signed_thinking: Final = bedrock_converse_rejects_signed_thinking(model, base_model)
|
||||
|
||||
messages = BedrockConverseMessagesProcessor._initial_message_setup(
|
||||
messages, model, llm_provider, user_continue_message
|
||||
|
|
@ -5194,7 +5198,9 @@ class BedrockConverseMessagesProcessor:
|
|||
thinking_blocks = assistant_message_block.get("thinking_blocks")
|
||||
converted_thinking_blocks = (
|
||||
BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks(
|
||||
_thinking_in_replay_order(_assistant_content, thinking_blocks), to_claude=to_claude
|
||||
_thinking_in_replay_order(_assistant_content, thinking_blocks),
|
||||
to_claude=to_claude,
|
||||
drop_signed=drop_signed_thinking,
|
||||
)
|
||||
)
|
||||
assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content(
|
||||
|
|
@ -5226,8 +5232,8 @@ class BedrockConverseMessagesProcessor:
|
|||
deployment_model=deployment_model,
|
||||
model_info=model_info,
|
||||
)
|
||||
if _cache_point_block is not None and not (
|
||||
drop_params and _as_thinking_block(element) is not None
|
||||
if _cache_point_block is not None and BedrockConverseMessagesProcessor.keeps_cache_point(
|
||||
element, drop_params=drop_params, drop_signed=drop_signed_thinking
|
||||
):
|
||||
assistants_parts.append(_cache_point_block)
|
||||
assistant_content.extend(assistants_parts)
|
||||
|
|
@ -5286,6 +5292,7 @@ class BedrockConverseMessagesProcessor:
|
|||
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
|
||||
*,
|
||||
to_claude: bool = False,
|
||||
drop_signed: bool = False,
|
||||
) -> list[BedrockContentBlock]:
|
||||
return [
|
||||
BedrockContentBlock(
|
||||
|
|
@ -5295,8 +5302,27 @@ class BedrockConverseMessagesProcessor:
|
|||
)
|
||||
for thinking_block in thinking_blocks
|
||||
if not (to_claude and is_encrypted_reasoning_block(thinking_block))
|
||||
and not (drop_signed and BedrockConverseMessagesProcessor._is_signed(thinking_block))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def keeps_cache_point(element: object, *, drop_params: bool, drop_signed: bool) -> bool:
|
||||
match _as_thinking_block(element):
|
||||
case None:
|
||||
return True
|
||||
case thinking_block:
|
||||
return not (
|
||||
drop_params or (drop_signed and BedrockConverseMessagesProcessor._is_signed(thinking_block))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_signed(thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock) -> bool:
|
||||
match thinking_block:
|
||||
case {"signature": signature} if signature:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _to_reasoning_content_block(
|
||||
thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
|
||||
|
|
@ -5455,11 +5481,15 @@ def _bedrock_converse_messages_pt(
|
|||
- Please ensure that function response turn comes immediately after a function call turn
|
||||
- Conversation blocks and tool result blocks cannot be provided in the same turn. Issue: https://github.com/BerriAI/litellm/issues/6053
|
||||
"""
|
||||
from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_converse_accepts_redacted_thinking,
|
||||
bedrock_converse_rejects_signed_thinking,
|
||||
)
|
||||
|
||||
contents: list[BedrockMessageBlock] = []
|
||||
msg_i = 0
|
||||
to_claude: Final = bedrock_converse_accepts_redacted_thinking(model, base_model)
|
||||
drop_signed_thinking: Final = bedrock_converse_rejects_signed_thinking(model, base_model)
|
||||
|
||||
messages = BedrockConverseMessagesProcessor._initial_message_setup(
|
||||
messages, model, llm_provider, user_continue_message
|
||||
|
|
@ -5616,7 +5646,9 @@ def _bedrock_converse_messages_pt(
|
|||
thinking_blocks = assistant_message_block.get("thinking_blocks")
|
||||
converted_thinking_blocks = (
|
||||
BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks(
|
||||
_thinking_in_replay_order(_assistant_content, thinking_blocks), to_claude=to_claude
|
||||
_thinking_in_replay_order(_assistant_content, thinking_blocks),
|
||||
to_claude=to_claude,
|
||||
drop_signed=drop_signed_thinking,
|
||||
)
|
||||
)
|
||||
assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content(
|
||||
|
|
@ -5649,8 +5681,8 @@ def _bedrock_converse_messages_pt(
|
|||
deployment_model=deployment_model,
|
||||
model_info=model_info,
|
||||
)
|
||||
if _cache_point_block is not None and not (
|
||||
drop_params and _as_thinking_block(element) is not None
|
||||
if _cache_point_block is not None and BedrockConverseMessagesProcessor.keeps_cache_point(
|
||||
element, drop_params=drop_params, drop_signed=drop_signed_thinking
|
||||
):
|
||||
assistants_parts.append(_cache_point_block)
|
||||
assistant_content.extend(assistants_parts)
|
||||
|
|
|
|||
|
|
@ -1560,15 +1560,29 @@ def bedrock_converse_accepts_redacted_thinking(model: str, base_model: str | Non
|
|||
its own. An ARN that hides its model, like an application inference profile, falls back to the
|
||||
deployment's ``base_model`` and counts as non-Claude without one.
|
||||
"""
|
||||
return _is_claude(_converse_target_model(model, base_model))
|
||||
|
||||
|
||||
def bedrock_converse_rejects_signed_thinking(model: str, base_model: str | None) -> bool:
|
||||
if base_model is None and "." not in get_bedrock_base_model(model):
|
||||
return False
|
||||
return not _is_claude_behind_any_region_prefix(_converse_target_model(model, base_model))
|
||||
|
||||
|
||||
def _converse_target_model(model: str, base_model: str | None) -> str:
|
||||
if base_model is None or "." in get_bedrock_base_model(model):
|
||||
return _is_claude(model)
|
||||
return _is_claude(base_model)
|
||||
return model
|
||||
return base_model
|
||||
|
||||
|
||||
def _is_claude(model: str) -> bool:
|
||||
return get_bedrock_base_model(model).startswith(("anthropic", "claude"))
|
||||
|
||||
|
||||
def _is_claude_behind_any_region_prefix(model: str) -> bool:
|
||||
return _is_claude(model) or get_bedrock_base_model(model).partition(".")[2].startswith("anthropic.")
|
||||
|
||||
|
||||
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
|
||||
return any(
|
||||
(litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True
|
||||
|
|
|
|||
|
|
@ -77,7 +77,9 @@ _NO_TOOL_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
|
|||
_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3.5", "chatgpt-4o", "o1", "o3", "o4")
|
||||
_PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI})
|
||||
_PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI})
|
||||
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
|
||||
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = frozenset(
|
||||
{LlmProviders.AZURE, LlmProviders.BEDROCK, LlmProviders.BEDROCK_MANTLE, LlmProviders.OPENAI}
|
||||
)
|
||||
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
|
||||
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
|
||||
_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"})
|
||||
|
|
|
|||
|
|
@ -429,6 +429,65 @@ _CONVERSE_SIGNED_PART: Final[dict[str, JsonValue]] = {
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment", "replayed_thinking"),
|
||||
[
|
||||
pytest.param({"model": f"bedrock/converse/{NOVA_MODEL_ID}"}, (), id="nova"),
|
||||
pytest.param(
|
||||
{
|
||||
"model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}",
|
||||
"model_info": {"base_model": f"bedrock/{NOVA_MODEL_ID}"},
|
||||
},
|
||||
(),
|
||||
id="application-profile-arn-with-nova-base-model",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}"},
|
||||
(_CONVERSE_SIGNED_PART,),
|
||||
id="application-profile-arn-without-base-model",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}",
|
||||
"model_info": {"base_model": f"bedrock/{CLAUDE_MODEL_ID}"},
|
||||
},
|
||||
(_CONVERSE_SIGNED_PART,),
|
||||
id="application-profile-arn-with-claude-base-model",
|
||||
),
|
||||
pytest.param(_CLAUDE, (_CONVERSE_SIGNED_PART,), id="claude"),
|
||||
],
|
||||
)
|
||||
def test_signed_history_keeps_its_thinking_unless_the_converse_target_is_known_not_to_be_claude(
|
||||
gateway: Gateway, deployment: dict[str, JsonValue], replayed_thinking: tuple[dict[str, JsonValue], ...]
|
||||
) -> None:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
parameters: Final = {key: value for key, value in deployment.items() if key != "model_info"}
|
||||
with wire_server(_bedrock_peer()) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model_info=model_info if isinstance(model_info, dict) else None,
|
||||
api_key=TOKEN,
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
reply: Final = _chat(
|
||||
gateway,
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 256,
|
||||
"messages": [
|
||||
{"role": "user", "content": QUESTION},
|
||||
{"role": "assistant", "content": FIRST_ANSWER, "thinking_blocks": [_SIGNED_THINKING]},
|
||||
{"role": "user", "content": FOLLOW_UP},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert reply.content == FINAL_ANSWER
|
||||
assert _replayed_assistant_turn(wire).content == (*replayed_thinking, {"text": FIRST_ANSWER})
|
||||
|
||||
|
||||
def _replayed_to_converse(
|
||||
gateway: Gateway, wire: Wire, model: str, assistants: list[JsonValue], **fields: JsonValue
|
||||
) -> tuple[dict[str, JsonValue], ...]:
|
||||
|
|
|
|||
|
|
@ -77,7 +77,18 @@ _AZURE_AI_OPENAI_HOST: Final = _Deployment(
|
|||
"/openai/v1/responses?api-version=preview",
|
||||
strips_message_status=True,
|
||||
)
|
||||
_DROPPING: Final = (_OPENAI, _AZURE, _AZURE_AI_OPENAI_HOST)
|
||||
_BEDROCK_MANTLE: Final = _Deployment(
|
||||
"bedrock_mantle", "bedrock_mantle/openai.gpt-oss-120b", "synthetic-mantle-key", "/v1/responses"
|
||||
)
|
||||
_BEDROCK: Final = _Deployment(
|
||||
"bedrock",
|
||||
"bedrock/openai.gpt-oss-120b-1:0",
|
||||
"synthetic-bedrock-key",
|
||||
"/openai/v1/responses",
|
||||
MappingProxyType({"aws_region_name": "us-east-1"}),
|
||||
model_info=MappingProxyType({"supported_endpoints": ["/v1/responses"]}),
|
||||
)
|
||||
_DROPPING: Final = (_OPENAI, _AZURE, _AZURE_AI_OPENAI_HOST, _BEDROCK_MANTLE, _BEDROCK)
|
||||
_KEEPING: Final = (
|
||||
_Deployment("litellm_proxy", f"litellm_proxy/{_GPT}", "synthetic-proxy-key", "/responses"),
|
||||
_Deployment("databricks", "databricks/gpt-5.6", "synthetic-databricks-key", "/responses"),
|
||||
|
|
@ -95,15 +106,6 @@ _KEEPING: Final = (
|
|||
"/v1/responses",
|
||||
types_untyped_items_as_messages=True,
|
||||
),
|
||||
_Deployment("bedrock_mantle", "bedrock_mantle/openai.gpt-oss-120b", "synthetic-mantle-key", "/v1/responses"),
|
||||
_Deployment(
|
||||
"bedrock",
|
||||
"bedrock/openai.gpt-oss-120b-1:0",
|
||||
"synthetic-bedrock-key",
|
||||
"/openai/v1/responses",
|
||||
MappingProxyType({"aws_region_name": "us-east-1"}),
|
||||
model_info=MappingProxyType({"supported_endpoints": ["/v1/responses"]}),
|
||||
),
|
||||
*(
|
||||
_Deployment(slug, f"{slug}/{model}", f"synthetic-{slug}-key", "/responses")
|
||||
for slug, model in (
|
||||
|
|
|
|||
|
|
@ -74,7 +74,12 @@ from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages
|
|||
from litellm.llms.vllm.completion import handler as vllm_handler
|
||||
from litellm.llms.watsonx.common_utils import _convert_watsonx_messages_core
|
||||
from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionToolMessage
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionToolMessage,
|
||||
ChatCompletionUserMessage,
|
||||
)
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall, Function, ModelResponse, TextCompletionResponse
|
||||
from litellm.utils import (
|
||||
Rules,
|
||||
|
|
@ -435,6 +440,46 @@ def test_anthropic_messages_pt_drops_the_stream_index_from_replayed_thinking_blo
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_content"),
|
||||
(
|
||||
(
|
||||
"global.anthropic.claude-haiku-5-5",
|
||||
[
|
||||
{"reasoningContent": {"reasoningText": {"text": "pondering", "signature": "signed-reasoning"}}},
|
||||
{"text": "hello"},
|
||||
],
|
||||
),
|
||||
("global.zai.glm-5.3", [{"text": "hello"}]),
|
||||
("amazon.nova-pro-v1:0", [{"text": "hello"}]),
|
||||
("converse/us.openai.gpt-6-luna", [{"text": "hello"}]),
|
||||
),
|
||||
)
|
||||
def test_bedrock_converse_replays_signed_thinking_only_to_claude(
|
||||
model: str, expected_content: Sequence[Mapping[str, object]]
|
||||
) -> None:
|
||||
body: Final = litellm.AmazonConverseConfig().transform_request(
|
||||
model=model,
|
||||
messages=[
|
||||
ChatCompletionUserMessage(role="user", content="hi"),
|
||||
ChatCompletionAssistantMessage(
|
||||
role="assistant",
|
||||
content="hello",
|
||||
thinking_blocks=[{"type": "thinking", "thinking": "pondering", "signature": "signed-reasoning"}],
|
||||
),
|
||||
ChatCompletionUserMessage(role="user", content="continue"),
|
||||
],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert body["messages"] == [
|
||||
{"role": "user", "content": [{"text": "hi"}]},
|
||||
{"role": "assistant", "content": expected_content},
|
||||
{"role": "user", "content": [{"text": "continue"}]},
|
||||
]
|
||||
|
||||
|
||||
async def test_anthropic_bedrock_thinking_blocks_with_none_content():
|
||||
"""
|
||||
Test the specific function that processes thinking_blocks when content is None
|
||||
|
|
@ -1032,6 +1077,192 @@ async def test_bedrock_converse_keeps_dropping_redacted_thinking_for_non_claude_
|
|||
assert assistant_content == [{"text": "hello"}]
|
||||
|
||||
|
||||
_HIDDEN_MODEL_ARN: Final = "arn:aws:bedrock:us-west-2:123456789012:application-inference-profile/abc123"
|
||||
|
||||
|
||||
async def _converse_messages(
|
||||
messages: list[dict[str, object]],
|
||||
model: str,
|
||||
base_model: str | None,
|
||||
use_async: bool,
|
||||
assistant_continue_message: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
return (
|
||||
await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
|
||||
messages=messages,
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
base_model=base_model,
|
||||
assistant_continue_message=assistant_continue_message,
|
||||
)
|
||||
if use_async
|
||||
else _bedrock_converse_messages_pt(
|
||||
messages=messages,
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
base_model=base_model,
|
||||
assistant_continue_message=assistant_continue_message,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _assistant_with_thinking(placement: str, block: dict[str, object]) -> dict[str, object]:
|
||||
return (
|
||||
{"role": "assistant", "content": "hello", "thinking_blocks": [block]}
|
||||
if placement == "thinking_blocks"
|
||||
else {"role": "assistant", "content": [block, {"type": "text", "text": "hello"}]}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize("placement", ["thinking_blocks", "content"])
|
||||
@pytest.mark.parametrize(
|
||||
"signature", ["claude-signature", encrypted_reasoning_signature("gAAAA_1")], ids=["claude", "encrypted"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[
|
||||
("converse/us.openai.gpt-6-luna", None),
|
||||
("us.amazon.nova-pro-v1:0", None),
|
||||
("us.amazon.nova-pro-v1:0", "anthropic.claude-sonnet-4-6"),
|
||||
(_HIDDEN_MODEL_ARN, "openai.gpt-oss-120b-1:0"),
|
||||
],
|
||||
)
|
||||
async def test_bedrock_converse_drops_signed_thinking_for_a_known_non_claude_target(
|
||||
use_async: bool, placement: str, signature: str, model: str, base_model: str | None
|
||||
):
|
||||
block: Final = {"type": "thinking", "thinking": "private plan", "signature": signature}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_with_thinking(placement, block),
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
assert await _converse_messages(messages, model, base_model, use_async) == [
|
||||
{"role": "user", "content": [{"text": "hi"}]},
|
||||
{"role": "assistant", "content": [{"text": "hello"}]},
|
||||
{"role": "user", "content": [{"text": "again"}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize("placement", ["thinking_blocks", "content"])
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[
|
||||
("us.anthropic.claude-sonnet-4-6", None),
|
||||
("ca.anthropic.claude-haiku-5-5", None),
|
||||
("converse/in.anthropic.claude-sonnet-5", None),
|
||||
("claude-sonnet-4-6", None),
|
||||
(_HIDDEN_MODEL_ARN, None),
|
||||
(_HIDDEN_MODEL_ARN, "anthropic.claude-sonnet-4-6"),
|
||||
],
|
||||
)
|
||||
async def test_bedrock_converse_keeps_signed_thinking_for_a_target_that_may_be_claude(
|
||||
use_async: bool, placement: str, model: str, base_model: str | None
|
||||
):
|
||||
block: Final = {"type": "thinking", "thinking": "private plan", "signature": "claude-signature"}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_with_thinking(placement, block),
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
result: Final = await _converse_messages(messages, model, base_model, use_async)
|
||||
|
||||
assistant_content: Final = next(m["content"] for m in result if m["role"] == "assistant")
|
||||
assert assistant_content == [
|
||||
{"reasoningContent": {"reasoningText": {"text": "private plan", "signature": "claude-signature"}}},
|
||||
{"text": "hello"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
async def test_bedrock_converse_keeps_the_tool_call_when_dropping_signed_thinking(use_async: bool):
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "weather?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "private plan", "signature": "claude-signature"}],
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
|
||||
result: Final = await _converse_messages(messages, "converse/us.openai.gpt-6-luna", None, use_async)
|
||||
|
||||
assert [m["role"] for m in result] == ["user", "assistant", "user"]
|
||||
assert [list(part) for part in result[1]["content"]] == [["toolUse"]]
|
||||
assert result[1]["content"][0]["toolUse"]["toolUseId"] == "call_1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("assistant_continue_message", "expected"),
|
||||
[
|
||||
(None, [{"role": "user", "content": [{"text": "hi"}, {"text": "again"}]}]),
|
||||
(
|
||||
"Please continue.",
|
||||
[
|
||||
{"role": "user", "content": [{"text": "hi"}]},
|
||||
{"role": "assistant", "content": [{"text": "Please continue."}]},
|
||||
{"role": "user", "content": [{"text": "again"}]},
|
||||
],
|
||||
),
|
||||
],
|
||||
ids=["merged_into_one_user_turn", "split_by_the_continue_message"],
|
||||
)
|
||||
async def test_bedrock_converse_drops_a_turn_that_held_only_signed_thinking(
|
||||
use_async: bool, assistant_continue_message: str | None, expected: list[dict[str, object]]
|
||||
):
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "private plan", "signature": "claude-signature"}],
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
result: Final = await _converse_messages(
|
||||
messages, "converse/us.openai.gpt-6-luna", None, use_async, assistant_continue_message
|
||||
)
|
||||
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
async def test_bedrock_converse_drops_the_cache_point_of_a_dropped_signed_thinking_block(use_async: bool):
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "private plan",
|
||||
"signature": "claude-signature",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
result: Final = await _converse_messages(messages, "converse/us.amazon.nova-2-lite-v1:0", None, use_async)
|
||||
|
||||
assert result == [{"role": "user", "content": [{"text": "hi"}, {"text": "again"}]}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_block",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -637,8 +637,7 @@ class TestNova2MultiTurnMessageTranslation:
|
|||
assert "Simple addition" in texts
|
||||
assert "4." in texts
|
||||
|
||||
def test_should_keep_signed_thinking_blocks_as_reasoning_content(self):
|
||||
"""With a signature, reasoning is preserved as a reasoningContent block."""
|
||||
def test_should_drop_signed_thinking_blocks_from_another_model(self):
|
||||
bedrock_msgs = self._to_bedrock(
|
||||
[
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
|
|
@ -653,13 +652,9 @@ class TestNova2MultiTurnMessageTranslation:
|
|||
]
|
||||
)
|
||||
assistant = next(m for m in bedrock_msgs if m["role"] == "assistant")
|
||||
rc_blocks = [b for b in assistant["content"] if "reasoningContent" in b]
|
||||
assert len(rc_blocks) >= 1
|
||||
assert rc_blocks[0]["reasoningContent"]["reasoningText"]["text"] == "math"
|
||||
assert rc_blocks[0]["reasoningContent"]["reasoningText"]["signature"] == "sig-1"
|
||||
assert assistant["content"] == [{"text": "4."}]
|
||||
|
||||
def test_should_translate_inline_content_list_thinking_type(self):
|
||||
"""content=[{type:'thinking',...},{type:'text',...}] should also round-trip."""
|
||||
def test_should_drop_inline_signed_thinking_and_keep_the_answer(self):
|
||||
bedrock_msgs = self._to_bedrock(
|
||||
[
|
||||
{"role": "user", "content": "Hi"},
|
||||
|
|
@ -674,11 +669,4 @@ class TestNova2MultiTurnMessageTranslation:
|
|||
]
|
||||
)
|
||||
assistant = next(m for m in bedrock_msgs if m["role"] == "assistant")
|
||||
rc_blocks = [b for b in assistant["content"] if "reasoningContent" in b]
|
||||
text_blocks = [
|
||||
b
|
||||
for b in assistant["content"]
|
||||
if "text" in b and "reasoningContent" not in b
|
||||
]
|
||||
assert len(rc_blocks) >= 1
|
||||
assert any("Hello!" in b["text"] for b in text_blocks)
|
||||
assert assistant["content"] == [{"text": "Hello!"}]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
|
||||
from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig
|
||||
from litellm.llms.bedrock_mantle.responses.transformation import BedrockMantleResponsesAPIConfig
|
||||
from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponsesAPIConfig
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
|
||||
|
|
@ -1350,7 +1352,15 @@ class TestOpenAIResponsesAPIConfig:
|
|||
)
|
||||
return list(bridged.output)
|
||||
|
||||
@pytest.mark.parametrize("config", [OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig()])
|
||||
@pytest.mark.parametrize(
|
||||
"config",
|
||||
[
|
||||
OpenAIResponsesAPIConfig(),
|
||||
AzureOpenAIResponsesAPIConfig(),
|
||||
BedrockOpenAIResponsesConfig(),
|
||||
BedrockMantleResponsesAPIConfig(),
|
||||
],
|
||||
)
|
||||
def test_claude_reasoning_minted_by_the_bridge_is_dropped_before_the_history_reaches_openai(self, config):
|
||||
saved_claude_turn = json.loads(
|
||||
json.dumps([item.model_dump() for item in self._claude_turn_bridged_to_responses_output()])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue