diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index f7a81082f80..907f8e50ac7 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime. AWS serves this surface at ``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions`` -for Grok 4.6, gpt-oss and GPT 5.6 and newer. GPT 5.6 and newer take it by default +for Grok, gpt-oss and GPT 5.6 and newer. Grok and GPT 5.6 and newer take it by default (``bedrock_runtime_chat_completions_is_default`` in ``common_utils``), so their chat completions stay chat completions instead of being rewritten to Converse; the ``chat_completions/`` route prefix opts any other model in, and ``converse/`` pins a @@ -15,6 +15,8 @@ Converse-only feature (``bedrock_request_needs_converse`` in ``common_utils``) i still served by Converse. """ +import re +import uuid from collections.abc import AsyncIterator, Iterator, Mapping from dataclasses import dataclass, replace from types import MappingProxyType @@ -36,6 +38,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token from litellm.llms.bedrock.common_utils import ( BedrockError, bedrock_model_is_openai_gpt, + bedrock_rejects_stop_sequences, bedrock_runtime_chat_completions_serves_reasoning_inline, split_bedrock_region_path, ) @@ -51,6 +54,7 @@ if TYPE_CHECKING: REASONING_OPEN_TAG: Final = "" REASONING_CLOSE_TAG: Final = "" +POSITIONAL_TOOL_CALL_ID: Final = re.compile(r"call_\d+") _PARAMS_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _PARAMS_LIST_ADAPTER: Final = TypeAdapter(list[str]) @@ -191,6 +195,30 @@ class ReasoningTagSplitter: return drained, "", self.pending +def is_positional_tool_call_id(tool_call_id: str | None) -> bool: + return tool_call_id is not None and POSITIONAL_TOOL_CALL_ID.fullmatch(tool_call_id) is not None + + +def mint_tool_call_id() -> str: + return f"call_{uuid.uuid4().hex}" + + +def _mint_unique_tool_call_ids(response: ModelResponse) -> None: + """Replace AWS's positional tool call ids with unique ones. + + bedrock-runtime's native Chat Completions numbers the tool calls of a response from zero (``call_0``, + ``call_1``), seen live on Grok 4.6, Grok 4.7 and GPT 5.6 (2026-10-09; GPT 5.6 returned unique ids + earlier the same day, so the format is not fixed per model). A multi-turn history then holds the same + id for different calls, and clients that match ``tool_result`` ids against it, Claude Code among them, + pair results with the wrong call. Ids that are already unique (``call_<32 hex>``, ``chatcmpl-tool-...``) + do not match ``POSITIONAL_TOOL_CALL_ID`` and pass through. + """ + for choice in response.choices: + for tool_call in choice.message.tool_calls or (): + if is_positional_tool_call_id(tool_call.id): + tool_call.id = mint_tool_call_id() + + def _split_streamed_content( splitter: ReasoningTagSplitter, content: str | None, finished: bool ) -> tuple[ReasoningTagSplitter, str, str]: @@ -229,9 +257,23 @@ class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamin ) -> None: super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode) self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({}) + self._minted_tool_call_ids: Mapping[tuple[int, int], str] = MappingProxyType({}) + + def _minted_tool_call_id(self, choice_index: int, tool_call_index: int) -> str: + key: Final = (choice_index, tool_call_index) + minted: Final = self._minted_tool_call_ids.get(key) or mint_tool_call_id() + self._minted_tool_call_ids = MappingProxyType({**self._minted_tool_call_ids, key: minted}) + return minted + + def _mint_streamed_tool_call_ids(self, parsed: ModelResponseStream) -> None: + for choice in parsed.choices: + for tool_call in choice.delta.tool_calls or (): + if is_positional_tool_call_id(tool_call.id): + tool_call.id = self._minted_tool_call_id(choice.index, tool_call.index) def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseModelResponseIterator signature parsed: Final = super().chunk_parser(chunk) + self._mint_streamed_tool_call_ids(parsed) if not bedrock_runtime_chat_completions_serves_reasoning_inline(parsed.model or ""): return parsed for choice in parsed.choices: @@ -405,10 +447,13 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ), status_code=400, ) + rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() return dict( without_refused_reasoning_effort( model, - with_max_completion_tokens(_without_params(mapped, refused_while_reasoning | malformed_effort)), + with_max_completion_tokens( + _without_params(mapped, refused_while_reasoning | malformed_effort | rejected_stop) + ), ) ) @@ -483,6 +528,7 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): json_mode=json_mode, ) set_provider_response_headers_in_hidden_params(response, raw_response.headers) + _mint_unique_tool_call_ids(response) if not bedrock_runtime_chat_completions_serves_reasoning_inline(model): return response for choice in response.choices: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d78936f6fbc..c0d0e618578 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -105,6 +105,7 @@ from ..common_utils import ( bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, bedrock_reasoning_effort_disabled, + bedrock_rejects_stop_sequences, get_anthropic_beta_from_headers, get_bedrock_tool_name, is_bedrock_application_inference_profile_arn, @@ -1054,7 +1055,7 @@ class AmazonConverseConfig(BaseConfig): ) if param == "stream": optional_params["stream"] = value - if param == "stop": + if param == "stop" and not bedrock_rejects_stop_sequences(model): if isinstance(value, str): if len(value) == 0: # converse raises error for empty strings continue diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index a3f80001868..e046b0f3616 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -45,6 +45,7 @@ _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs" _OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.") _OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?") _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6) +_XAI_GROK_MODEL_RE: Final = re.compile(r"(^|[./])xai\.grok-") _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT: Final = "/v1/chat/completions" BedrockRoute = Literal[ "converse", @@ -869,15 +870,21 @@ def _openai_gpt_version(model: str) -> tuple[int, int] | None: return int(match.group(2)), int(match.group(3) or 0) +def _bedrock_runtime_chat_completions_default_family(model: str) -> bool: + if _XAI_GROK_MODEL_RE.search(model) is not None: + return True + gpt_version: Final = _openai_gpt_version(model) + return gpt_version is not None and gpt_version >= _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE + + def bedrock_runtime_chat_completions_is_default(model: str) -> bool: """Whether a model with no route prefix goes to bedrock-runtime's native Chat Completions by default. - GPT 5.6 and newer (``openai.gpt-[.]`` at or above 5.6, which gpt-oss never matches) whose - price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``. Older GPT rows, gpt-oss and Grok - stay on Converse unless the ``chat_completions/`` prefix opts them in. + Grok (``xai.grok-*``) and GPT 5.6 and newer (``openai.gpt-[.]`` at or above 5.6, which + gpt-oss never matches) whose price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``. + Older GPT rows and gpt-oss stay on Converse unless the ``chat_completions/`` prefix opts them in. """ - version: Final = _openai_gpt_version(model) - if version is None or version < _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: + if not _bedrock_runtime_chat_completions_default_family(model): return False return any( _price_map_entry_lists_endpoint(entry, _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT) @@ -885,6 +892,16 @@ def bedrock_runtime_chat_completions_is_default(model: str) -> bool: ) +def bedrock_rejects_stop_sequences(model: str) -> bool: + """Whether AWS refuses stop sequences for this model on every Bedrock route. + + Grok answers ``stopSequences`` on Converse and ``stop`` on native Chat Completions alike with + ``This model doesn't support the stopSequences field`` (Grok 4.6 and 4.7 checked live on 2026-10-09), + so litellm drops ``stop`` for it instead of forwarding it to a 400. + """ + return _XAI_GROK_MODEL_RE.search(model) is not None + + def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool: """Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``. @@ -961,7 +978,9 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje AWS's native OpenAI surface, a ``model_id`` override (an application inference profile or provisioned throughput ARN) is only encoded into Converse's request URL and so stays on Converse like the ``bedrock/arn:...`` model form, ``stop`` stays on Converse where it fails loudly instead of silently - stopping hidden reasoning, operator-owned request metadata is only written onto the Converse body, + stopping hidden reasoning (except on models that reject stop sequences everywhere, + ``bedrock_rejects_stop_sequences``, where both routes drop it), operator-owned request metadata is only + written onto the Converse body, function tools (``tools`` or legacy ``functions``) on a model without ``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless ``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as @@ -971,7 +990,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje handling everywhere, since AWS's native surface rejects that type with a 400 unless the prompt mentions json. """ - if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS): + converse_only_keys: Final = ( + BEDROCK_CONVERSE_ONLY_REQUEST_KEYS - {"stop"} + if bedrock_rejects_stop_sequences(model) + else BEDROCK_CONVERSE_ONLY_REQUEST_KEYS + ) + if any(request_params.get(key) is not None for key in converse_only_keys): return True if bedrock_request_metadata_is_owned(): return True @@ -1421,8 +1445,8 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Get the bedrock route for the given model. - GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default - (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; + Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by + default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family model stays on Converse without the prefix. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 81768d630be..6fffc109bef 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -80425,6 +80425,9 @@ }, "global.xai.grok-4.7": { "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -80448,6 +80451,9 @@ }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_flex": 2.75e-07, "cache_read_input_token_cost_priority": 9.625e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 81768d630be..6fffc109bef 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -80425,6 +80425,9 @@ }, "global.xai.grok-4.7": { "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -80448,6 +80451,9 @@ }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_flex": 2.75e-07, "cache_read_input_token_cost_priority": 9.625e-07, diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 46f2beb5d20..79c02b0970d 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -110,6 +110,12 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-bedrock-key aws_region_name: us-east-1 + - model_name: bedrock/global.xai.grok-4.7 + litellm_params: + model: bedrock/global.xai.grok-4.7 + api_base: http://127.0.0.1:8191 + api_key: synthetic-bedrock-key + aws_region_name: us-east-1 - model_name: openai/gpt-5.4 litellm_params: model: openai/gpt-5.4 diff --git a/tests/integration/translation/messages/bases/bedrock_chat_completions.py b/tests/integration/translation/messages/bases/bedrock_chat_completions.py new file mode 100644 index 00000000000..9a59f40a09d --- /dev/null +++ b/tests/integration/translation/messages/bases/bedrock_chat_completions.py @@ -0,0 +1,63 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +GROK_4_7_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/messages", + litellm_request={ + "model": "bedrock/global.xai.grok-4.7", + "max_tokens": 1024, + "system": "You are a terse assistant.", + "messages": [{"role": "user", "content": "Say hello."}], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/openai/v1/chat/completions", + expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "global.xai.grok-4.7", + "messages": [ + {"role": "system", "content": "You are a terse assistant."}, + {"role": "user", "content": "Say hello."}, + ], + "max_completion_tokens": 1024, + }, + mock_provider_response={ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"}, + } + ], + "created": 1791505413, + "id": "chatcmpl-3xfhgmhrhhwplbepas3pjzbgjcccj4hx5tzplx7fw35giw3efznq", + "model": "global.xai.grok-4.7", + "object": "chat.completion", + "service_tier": "default", + "usage": { + "completion_tokens": 37, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 27, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens": 30, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 67, + }, + "system_fingerprint": None, + }, + expected_litellm_response={ + "id": "chatcmpl-3xfhgmhrhhwplbepas3pjzbgjcccj4hx5tzplx7fw35giw3efznq", + "type": "message", + "role": "assistant", + "model": "bedrock/global.xai.grok-4.7", + "stop_sequence": None, + "usage": {"input_tokens": 30, "output_tokens": 37}, + "content": [{"type": "text", "text": "Hello."}], + "stop_reason": "end_turn", + "stop_details": None, + }, +) diff --git a/tests/integration/translation/messages/basic/test_messages_basic_bedrock_chat_completions.py b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_chat_completions.py new file mode 100644 index 00000000000..5cfe18a243d --- /dev/null +++ b/tests/integration/translation/messages/basic/test_messages_basic_bedrock_chat_completions.py @@ -0,0 +1,13 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [GROK_4_7_TEST_CASE], ids=lambda case: case.id) +def test_messages_basic_bedrock_chat_completions( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/tool_search/__init__.py b/tests/integration/translation/messages/tool_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py new file mode 100644 index 00000000000..b457d9920bd --- /dev/null +++ b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py @@ -0,0 +1,147 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE +from integration.translation.runner import assert_translation + +GROK_4_7_TOOL_SEARCH_TEST_CASE: Final = replace( + GROK_4_7_TEST_CASE, + scenario="tool_search", + litellm_request={ + "model": "bedrock/global.xai.grok-4.7", + "max_tokens": 1024, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "low"}, + "tools": [ + { + "name": "ToolSearch", + "description": "Load deferred tools by name.", + "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, + }, + { + "name": "WebSearch", + "description": "Search the web.", + "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, + }, + ], + "messages": [ + {"role": "user", "content": "Load the web search tool."}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "call_tool_search_1", + "name": "ToolSearch", + "input": {"query": "select:WebSearch"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_tool_search_1", + "content": [{"type": "tool_reference", "tool_name": "WebSearch"}], + } + ], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_request={ + "model": "global.xai.grok-4.7", + "messages": [ + {"role": "user", "content": "Load the web search tool."}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_tool_search_1", + "type": "function", + "function": {"name": "ToolSearch", "arguments": '{"query": "select:WebSearch"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_tool_search_1", "content": ""}, + ], + "max_completion_tokens": 1024, + "tools": [ + { + "type": "function", + "function": { + "name": "ToolSearch", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + "description": "Load deferred tools by name.", + }, + }, + { + "type": "function", + "function": { + "name": "WebSearch", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + "description": "Search the web.", + }, + }, + ], + "reasoning_effort": "low", + }, + mock_provider_response={ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "annotations": [], + "content": "The web search tool is now loaded and available for use.", + "refusal": None, + "role": "assistant", + }, + } + ], + "created": 1791505405, + "id": "chatcmpl-xevjeh2pxs6gjvcn2yz76piwdc4pi6fb2sqwb7psnwitm5higvvq", + "model": "global.xai.grok-4.7", + "object": "chat.completion", + "service_tier": "default", + "usage": { + "completion_tokens": 76, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 57, + "rejected_prediction_tokens": 0, + }, + "prompt_tokens": 162, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + "total_tokens": 238, + }, + "system_fingerprint": None, + }, + expected_litellm_response={ + **GROK_4_7_TEST_CASE.expected_litellm_response, + "id": "chatcmpl-xevjeh2pxs6gjvcn2yz76piwdc4pi6fb2sqwb7psnwitm5higvvq", + "usage": {"input_tokens": 162, "output_tokens": 76}, + "content": [{"type": "text", "text": "The web search tool is now loaded and available for use."}], + }, +) + + +@pytest.mark.parametrize("case", [GROK_4_7_TOOL_SEARCH_TEST_CASE], ids=lambda case: case.id) +def test_messages_tool_search_bedrock_chat_completions( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/tool_use/__init__.py b/tests/integration/translation/messages/tool_use/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/messages/tool_use/test_messages_tool_use_bedrock_chat_completions.py b/tests/integration/translation/messages/tool_use/test_messages_tool_use_bedrock_chat_completions.py new file mode 100644 index 00000000000..9420e51fde6 --- /dev/null +++ b/tests/integration/translation/messages/tool_use/test_messages_tool_use_bedrock_chat_completions.py @@ -0,0 +1,107 @@ +import json +import re +from dataclasses import replace +from typing import Final +from unittest.mock import ANY + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE +from integration.translation.runner import assert_translation + +GROK_4_7_TOOL_USE_TEST_CASE: Final = replace( + GROK_4_7_TEST_CASE, + scenario="tool_use", + litellm_request={ + "model": "bedrock/global.xai.grok-4.7", + "max_tokens": 1024, + "tools": [ + { + "name": "Read", + "description": "Read a file.", + "input_schema": { + "type": "object", + "properties": {"file_path": {"type": "string"}}, + "required": ["file_path"], + }, + } + ], + "messages": [{"role": "user", "content": "Read notes.txt."}], + "cache": {"no-cache": True}, + }, + expected_provider_request={ + "model": "global.xai.grok-4.7", + "messages": [{"role": "user", "content": "Read notes.txt."}], + "max_completion_tokens": 1024, + "tools": [ + { + "type": "function", + "function": { + "name": "Read", + "parameters": { + "type": "object", + "properties": {"file_path": {"type": "string"}}, + "required": ["file_path"], + }, + "description": "Read a file.", + }, + } + ], + }, + mock_provider_response={ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "annotations": [], + "content": None, + "refusal": None, + "role": "assistant", + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": {"name": "Read", "arguments": '{"file_path":"notes.txt"}'}, + } + ], + }, + } + ], + "created": 1791509000, + "id": "chatcmpl-toolusegrok47", + "model": "global.xai.grok-4.7", + "object": "chat.completion", + "usage": {"completion_tokens": 20, "prompt_tokens": 40, "total_tokens": 60}, + }, + expected_litellm_response={ + **GROK_4_7_TEST_CASE.expected_litellm_response, + "id": "chatcmpl-toolusegrok47", + "usage": {"input_tokens": 40, "output_tokens": 20}, + "content": [{"type": "tool_use", "id": ANY, "name": "Read", "input": {"file_path": "notes.txt"}}], + "stop_reason": "tool_use", + }, +) + + +@pytest.mark.parametrize("case", [GROK_4_7_TOOL_USE_TEST_CASE], ids=lambda case: case.id) +def test_messages_tool_use_bedrock_chat_completions( + case: TranslationTestCase, gateway: Gateway, provider: SharedProvider +) -> None: + assert_translation(case, gateway, provider) + + +def test_messages_tool_use_ids_are_unique_across_bedrock_chat_completions_turns( + gateway: Gateway, provider: SharedProvider +) -> None: + case: Final = GROK_4_7_TOOL_USE_TEST_CASE + reply: Final = Reply(body=json.dumps(case.mock_provider_response).encode()) + provider.expect(reply, reply) + responses: Final = tuple(gateway.request("POST", case.litellm_endpoint, case.litellm_request) for _ in range(2)) + assert len(provider.received()) == 2 + tool_use_ids: Final = tuple(response.json()["content"][0]["id"] for response in responses) + assert len(set(tool_use_ids)) == 2 + assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_use_id) for tool_use_id in tool_use_ids) diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 9d4c0fac094..a2d3ba2a1bb 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,6 +1,7 @@ """Bedrock Runtime Chat Completions: the default for GPT 5.6 and newer, ``bedrock/chat_completions/`` for the rest.""" import json +import re import httpx import pytest @@ -63,7 +64,6 @@ def test_claude_stays_on_converse(local_cost_map): @pytest.mark.parametrize( "model", [ - "us.xai.grok-4.6", "bedrock/openai.gpt-oss-20b-1:0", "openai.gpt-oss-120b-1:0", "global.openai.gpt-5.5", @@ -88,7 +88,9 @@ def test_cost_map_row_listing_chat_completions_leaves_the_default_route_alone(mo } monkeypatch.setattr(litellm, "model_cost", {"openai.gpt-oss-20b-1:0": entry}) assert BedrockModelInfo.get_bedrock_route("bedrock/openai.gpt-oss-20b-1:0", {}) == "converse" - assert BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {}) == "chat_completions" + assert ( + BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {}) == "chat_completions" + ) @pytest.mark.parametrize( @@ -103,10 +105,14 @@ def test_cost_map_row_listing_chat_completions_leaves_the_default_route_alone(mo ("us.openai.gpt-6.1-sol", ["/v1/chat/completions"], "chat_completions"), ("global.openai.gpt-10-sol", ["/v1/chat/completions"], "chat_completions"), ("openai.gpt-oss-120b-1:0", ["/v1/chat/completions"], "converse"), - ("us.xai.grok-4.6", ["/v1/chat/completions"], "converse"), + ("us.xai.grok-4.6", ["/v1/chat/completions"], "chat_completions"), + ("global.xai.grok-4.7", ["/v1/chat/completions"], "chat_completions"), + ("global.xai.grok-4.7", ["/v1/responses"], "converse"), + ("global.xai.grok-4.7", [], "converse"), + ("us-gov.xai.grok-4.6", ["/v1/chat/completions"], "chat_completions"), ], ) -def test_default_route_needs_gpt_56_or_newer_and_a_row_listing_chat_completions( +def test_default_route_needs_grok_or_gpt_56_or_newer_and_a_row_listing_chat_completions( monkeypatch, model, supported_endpoints, expected_route ): entry = {"litellm_provider": "bedrock_converse", "supported_endpoints": supported_endpoints} @@ -257,7 +263,7 @@ def _recording_client(**response_kwargs): @pytest.mark.parametrize( "model, model_path", [ - ("bedrock/us.xai.grok-4.6", b"/model/us.xai.grok-4.6/converse"), + ("bedrock/converse/us.xai.grok-4.6", b"/model/us.xai.grok-4.6/converse"), ("bedrock/openai.gpt-oss-20b-1:0", b"/model/openai.gpt-oss-20b-1%3A0/converse"), ("bedrock/global.openai.gpt-5.5", b"/model/global.openai.gpt-5.5/converse"), ], @@ -299,6 +305,7 @@ def test_completion_keeps_the_aws_request_id_as_a_provider_header(local_cost_map assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-native-1" + def test_region_path_sends_the_bare_model_id_to_the_path_region(local_cost_map, fake_aws_env): requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0")) litellm.completion( @@ -384,6 +391,10 @@ GPT_56_AND_NEWER_MODELS = ( "global.openai.gpt-6-luna", "bedrock/global.openai.gpt-6.1-sol", "us.openai.gpt-6.1-sol", + "global.xai.grok-4.6", + "bedrock/us.xai.grok-4.6", + "global.xai.grok-4.7", + "bedrock/us.xai.grok-4.7", ) @@ -430,10 +441,9 @@ def test_guardrail_config_falls_back_to_converse(local_cost_map, model): [ {"additionalModelRequestFields": {"reasoning_effort": "high"}}, {"top_k": 40}, - {"stop": ["END"]}, {"model_id": APPLICATION_INFERENCE_PROFILE_ARN}, ], - ids=["additionalModelRequestFields", "top_k", "stop", "model_id"], + ids=["additionalModelRequestFields", "top_k", "model_id"], ) def test_converse_extension_params_fall_back_to_converse(local_cost_map, model, request_params): assert bedrock_request_needs_converse(model, request_params) is True @@ -441,6 +451,43 @@ def test_converse_extension_params_fall_back_to_converse(local_cost_map, model, assert BedrockModelInfo.get_bedrock_route(model, {key: None for key in request_params}) == "chat_completions" +@pytest.mark.parametrize( + "model", + ["chat_completions/openai.gpt-oss-20b-1:0", "bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6.1-sol"], +) +def test_stop_keeps_other_models_on_converse(local_cost_map, model): + assert bedrock_request_needs_converse(model, {"stop": ["END"]}) is True + assert BedrockModelInfo.get_bedrock_route(model, {"stop": ["END"]}) == "converse" + + +@pytest.mark.parametrize("model", ["bedrock/global.xai.grok-4.7", "bedrock/us.xai.grok-4.6"]) +def test_stop_is_dropped_on_chat_completions_for_grok(local_cost_map, fake_aws_env, model): + requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/"))) + response = litellm.completion( + model=model, messages=[{"role": "user", "content": "hello"}], stop=[""], max_tokens=64, client=client + ) + + assert requests[0].url.raw_path == b"/openai/v1/chat/completions" + body = json.loads(requests[0].content) + assert "stop" not in body + assert body["max_completion_tokens"] == 64 + assert response.choices[0].message.content == "ok" + + +def test_stop_is_dropped_on_converse_for_grok(local_cost_map, fake_aws_env): + requests, client = _recording_client(json=CONVERSE_JSON) + litellm.completion( + model="bedrock/converse/global.xai.grok-4.7", + messages=[{"role": "user", "content": "hello"}], + stop=[""], + max_tokens=64, + client=client, + ) + + assert requests[0].url.raw_path == b"/model/global.xai.grok-4.7/converse" + assert json.loads(requests[0].content)["inferenceConfig"] == {"maxTokens": 64} + + @pytest.mark.parametrize( "model", ["bedrock/us.openai.gpt-5.6-sol", "global.openai.gpt-6-sol", "bedrock/chat_completions/us.xai.grok-4.6"] ) @@ -508,7 +555,10 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch): ], ) def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route): - assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route + assert ( + BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) + == expected_route + ) assert ( BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/us.openai.gpt-5.6-terra", request_params) == expected_route @@ -535,14 +585,22 @@ def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_ ], ) def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params, expected_route): - assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route - assert BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", request_params) == "chat_completions" + assert ( + BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) + == expected_route + ) + assert ( + BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", request_params) + == "chat_completions" + ) def test_thinking_block_goes_to_converse(local_cost_map): thinking = {"type": "enabled", "budget_tokens": 1024} assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse" - assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": None}) == "chat_completions" + assert ( + BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": None}) == "chat_completions" + ) def test_explicit_converse_prefix_wins_for_openai_models(local_cost_map): @@ -592,9 +650,7 @@ def _assert_remote_images_inlined(content): def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling - monkeypatch.setattr( - image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}" - ) + monkeypatch.setattr(image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}") body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request( model="us.xai.grok-4.6", messages=IMAGE_MESSAGES, @@ -760,7 +816,9 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod ids=lambda value: value if isinstance(value, str) else next(iter(value)), ) def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_map, fake_aws_env, model, param): - requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/chat_completions/"))) + requests, client = _recording_client( + json=_chat_completion_json("ok", model.removeprefix("bedrock/chat_completions/")) + ) with pytest.raises(litellm.UnsupportedParamsError, match=next(iter(param))): litellm.completion(model=model, messages=[{"role": "user", "content": "hello"}], client=client, **param) litellm.completion( @@ -901,12 +959,12 @@ def test_reasoning_tag_splitter_releases_a_false_tag_prefix(): assert _run_splitter(["<", "b>x"]) == ("", "x") -def _stream_chunk(delta, finish_reason=None, index=0): +def _stream_chunk(delta, finish_reason=None, index=0, model="openai.gpt-oss-20b-1:0"): return { "id": "chatcmpl-test", "object": "chat.completion.chunk", "created": 1733529600, - "model": "openai.gpt-oss-20b-1:0", + "model": model, "choices": [{"index": index, "delta": delta, "finish_reason": finish_reason}], } @@ -1480,7 +1538,9 @@ def test_streaming_handler_keeps_a_literal_reasoning_tag_outside_gpt_oss(local_c assert opened.choices[0].delta.content == "not thinking" assert _reasoning_of(opened) is None - closed = handler.chunk_parser({**_stream_chunk({"content": " Hello"}, finish_reason="stop"), "model": model}) + closed = handler.chunk_parser( + {**_stream_chunk({"content": " Hello"}, finish_reason="stop"), "model": model} + ) assert closed.choices[0].delta.content == " Hello" assert _reasoning_of(closed) is None @@ -1540,3 +1600,99 @@ def test_reasoning_tag_split_is_read_from_the_cost_map( ) assert chunk.choices[0].delta.content == expected_content assert _reasoning_of(chunk) == expected_reasoning + + +def _tool_call(tool_call_id, name): + return {"id": tool_call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + +def test_positional_tool_call_ids_are_minted_unique_per_response(local_cost_map, fake_aws_env): + reply = _chat_completion_json( + None, + "global.xai.grok-4.7", + tool_calls=[_tool_call("call_0", "read_a"), _tool_call("call_1", "read_b")], + ) + _, client = _recording_client(json=reply) + responses = [ + litellm.completion( + model="bedrock/chat_completions/global.xai.grok-4.7", + messages=[{"role": "user", "content": "hello"}], + client=client, + ) + for _ in range(2) + ] + + ids = [tool_call.id for response in responses for tool_call in response.choices[0].message.tool_calls] + assert len(set(ids)) == 4 + assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids) + assert [tool_call.function.name for tool_call in responses[0].choices[0].message.tool_calls] == ["read_a", "read_b"] + + +def test_provider_unique_tool_call_ids_pass_through(local_cost_map, fake_aws_env): + reply = _chat_completion_json( + None, "openai.gpt-oss-20b-1:0", tool_calls=[_tool_call("chatcmpl-tool-90090f0c1c521528", "read_a")] + ) + _, client = _recording_client(json=reply) + response = litellm.completion( + model="bedrock/chat_completions/openai.gpt-oss-20b-1:0", + messages=[{"role": "user", "content": "hello"}], + client=client, + ) + + assert response.choices[0].message.tool_calls[0].id == "chatcmpl-tool-90090f0c1c521528" + + +def _streamed_tool_call_ids(tool_call_deltas, model="global.xai.grok-4.7"): + handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True) + return [ + tool_call.id + for delta in tool_call_deltas + for tool_call in handler.chunk_parser(_stream_chunk({"tool_calls": [delta]}, model=model)) + .choices[0] + .delta.tool_calls + ] + + +def test_streamed_positional_tool_call_ids_are_minted_once_per_tool_call(local_cost_map): + deltas = [ + {"index": 0, **_tool_call("call_0", "read_a")}, + {"index": 0, "function": {"arguments": '{"x":1}'}}, + {"index": 0, "id": "call_0", "function": {"arguments": "}"}}, + {"index": 1, **_tool_call("call_1", "read_b")}, + ] + first_stream = _streamed_tool_call_ids(deltas) + second_stream = _streamed_tool_call_ids(deltas) + + assert first_stream[0] == first_stream[2] + assert first_stream[1] is None + assert len({first_stream[0], first_stream[3], second_stream[0], second_stream[3]}) == 4 + assert all(re.fullmatch(r"call_[0-9a-f]{32}", first_stream[i]) for i in (0, 3)) + + +def test_streamed_provider_unique_tool_call_ids_pass_through(local_cost_map): + assert _streamed_tool_call_ids( + [{"index": 0, **_tool_call("chatcmpl-tool-8c9232df5019ff4f", "read_a")}], model="openai.gpt-oss-20b-1:0" + ) == ["chatcmpl-tool-8c9232df5019ff4f"] + + +def test_streamed_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map): + deltas = [{"index": 0, **_tool_call("call_0", "read_a")}, {"index": 1, **_tool_call("call_1", "read_b")}] + + ids = _streamed_tool_call_ids(deltas, model="global.openai.gpt-5.6-sol") + + assert len(set(ids)) == 2 + assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids) + + +def test_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map, fake_aws_env): + reply = _chat_completion_json( + None, "global.openai.gpt-5.6-sol", tool_calls=[_tool_call("call_0", "read_a"), _tool_call("call_1", "read_b")] + ) + _, client = _recording_client(json=reply) + response = litellm.completion( + model="bedrock/global.openai.gpt-5.6-sol", messages=[{"role": "user", "content": "hello"}], client=client + ) + + ids = [tool_call.id for tool_call in response.choices[0].message.tool_calls] + assert len(set(ids)) == 2 + assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids)