scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

feat(bedrock): default Grok on Bedrock to native Chat Completions, mint unique tool call ids and drop stop (#45473)

Browse source 
* fix(bedrock): route Grok 4.7 tools with reasoning to native Chat Completions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): mint unique tool call ids on native Chat Completions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): send a blank text block for Converse tool results with no supported content

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(bedrock): default Grok on Bedrock to native Chat Completions and leave Converse unchanged

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): gate tool call id minting to Grok on native Chat Completions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(bedrock): check the Grok tool call id gate at both call sites

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): remint positional tool call ids on native Chat Completions for every model

AWS returned call_0, call_1 for GPT 5.6 as well as Grok on 2026-10-09 (and unique ids for GPT 5.6 earlier the same day), so the id format is not fixed per model. Drop the Grok-only gate and remint any call_<digits> id on the route

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(bedrock): drop stop sequences for Grok instead of forwarding them to a 400

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-09 20:56:03 +00:00 • committed by GitHub
parent 097d018dbf
commit 8080601a95
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 605 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime.
AWS serves this surface at
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
for Grok 4.6, gpt-oss and GPT 5.6 and newer. GPT 5.6 and newer take it by default
for Grok, gpt-oss and GPT 5.6 and newer. Grok and GPT 5.6 and newer take it by default
(``bedrock_runtime_chat_completions_is_default`` in ``common_utils``), so their chat
completions stay chat completions instead of being rewritten to Converse; the
``chat_completions/`` route prefix opts any other model in, and ``converse/`` pins a
@ -15,6 +15,8 @@ Converse-only feature (``bedrock_request_needs_converse`` in ``common_utils``) i
still served by Converse.
"""
import re
import uuid
from collections.abc import AsyncIterator, Iterator, Mapping
from dataclasses import dataclass, replace
from types import MappingProxyType
@ -36,6 +38,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token
from litellm.llms.bedrock.common_utils import (
BedrockError,
bedrock_model_is_openai_gpt,
bedrock_rejects_stop_sequences,
bedrock_runtime_chat_completions_serves_reasoning_inline,
split_bedrock_region_path,
)
@ -51,6 +54,7 @@ if TYPE_CHECKING:
REASONING_OPEN_TAG: Final = "<reasoning>"
REASONING_CLOSE_TAG: Final = "</reasoning>"
POSITIONAL_TOOL_CALL_ID: Final = re.compile(r"call_\d+")
_PARAMS_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
_PARAMS_LIST_ADAPTER: Final = TypeAdapter(list[str])
@ -191,6 +195,30 @@ class ReasoningTagSplitter:
return drained, "", self.pending
def is_positional_tool_call_id(tool_call_id: str | None) -> bool:
return tool_call_id is not None and POSITIONAL_TOOL_CALL_ID.fullmatch(tool_call_id) is not None
def mint_tool_call_id() -> str:
return f"call_{uuid.uuid4().hex}"
def _mint_unique_tool_call_ids(response: ModelResponse) -> None:
"""Replace AWS's positional tool call ids with unique ones.
bedrock-runtime's native Chat Completions numbers the tool calls of a response from zero (``call_0``,
``call_1``), seen live on Grok 4.6, Grok 4.7 and GPT 5.6 (2026-10-09; GPT 5.6 returned unique ids
earlier the same day, so the format is not fixed per model). A multi-turn history then holds the same
id for different calls, and clients that match ``tool_result`` ids against it, Claude Code among them,
pair results with the wrong call. Ids that are already unique (``call_<32 hex>``, ``chatcmpl-tool-...``)
do not match ``POSITIONAL_TOOL_CALL_ID`` and pass through.
"""
for choice in response.choices:
for tool_call in choice.message.tool_calls or ():
if is_positional_tool_call_id(tool_call.id):
tool_call.id = mint_tool_call_id()
def _split_streamed_content(
splitter: ReasoningTagSplitter, content: str | None, finished: bool
) -> tuple[ReasoningTagSplitter, str, str]:
@ -229,9 +257,23 @@ class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamin
) -> None:
super().__init__(streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode)
self._splitters: Mapping[int, ReasoningTagSplitter] = MappingProxyType({})
self._minted_tool_call_ids: Mapping[tuple[int, int], str] = MappingProxyType({})
def _minted_tool_call_id(self, choice_index: int, tool_call_index: int) -> str:
key: Final = (choice_index, tool_call_index)
minted: Final = self._minted_tool_call_ids.get(key) or mint_tool_call_id()
self._minted_tool_call_ids = MappingProxyType({**self._minted_tool_call_ids, key: minted})
return minted
def _mint_streamed_tool_call_ids(self, parsed: ModelResponseStream) -> None:
for choice in parsed.choices:
for tool_call in choice.delta.tool_calls or ():
if is_positional_tool_call_id(tool_call.id):
tool_call.id = self._minted_tool_call_id(choice.index, tool_call.index)
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: BaseModelResponseIterator signature
parsed: Final = super().chunk_parser(chunk)
self._mint_streamed_tool_call_ids(parsed)
if not bedrock_runtime_chat_completions_serves_reasoning_inline(parsed.model or ""):
return parsed
for choice in parsed.choices:
@ -405,10 +447,13 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
),
status_code=400,
)
rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]()
return dict(
without_refused_reasoning_effort(
model,
with_max_completion_tokens(_without_params(mapped, refused_while_reasoning | malformed_effort)),
with_max_completion_tokens(
_without_params(mapped, refused_while_reasoning | malformed_effort | rejected_stop)
),
)
)
@ -483,6 +528,7 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
json_mode=json_mode,
)
set_provider_response_headers_in_hidden_params(response, raw_response.headers)
_mint_unique_tool_call_ids(response)
if not bedrock_runtime_chat_completions_serves_reasoning_inline(model):
return response
for choice in response.choices:

3
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -105,6 +105,7 @@ from ..common_utils import (
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
bedrock_reasoning_effort_disabled,
bedrock_rejects_stop_sequences,
get_anthropic_beta_from_headers,
get_bedrock_tool_name,
is_bedrock_application_inference_profile_arn,
@ -1054,7 +1055,7 @@ class AmazonConverseConfig(BaseConfig):
)
if param == "stream":
optional_params["stream"] = value
if param == "stop":
if param == "stop" and not bedrock_rejects_stop_sequences(model):
if isinstance(value, str):
if len(value) == 0: # converse raises error for empty strings
continue

42
litellm/llms/bedrock/common_utils.py
View file

@ -45,6 +45,7 @@ _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
_OPENAI_FAMILY_MODEL_RE: Final = re.compile(r"(^|[./])openai\.")
_OPENAI_GPT_VERSION_RE: Final = re.compile(r"(^|[./])openai\.gpt-(\d{1,3})(?!\d)(?:\.(\d{1,3})(?!\d))?")
_BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE: Final = (5, 6)
_XAI_GROK_MODEL_RE: Final = re.compile(r"(^|[./])xai\.grok-")
_BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT: Final = "/v1/chat/completions"
BedrockRoute = Literal[
"converse",
@ -869,15 +870,21 @@ def _openai_gpt_version(model: str) -> tuple[int, int] | None:
return int(match.group(2)), int(match.group(3) or 0)
def _bedrock_runtime_chat_completions_default_family(model: str) -> bool:
if _XAI_GROK_MODEL_RE.search(model) is not None:
return True
gpt_version: Final = _openai_gpt_version(model)
return gpt_version is not None and gpt_version >= _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE
def bedrock_runtime_chat_completions_is_default(model: str) -> bool:
"""Whether a model with no route prefix goes to bedrock-runtime's native Chat Completions by default.
GPT 5.6 and newer (``openai.gpt-<major>[.<minor>]`` at or above 5.6, which gpt-oss never matches) whose
price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``. Older GPT rows, gpt-oss and Grok
stay on Converse unless the ``chat_completions/`` prefix opts them in.
Grok (``xai.grok-*``) and GPT 5.6 and newer (``openai.gpt-<major>[.<minor>]`` at or above 5.6, which
gpt-oss never matches) whose price-map row lists ``/v1/chat/completions`` in ``supported_endpoints``.
Older GPT rows and gpt-oss stay on Converse unless the ``chat_completions/`` prefix opts them in.
"""
version: Final = _openai_gpt_version(model)
if version is None or version < _BEDROCK_RUNTIME_CHAT_COMPLETIONS_DEFAULT_SINCE:
if not _bedrock_runtime_chat_completions_default_family(model):
return False
return any(
_price_map_entry_lists_endpoint(entry, _BEDROCK_RUNTIME_CHAT_COMPLETIONS_ENDPOINT)
@ -885,6 +892,16 @@ def bedrock_runtime_chat_completions_is_default(model: str) -> bool:
)
def bedrock_rejects_stop_sequences(model: str) -> bool:
"""Whether AWS refuses stop sequences for this model on every Bedrock route.
Grok answers ``stopSequences`` on Converse and ``stop`` on native Chat Completions alike with
``This model doesn't support the stopSequences field`` (Grok 4.6 and 4.7 checked live on 2026-10-09),
so litellm drops ``stop`` for it instead of forwarding it to a 400.
"""
return _XAI_GROK_MODEL_RE.search(model) is not None
def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:
"""Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``.
@ -961,7 +978,9 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
AWS's native OpenAI surface, a ``model_id`` override (an application inference profile or provisioned
throughput ARN) is only encoded into Converse's request URL and so stays on Converse like the
``bedrock/arn:...`` model form, ``stop`` stays on Converse where it fails loudly instead of silently
stopping hidden reasoning, operator-owned request metadata is only written onto the Converse body,
stopping hidden reasoning (except on models that reject stop sequences everywhere,
``bedrock_rejects_stop_sequences``, where both routes drop it), operator-owned request metadata is only
written onto the Converse body,
function tools (``tools`` or legacy ``functions``) on a model without
``supports_bedrock_runtime_chat_completions_tools_with_reasoning`` are rejected there unless
``reasoning_effort`` is exactly ``"none"``, and a ``response_format`` goes native only as
@ -971,7 +990,12 @@ def bedrock_request_needs_converse(model: str, request_params: Mapping[str, obje
handling everywhere, since AWS's native surface rejects that type with a 400 unless the prompt
mentions json.
"""
if any(request_params.get(key) is not None for key in BEDROCK_CONVERSE_ONLY_REQUEST_KEYS):
converse_only_keys: Final = (
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS - {"stop"}
if bedrock_rejects_stop_sequences(model)
else BEDROCK_CONVERSE_ONLY_REQUEST_KEYS
)
if any(request_params.get(key) is not None for key in converse_only_keys):
return True
if bedrock_request_metadata_is_owned():
return True
@ -1421,8 +1445,8 @@ class BedrockModelInfo(BaseLLMModelInfo):
"""
Get the bedrock route for the given model.
GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by default
(``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
Grok and GPT 5.6 and newer go to bedrock-runtime's native OpenAI Chat Completions by
default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in;
``request_params`` (the caller's chat params) sends such a request to Converse when it needs a
feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family
model stays on Converse without the prefix.

6
litellm/model_prices_and_context_window_backup.json
View file

@ -80425,6 +80425,9 @@
},
"global.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
@ -80448,6 +80451,9 @@
},
"us.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_flex": 2.75e-07,
"cache_read_input_token_cost_priority": 9.625e-07,

6
model_prices_and_context_window.json
View file

@ -80425,6 +80425,9 @@
},
"global.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
@ -80448,6 +80451,9 @@
},
"us.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_flex": 2.75e-07,
"cache_read_input_token_cost_priority": 9.625e-07,

6
tests/integration/proxy_config.yaml
View file

@ -110,6 +110,12 @@ model_list:
api_base: http://127.0.0.1:8191
api_key: synthetic-bedrock-key
aws_region_name: us-east-1
- model_name: bedrock/global.xai.grok-4.7
litellm_params:
model: bedrock/global.xai.grok-4.7
api_base: http://127.0.0.1:8191
api_key: synthetic-bedrock-key
aws_region_name: us-east-1
- model_name: openai/gpt-5.4
litellm_params:
model: openai/gpt-5.4

63
tests/integration/translation/messages/bases/bedrock_chat_completions.py Normal file
View file

@ -0,0 +1,63 @@
from typing import Final
from integration.translation.case import TranslationTestCase
GROK_4_7_TEST_CASE: Final = TranslationTestCase(
scenario="basic",
litellm_endpoint="/v1/messages",
litellm_request={
"model": "bedrock/global.xai.grok-4.7",
"max_tokens": 1024,
"system": "You are a terse assistant.",
"messages": [{"role": "user", "content": "Say hello."}],
"cache": {"no-cache": True},
},
expected_provider_endpoint="/openai/v1/chat/completions",
expected_provider_headers={"authorization": "Bearer synthetic-bedrock-key", "content-type": "application/json"},
expected_provider_request={
"model": "global.xai.grok-4.7",
"messages": [
{"role": "system", "content": "You are a terse assistant."},
{"role": "user", "content": "Say hello."},
],
"max_completion_tokens": 1024,
},
mock_provider_response={
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {"annotations": [], "content": "Hello.", "refusal": None, "role": "assistant"},
}
],
"created": 1791505413,
"id": "chatcmpl-3xfhgmhrhhwplbepas3pjzbgjcccj4hx5tzplx7fw35giw3efznq",
"model": "global.xai.grok-4.7",
"object": "chat.completion",
"service_tier": "default",
"usage": {
"completion_tokens": 37,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 27,
"rejected_prediction_tokens": 0,
},
"prompt_tokens": 30,
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0},
"total_tokens": 67,
},
"system_fingerprint": None,
},
expected_litellm_response={
"id": "chatcmpl-3xfhgmhrhhwplbepas3pjzbgjcccj4hx5tzplx7fw35giw3efznq",
"type": "message",
"role": "assistant",
"model": "bedrock/global.xai.grok-4.7",
"stop_sequence": None,
"usage": {"input_tokens": 30, "output_tokens": 37},
"content": [{"type": "text", "text": "Hello."}],
"stop_reason": "end_turn",
"stop_details": None,
},
)

13
tests/integration/translation/messages/basic/test_messages_basic_bedrock_chat_completions.py Normal file
View file

@ -0,0 +1,13 @@
import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE
from integration.translation.runner import assert_translation
@pytest.mark.parametrize("case", [GROK_4_7_TEST_CASE], ids=lambda case: case.id)
def test_messages_basic_bedrock_chat_completions(
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
) -> None:
assert_translation(case, gateway, provider)

0
tests/integration/translation/messages/tool_search/__init__.py Normal file
View file

147
tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py Normal file
View file

@ -0,0 +1,147 @@
from dataclasses import replace
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE
from integration.translation.runner import assert_translation
GROK_4_7_TOOL_SEARCH_TEST_CASE: Final = replace(
GROK_4_7_TEST_CASE,
scenario="tool_search",
litellm_request={
"model": "bedrock/global.xai.grok-4.7",
"max_tokens": 1024,
"thinking": {"type": "adaptive"},
"output_config": {"effort": "low"},
"tools": [
{
"name": "ToolSearch",
"description": "Load deferred tools by name.",
"input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
},
{
"name": "WebSearch",
"description": "Search the web.",
"input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
},
],
"messages": [
{"role": "user", "content": "Load the web search tool."},
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "call_tool_search_1",
"name": "ToolSearch",
"input": {"query": "select:WebSearch"},
}
],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_tool_search_1",
"content": [{"type": "tool_reference", "tool_name": "WebSearch"}],
}
],
},
],
"cache": {"no-cache": True},
},
expected_provider_request={
"model": "global.xai.grok-4.7",
"messages": [
{"role": "user", "content": "Load the web search tool."},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_tool_search_1",
"type": "function",
"function": {"name": "ToolSearch", "arguments": '{"query": "select:WebSearch"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_tool_search_1", "content": ""},
],
"max_completion_tokens": 1024,
"tools": [
{
"type": "function",
"function": {
"name": "ToolSearch",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
"description": "Load deferred tools by name.",
},
},
{
"type": "function",
"function": {
"name": "WebSearch",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
"description": "Search the web.",
},
},
],
"reasoning_effort": "low",
},
mock_provider_response={
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"annotations": [],
"content": "The web search tool is now loaded and available for use.",
"refusal": None,
"role": "assistant",
},
}
],
"created": 1791505405,
"id": "chatcmpl-xevjeh2pxs6gjvcn2yz76piwdc4pi6fb2sqwb7psnwitm5higvvq",
"model": "global.xai.grok-4.7",
"object": "chat.completion",
"service_tier": "default",
"usage": {
"completion_tokens": 76,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 57,
"rejected_prediction_tokens": 0,
},
"prompt_tokens": 162,
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0},
"total_tokens": 238,
},
"system_fingerprint": None,
},
expected_litellm_response={
**GROK_4_7_TEST_CASE.expected_litellm_response,
"id": "chatcmpl-xevjeh2pxs6gjvcn2yz76piwdc4pi6fb2sqwb7psnwitm5higvvq",
"usage": {"input_tokens": 162, "output_tokens": 76},
"content": [{"type": "text", "text": "The web search tool is now loaded and available for use."}],
},
)
@pytest.mark.parametrize("case", [GROK_4_7_TOOL_SEARCH_TEST_CASE], ids=lambda case: case.id)
def test_messages_tool_search_bedrock_chat_completions(
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
) -> None:
assert_translation(case, gateway, provider)

0
tests/integration/translation/messages/tool_use/__init__.py Normal file
View file

107
tests/integration/translation/messages/tool_use/test_messages_tool_use_bedrock_chat_completions.py Normal file
View file

@ -0,0 +1,107 @@
import json
import re
from dataclasses import replace
from typing import Final
from unittest.mock import ANY
import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration._support.wire import Reply
from integration.translation.case import TranslationTestCase
from integration.translation.messages.bases.bedrock_chat_completions import GROK_4_7_TEST_CASE
from integration.translation.runner import assert_translation
GROK_4_7_TOOL_USE_TEST_CASE: Final = replace(
GROK_4_7_TEST_CASE,
scenario="tool_use",
litellm_request={
"model": "bedrock/global.xai.grok-4.7",
"max_tokens": 1024,
"tools": [
{
"name": "Read",
"description": "Read a file.",
"input_schema": {
"type": "object",
"properties": {"file_path": {"type": "string"}},
"required": ["file_path"],
},
}
],
"messages": [{"role": "user", "content": "Read notes.txt."}],
"cache": {"no-cache": True},
},
expected_provider_request={
"model": "global.xai.grok-4.7",
"messages": [{"role": "user", "content": "Read notes.txt."}],
"max_completion_tokens": 1024,
"tools": [
{
"type": "function",
"function": {
"name": "Read",
"parameters": {
"type": "object",
"properties": {"file_path": {"type": "string"}},
"required": ["file_path"],
},
"description": "Read a file.",
},
}
],
},
mock_provider_response={
"choices": [
{
"finish_reason": "tool_calls",
"index": 0,
"message": {
"annotations": [],
"content": None,
"refusal": None,
"role": "assistant",
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {"name": "Read", "arguments": '{"file_path":"notes.txt"}'},
}
],
},
}
],
"created": 1791509000,
"id": "chatcmpl-toolusegrok47",
"model": "global.xai.grok-4.7",
"object": "chat.completion",
"usage": {"completion_tokens": 20, "prompt_tokens": 40, "total_tokens": 60},
},
expected_litellm_response={
**GROK_4_7_TEST_CASE.expected_litellm_response,
"id": "chatcmpl-toolusegrok47",
"usage": {"input_tokens": 40, "output_tokens": 20},
"content": [{"type": "tool_use", "id": ANY, "name": "Read", "input": {"file_path": "notes.txt"}}],
"stop_reason": "tool_use",
},
)
@pytest.mark.parametrize("case", [GROK_4_7_TOOL_USE_TEST_CASE], ids=lambda case: case.id)
def test_messages_tool_use_bedrock_chat_completions(
case: TranslationTestCase, gateway: Gateway, provider: SharedProvider
) -> None:
assert_translation(case, gateway, provider)
def test_messages_tool_use_ids_are_unique_across_bedrock_chat_completions_turns(
gateway: Gateway, provider: SharedProvider
) -> None:
case: Final = GROK_4_7_TOOL_USE_TEST_CASE
reply: Final = Reply(body=json.dumps(case.mock_provider_response).encode())
provider.expect(reply, reply)
responses: Final = tuple(gateway.request("POST", case.litellm_endpoint, case.litellm_request) for _ in range(2))
assert len(provider.received()) == 2
tool_use_ids: Final = tuple(response.json()["content"][0]["id"] for response in responses)
assert len(set(tool_use_ids)) == 2
assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_use_id) for tool_use_id in tool_use_ids)

192
tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py
View file

@ -1,6 +1,7 @@
"""Bedrock Runtime Chat Completions: the default for GPT 5.6 and newer, ``bedrock/chat_completions/<model>`` for the rest."""
import json
import re
import httpx
import pytest
@ -63,7 +64,6 @@ def test_claude_stays_on_converse(local_cost_map):
@pytest.mark.parametrize(
"model",
[
"us.xai.grok-4.6",
"bedrock/openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"global.openai.gpt-5.5",
@ -88,7 +88,9 @@ def test_cost_map_row_listing_chat_completions_leaves_the_default_route_alone(mo
}
monkeypatch.setattr(litellm, "model_cost", {"openai.gpt-oss-20b-1:0": entry})
assert BedrockModelInfo.get_bedrock_route("bedrock/openai.gpt-oss-20b-1:0", {}) == "converse"
assert BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {}) == "chat_completions"
assert (
BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {}) == "chat_completions"
)
@pytest.mark.parametrize(
@ -103,10 +105,14 @@ def test_cost_map_row_listing_chat_completions_leaves_the_default_route_alone(mo
("us.openai.gpt-6.1-sol", ["/v1/chat/completions"], "chat_completions"),
("global.openai.gpt-10-sol", ["/v1/chat/completions"], "chat_completions"),
("openai.gpt-oss-120b-1:0", ["/v1/chat/completions"], "converse"),
("us.xai.grok-4.6", ["/v1/chat/completions"], "converse"),
("us.xai.grok-4.6", ["/v1/chat/completions"], "chat_completions"),
("global.xai.grok-4.7", ["/v1/chat/completions"], "chat_completions"),
("global.xai.grok-4.7", ["/v1/responses"], "converse"),
("global.xai.grok-4.7", [], "converse"),
("us-gov.xai.grok-4.6", ["/v1/chat/completions"], "chat_completions"),
],
)
def test_default_route_needs_gpt_56_or_newer_and_a_row_listing_chat_completions(
def test_default_route_needs_grok_or_gpt_56_or_newer_and_a_row_listing_chat_completions(
monkeypatch, model, supported_endpoints, expected_route
):
entry = {"litellm_provider": "bedrock_converse", "supported_endpoints": supported_endpoints}
@ -257,7 +263,7 @@ def _recording_client(**response_kwargs):
@pytest.mark.parametrize(
"model, model_path",
[
("bedrock/us.xai.grok-4.6", b"/model/us.xai.grok-4.6/converse"),
("bedrock/converse/us.xai.grok-4.6", b"/model/us.xai.grok-4.6/converse"),
("bedrock/openai.gpt-oss-20b-1:0", b"/model/openai.gpt-oss-20b-1%3A0/converse"),
("bedrock/global.openai.gpt-5.5", b"/model/global.openai.gpt-5.5/converse"),
],
@ -299,6 +305,7 @@ def test_completion_keeps_the_aws_request_id_as_a_provider_header(local_cost_map
assert response._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-native-1"
def test_region_path_sends_the_bare_model_id_to_the_path_region(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
litellm.completion(
@ -384,6 +391,10 @@ GPT_56_AND_NEWER_MODELS = (
"global.openai.gpt-6-luna",
"bedrock/global.openai.gpt-6.1-sol",
"us.openai.gpt-6.1-sol",
"global.xai.grok-4.6",
"bedrock/us.xai.grok-4.6",
"global.xai.grok-4.7",
"bedrock/us.xai.grok-4.7",
)
@ -430,10 +441,9 @@ def test_guardrail_config_falls_back_to_converse(local_cost_map, model):
[
{"additionalModelRequestFields": {"reasoning_effort": "high"}},
{"top_k": 40},
{"stop": ["END"]},
{"model_id": APPLICATION_INFERENCE_PROFILE_ARN},
],
ids=["additionalModelRequestFields", "top_k", "stop", "model_id"],
ids=["additionalModelRequestFields", "top_k", "model_id"],
)
def test_converse_extension_params_fall_back_to_converse(local_cost_map, model, request_params):
assert bedrock_request_needs_converse(model, request_params) is True
@ -441,6 +451,43 @@ def test_converse_extension_params_fall_back_to_converse(local_cost_map, model,
assert BedrockModelInfo.get_bedrock_route(model, {key: None for key in request_params}) == "chat_completions"
@pytest.mark.parametrize(
"model",
["chat_completions/openai.gpt-oss-20b-1:0", "bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6.1-sol"],
)
def test_stop_keeps_other_models_on_converse(local_cost_map, model):
assert bedrock_request_needs_converse(model, {"stop": ["END"]}) is True
assert BedrockModelInfo.get_bedrock_route(model, {"stop": ["END"]}) == "converse"
@pytest.mark.parametrize("model", ["bedrock/global.xai.grok-4.7", "bedrock/us.xai.grok-4.6"])
def test_stop_is_dropped_on_chat_completions_for_grok(local_cost_map, fake_aws_env, model):
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
response = litellm.completion(
model=model, messages=[{"role": "user", "content": "hello"}], stop=["</block>"], max_tokens=64, client=client
)
assert requests[0].url.raw_path == b"/openai/v1/chat/completions"
body = json.loads(requests[0].content)
assert "stop" not in body
assert body["max_completion_tokens"] == 64
assert response.choices[0].message.content == "ok"
def test_stop_is_dropped_on_converse_for_grok(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/converse/global.xai.grok-4.7",
messages=[{"role": "user", "content": "hello"}],
stop=["</block>"],
max_tokens=64,
client=client,
)
assert requests[0].url.raw_path == b"/model/global.xai.grok-4.7/converse"
assert json.loads(requests[0].content)["inferenceConfig"] == {"maxTokens": 64}
@pytest.mark.parametrize(
"model", ["bedrock/us.openai.gpt-5.6-sol", "global.openai.gpt-6-sol", "bedrock/chat_completions/us.xai.grok-4.6"]
)
@ -508,7 +555,10 @@ def test_bearer_api_key_is_sent_as_the_authorization_header(monkeypatch):
],
)
def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route):
assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params)
== expected_route
)
assert (
BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/us.openai.gpt-5.6-terra", request_params)
== expected_route
@ -535,14 +585,22 @@ def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_
],
)
def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params, expected_route):
assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route
assert BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", request_params) == "chat_completions"
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params)
== expected_route
)
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", request_params)
== "chat_completions"
)
def test_thinking_block_goes_to_converse(local_cost_map):
thinking = {"type": "enabled", "budget_tokens": 1024}
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse"
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": None}) == "chat_completions"
assert (
BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": None}) == "chat_completions"
)
def test_explicit_converse_prefix_wins_for_openai_models(local_cost_map):
@ -592,9 +650,7 @@ def _assert_remote_images_inlined(content):
def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch):
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
monkeypatch.setattr(
image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}"
)
monkeypatch.setattr(image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}")
body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request(
model="us.xai.grok-4.6",
messages=IMAGE_MESSAGES,
@ -760,7 +816,9 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod
ids=lambda value: value if isinstance(value, str) else next(iter(value)),
)
def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_map, fake_aws_env, model, param):
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/chat_completions/")))
requests, client = _recording_client(
json=_chat_completion_json("ok", model.removeprefix("bedrock/chat_completions/"))
)
with pytest.raises(litellm.UnsupportedParamsError, match=next(iter(param))):
litellm.completion(model=model, messages=[{"role": "user", "content": "hello"}], client=client, **param)
litellm.completion(
@ -901,12 +959,12 @@ def test_reasoning_tag_splitter_releases_a_false_tag_prefix():
assert _run_splitter(["<", "b>x"]) == ("", "<b>x")
def _stream_chunk(delta, finish_reason=None, index=0):
def _stream_chunk(delta, finish_reason=None, index=0, model="openai.gpt-oss-20b-1:0"):
return {
"id": "chatcmpl-test",
"object": "chat.completion.chunk",
"created": 1733529600,
"model": "openai.gpt-oss-20b-1:0",
"model": model,
"choices": [{"index": index, "delta": delta, "finish_reason": finish_reason}],
}
@ -1480,7 +1538,9 @@ def test_streaming_handler_keeps_a_literal_reasoning_tag_outside_gpt_oss(local_c
assert opened.choices[0].delta.content == "<reasoning>not thinking"
assert _reasoning_of(opened) is None
closed = handler.chunk_parser({**_stream_chunk({"content": "</reasoning> Hello"}, finish_reason="stop"), "model": model})
closed = handler.chunk_parser(
{**_stream_chunk({"content": "</reasoning> Hello"}, finish_reason="stop"), "model": model}
)
assert closed.choices[0].delta.content == "</reasoning> Hello"
assert _reasoning_of(closed) is None
@ -1540,3 +1600,99 @@ def test_reasoning_tag_split_is_read_from_the_cost_map(
)
assert chunk.choices[0].delta.content == expected_content
assert _reasoning_of(chunk) == expected_reasoning
def _tool_call(tool_call_id, name):
return {"id": tool_call_id, "type": "function", "function": {"name": name, "arguments": "{}"}}
def test_positional_tool_call_ids_are_minted_unique_per_response(local_cost_map, fake_aws_env):
reply = _chat_completion_json(
None,
"global.xai.grok-4.7",
tool_calls=[_tool_call("call_0", "read_a"), _tool_call("call_1", "read_b")],
)
_, client = _recording_client(json=reply)
responses = [
litellm.completion(
model="bedrock/chat_completions/global.xai.grok-4.7",
messages=[{"role": "user", "content": "hello"}],
client=client,
)
for _ in range(2)
]
ids = [tool_call.id for response in responses for tool_call in response.choices[0].message.tool_calls]
assert len(set(ids)) == 4
assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids)
assert [tool_call.function.name for tool_call in responses[0].choices[0].message.tool_calls] == ["read_a", "read_b"]
def test_provider_unique_tool_call_ids_pass_through(local_cost_map, fake_aws_env):
reply = _chat_completion_json(
None, "openai.gpt-oss-20b-1:0", tool_calls=[_tool_call("chatcmpl-tool-90090f0c1c521528", "read_a")]
)
_, client = _recording_client(json=reply)
response = litellm.completion(
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
client=client,
)
assert response.choices[0].message.tool_calls[0].id == "chatcmpl-tool-90090f0c1c521528"
def _streamed_tool_call_ids(tool_call_deltas, model="global.xai.grok-4.7"):
handler = BedrockRuntimeChatCompletionsStreamingHandler(streaming_response=iter(()), sync_stream=True)
return [
tool_call.id
for delta in tool_call_deltas
for tool_call in handler.chunk_parser(_stream_chunk({"tool_calls": [delta]}, model=model))
.choices[0]
.delta.tool_calls
]
def test_streamed_positional_tool_call_ids_are_minted_once_per_tool_call(local_cost_map):
deltas = [
{"index": 0, **_tool_call("call_0", "read_a")},
{"index": 0, "function": {"arguments": '{"x":1}'}},
{"index": 0, "id": "call_0", "function": {"arguments": "}"}},
{"index": 1, **_tool_call("call_1", "read_b")},
]
first_stream = _streamed_tool_call_ids(deltas)
second_stream = _streamed_tool_call_ids(deltas)
assert first_stream[0] == first_stream[2]
assert first_stream[1] is None
assert len({first_stream[0], first_stream[3], second_stream[0], second_stream[3]}) == 4
assert all(re.fullmatch(r"call_[0-9a-f]{32}", first_stream[i]) for i in (0, 3))
def test_streamed_provider_unique_tool_call_ids_pass_through(local_cost_map):
assert _streamed_tool_call_ids(
[{"index": 0, **_tool_call("chatcmpl-tool-8c9232df5019ff4f", "read_a")}], model="openai.gpt-oss-20b-1:0"
) == ["chatcmpl-tool-8c9232df5019ff4f"]
def test_streamed_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map):
deltas = [{"index": 0, **_tool_call("call_0", "read_a")}, {"index": 1, **_tool_call("call_1", "read_b")}]
ids = _streamed_tool_call_ids(deltas, model="global.openai.gpt-5.6-sol")
assert len(set(ids)) == 2
assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids)
def test_positional_tool_call_ids_are_minted_for_gpt_too(local_cost_map, fake_aws_env):
reply = _chat_completion_json(
None, "global.openai.gpt-5.6-sol", tool_calls=[_tool_call("call_0", "read_a"), _tool_call("call_1", "read_b")]
)
_, client = _recording_client(json=reply)
response = litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol", messages=[{"role": "user", "content": "hello"}], client=client
)
ids = [tool_call.id for tool_call in response.choices[0].message.tool_calls]
assert len(set(ids)) == 2
assert all(re.fullmatch(r"call_[0-9a-f]{32}", tool_call_id) for tool_call_id in ids)