scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(bedrock): make native chat completions an opt-in bedrock/chat_completions/ route

Browse source 
Bare Bedrock OpenAI and Grok model ids stay on Converse as on main. The
bedrock/chat_completions/<model> prefix opts a deployment into bedrock-runtime's
/openai/v1/chat/completions, and a request carrying a Converse-only param
still falls back to Converse. The cost map no longer decides the route.
This commit is contained in:
mateo-berri 2026-09-28 21:29:04 -07:00
parent ef03d19aae
commit 3a390e6206
5 changed files with 156 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -3,14 +3,14 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime.
AWS serves this surface at
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
for the models whose price-map ``supported_endpoints`` lists ``/v1/chat/completions``
(Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions
instead of being rewritten to Converse.
for Grok 4.6, gpt-oss and the GPT-5.6 family. The ``chat_completions/`` route prefix
opts a model in, so chat completions stay chat completions instead of being rewritten
to Converse; without it these models stay on Converse.
Usage: model="us.xai.grok-4.6", model="bedrock/openai.gpt-oss-20b-1:0" or
model="bedrock/global.openai.gpt-5.6-sol". Explicit ``bedrock/converse/...``
still uses Converse, and so does a request that needs a Converse-only feature
(``bedrock_request_needs_converse`` in ``common_utils``).
Usage: model="bedrock/chat_completions/openai.gpt-oss-20b-1:0" or
model="bedrock/chat_completions/global.openai.gpt-5.6-sol". A request that needs a
Converse-only feature (``bedrock_request_needs_converse`` in ``common_utils``) is
still served by Converse.
"""
from collections.abc import AsyncIterator, Iterator, Mapping

48
litellm/llms/bedrock/common_utils.py
View file

@ -801,7 +801,7 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool:
def strip_bedrock_routing_prefix(model: str) -> str:
"""Strip LiteLLM routing prefixes from model name."""
for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
for prefix in ["bedrock/", "chat_completions/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]:
if model.startswith(prefix):
model = model.split("/", 1)[1]
return model
@ -820,9 +820,6 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]:
return None, stripped
BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse"))
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
litellm.model_cost.get(key)
@ -834,32 +831,6 @@ def _bedrock_price_map_flag(model: str, flag: str) -> bool:
return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model))
def _bedrock_runtime_row_lists_chat_completions(entry: Mapping[str, object]) -> bool:
endpoints: Final = entry.get("supported_endpoints")
return (
entry.get("litellm_provider") in BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS
and isinstance(endpoints, (list, tuple))
and "/v1/chat/completions" in endpoints
)
def uses_bedrock_runtime_chat_completions(model: str) -> bool:
"""Whether this Bedrock model should use runtime native Chat Completions.
Data-driven from ``/v1/chat/completions`` in the price-map row's ``supported_endpoints``,
the same per-model signal ``bedrock_supports_openai_responses`` reads for ``/v1/responses``,
so onboarding a model is a JSON change. Explicit ``converse/`` still wins in
``get_bedrock_route`` because prefix routes are checked first, and a request
that needs a Converse-only feature (``bedrock_request_needs_converse``) is
served by Converse even on a listed model. Only a bedrock-runtime row counts: a
``bedrock_mantle`` row lists the endpoints of the Mantle host, not this one.
"""
return any(
entry is not None and _bedrock_runtime_row_lists_chat_completions(entry)
for entry in _bedrock_price_map_entries(model)
)
def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:
"""Whether AWS's native Chat Completions serves this model's function tools with any ``reasoning_effort``.
@ -908,7 +879,7 @@ def _response_format_needs_converse(model: str, response_format: object) -> bool
def bedrock_request_needs_converse(model: str, request_params: Mapping[str, object]) -> bool:
"""Whether a request on a runtime-Chat-Completions model must still be served by Converse.
"""Whether a request on the opt-in ``chat_completions/`` route must still be served by Converse.
Converse-shaped body keys (``BEDROCK_CONVERSE_ONLY_REQUEST_KEYS``, the Anthropic-style ``thinking``
block and the ``additionalModelRequestFields`` / ``top_k`` extension params included, which only Converse
@ -1314,8 +1285,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
"""
Get the bedrock route for the given model.
``request_params`` (the caller's chat params) lets a runtime Chat Completions
model fall back to Converse for the requests only Converse can serve.
``chat_completions/`` opts a model into bedrock-runtime's native OpenAI Chat Completions;
``request_params`` (the caller's chat params) sends such a request to Converse when it
needs a feature only Converse serves. Without the prefix, OpenAI-family models stay on Converse.
"""
route_mappings: dict[
str,
@ -1351,6 +1323,11 @@ class BedrockModelInfo(BaseLLMModelInfo):
if BedrockModelInfo._model_has_route_prefix(model, prefix):
return route_type
if BedrockModelInfo._model_has_route_prefix(model, "chat_completions/"):
if request_params is not None and bedrock_request_needs_converse(model, request_params):
return "converse"
return "chat_completions"
# Check for nova spec prefixes (nova/ and nova-2/)
_model_after_bedrock: Final = model.replace("bedrock/", "", 1)
if _model_after_bedrock.startswith("nova-2/") or _model_after_bedrock.startswith("nova/"):
@ -1359,11 +1336,6 @@ class BedrockModelInfo(BaseLLMModelInfo):
if is_bedrock_application_inference_profile_arn(model):
return "converse"
if uses_bedrock_runtime_chat_completions(model) and not (
request_params is not None and bedrock_request_needs_converse(model, request_params)
):
return "chat_completions"
base_model: Final = BedrockModelInfo.get_base_model(model)
alt_model: Final = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
if base_model in litellm.bedrock_converse_models or alt_model in litellm.bedrock_converse_models:

2
litellm/main.py
View file

@ -4234,7 +4234,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchRes
provider_config=provider_config,
)
elif bedrock_route == "converse":
model = model.replace("converse/", "")
model = model.replace("converse/", "").replace("chat_completions/", "")
response = bedrock_converse_chat_completion.completion(
model=model,
messages=messages,

220
tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py
View file

@ -1,4 +1,4 @@
"""Native Bedrock Runtime Chat Completions: Grok, gpt-oss and GPT-5.6 stay on /openai/v1/chat/completions."""
"""Opt-in Bedrock Runtime Chat Completions: ``bedrock/chat_completions/<model>`` posts to /openai/v1/chat/completions."""
import json
@ -21,7 +21,6 @@ from litellm.llms.bedrock.common_utils import (
bedrock_request_needs_converse,
bedrock_route_for_request,
get_bedrock_chat_config,
uses_bedrock_runtime_chat_completions,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
@ -38,14 +37,13 @@ def local_cost_map(monkeypatch):
@pytest.mark.parametrize(
"model",
[
"us.xai.grok-4.6",
"global.xai.grok-4.6",
"us-gov.xai.grok-4.6",
"bedrock/us.xai.grok-4.6",
"chat_completions/us.xai.grok-4.6",
"chat_completions/global.xai.grok-4.6",
"chat_completions/us-gov.xai.grok-4.6",
"bedrock/chat_completions/us.xai.grok-4.6",
],
)
def test_grok_runtime_models_use_chat_completions_route(local_cost_map, model):
assert uses_bedrock_runtime_chat_completions(model) is True
def test_chat_completions_prefix_opts_grok_into_the_native_route(local_cost_map, model):
assert BedrockModelInfo.get_bedrock_route(model) == "chat_completions"
assert isinstance(get_bedrock_chat_config(model), AmazonBedrockRuntimeChatCompletionsConfig)
@ -56,31 +54,44 @@ def test_explicit_converse_prefix_still_uses_converse(local_cost_map):
def test_claude_stays_on_converse(local_cost_map):
assert uses_bedrock_runtime_chat_completions("us.anthropic.claude-3-sonnet-20240229-v1:0") is False
assert BedrockModelInfo.get_bedrock_route("us.anthropic.claude-3-sonnet-20240229-v1:0") == "converse"
@pytest.mark.parametrize(
"entry",
"model",
[
{"litellm_provider": "bedrock_converse"},
{"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/responses"]},
{"litellm_provider": "bedrock_converse", "supports_bedrock_runtime_chat_completions": True},
{"litellm_provider": "bedrock_mantle", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]},
{"litellm_provider": "openai", "supported_endpoints": ["/v1/chat/completions"]},
"us.xai.grok-4.6",
"bedrock/openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"global.openai.gpt-5.6-sol",
"bedrock/us.openai.gpt-5.6-terra",
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0",
],
)
def test_chat_completions_missing_from_supported_endpoints_means_no_chat_completions_route(monkeypatch, entry):
monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry})
assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is False
assert BedrockModelInfo.get_bedrock_route("us.xai.grok-4.6") == "converse"
def test_models_without_the_prefix_stay_on_converse(local_cost_map, model):
assert BedrockModelInfo.get_bedrock_route(model) == "converse"
assert BedrockModelInfo.get_bedrock_route(model, {}) == "converse"
assert isinstance(get_bedrock_chat_config(model), litellm.AmazonConverseConfig)
def test_chat_completions_in_supported_endpoints_opts_into_the_native_route(monkeypatch):
entry = {"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}
monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry})
assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is True
assert BedrockModelInfo.get_bedrock_route("bedrock/us.xai.grok-4.6") == "chat_completions"
def test_cost_map_row_listing_chat_completions_leaves_the_default_route_alone(monkeypatch):
entry = {
"litellm_provider": "bedrock_converse",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": True,
"supports_bedrock_runtime_chat_completions_response_format": True,
}
monkeypatch.setattr(litellm, "model_cost", {"openai.gpt-oss-20b-1:0": entry})
assert BedrockModelInfo.get_bedrock_route("bedrock/openai.gpt-oss-20b-1:0", {}) == "converse"
assert BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {}) == "chat_completions"
@pytest.mark.parametrize("model", ["global.openai.gpt-5.6-sol", "openai.gpt-oss-20b-1:0", "us.xai.grok-4.6"])
def test_chat_completions_prefix_prices_like_the_bare_model(local_cost_map, model):
prefixed = litellm.get_model_info(model=f"bedrock/chat_completions/{model}")
bare = litellm.get_model_info(model=f"bedrock/{model}")
assert prefixed["input_cost_per_token"] == bare["input_cost_per_token"] > 0
assert prefixed["output_cost_per_token"] == bare["output_cost_per_token"] > 0
def test_complete_url_is_runtime_openai_chat_completions(monkeypatch):
@ -113,7 +124,7 @@ def test_project_id_is_not_sent_as_openai_project_header():
cfg = AmazonBedrockRuntimeChatCompletionsConfig()
headers = cfg.validate_environment(
headers={},
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
optional_params={},
litellm_params={"aws_bedrock_project_id": "proj_from_config"},
@ -125,7 +136,7 @@ def test_project_id_is_not_sent_as_openai_project_header():
def test_transform_request_is_openai_chat_body_not_converse():
cfg = AmazonBedrockRuntimeChatCompletionsConfig()
body = cfg.transform_request(
model="bedrock/us.xai.grok-4.6",
model="bedrock/chat_completions/us.xai.grok-4.6",
messages=[{"role": "user", "content": "hello"}],
optional_params={"temperature": 0.2, "aws_region_name": "us-east-1"},
litellm_params={},
@ -178,10 +189,26 @@ def _recording_client(**response_kwargs):
return requests, HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle)))
@pytest.mark.parametrize(
"model, model_path",
[
("bedrock/us.xai.grok-4.6", b"/model/us.xai.grok-4.6/converse"),
("bedrock/openai.gpt-oss-20b-1:0", b"/model/openai.gpt-oss-20b-1%3A0/converse"),
("bedrock/global.openai.gpt-5.6-sol", b"/model/global.openai.gpt-5.6-sol/converse"),
],
)
def test_completion_without_the_prefix_posts_converse(local_cost_map, fake_aws_env, model, model_path):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(model=model, messages=[{"role": "user", "content": "hello"}], client=client)
assert response.choices[0].message.content == "ok"
assert [request.url.raw_path for request in requests] == [model_path]
def test_completion_posts_runtime_chat_completions(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "us.xai.grok-4.6"))
response = litellm.completion(
model="us.xai.grok-4.6",
model="bedrock/chat_completions/us.xai.grok-4.6",
messages=[{"role": "user", "content": "hello"}],
client=client,
)
@ -198,7 +225,7 @@ def test_completion_posts_runtime_chat_completions(local_cost_map, fake_aws_env)
def test_region_path_sends_the_bare_model_id_to_the_path_region(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
litellm.completion(
model="bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/us-gov-west-1/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
client=client,
)
@ -211,7 +238,7 @@ def test_region_path_sends_the_bare_model_id_to_the_path_region(local_cost_map,
def test_explicit_aws_region_name_wins_over_the_region_path(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
litellm.completion(
model="bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/us-gov-west-1/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
aws_region_name="us-gov-east-1",
client=client,
@ -222,6 +249,21 @@ def test_explicit_aws_region_name_wins_over_the_region_path(local_cost_map, fake
assert "/us-gov-east-1/bedrock/aws4_request" in requests[0].headers["Authorization"]
def test_region_path_falls_back_to_converse_in_the_path_region(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/chat_completions/us-gov-west-1/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
stop=["END"],
client=client,
)
assert requests[0].url.host == "bedrock-runtime.us-gov-west-1.amazonaws.com"
assert requests[0].url.raw_path == b"/model/openai.gpt-oss-20b-1%3A0/converse"
assert json.loads(requests[0].content)["inferenceConfig"]["stopSequences"] == ["END"]
assert "/us-gov-west-1/bedrock/aws4_request" in requests[0].headers["Authorization"]
OPENAI_RUNTIME_MODELS = (
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
@ -244,26 +286,26 @@ GET_WEATHER_TOOL = {
@pytest.mark.parametrize(
"model",
[
*OPENAI_RUNTIME_MODELS,
"bedrock/openai.gpt-oss-20b-1:0",
"us-gov.openai.gpt-oss-20b-1:0",
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0",
"us-gov-east-1/openai.gpt-oss-120b-1:0",
*(f"chat_completions/{model}" for model in OPENAI_RUNTIME_MODELS),
"bedrock/chat_completions/openai.gpt-oss-20b-1:0",
"chat_completions/us-gov.openai.gpt-oss-20b-1:0",
"bedrock/chat_completions/us-gov-west-1/openai.gpt-oss-20b-1:0",
"chat_completions/us-gov-east-1/openai.gpt-oss-120b-1:0",
],
)
def test_openai_runtime_models_use_chat_completions_route(local_cost_map, model):
assert uses_bedrock_runtime_chat_completions(model) is True
assert BedrockModelInfo.get_bedrock_route(model) == "chat_completions"
assert isinstance(get_bedrock_chat_config(model), AmazonBedrockRuntimeChatCompletionsConfig)
@pytest.mark.parametrize("model", ["us.amazon.nova-micro-v1:0", "us.anthropic.claude-haiku-4-5-20251001-v1:0"])
def test_nova_and_claude_stay_on_converse(local_cost_map, model):
assert uses_bedrock_runtime_chat_completions(model) is False
assert BedrockModelInfo.get_bedrock_route(model, {"tools": [GET_WEATHER_TOOL]}) == "converse"
@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "global.openai.gpt-5.6-sol"])
@pytest.mark.parametrize(
"model", ["chat_completions/openai.gpt-oss-20b-1:0", "bedrock/chat_completions/global.openai.gpt-5.6-sol"]
)
def test_guardrail_config_falls_back_to_converse(local_cost_map, model):
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
assert bedrock_request_needs_converse(model, {"guardrailConfig": guardrail}) is True
@ -272,7 +314,12 @@ def test_guardrail_config_falls_back_to_converse(local_cost_map, model):
@pytest.mark.parametrize(
"model", ["openai.gpt-oss-20b-1:0", "us.xai.grok-4.6", "global.openai.gpt-5.6-sol"]
"model",
[
"chat_completions/openai.gpt-oss-20b-1:0",
"chat_completions/us.xai.grok-4.6",
"bedrock/chat_completions/global.openai.gpt-5.6-sol",
],
)
@pytest.mark.parametrize(
"request_params",
@ -299,15 +346,18 @@ def test_converse_extension_params_fall_back_to_converse(local_cost_map, model,
],
)
def test_gpt56_tools_need_reasoning_none_on_chat_completions(local_cost_map, request_params, expected_route):
assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == expected_route
assert BedrockModelInfo.get_bedrock_route("bedrock/us.openai.gpt-5.6-terra", request_params) == expected_route
assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route
assert (
BedrockModelInfo.get_bedrock_route("bedrock/chat_completions/us.openai.gpt-5.6-terra", request_params)
== expected_route
)
@pytest.mark.parametrize("reasoning_effort", ["low", "high", None])
def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_cost_map, reasoning_effort):
params = {"tools": [GET_WEATHER_TOOL], "reasoning_effort": reasoning_effort}
assert bedrock_request_needs_converse("openai.gpt-oss-120b-1:0", params) is False
assert BedrockModelInfo.get_bedrock_route("openai.gpt-oss-120b-1:0", params) == "chat_completions"
assert BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", params) == "chat_completions"
@pytest.mark.parametrize(
@ -320,14 +370,14 @@ def test_gpt_oss_tools_with_any_reasoning_effort_stay_on_chat_completions(local_
],
)
def test_gpt56_legacy_functions_route_like_tools(local_cost_map, request_params, expected_route):
assert BedrockModelInfo.get_bedrock_route("global.openai.gpt-5.6-sol", request_params) == expected_route
assert BedrockModelInfo.get_bedrock_route("openai.gpt-oss-120b-1:0", request_params) == "chat_completions"
assert BedrockModelInfo.get_bedrock_route("chat_completions/global.openai.gpt-5.6-sol", request_params) == expected_route
assert BedrockModelInfo.get_bedrock_route("chat_completions/openai.gpt-oss-120b-1:0", request_params) == "chat_completions"
def test_thinking_block_goes_to_converse(local_cost_map):
thinking = {"type": "enabled", "budget_tokens": 1024}
assert BedrockModelInfo.get_bedrock_route("us.xai.grok-4.6", {"thinking": thinking}) == "converse"
assert BedrockModelInfo.get_bedrock_route("us.xai.grok-4.6", {"thinking": None}) == "chat_completions"
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": thinking}) == "converse"
assert BedrockModelInfo.get_bedrock_route("chat_completions/us.xai.grok-4.6", {"thinking": None}) == "chat_completions"
def test_explicit_converse_prefix_wins_for_openai_models(local_cost_map):
@ -500,15 +550,15 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod
@pytest.mark.parametrize(
"model, param",
[
("bedrock/global.openai.gpt-5.6-sol", {"frequency_penalty": 0.5}),
("bedrock/global.openai.gpt-5.6-sol", {"logprobs": True, "top_logprobs": 2}),
("bedrock/us.xai.grok-4.6", {"presence_penalty": 0.5}),
("bedrock/openai.gpt-oss-20b-1:0", {"logit_bias": {"1": 1}}),
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"frequency_penalty": 0.5}),
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"logprobs": True, "top_logprobs": 2}),
("bedrock/chat_completions/us.xai.grok-4.6", {"presence_penalty": 0.5}),
("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {"logit_bias": {"1": 1}}),
],
ids=lambda value: value if isinstance(value, str) else next(iter(value)),
)
def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_map, fake_aws_env, model, param):
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/chat_completions/")))
with pytest.raises(litellm.UnsupportedParamsError, match=next(iter(param))):
litellm.completion(model=model, messages=[{"role": "user", "content": "hello"}], client=client, **param)
litellm.completion(
@ -651,7 +701,7 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos
json=_chat_completion_json("<reasoning>plan</reasoning>\n\nHi", "openai.gpt-oss-20b-1:0")
)
response = litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
max_tokens=64,
reasoning_effort="low",
@ -673,7 +723,7 @@ def test_gpt_oss_completion_hits_chat_completions_and_splits_reasoning(local_cos
def test_gpt56_tools_with_reasoning_effort_go_to_converse(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
response = litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
@ -691,7 +741,7 @@ def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map
]
requests, client = _recording_client(json=_chat_completion_json(None, "global.openai.gpt-5.6-sol", tool_calls))
response = litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "weather in Paris"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="none",
@ -720,7 +770,7 @@ def test_gpt56_tools_with_reasoning_none_stay_on_chat_completions(local_cost_map
def test_converse_only_request_keys_go_to_converse(local_cost_map, fake_aws_env, converse_only_param):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
client=client,
**converse_only_param,
@ -739,7 +789,7 @@ def test_operator_owned_request_metadata_goes_to_converse(local_cost_map, fake_a
monkeypatch.setattr(litellm, "bedrock_request_metadata_fields", ["user_api_key_team_alias"])
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
metadata={"user_api_key_team_alias": "search"},
client=client,
@ -752,7 +802,7 @@ def test_operator_owned_request_metadata_goes_to_converse(local_cost_map, fake_a
def test_dropped_converse_only_key_keeps_the_request_on_chat_completions(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
guardrailConfig={"guardrailIdentifier": "gr-1", "guardrailVersion": "1"},
additional_drop_params=["guardrailConfig"],
@ -770,7 +820,7 @@ def test_dropped_converse_only_key_keeps_the_request_on_chat_completions(local_c
def test_dropped_tools_keep_gpt56_reasoning_request_on_chat_completions(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "global.openai.gpt-5.6-sol"))
litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
tools=[GET_WEATHER_TOOL],
reasoning_effort="low",
@ -787,7 +837,7 @@ def test_dropped_tools_keep_gpt56_reasoning_request_on_chat_completions(local_co
def test_legacy_functions_stay_on_chat_completions(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
functions=[GET_WEATHER_TOOL["function"]],
client=client,
@ -801,14 +851,14 @@ def test_gpt56_legacy_functions_with_reasoning_fall_back_to_converse(local_cost_
requests, client = _recording_client(json=CONVERSE_JSON)
with pytest.raises(litellm.UnsupportedParamsError, match="functions"):
litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
functions=[GET_WEATHER_TOOL["function"]],
reasoning_effort="low",
client=client,
)
litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "hello"}],
functions=[GET_WEATHER_TOOL["function"]],
reasoning_effort="low",
@ -826,7 +876,7 @@ def test_grok_thinking_block_is_served_by_converse(local_cost_map, fake_aws_env)
requests, client = _recording_client(json=CONVERSE_JSON)
thinking = {"type": "enabled", "budget_tokens": 1024}
litellm.completion(
model="bedrock/us.xai.grok-4.6",
model="bedrock/chat_completions/us.xai.grok-4.6",
messages=[{"role": "user", "content": "hello"}],
thinking=thinking,
client=client,
@ -841,14 +891,14 @@ def test_converse_fallback_validates_against_converse_params(local_cost_map, fak
guardrail = {"guardrailIdentifier": "gr-1", "guardrailVersion": "1"}
with pytest.raises(litellm.UnsupportedParamsError, match="seed"):
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
guardrailConfig=guardrail,
seed=7,
client=client,
)
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
guardrailConfig=guardrail,
seed=7,
@ -864,13 +914,13 @@ def test_n_is_rejected_before_reaching_chat_completions(local_cost_map, fake_aws
requests, client = _recording_client(json=_chat_completion_json("ok", "openai.gpt-oss-20b-1:0"))
with pytest.raises(litellm.UnsupportedParamsError, match="'n'"):
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
n=2,
client=client,
)
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
n=2,
drop_params=True,
@ -892,7 +942,7 @@ def test_gpt_oss_streaming_completion_splits_reasoning(local_cost_map, fake_aws_
)
requests, client = _recording_client(content=_sse(chunks), headers={"content-type": "text/event-stream"})
stream = litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "hello"}],
stream=True,
client=client,
@ -931,7 +981,9 @@ class Answer(BaseModel):
word: str
@pytest.mark.parametrize("model", ["openai.gpt-oss-20b-1:0", "bedrock/openai.gpt-oss-120b-1:0"])
@pytest.mark.parametrize(
"model", ["chat_completions/openai.gpt-oss-20b-1:0", "bedrock/chat_completions/openai.gpt-oss-120b-1:0"]
)
@pytest.mark.parametrize(
"response_format, expected_route",
[
@ -949,7 +1001,11 @@ def test_gpt_oss_response_format_falls_back_to_converse(local_cost_map, model, r
assert BedrockModelInfo.get_bedrock_route(model, params) == expected_route
RESPONSE_FORMAT_ENFORCING_MODELS = ["global.openai.gpt-5.6-sol", "us.xai.grok-4.6", "bedrock/us-gov.xai.grok-4.6"]
RESPONSE_FORMAT_ENFORCING_MODELS = [
"chat_completions/global.openai.gpt-5.6-sol",
"chat_completions/us.xai.grok-4.6",
"bedrock/chat_completions/us-gov.xai.grok-4.6",
]
JSON_OBJECT_WITH_RESPONSE_SCHEMA = {
@ -980,7 +1036,7 @@ def test_json_object_keeps_converse_where_aws_would_demand_the_word_json(local_c
assert BedrockModelInfo.get_bedrock_route(model, params) == "converse"
SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0"
SYNTHETIC_NATIVE_MODEL = "chat_completions/vendor.native-model-v1:0"
@pytest.mark.parametrize(
@ -1008,12 +1064,8 @@ SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0"
],
)
def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_flags, request_params, needs_converse):
entry = {
"litellm_provider": "bedrock_converse",
"supported_endpoints": ["/v1/chat/completions"],
**capability_flags,
}
monkeypatch.setattr(litellm, "model_cost", {SYNTHETIC_NATIVE_MODEL: entry})
entry = {"litellm_provider": "bedrock_converse", **capability_flags}
monkeypatch.setattr(litellm, "model_cost", {"vendor.native-model-v1:0": entry})
assert bedrock_request_needs_converse(SYNTHETIC_NATIVE_MODEL, request_params) is needs_converse
route = bedrock_route_for_request(SYNTHETIC_NATIVE_MODEL, request_params, None)
assert (route == "chat_completions") is (not needs_converse)
@ -1021,18 +1073,16 @@ def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_fla
def test_route_for_request_ignores_dropped_params(local_cost_map):
params = {"response_format": RESPONSE_FORMAT_JSON_SCHEMA, "guardrailConfig": {"guardrailIdentifier": "gr-1"}}
assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, None) == "converse"
assert bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig"]) == "converse"
assert (
bedrock_route_for_request("openai.gpt-oss-20b-1:0", params, ["guardrailConfig", "response_format"])
== "chat_completions"
)
model = "chat_completions/openai.gpt-oss-20b-1:0"
assert bedrock_route_for_request(model, params, None) == "converse"
assert bedrock_route_for_request(model, params, ["guardrailConfig"]) == "converse"
assert bedrock_route_for_request(model, params, ["guardrailConfig", "response_format"]) == "chat_completions"
def test_gpt_oss_response_format_goes_to_converse_with_json_tool_call(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/openai.gpt-oss-20b-1:0",
model="bedrock/chat_completions/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "Reply with the single word pong."}],
response_format=RESPONSE_FORMAT_JSON_SCHEMA,
max_tokens=64,
@ -1051,7 +1101,7 @@ def test_gpt_oss_response_format_goes_to_converse_with_json_tool_call(local_cost
def test_gpt56_response_format_is_sent_as_is_on_chat_completions(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=_chat_completion_json('{"word": "pong"}', "global.openai.gpt-5.6-sol"))
response = litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "Reply with the single word pong."}],
response_format=RESPONSE_FORMAT_JSON_SCHEMA,
client=client,
@ -1065,7 +1115,7 @@ def test_gpt56_response_format_is_sent_as_is_on_chat_completions(local_cost_map,
def test_gpt56_schema_less_json_object_goes_to_converse_without_a_schema_tool(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "Reply with the single word pong."}],
response_format={"type": "json_object"},
max_tokens=64,
@ -1082,7 +1132,7 @@ def test_gpt56_schema_less_json_object_goes_to_converse_without_a_schema_tool(lo
def test_gpt56_json_object_with_response_schema_goes_to_converse_as_a_json_tool(local_cost_map, fake_aws_env):
requests, client = _recording_client(json=CONVERSE_JSON)
litellm.completion(
model="bedrock/global.openai.gpt-5.6-sol",
model="bedrock/chat_completions/global.openai.gpt-5.6-sol",
messages=[{"role": "user", "content": "Reply with the single word pong."}],
response_format=JSON_OBJECT_WITH_RESPONSE_SCHEMA,
max_tokens=64,

9
tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -138,12 +138,9 @@ def _bedrock_response(model, usage):
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_profiles_never_route_to_invoke(profile, local_model_cost_map):
"""GPT-5.6 is served by bedrock-runtime's native Chat Completions, and by Converse when
the request carries function tools without reasoning_effort "none", never by Invoke."""
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "chat_completions"
tools_with_reasoning = {"tools": [{"type": "function", "function": {"name": "f"}}], "reasoning_effort": "low"}
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}", tools_with_reasoning) == "converse"
def test_bedrock_gpt_5_6_profiles_route_to_converse(profile, local_model_cost_map):
"""GPT-5.6 is served by Converse on bedrock-runtime, never by Invoke."""
assert BedrockModelInfo.get_bedrock_route(f"bedrock/{profile.model_id}") == "converse"
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)