diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 0b52bea94de..23e69f66492 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -582,8 +582,6 @@ pub struct ModelInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub supports_audio_output: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub supports_bedrock_runtime_chat_completions: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub supports_bedrock_runtime_chat_completions_response_format: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option, diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 11e2467b093..4c5e5768119 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime. AWS serves this surface at ``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions`` -for the models whose price-map entry sets ``supports_bedrock_runtime_chat_completions`` +for the models whose price-map ``supported_endpoints`` lists ``/v1/chat/completions`` (Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions instead of being rewritten to Converse. diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 09e62dc393b..4492052b3c4 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -829,24 +829,44 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]: return None, stripped -def _bedrock_price_map_flag(model: str, flag: str) -> bool: - entries: Final = ( +BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse")) + + +def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: + return tuple( litellm.model_cost.get(key) for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1]) ) - return any(entry is not None and entry.get(flag) is True for entry in entries) + + +def _bedrock_price_map_flag(model: str, flag: str) -> bool: + return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model)) + + +def _bedrock_runtime_row_lists_chat_completions(entry: Mapping[str, object]) -> bool: + endpoints: Final = entry.get("supported_endpoints") + return ( + entry.get("litellm_provider") in BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS + and isinstance(endpoints, (list, tuple)) + and "/v1/chat/completions" in endpoints + ) def uses_bedrock_runtime_chat_completions(model: str) -> bool: """Whether this Bedrock model should use runtime native Chat Completions. - Data-driven from the price-map ``supports_bedrock_runtime_chat_completions`` flag + Data-driven from ``/v1/chat/completions`` in the price-map row's ``supported_endpoints``, + the same per-model signal ``bedrock_supports_openai_responses`` reads for ``/v1/responses``, so onboarding a model is a JSON change. Explicit ``converse/`` still wins in ``get_bedrock_route`` because prefix routes are checked first, and a request that needs a Converse-only feature (``bedrock_request_needs_converse``) is - served by Converse even on a flagged model. + served by Converse even on a listed model. Only a bedrock-runtime row counts: a + ``bedrock_mantle`` row lists the endpoints of the Mantle host, not this one. """ - return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions") + return any( + entry is not None and _bedrock_runtime_row_lists_chat_completions(entry) + for entry in _bedrock_price_map_entries(model) + ) def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d58dd4d105a..7fdd8c85ddc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -40834,7 +40834,9 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", @@ -40850,7 +40852,9 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", @@ -46591,7 +46595,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -46606,7 +46612,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -46617,7 +46625,9 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -57156,7 +57166,6 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html" }, "us.openai.gpt-5.6-sol": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, @@ -57188,11 +57197,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "global.openai.gpt-5.6-sol": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, @@ -57224,11 +57233,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "us.openai.gpt-5.6-terra": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -57260,11 +57269,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "global.openai.gpt-5.6-terra": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -57296,11 +57305,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "us.openai.gpt-5.6-luna": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -57332,6 +57341,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57460,7 +57470,6 @@ ] }, "global.openai.gpt-5.6-luna": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, @@ -57492,6 +57501,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58095,7 +58105,9 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -58114,7 +58126,9 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -63818,7 +63832,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -63833,7 +63849,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -64076,7 +64094,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -64091,7 +64111,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d58dd4d105a..7fdd8c85ddc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -40834,7 +40834,9 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", @@ -40850,7 +40852,9 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", @@ -46591,7 +46595,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -46606,7 +46612,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -46617,7 +46625,9 @@ "input_cost_per_token": 2.64e-06, "output_cost_per_token": 7.92e-06, "cache_read_input_token_cost": 6.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -57156,7 +57166,6 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html" }, "us.openai.gpt-5.6-sol": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4.4e-06, "input_cost_per_token_above_272k_tokens": 8.8e-06, @@ -57188,11 +57197,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "global.openai.gpt-5.6-sol": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, @@ -57224,11 +57233,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "us.openai.gpt-5.6-terra": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -57260,11 +57269,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "global.openai.gpt-5.6-terra": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -57296,11 +57305,11 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, "us.openai.gpt-5.6-luna": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2.2e-07, "input_cost_per_token_above_272k_tokens": 4.4e-07, @@ -57332,6 +57341,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -57460,7 +57470,6 @@ ] }, "global.openai.gpt-5.6-luna": { - "supports_bedrock_runtime_chat_completions": true, "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 2e-07, "input_cost_per_token_above_272k_tokens": 4e-07, @@ -57492,6 +57501,7 @@ "supports_vision": true, "supports_sampling_params": false, "supported_endpoints": [ + "/v1/chat/completions", "/v1/responses" ] }, @@ -58095,7 +58105,9 @@ "input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -58114,7 +58126,9 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_bedrock_runtime_chat_completions_response_format": true, "litellm_provider": "bedrock_converse", @@ -63818,7 +63832,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -63833,7 +63849,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -64076,7 +64094,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.6e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, @@ -64091,7 +64111,9 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.2e-07, - "supports_bedrock_runtime_chat_completions": true, + "supported_endpoints": [ + "/v1/chat/completions" + ], "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, "supports_function_calling": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 89ebedc3a0a..bcb0509f25c 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -902,9 +902,6 @@ "supports_audio_output": { "type": "boolean" }, - "supports_bedrock_runtime_chat_completions": { - "type": "boolean" - }, "supports_bedrock_runtime_chat_completions_response_format": { "type": "boolean" }, diff --git a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 2f0d384fcdd..3c347e4bed2 100644 --- a/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -59,9 +59,27 @@ def test_claude_stays_on_converse(local_cost_map): assert BedrockModelInfo.get_bedrock_route("us.anthropic.claude-3-sonnet-20240229-v1:0") == "converse" -def test_flag_absent_means_no_chat_completions_route(monkeypatch): - monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": {"litellm_provider": "bedrock_converse"}}) +@pytest.mark.parametrize( + "entry", + [ + {"litellm_provider": "bedrock_converse"}, + {"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/responses"]}, + {"litellm_provider": "bedrock_converse", "supports_bedrock_runtime_chat_completions": True}, + {"litellm_provider": "bedrock_mantle", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}, + {"litellm_provider": "openai", "supported_endpoints": ["/v1/chat/completions"]}, + ], +) +def test_chat_completions_missing_from_supported_endpoints_means_no_chat_completions_route(monkeypatch, entry): + monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry}) assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is False + assert BedrockModelInfo.get_bedrock_route("us.xai.grok-4.6") == "converse" + + +def test_chat_completions_in_supported_endpoints_opts_into_the_native_route(monkeypatch): + entry = {"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]} + monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry}) + assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is True + assert BedrockModelInfo.get_bedrock_route("bedrock/us.xai.grok-4.6") == "chat_completions" def test_complete_url_is_runtime_openai_chat_completions(monkeypatch): @@ -860,7 +878,7 @@ SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0" def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_flags, request_params, needs_converse): entry = { "litellm_provider": "bedrock_converse", - "supports_bedrock_runtime_chat_completions": True, + "supported_endpoints": ["/v1/chat/completions"], **capability_flags, } monkeypatch.setattr(litellm, "model_cost", {SYNTHETIC_NATIVE_MODEL: entry}) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 98e46016401..39cb25a0f51 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -927,7 +927,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_video_input": {"type": "boolean"}, "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, - "supports_bedrock_runtime_chat_completions": {"type": "boolean"}, "supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"}, "supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"}, "supports_url_context": {"type": "boolean"},