scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-01 02:02:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 92

fix(bedrock): opt into the native chat completions route through supported_endpoints
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details
LiteLLM Rust / rust-wheel (push) Has been cancelled
Details
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Details
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Details
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details

Browse source
This commit is contained in:
mateo-berri 2026-09-24 15:21:07 -07:00
parent c4c24dd986
commit ff8e15d844
8 changed files with 126 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -582,8 +582,6 @@ pub struct ModelInfo {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_audio_output: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,

2
litellm/llms/bedrock/chat/chat_completions/transformation.py
View file

@ -3,7 +3,7 @@ Native OpenAI Chat Completions on Amazon Bedrock Runtime.
AWS serves this surface at
``https://bedrock-runtime.{region}.amazonaws.com/openai/v1/chat/completions``
for the models whose price-map entry sets ``supports_bedrock_runtime_chat_completions``
for the models whose price-map ``supported_endpoints`` lists ``/v1/chat/completions``
(Grok 4.6, gpt-oss, the GPT-5.6 family): chat completions stay chat completions
instead of being rewritten to Converse.

32
litellm/llms/bedrock/common_utils.py
View file

@ -829,24 +829,44 @@ def split_bedrock_region_path(model: str) -> tuple[str | None, str]:
return None, stripped
def _bedrock_price_map_flag(model: str, flag: str) -> bool:
entries: Final = (
BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse"))
def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]:
return tuple(
litellm.model_cost.get(key)
for key in (model, strip_bedrock_routing_prefix(model), split_bedrock_region_path(model)[1])
)
return any(entry is not None and entry.get(flag) is True for entry in entries)
def _bedrock_price_map_flag(model: str, flag: str) -> bool:
return any(entry is not None and entry.get(flag) is True for entry in _bedrock_price_map_entries(model))
def _bedrock_runtime_row_lists_chat_completions(entry: Mapping[str, object]) -> bool:
endpoints: Final = entry.get("supported_endpoints")
return (
entry.get("litellm_provider") in BEDROCK_RUNTIME_PRICE_MAP_PROVIDERS
and isinstance(endpoints, (list, tuple))
and "/v1/chat/completions" in endpoints
)
def uses_bedrock_runtime_chat_completions(model: str) -> bool:
"""Whether this Bedrock model should use runtime native Chat Completions.
Data-driven from the price-map ``supports_bedrock_runtime_chat_completions`` flag
Data-driven from ``/v1/chat/completions`` in the price-map row's ``supported_endpoints``,
the same per-model signal ``bedrock_supports_openai_responses`` reads for ``/v1/responses``,
so onboarding a model is a JSON change. Explicit ``converse/`` still wins in
``get_bedrock_route`` because prefix routes are checked first, and a request
that needs a Converse-only feature (``bedrock_request_needs_converse``) is
served by Converse even on a flagged model.
served by Converse even on a listed model. Only a bedrock-runtime row counts: a
``bedrock_mantle`` row lists the endpoints of the Mantle host, not this one.
"""
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions")
return any(
entry is not None and _bedrock_runtime_row_lists_chat_completions(entry)
for entry in _bedrock_price_map_entries(model)
)
def bedrock_runtime_chat_completions_serves_tools_with_reasoning(model: str) -> bool:

56
litellm/model_prices_and_context_window_backup.json
View file

@ -40834,7 +40834,9 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
@ -40850,7 +40852,9 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 7e-08,
"litellm_provider": "bedrock_converse",
@ -46591,7 +46595,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -46606,7 +46612,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -46617,7 +46625,9 @@
"input_cost_per_token": 2.64e-06,
"output_cost_per_token": 7.92e-06,
"cache_read_input_token_cost": 6.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -57156,7 +57166,6 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
},
"us.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4.4e-06,
"input_cost_per_token_above_272k_tokens": 8.8e-06,
@ -57188,11 +57197,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
@ -57224,11 +57233,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
@ -57260,11 +57269,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
@ -57296,11 +57305,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
@ -57332,6 +57341,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -57460,7 +57470,6 @@
]
},
"global.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
@ -57492,6 +57501,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58095,7 +58105,9 @@
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 6.6e-06,
"cache_read_input_token_cost": 5.5e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -58114,7 +58126,9 @@
"input_cost_per_token": 2e-06,
"output_cost_per_token": 6e-06,
"cache_read_input_token_cost": 5e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -63818,7 +63832,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -63833,7 +63849,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -64076,7 +64094,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -64091,7 +64111,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,

56
model_prices_and_context_window.json
View file

@ -40834,7 +40834,9 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
@ -40850,7 +40852,9 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"input_cost_per_token": 7e-08,
"litellm_provider": "bedrock_converse",
@ -46591,7 +46595,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -46606,7 +46612,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -46617,7 +46625,9 @@
"input_cost_per_token": 2.64e-06,
"output_cost_per_token": 7.92e-06,
"cache_read_input_token_cost": 6.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -57156,7 +57166,6 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
},
"us.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4.4e-06,
"input_cost_per_token_above_272k_tokens": 8.8e-06,
@ -57188,11 +57197,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-sol": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
@ -57224,11 +57233,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
@ -57260,11 +57269,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"global.openai.gpt-5.6-terra": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
@ -57296,11 +57305,11 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
"us.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
@ -57332,6 +57341,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -57460,7 +57470,6 @@
]
},
"global.openai.gpt-5.6-luna": {
"supports_bedrock_runtime_chat_completions": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
@ -57492,6 +57501,7 @@
"supports_vision": true,
"supports_sampling_params": false,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
]
},
@ -58095,7 +58105,9 @@
"input_cost_per_token": 2.2e-06,
"output_cost_per_token": 6.6e-06,
"cache_read_input_token_cost": 5.5e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -58114,7 +58126,9 @@
"input_cost_per_token": 2e-06,
"output_cost_per_token": 6e-06,
"cache_read_input_token_cost": 5e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_bedrock_runtime_chat_completions_response_format": true,
"litellm_provider": "bedrock_converse",
@ -63818,7 +63832,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -63833,7 +63849,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -64076,7 +64094,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,
@ -64091,7 +64111,9 @@
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_bedrock_runtime_chat_completions": true,
"supported_endpoints": [
"/v1/chat/completions"
],
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": true,
"supports_function_calling": true,
"supports_reasoning": true,

3
model_prices_and_context_window.schema.json
View file

@ -902,9 +902,6 @@
"supports_audio_output": {
"type": "boolean"
},
"supports_bedrock_runtime_chat_completions": {
"type": "boolean"
},
"supports_bedrock_runtime_chat_completions_response_format": {
"type": "boolean"
},

24
tests/test_litellm/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py
View file

@ -59,9 +59,27 @@ def test_claude_stays_on_converse(local_cost_map):
assert BedrockModelInfo.get_bedrock_route("us.anthropic.claude-3-sonnet-20240229-v1:0") == "converse"
def test_flag_absent_means_no_chat_completions_route(monkeypatch):
monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": {"litellm_provider": "bedrock_converse"}})
@pytest.mark.parametrize(
"entry",
[
{"litellm_provider": "bedrock_converse"},
{"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/responses"]},
{"litellm_provider": "bedrock_converse", "supports_bedrock_runtime_chat_completions": True},
{"litellm_provider": "bedrock_mantle", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]},
{"litellm_provider": "openai", "supported_endpoints": ["/v1/chat/completions"]},
],
)
def test_chat_completions_missing_from_supported_endpoints_means_no_chat_completions_route(monkeypatch, entry):
monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry})
assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is False
assert BedrockModelInfo.get_bedrock_route("us.xai.grok-4.6") == "converse"
def test_chat_completions_in_supported_endpoints_opts_into_the_native_route(monkeypatch):
entry = {"litellm_provider": "bedrock_converse", "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}
monkeypatch.setattr(litellm, "model_cost", {"us.xai.grok-4.6": entry})
assert uses_bedrock_runtime_chat_completions("us.xai.grok-4.6") is True
assert BedrockModelInfo.get_bedrock_route("bedrock/us.xai.grok-4.6") == "chat_completions"
def test_complete_url_is_runtime_openai_chat_completions(monkeypatch):
@ -860,7 +878,7 @@ SYNTHETIC_NATIVE_MODEL = "vendor.native-model-v1:0"
def test_capability_flags_are_read_from_the_cost_map(monkeypatch, capability_flags, request_params, needs_converse):
entry = {
"litellm_provider": "bedrock_converse",
"supports_bedrock_runtime_chat_completions": True,
"supported_endpoints": ["/v1/chat/completions"],
**capability_flags,
}
monkeypatch.setattr(litellm, "model_cost", {SYNTHETIC_NATIVE_MODEL: entry})

1
tests/test_litellm/test_utils.py
View file

@ -927,7 +927,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_video_input": {"type": "boolean"},
"supports_vision": {"type": "boolean"},
"supports_web_search": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_tools_with_reasoning": {"type": "boolean"},
"supports_bedrock_runtime_chat_completions_response_format": {"type": "boolean"},
"supports_url_context": {"type": "boolean"},