scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

fix: repair seven regressions caught by CircleCI on main (#42640)

Browse source 
* fix: repair seven regressions caught by CircleCI on main

- vertex_ai: stop treating fine-tuned endpoint ids (numeric or
  vertex_ai/gemini/<id>) and gemma models as Gemini 3+, which injected
  temperature=1.0 and Gemini 3 thinking config into their requests (#42465)
- cost: price Azure DALL-E 3 from its azure/<quality>/<size>/dall-e-3 rows;
  it only worked through the OpenAI rows that #42435 removed
- bedrock: stream bedrock/invoke/moonshot through an OpenAI-shaped chunk
  decoder; the generic decoder dropped every chunk, which the
  supports_response_schema flag from #42338 un-skipped in CI
- proxy: keep the public model_group on pre-routing rejections so the Usage
  page groups them under the model name, not the deployment (#41077)
- cost map: mirror the base rows' capability flags onto Bedrock regional and
  cross-region copies (#42254 and later syncs)
- whitelist the new regional Bedrock rows from #42543 and #42588 for the
  converse routing check, following the existing regional-row convention

* fix(model-prices): mirror capability flags onto ap-southeast-3 bedrock rows

* refactor(bedrock): tighten types on the moonshot stream decoder and its tests
This commit is contained in:
yuneng-jiang 2026-09-22 19:26:36 -07:00 • committed by GitHub
parent 87a7052fe8
commit b395bfefdd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 726 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/cost_calculator.py
View file

@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
@ -2386,6 +2391,7 @@ def default_image_cost_calculator(
models_to_check: Final = (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,

30
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -231,6 +231,12 @@ async def make_call(
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
@ -329,6 +335,12 @@ def make_sync_call(
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
@ -795,6 +807,24 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def __init__(
self,
model: str,
sync_stream: bool,
) -> None:
super().__init__(model=model)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
streaming_response=None,
sync_stream=sync_stream,
)
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response

5
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -287,7 +287,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Check if the model is Gemini 3 or newer.
"""
model_name = model.split("/")[-1].lower()
if not model_name:
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
model.startswith("gemini/") and not model_name.startswith("gemini-")
)
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
return False
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):

261
litellm/model_prices_and_context_window_backup.json
View file

@ -11845,6 +11845,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
@ -11858,6 +11861,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
@ -11872,6 +11878,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
@ -11897,6 +11906,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/qwen.qwen3-coder-next": {
@ -11910,6 +11921,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/moonshotai.kimi-k2-thinking": {
@ -11937,7 +11951,9 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true
},
"bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.18e-06,
@ -11969,6 +11985,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
@ -11982,6 +12001,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
@ -11996,6 +12018,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
@ -12021,6 +12046,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/qwen.qwen3-coder-next": {
@ -12034,6 +12061,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
@ -12048,6 +12078,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
@ -12062,6 +12095,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.1": {
@ -12075,6 +12111,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
@ -12089,6 +12128,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
@ -12103,6 +12145,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/qwen.qwen3-coder-next": {
@ -12116,6 +12160,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
@ -12148,6 +12195,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
@ -12161,6 +12211,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
@ -12175,6 +12228,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
@ -12189,6 +12245,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": {
@ -12289,6 +12347,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
@ -12303,6 +12364,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
@ -12316,6 +12380,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
@ -12347,6 +12414,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
@ -12361,6 +12431,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
@ -12374,6 +12447,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
@ -12405,6 +12481,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
@ -12419,6 +12498,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/nvidia.nemotron-super-3-120b": {
@ -12449,6 +12531,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
@ -12492,6 +12577,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
@ -12506,6 +12594,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
@ -12519,6 +12610,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": {
@ -12569,6 +12663,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
@ -12582,6 +12679,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
@ -12596,6 +12696,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
@ -12621,6 +12724,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/qwen.qwen3-coder-next": {
@ -12634,6 +12739,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
@ -12784,6 +12892,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
@ -12797,6 +12908,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
@ -12810,6 +12924,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
@ -12835,6 +12952,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/qwen.qwen3-coder-next": {
@ -12848,6 +12967,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/deepseek.v3.2": {
@ -12862,6 +12984,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
@ -12875,6 +13000,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
@ -12889,6 +13017,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
@ -12914,6 +13045,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/qwen.qwen3-coder-next": {
@ -12927,6 +13060,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
@ -13385,6 +13521,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
@ -13398,6 +13537,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
@ -13411,6 +13553,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
@ -13436,6 +13581,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/qwen.qwen3-coder-next": {
@ -13449,6 +13596,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
@ -42315,7 +42465,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42328,7 +42481,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.545e-07,
@ -42341,7 +42497,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42354,7 +42513,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 2.3e-07,
@ -42367,7 +42529,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42380,7 +42545,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"qwen.qwen3-vl-235b-a22b": {
"input_cost_per_token": 5.3e-07,
@ -44718,7 +44886,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -44729,7 +44900,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"us-gov.nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -44739,7 +44913,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -44753,7 +44931,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,
@ -44999,7 +45180,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"eu.deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
@ -45012,7 +45196,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us.meta.llama3-1-405b-instruct-v1:0": {
"input_cost_per_token": 5.32e-06,
@ -55772,6 +55959,9 @@
"supports_system_messages": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/zai.glm-5": {
@ -55787,6 +55977,9 @@
"supports_system_messages": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
@ -61011,7 +61204,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -61022,7 +61218,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -61032,7 +61231,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -61046,7 +61249,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,
@ -61252,7 +61458,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -61263,7 +61472,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -61273,7 +61485,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -61287,7 +61503,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,

2
litellm/proxy/utils.py
View file

@ -1004,6 +1004,8 @@ def _stamp_deployment_attribution(
return attribution
metadata.setdefault("model_info", attribution["model_info"])
metadata.setdefault("deployment", attribution["deployment"])
if isinstance(model_group, str):
metadata.setdefault("model_group", model_group)
return attribution

261
model_prices_and_context_window.json
View file

@ -11845,6 +11845,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
@ -11858,6 +11861,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
@ -11872,6 +11878,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
@ -11897,6 +11906,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/qwen.qwen3-coder-next": {
@ -11910,6 +11921,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/moonshotai.kimi-k2-thinking": {
@ -11937,7 +11951,9 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true
},
"bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.18e-06,
@ -11969,6 +11985,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
@ -11982,6 +12001,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
@ -11996,6 +12018,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
@ -12021,6 +12046,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/qwen.qwen3-coder-next": {
@ -12034,6 +12061,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
@ -12048,6 +12078,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
@ -12062,6 +12095,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.1": {
@ -12075,6 +12111,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
@ -12089,6 +12128,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
@ -12103,6 +12145,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/qwen.qwen3-coder-next": {
@ -12116,6 +12160,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
@ -12148,6 +12195,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
@ -12161,6 +12211,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
@ -12175,6 +12228,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
@ -12189,6 +12245,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": {
@ -12289,6 +12347,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
@ -12303,6 +12364,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
@ -12316,6 +12380,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
@ -12347,6 +12414,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
@ -12361,6 +12431,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
@ -12374,6 +12447,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
@ -12405,6 +12481,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
@ -12419,6 +12498,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/nvidia.nemotron-super-3-120b": {
@ -12449,6 +12531,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
@ -12492,6 +12577,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
@ -12506,6 +12594,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
@ -12519,6 +12610,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": {
@ -12569,6 +12663,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
@ -12582,6 +12679,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
@ -12596,6 +12696,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
@ -12621,6 +12724,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/qwen.qwen3-coder-next": {
@ -12634,6 +12739,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
@ -12784,6 +12892,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
@ -12797,6 +12908,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
@ -12810,6 +12924,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
@ -12835,6 +12952,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/qwen.qwen3-coder-next": {
@ -12848,6 +12967,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/deepseek.v3.2": {
@ -12862,6 +12984,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
@ -12875,6 +13000,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
@ -12889,6 +13017,9 @@
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
@ -12914,6 +13045,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/qwen.qwen3-coder-next": {
@ -12927,6 +13060,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
@ -13385,6 +13521,9 @@
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
@ -13398,6 +13537,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
@ -13411,6 +13553,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
@ -13436,6 +13581,8 @@
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_audio_input": false,
"supports_response_schema": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/qwen.qwen3-coder-next": {
@ -13449,6 +13596,9 @@
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
@ -42315,7 +42465,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42328,7 +42481,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.545e-07,
@ -42341,7 +42497,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42354,7 +42513,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 2.3e-07,
@ -42367,7 +42529,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": {
"input_cost_per_token": 1.8e-07,
@ -42380,7 +42545,10 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"qwen.qwen3-vl-235b-a22b": {
"input_cost_per_token": 5.3e-07,
@ -44718,7 +44886,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -44729,7 +44900,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"us-gov.nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -44739,7 +44913,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -44753,7 +44931,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,
@ -44999,7 +45180,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"eu.deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
@ -45012,7 +45196,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_native_structured_output": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"us.meta.llama3-1-405b-instruct-v1:0": {
"input_cost_per_token": 5.32e-06,
@ -55772,6 +55959,9 @@
"supports_system_messages": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/zai.glm-5": {
@ -55787,6 +55977,9 @@
"supports_system_messages": true,
"supports_native_structured_output": true,
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
@ -61011,7 +61204,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -61022,7 +61218,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -61032,7 +61231,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -61046,7 +61249,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,
@ -61252,7 +61458,10 @@
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": {
"input_cost_per_token": 2.4e-07,
@ -61263,7 +61472,10 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"supports_system_messages": true,
"supports_vision": true
"supports_vision": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true
},
"bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 7.2e-08,
@ -61273,7 +61485,11 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.76e-07,
"supports_system_messages": true
"supports_system_messages": true,
"supports_audio_input": false,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.8e-07,
@ -61287,7 +61503,10 @@
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_audio_input": false,
"supports_response_schema": true,
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"input_cost_per_token": 8.4e-08,

4
tests/local_testing/test_amazing_vertex_completion.py
View file

@ -3331,7 +3331,7 @@ def test_gemini_fine_tuned_model_request_consistency():
Assert the same transformation is applied to Fine tuned gemini 2.0 flash and gemini 2.0 flash
- Request 1: Fine tuned: vertex_ai/gemini/ft-uuid
- Request 2: vertex_ai/gemini-2.0-flash-001
- Request 2: vertex_ai/gemini-2.5-flash
"""
litellm.set_verbose = True
load_vertex_ai_credentials()
@ -3403,7 +3403,7 @@ def test_gemini_fine_tuned_model_request_consistency():
with patch.object(client, "post", new=MagicMock()) as mock_post_2:
try:
response_2 = completion(
model="vertex_ai/gemini-2.0-flash-001",
model="vertex_ai/gemini-2.5-flash",
messages=messages,
tools=tools,
tool_choice="auto",

1
tests/local_testing/whitelisted_bedrock_models.txt
View file

@ -139,3 +139,4 @@ bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b
bedrock/eu-west-1/qwen.qwen3-next-80b-a3b
bedrock/eu-west-2/qwen.qwen3-next-80b-a3b
bedrock/sa-east-1/qwen.qwen3-next-80b-a3b
bedrock/eu-west-2/nvidia.nemotron-super-3-120b

95
tests/test_litellm/llms/bedrock/chat/test_invoke_handler.py
View file

@ -1,4 +1,10 @@
import base64
import binascii
import datetime
import json
import struct
from collections.abc import AsyncIterator, Mapping, Sequence
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import httpx
@ -13,6 +19,7 @@ from litellm.llms.bedrock.chat.invoke_handler import (
make_sync_call,
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.utils import ModelResponseStream
def test_transform_thinking_blocks_with_redacted_content():
@ -704,3 +711,91 @@ async def test_async_invoke_streaming_non_200_forwards_bedrock_response_headers(
)
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-non200-async"
def _bedrock_event_stream_frame(chunk: Mapping[str, object]) -> bytes:
def header(name: str, value: str) -> bytes:
return bytes([len(name)]) + name.encode() + bytes([7]) + struct.pack(">H", len(value)) + value.encode()
headers: Final = header(":event-type", "chunk") + header(":content-type", "application/json") + header(
":message-type", "event"
)
payload: Final = json.dumps({"bytes": base64.b64encode(json.dumps(chunk).encode()).decode()}).encode()
prelude: Final = struct.pack(">II", 12 + len(headers) + len(payload) + 4, len(headers))
body: Final = prelude + struct.pack(">I", binascii.crc32(prelude)) + headers + payload
return body + struct.pack(">I", binascii.crc32(body))
def _openai_stream_chunk(delta: Mapping[str, str], finish_reason: str | None = None) -> Mapping[str, object]:
return {
"id": "chatcmpl-1",
"object": "chat.completion.chunk",
"created": 1,
"model": "moonshot.kimi-k2-thinking",
"choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
}
_MOONSHOT_RAW_STREAM: Final = b"".join(
_bedrock_event_stream_frame(chunk)
for chunk in (
_openai_stream_chunk({"role": "assistant", "reasoning_content": "thinking"}),
_openai_stream_chunk({"content": '{"city": '}),
_openai_stream_chunk({"content": '"San Francisco"}'}),
_openai_stream_chunk({}, "stop"),
)
)
def _assert_moonshot_stream_content(chunks: Sequence[ModelResponseStream]) -> None:
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == '{"city": "San Francisco"}'
assert "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) == "thinking"
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"]
@pytest.fixture
def _aws_test_credentials(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIATEST")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
monkeypatch.setenv("AWS_REGION_NAME", "us-east-1")
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
def test_moonshot_invoke_stream_yields_openai_shaped_chunks(
_aws_test_credentials: None, response_format: Mapping[str, str] | None
) -> None:
raw_stream: Final = _MOONSHOT_RAW_STREAM
response: Final = MagicMock(status_code=200, headers={})
response.iter_bytes = lambda chunk_size=None: iter([raw_stream])
client: Final = HTTPHandler()
client.post = MagicMock(return_value=response)
stream: Final = litellm.completion(
model="bedrock/invoke/moonshot.kimi-k2-thinking",
messages=[{"role": "user", "content": "weather as json"}],
stream=True,
client=client,
**({"response_format": response_format} if response_format else {}),
)
_assert_moonshot_stream_content(list(stream))
@pytest.mark.asyncio
async def test_moonshot_invoke_async_stream_yields_openai_shaped_chunks(_aws_test_credentials: None) -> None:
async def _aiter_bytes(chunk_size: int | None = None) -> AsyncIterator[bytes]:
yield _MOONSHOT_RAW_STREAM
response: Final = MagicMock(status_code=200, headers={})
response.aiter_bytes = _aiter_bytes
client: Final = AsyncHTTPHandler()
client.post = AsyncMock(return_value=response)
stream: Final = await litellm.acompletion(
model="bedrock/invoke/moonshot.kimi-k2-thinking",
messages=[{"role": "user", "content": "weather as json"}],
stream=True,
response_format={"type": "json_object"},
client=client,
)
_assert_moonshot_stream_content([chunk async for chunk in stream])

26
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -2461,6 +2461,12 @@ def test_is_gemini_3_or_newer():
assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("4965075652664360960") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/4965075652664360960") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/ft-uuid") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("gemma-3-27b-it") == False
assert VertexGeminiConfig._is_gemini_3_or_newer("gemini/gemma-3-27b-it") == False
# Edge cases
assert VertexGeminiConfig._is_gemini_3_or_newer("") == False
@ -2504,6 +2510,26 @@ def test_gemini_3_reasoning_effort_maps_to_thinking_level(model: str):
assert "thinkingBudget" not in mapped["thinkingConfig"]
@pytest.mark.parametrize(
"model",
["4965075652664360960", "gemini/4965075652664360960", "gemini/ft-uuid", "gemma-3-27b-it"],
)
def test_fine_tuned_endpoint_and_gemma_get_no_gemini_3_default_temperature(model: str):
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
mapped = VertexGeminiConfig().map_openai_params(
non_default_params={"max_tokens": 10},
optional_params={},
model=model,
drop_params=False,
)
assert mapped["max_output_tokens"] == 10
assert "temperature" not in mapped
def _tool_call_messages(tool_call_id: str):

43
tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py
View file

@ -149,6 +149,49 @@ async def test_post_call_failure_hook_attributes_single_router_deployment(
)
@pytest.mark.asyncio
async def test_pre_routing_reject_spend_log_keeps_public_model_group(proxy_logging, make_user_api_key_auth, monkeypatch):
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
recorded: list[dict] = []
class _RecordingLogger(CustomLogger):
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
recorded.append(kwargs)
monkeypatch.setattr(
proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "internal-model",
"litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"},
}
]
),
)
monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()])
proxy_logging.alert_types = []
await proxy_logging.post_call_failure_hook(
request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]},
original_exception=HTTPException(status_code=401, detail="blocked key"),
user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"),
route="/chat/completions",
)
assert len(recorded) == 1
assert recorded[0]["standard_logging_object"]["model_group"] == "internal-model"
now: Final = datetime.now()
payload = get_logging_payload(
kwargs={**recorded[0], "completion_start_time": now}, response_obj=None, start_time=now, end_time=now
)
assert payload["model"] == "openai/gpt-4.1"
assert payload["model_group"] == "internal-model"
@pytest.mark.asyncio
async def test_post_call_failure_hook_keeps_router_stamped_metadata_for_post_call_failures(
proxy_logging, make_user_api_key_auth, monkeypatch

34
tests/test_litellm/test_cost_calculator.py
View file

@ -872,6 +872,40 @@ def test_default_image_cost_calculator(monkeypatch):
assert cost == 10485760
@pytest.mark.parametrize(
("model", "quality", "size", "priced_key", "pixels"),
[
("azure/dall-e-3", "standard", "1024x1024", "azure/standard/1024-x-1024/dall-e-3", 1024 * 1024),
("azure/dall-e-3", "hd", "1024x1792", "azure/hd/1024-x-1792/dall-e-3", 1024 * 1792),
("dall-e-3", "hd", "1024x1792", "azure/hd/1024-x-1792/dall-e-3", 1024 * 1792),
],
)
def test_default_image_cost_calculator_matches_provider_first_quality_key(
monkeypatch, model: str, quality: str, size: str, priced_key: str, pixels: int
):
from litellm.cost_calculator import default_image_cost_calculator
monkeypatch.setattr(
litellm,
"model_cost",
{
"azure/standard/1024-x-1024/dall-e-3": {"litellm_provider": "azure", "input_cost_per_pixel": 1e-08},
"azure/hd/1024-x-1792/dall-e-3": {"litellm_provider": "azure", "input_cost_per_pixel": 3e-08},
},
)
cost = default_image_cost_calculator(
model=model,
custom_llm_provider="azure",
quality=quality,
n=1,
size=size,
optional_params={},
)
assert cost == litellm.model_cost[priced_key]["input_cost_per_pixel"] * pixels
def test_cost_calculator_with_cache_creation():
from litellm import completion_cost
from litellm.types.utils import Choices, Message, Usage

3
whitelisted_bedrock_models.txt
View file

@ -90,6 +90,7 @@ bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0
bedrock/eu-west-2/meta.llama3-8b-instruct-v1:0
bedrock/eu-west-2/minimax.minimax-m2.1
bedrock/eu-west-2/minimax.minimax-m2.5
bedrock/eu-west-2/nvidia.nemotron-super-3-120b
bedrock/eu-west-2/qwen.qwen3-coder-next
bedrock/eu-west-2/qwen.qwen3-next-80b-a3b
bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2
@ -234,6 +235,7 @@ bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0
bedrock/us-gov-west-1/anthropic.claude-sonnet-5
bedrock/us-gov-west-1/anthropic.claude-opus-4-8
bedrock/us-gov-west-1/anthropic.claude-opus-5
bedrock/us-gov-west-1/anthropic.claude-opus-5-5
bedrock/us-gov-west-1/anthropic.claude-fable-5-1
bedrock/us-gov-east-1/nvidia.nemotron-nano-3-30b
bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2
@ -244,5 +246,6 @@ bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0
bedrock/us-gov-east-1/anthropic.claude-sonnet-5
bedrock/us-gov-east-1/anthropic.claude-opus-4-8
bedrock/us-gov-east-1/anthropic.claude-opus-5
bedrock/us-gov-east-1/anthropic.claude-opus-5-5
bedrock/us-gov-east-1/anthropic.claude-fable-5-1
bedrock/ap-southeast-2/qwen.qwen3-next-80b-a3b