mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(bedrock): route gpt-6-astra reasoning_effort to reasoning.effort and mark Nova 2 tool_choice
The converse reasoning gate only matched openai.gpt-5, so gpt-6-astra fell through to Anthropic's thinking block and Bedrock rejected the call with 400 Unknown parameter: 'thinking'. Match any openai.gpt-<digit> model at the three gate sites instead. Nova 2 lite and pro accept forced tool_choice on Converse (verified live on us.amazon.nova-2-lite-v1:0), so the nine Nova 2 registry keys now advertise supports_tool_choice. The invoke dispatcher also forwards json_mode to Nova like it already does for Anthropic and TwelveLabs.
This commit is contained in:
parent
b89c32407f
commit
fbc6fb56ae
6 changed files with 79 additions and 4 deletions
|
|
@ -4,6 +4,7 @@ Translating between OpenAI's `/chat/completion` format and Amazon's `/converse`
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
|
import re
|
||||||
import time
|
import time
|
||||||
import types
|
import types
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
|
|
@ -293,6 +294,10 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
llm_provider="bedrock",
|
llm_provider="bedrock",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_openai_gpt_reasoning_model(model: str) -> bool:
|
||||||
|
return re.search(r"openai\.gpt-\d", model) is not None
|
||||||
|
|
||||||
def _is_nova_2_model(self, model: str) -> bool:
|
def _is_nova_2_model(self, model: str) -> bool:
|
||||||
"""
|
"""
|
||||||
Check if the model is a Nova 2 model that supports reasoningConfig.
|
Check if the model is a Nova 2 model that supports reasoningConfig.
|
||||||
|
|
@ -423,14 +428,14 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
Handle the reasoning_effort parameter based on the model type.
|
Handle the reasoning_effort parameter based on the model type.
|
||||||
|
|
||||||
- GPT-OSS models: passed through unchanged via additionalModelRequestFields.
|
- GPT-OSS models: passed through unchanged via additionalModelRequestFields.
|
||||||
- OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
- OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
|
||||||
- Nova 2 models: transformed to reasoningConfig.
|
- Nova 2 models: transformed to reasoningConfig.
|
||||||
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
|
||||||
adaptive Claude 4.6 / 4.7).
|
adaptive Claude 4.6 / 4.7).
|
||||||
"""
|
"""
|
||||||
if "gpt-oss" in model:
|
if "gpt-oss" in model:
|
||||||
optional_params["reasoning_effort"] = reasoning_effort
|
optional_params["reasoning_effort"] = reasoning_effort
|
||||||
elif "openai.gpt-5" in model:
|
elif self._is_openai_gpt_reasoning_model(model):
|
||||||
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
|
||||||
optional_params["reasoning"] = reasoning
|
optional_params["reasoning"] = reasoning
|
||||||
elif self._is_nova_2_model(model):
|
elif self._is_nova_2_model(model):
|
||||||
|
|
@ -564,7 +569,11 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
|
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
|
||||||
supported_params.append("tool_choice")
|
supported_params.append("tool_choice")
|
||||||
|
|
||||||
if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model:
|
if (
|
||||||
|
"gpt-oss" in model
|
||||||
|
or self._is_openai_gpt_reasoning_model(model)
|
||||||
|
or self._is_openai_gpt_reasoning_model(base_model)
|
||||||
|
):
|
||||||
supported_params.append("reasoning_effort")
|
supported_params.append("reasoning_effort")
|
||||||
elif self._is_nova_2_model(model):
|
elif self._is_nova_2_model(model):
|
||||||
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
|
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
|
||||||
|
|
@ -920,7 +929,7 @@ class AmazonConverseConfig(BaseConfig):
|
||||||
optional_params["_parallel_tool_use_config"] = {
|
optional_params["_parallel_tool_use_config"] = {
|
||||||
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
|
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
|
||||||
}
|
}
|
||||||
if param == "thinking" and "openai.gpt-5" not in model:
|
if param == "thinking" and not self._is_openai_gpt_reasoning_model(model):
|
||||||
if (
|
if (
|
||||||
isinstance(value, dict)
|
isinstance(value, dict)
|
||||||
and value.get("type") == "adaptive"
|
and value.get("type") == "adaptive"
|
||||||
|
|
|
||||||
|
|
@ -340,6 +340,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
||||||
optional_params=optional_params,
|
optional_params=optional_params,
|
||||||
litellm_params=litellm_params,
|
litellm_params=litellm_params,
|
||||||
encoding=encoding,
|
encoding=encoding,
|
||||||
|
json_mode=json_mode,
|
||||||
)
|
)
|
||||||
elif provider == "twelvelabs":
|
elif provider == "twelvelabs":
|
||||||
return litellm.AmazonTwelveLabsPegasusConfig().transform_response(
|
return litellm.AmazonTwelveLabsPegasusConfig().transform_response(
|
||||||
|
|
|
||||||
|
|
@ -381,6 +381,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -400,6 +401,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -417,6 +419,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -436,6 +439,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -453,6 +457,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -472,6 +477,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -489,6 +495,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -508,6 +515,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -28578,6 +28586,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -381,6 +381,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -400,6 +401,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -417,6 +419,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -436,6 +439,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -453,6 +457,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -472,6 +477,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -489,6 +495,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -508,6 +515,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
@ -28578,6 +28586,7 @@
|
||||||
"supports_prompt_caching": true,
|
"supports_prompt_caching": true,
|
||||||
"supports_reasoning": true,
|
"supports_reasoning": true,
|
||||||
"supports_response_schema": true,
|
"supports_response_schema": true,
|
||||||
|
"supports_tool_choice": true,
|
||||||
"supports_video_input": true,
|
"supports_video_input": true,
|
||||||
"supports_vision": true
|
"supports_vision": true
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
import json
|
import json
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -190,3 +192,45 @@ def test_get_error_class_preserves_provider_headers():
|
||||||
assert isinstance(error, BedrockError)
|
assert isinstance(error, BedrockError)
|
||||||
assert error.headers == {"x-amzn-RequestId": "req-invoke-500"}
|
assert error.headers == {"x-amzn-RequestId": "req-invoke-500"}
|
||||||
assert error.response.headers["x-amzn-requestid"] == "req-invoke-500"
|
assert error.response.headers["x-amzn-requestid"] == "req-invoke-500"
|
||||||
|
|
||||||
|
|
||||||
|
def test_transform_response_hands_json_mode_to_nova():
|
||||||
|
"""The invoke dispatcher forwards its json_mode argument to Nova instead of dropping it."""
|
||||||
|
from litellm.types.utils import ModelResponse
|
||||||
|
|
||||||
|
response_json = {
|
||||||
|
"output": {
|
||||||
|
"message": {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"toolUse": {
|
||||||
|
"toolUseId": "tooluse_nova_json",
|
||||||
|
"name": "json_tool_call",
|
||||||
|
"input": {"city": "Paris", "temperature": 21},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"stopReason": "tool_use",
|
||||||
|
"usage": {"inputTokens": 5, "outputTokens": 4, "totalTokens": 9},
|
||||||
|
}
|
||||||
|
raw_response = httpx.Response(200, json=response_json, request=httpx.Request("POST", "https://bedrock"))
|
||||||
|
|
||||||
|
result = AmazonInvokeConfig().transform_response(
|
||||||
|
model="invoke/amazon.nova-lite-v1:0",
|
||||||
|
raw_response=raw_response,
|
||||||
|
model_response=ModelResponse(),
|
||||||
|
logging_obj=MagicMock(),
|
||||||
|
request_data={},
|
||||||
|
messages=[{"role": "user", "content": "weather"}],
|
||||||
|
optional_params={},
|
||||||
|
litellm_params={},
|
||||||
|
encoding=None,
|
||||||
|
api_key=None,
|
||||||
|
json_mode=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.choices[0].message.tool_calls is None
|
||||||
|
assert json.loads(result.choices[0].message.content) == {"city": "Paris", "temperature": 21}
|
||||||
|
|
|
||||||
|
|
@ -382,6 +382,8 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto():
|
||||||
"us.openai.gpt-5.6-sol",
|
"us.openai.gpt-5.6-sol",
|
||||||
"global.openai.gpt-5.6-terra",
|
"global.openai.gpt-5.6-terra",
|
||||||
"bedrock/converse/us.openai.gpt-5.6-luna",
|
"bedrock/converse/us.openai.gpt-5.6-luna",
|
||||||
|
"us.openai.gpt-6-astra",
|
||||||
|
"bedrock/converse/global.openai.gpt-6-astra",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(model, local_model_cost_map):
|
def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(model, local_model_cost_map):
|
||||||
|
|
@ -412,6 +414,7 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode
|
||||||
[
|
[
|
||||||
"us.openai.gpt-5.6-sol",
|
"us.openai.gpt-5.6-sol",
|
||||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||||
|
"us.openai.gpt-6-astra",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_openai_gpt5_converse_never_forwards_thinking(model, local_model_cost_map):
|
def test_openai_gpt5_converse_never_forwards_thinking(model, local_model_cost_map):
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue