mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(bedrock): clamp maxTokens to the 16-token minimum for OpenAI GPT and xAI Grok models on Converse (#41870)
Backport of #41870 to stable/1.100.x. Cherry-picked from a6e3a72ed8 (main) with -m 1.
converse_transformation.py conflicted because this line has no `import re` and no
_is_openai_gpt_reasoning_model helper next to the insertion point. The resolution adds
exactly the four hunks #41870 merged: the import, the 16-token constant,
_requires_min_max_tokens, and the clamped maxTokens assignment. The test file applied clean.
This commit is contained in:
parent
b7d81e98b6
commit
fff43dfc05
2 changed files with 39 additions and 1 deletions
|
|
@ -4,6 +4,7 @@ Translating between OpenAI's `/chat/completion` format and Amazon's `/converse`
|
|||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -104,6 +105,7 @@ BEDROCK_COMPUTER_USE_TOOLS: Final = [
|
|||
"bash_",
|
||||
"text_editor_",
|
||||
]
|
||||
BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS: Final = 16
|
||||
|
||||
# Beta header patterns that are not supported by Bedrock Converse API
|
||||
# These will be filtered out to prevent errors
|
||||
|
|
@ -292,6 +294,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
llm_provider="bedrock",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _requires_min_max_tokens(model: str) -> bool:
|
||||
return re.search(r"openai\.gpt-\d|xai\.grok-", model) is not None
|
||||
|
||||
def _is_nova_2_model(self, model: str) -> bool:
|
||||
"""
|
||||
Check if the model is a Nova 2 model that supports reasoningConfig.
|
||||
|
|
@ -874,7 +880,11 @@ class AmazonConverseConfig(BaseConfig):
|
|||
is_thinking_enabled=is_thinking_enabled,
|
||||
)
|
||||
if param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["maxTokens"] = value
|
||||
optional_params["maxTokens"] = (
|
||||
max(value, BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS)
|
||||
if isinstance(value, int) and self._requires_min_max_tokens(model)
|
||||
else value
|
||||
)
|
||||
if param == "stream":
|
||||
optional_params["stream"] = value
|
||||
if param == "stop":
|
||||
|
|
|
|||
|
|
@ -375,6 +375,34 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto():
|
|||
assert optional_params["tool_choice"] == {"auto": {}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, param, value, expected_max_tokens",
|
||||
[
|
||||
("us.openai.gpt-6-astra", "max_tokens", 1, 16),
|
||||
("us.openai.gpt-6-astra", "max_completion_tokens", 1, 16),
|
||||
("us.openai.gpt-6-astra", "max_tokens", 64, 64),
|
||||
("us.xai.grok-4.6", "max_tokens", 1, 16),
|
||||
("global.xai.grok-4.6", "max_completion_tokens", 1, 16),
|
||||
("us.xai.grok-4.6", "max_tokens", 32, 32),
|
||||
("anthropic.claude-sonnet-4-5-20250929-v1:0", "max_tokens", 1, 1),
|
||||
("arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.openai.gpt-6-astra", "max_tokens", 1, 16),
|
||||
("arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.xai.grok-4.6", "max_tokens", 1, 16),
|
||||
("arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123xyz", "max_tokens", 1, 1),
|
||||
],
|
||||
)
|
||||
def test_map_openai_params_enforces_minimum_max_tokens_for_openai_compat_models(
|
||||
model: str, param: str, value: int, expected_max_tokens: int
|
||||
):
|
||||
optional_params = AmazonConverseConfig().map_openai_params(
|
||||
non_default_params={param: value},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert optional_params["maxTokens"] == expected_max_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue