mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(bedrock): refuse GPT sampling and logprob params natively only while reasoning is on
On bedrock-runtime's native chat completions endpoint, GPT-5.x and GPT-6.x accept temperature, top_p, frequency_penalty, presence_penalty, logprobs, and top_logprobs once reasoning_effort is "none", and refuse them with any other effort or when the effort is unset. The previous commit refused the sampling params unconditionally from the cost map's supports_sampling_params flag, which lost the reasoning-off case and never covered the penalties or logprobs. The refusal now keys on the model being a GPT id and reasoning being active, raises a 400 UnsupportedParamsError naming the params unless drop_params drops them, and lets everything through under "none". Grok and gpt-oss keep their unconditional family refusals.
This commit is contained in:
parent
22e35c6b21
commit
952adfc025
3 changed files with 88 additions and 39 deletions
|
|
@ -34,7 +34,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BedrockError,
|
||||
bedrock_model_supports_sampling_params,
|
||||
bedrock_model_is_openai_gpt,
|
||||
split_bedrock_region_path,
|
||||
)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
|
||||
|
|
@ -50,37 +50,42 @@ if TYPE_CHECKING:
|
|||
REASONING_OPEN_TAG: Final = "<reasoning>"
|
||||
REASONING_CLOSE_TAG: Final = "</reasoning>"
|
||||
|
||||
GPT_CHAT_COMPLETIONS_REFUSED_PARAMS: Final = frozenset(
|
||||
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
|
||||
)
|
||||
CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType(
|
||||
{
|
||||
"openai.gpt-5": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS,
|
||||
"openai.gpt-6": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS,
|
||||
"openai.gpt-oss": frozenset(("logit_bias",)),
|
||||
"xai.": frozenset(("frequency_penalty", "presence_penalty")),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
CHAT_COMPLETIONS_SAMPLING_PARAMS: Final = frozenset(("temperature", "top_p"))
|
||||
GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING: Final = frozenset(
|
||||
("temperature", "top_p", "frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
|
||||
)
|
||||
|
||||
|
||||
def chat_completions_params_refused_for(model: str) -> frozenset[str]:
|
||||
"""The OpenAI params AWS's Chat Completions endpoint rejects for this model whatever else the request says.
|
||||
|
||||
Each family answers them with a 400 (GPT 5.6 and newer, gpt-oss) or a 503 (Grok), and a model whose price-map row
|
||||
says ``supports_sampling_params: false`` answers ``temperature`` and ``top_p`` with a 400 too, where
|
||||
Converse dropped the same params under ``drop_params``, so the native config leaves them out of its
|
||||
supported list and the usual drop-or-raise handling applies before the request reaches AWS.
|
||||
GPT-OSS answers ``logit_bias`` with a 400 and Grok answers the penalties with a 503, so the native config leaves
|
||||
them out of its supported params and litellm refuses them, or drops them under ``drop_params``, before sending.
|
||||
"""
|
||||
model_id: Final = split_bedrock_region_path(model)[1]
|
||||
family_refused: Final = frozenset().union(
|
||||
return frozenset().union(
|
||||
*(refused for family, refused in CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY.items() if family in model_id)
|
||||
)
|
||||
if bedrock_model_supports_sampling_params(model):
|
||||
return family_refused
|
||||
return family_refused | CHAT_COMPLETIONS_SAMPLING_PARAMS
|
||||
|
||||
|
||||
def chat_completions_params_refused_while_reasoning(model: str, params: Mapping[str, object]) -> frozenset[str]:
|
||||
"""The params of this request that AWS ties to ``reasoning_effort: "none"`` on the GPT-5.x and GPT-6.x families.
|
||||
|
||||
AWS answers ``temperature``, ``top_p``, the penalties, and logprobs with a 400 while the model reasons, which
|
||||
is every effort but ``"none"`` and the default when none is set, and accepts all of them under ``"none"``.
|
||||
"""
|
||||
if params.get("reasoning_effort") == "none" or not bedrock_model_is_openai_gpt(model):
|
||||
return frozenset()
|
||||
return GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING & frozenset(params)
|
||||
|
||||
|
||||
def _without_params(params: Mapping[str, object], dropped: frozenset[str]) -> Mapping[str, object]:
|
||||
return MappingProxyType({key: value for key, value in params.items() if key not in dropped})
|
||||
|
||||
|
||||
CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY: Final = MappingProxyType({"xai.": frozenset(("none",))})
|
||||
|
|
@ -105,7 +110,7 @@ def chat_completions_reasoning_efforts_refused_for(model: str) -> frozenset[str]
|
|||
def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if params.get("reasoning_effort") not in chat_completions_reasoning_efforts_refused_for(model):
|
||||
return params
|
||||
return MappingProxyType({key: value for key, value in params.items() if key != "reasoning_effort"})
|
||||
return _without_params(params, frozenset(("reasoning_effort",)))
|
||||
|
||||
|
||||
def _held_close_tag_prefix(text: str) -> int:
|
||||
|
|
@ -329,8 +334,20 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
|
|||
drop_params=drop_params,
|
||||
replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens,
|
||||
)
|
||||
refused_while_reasoning: Final = chat_completions_params_refused_while_reasoning(model, non_default_params)
|
||||
if refused_while_reasoning and not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} doesn't support {sorted(refused_while_reasoning)} while reasoning is active on "
|
||||
"Bedrock's Chat Completions endpoint. Set reasoning_effort to 'none' to send them, or set "
|
||||
"`litellm.drop_params = True` to drop them"
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
return dict( # mutable-ok: get_optional_params keeps filling this dict
|
||||
without_refused_reasoning_effort(model, with_max_completion_tokens(mapped))
|
||||
without_refused_reasoning_effort(
|
||||
model, with_max_completion_tokens(_without_params(mapped, refused_while_reasoning))
|
||||
)
|
||||
)
|
||||
|
||||
def _inference_params(
|
||||
|
|
|
|||
|
|
@ -882,17 +882,9 @@ def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> boo
|
|||
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format")
|
||||
|
||||
|
||||
def bedrock_model_supports_sampling_params(model: str) -> bool:
|
||||
"""Whether the model takes ``temperature`` and ``top_p``: false only when a price-map row says so.
|
||||
|
||||
The GPT 5.6 and newer rows carry ``supports_sampling_params: false`` because AWS answers either param with
|
||||
a 400 on Converse and on native Chat Completions alike, so both routes drop them under ``drop_params`` and
|
||||
refuse them otherwise.
|
||||
"""
|
||||
return not any(
|
||||
entry is not None and entry.get("supports_sampling_params") is False
|
||||
for entry in _bedrock_price_map_entries(model)
|
||||
)
|
||||
def bedrock_model_is_openai_gpt(model: str) -> bool:
|
||||
"""A GPT-5.x or GPT-6.x id, never GPT-OSS: the families whose sampling params AWS ties to reasoning being off."""
|
||||
return _openai_gpt_version(model) is not None
|
||||
|
||||
|
||||
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(
|
||||
|
|
|
|||
|
|
@ -599,13 +599,13 @@ def test_supported_params_include_reasoning_effort_for_gpt56(local_cost_map):
|
|||
[
|
||||
(
|
||||
"bedrock/global.openai.gpt-5.6-sol",
|
||||
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"),
|
||||
("logit_bias", "reasoning_effort", "tools", "functions", "stop"),
|
||||
("n",),
|
||||
("temperature", "top_p", "frequency_penalty", "logprobs", "logit_bias", "reasoning_effort", "stop"),
|
||||
),
|
||||
(
|
||||
"bedrock/us.openai.gpt-6.1-sol",
|
||||
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"),
|
||||
("logit_bias", "reasoning_effort", "tools", "functions", "stop"),
|
||||
("n",),
|
||||
("temperature", "top_p", "presence_penalty", "top_logprobs", "reasoning_effort", "tools", "functions"),
|
||||
),
|
||||
(
|
||||
"us.xai.grok-4.6",
|
||||
|
|
@ -628,11 +628,6 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod
|
|||
@pytest.mark.parametrize(
|
||||
"model, param",
|
||||
[
|
||||
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"frequency_penalty": 0.5}),
|
||||
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"logprobs": True, "top_logprobs": 2}),
|
||||
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"temperature": 0.2}),
|
||||
("bedrock/chat_completions/global.openai.gpt-6-sol", {"top_p": 0.9}),
|
||||
("bedrock/chat_completions/global.openai.gpt-6-sol", {"presence_penalty": 0.5}),
|
||||
("bedrock/chat_completions/us.xai.grok-4.6", {"presence_penalty": 0.5}),
|
||||
("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {"logit_bias": {"1": 1}}),
|
||||
],
|
||||
|
|
@ -650,6 +645,51 @@ def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_ma
|
|||
assert param.keys().isdisjoint(json.loads(requests[0].content))
|
||||
|
||||
|
||||
GPT_PARAMS_TIED_TO_REASONING_OFF = {
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.9,
|
||||
"frequency_penalty": 0.5,
|
||||
"presence_penalty": 0.5,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"])
|
||||
@pytest.mark.parametrize("reasoning", [{}, {"reasoning_effort": "low"}], ids=["effort_unset", "effort_low"])
|
||||
@pytest.mark.parametrize("param", list(GPT_PARAMS_TIED_TO_REASONING_OFF))
|
||||
def test_gpt_sampling_params_are_refused_or_dropped_while_reasoning(
|
||||
local_cost_map, fake_aws_env, model, reasoning, param
|
||||
):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
|
||||
request = {"model": model, "messages": [{"role": "user", "content": "hello"}], "client": client, **reasoning}
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=param):
|
||||
litellm.completion(**request, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]})
|
||||
litellm.completion(**request, drop_params=True, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]})
|
||||
|
||||
body = json.loads(requests[0].content)
|
||||
assert str(requests[0].url).endswith("/openai/v1/chat/completions")
|
||||
assert param not in body
|
||||
assert body.get("reasoning_effort") == reasoning.get("reasoning_effort")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"])
|
||||
def test_gpt_sampling_params_reach_aws_with_reasoning_effort_none(local_cost_map, fake_aws_env, model):
|
||||
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
reasoning_effort="none",
|
||||
client=client,
|
||||
**GPT_PARAMS_TIED_TO_REASONING_OFF,
|
||||
)
|
||||
|
||||
body = json.loads(requests[0].content)
|
||||
assert str(requests[0].url).endswith("/openai/v1/chat/completions")
|
||||
assert body["reasoning_effort"] == "none"
|
||||
assert {key: body[key] for key in GPT_PARAMS_TIED_TO_REASONING_OFF} == GPT_PARAMS_TIED_TO_REASONING_OFF
|
||||
|
||||
|
||||
def test_split_reasoning_tag_splits_leading_tag():
|
||||
assert split_reasoning_tag("<reasoning>plan it\n</reasoning>\n\nHello") == ("plan it\n", "Hello")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue