From 952adfc02519916e2a637c47af3467f9a2ccfcb2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:04:54 -0700 Subject: [PATCH] fix(bedrock): refuse GPT sampling and logprob params natively only while reasoning is on On bedrock-runtime's native chat completions endpoint, GPT-5.x and GPT-6.x accept temperature, top_p, frequency_penalty, presence_penalty, logprobs, and top_logprobs once reasoning_effort is "none", and refuse them with any other effort or when the effort is unset. The previous commit refused the sampling params unconditionally from the cost map's supports_sampling_params flag, which lost the reasoning-off case and never covered the penalties or logprobs. The refusal now keys on the model being a GPT id and reasoning being active, raises a 400 UnsupportedParamsError naming the params unless drop_params drops them, and lets everything through under "none". Grok and gpt-oss keep their unconditional family refusals. --- .../chat/chat_completions/transformation.py | 55 ++++++++++++------ litellm/llms/bedrock/common_utils.py | 14 +---- ...bedrock_chat_completions_transformation.py | 58 ++++++++++++++++--- 3 files changed, 88 insertions(+), 39 deletions(-) diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 34a756a54e2..1477a46dcad 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -34,7 +34,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BedrockError, - bedrock_model_supports_sampling_params, + bedrock_model_is_openai_gpt, split_bedrock_region_path, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler @@ -50,37 +50,42 @@ if TYPE_CHECKING: REASONING_OPEN_TAG: Final = "" REASONING_CLOSE_TAG: Final = "" -GPT_CHAT_COMPLETIONS_REFUSED_PARAMS: Final = frozenset( - ("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs") -) CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType( { - "openai.gpt-5": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS, - "openai.gpt-6": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS, "openai.gpt-oss": frozenset(("logit_bias",)), "xai.": frozenset(("frequency_penalty", "presence_penalty")), } ) - - -CHAT_COMPLETIONS_SAMPLING_PARAMS: Final = frozenset(("temperature", "top_p")) +GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING: Final = frozenset( + ("temperature", "top_p", "frequency_penalty", "presence_penalty", "logprobs", "top_logprobs") +) def chat_completions_params_refused_for(model: str) -> frozenset[str]: """The OpenAI params AWS's Chat Completions endpoint rejects for this model whatever else the request says. - Each family answers them with a 400 (GPT 5.6 and newer, gpt-oss) or a 503 (Grok), and a model whose price-map row - says ``supports_sampling_params: false`` answers ``temperature`` and ``top_p`` with a 400 too, where - Converse dropped the same params under ``drop_params``, so the native config leaves them out of its - supported list and the usual drop-or-raise handling applies before the request reaches AWS. + GPT-OSS answers ``logit_bias`` with a 400 and Grok answers the penalties with a 503, so the native config leaves + them out of its supported params and litellm refuses them, or drops them under ``drop_params``, before sending. """ model_id: Final = split_bedrock_region_path(model)[1] - family_refused: Final = frozenset().union( + return frozenset().union( *(refused for family, refused in CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY.items() if family in model_id) ) - if bedrock_model_supports_sampling_params(model): - return family_refused - return family_refused | CHAT_COMPLETIONS_SAMPLING_PARAMS + + +def chat_completions_params_refused_while_reasoning(model: str, params: Mapping[str, object]) -> frozenset[str]: + """The params of this request that AWS ties to ``reasoning_effort: "none"`` on the GPT-5.x and GPT-6.x families. + + AWS answers ``temperature``, ``top_p``, the penalties, and logprobs with a 400 while the model reasons, which + is every effort but ``"none"`` and the default when none is set, and accepts all of them under ``"none"``. + """ + if params.get("reasoning_effort") == "none" or not bedrock_model_is_openai_gpt(model): + return frozenset() + return GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING & frozenset(params) + + +def _without_params(params: Mapping[str, object], dropped: frozenset[str]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in params.items() if key not in dropped}) CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY: Final = MappingProxyType({"xai.": frozenset(("none",))}) @@ -105,7 +110,7 @@ def chat_completions_reasoning_efforts_refused_for(model: str) -> frozenset[str] def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -> Mapping[str, object]: if params.get("reasoning_effort") not in chat_completions_reasoning_efforts_refused_for(model): return params - return MappingProxyType({key: value for key, value in params.items() if key != "reasoning_effort"}) + return _without_params(params, frozenset(("reasoning_effort",))) def _held_close_tag_prefix(text: str) -> int: @@ -329,8 +334,20 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): drop_params=drop_params, replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens, ) + refused_while_reasoning: Final = chat_completions_params_refused_while_reasoning(model, non_default_params) + if refused_while_reasoning and not (litellm.drop_params or drop_params): + raise litellm.utils.UnsupportedParamsError( + message=( + f"{model} doesn't support {sorted(refused_while_reasoning)} while reasoning is active on " + "Bedrock's Chat Completions endpoint. Set reasoning_effort to 'none' to send them, or set " + "`litellm.drop_params = True` to drop them" + ), + status_code=400, + ) return dict( # mutable-ok: get_optional_params keeps filling this dict - without_refused_reasoning_effort(model, with_max_completion_tokens(mapped)) + without_refused_reasoning_effort( + model, with_max_completion_tokens(_without_params(mapped, refused_while_reasoning)) + ) ) def _inference_params( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index bf40e9e93be..b45d4f40934 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -882,17 +882,9 @@ def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> boo return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format") -def bedrock_model_supports_sampling_params(model: str) -> bool: - """Whether the model takes ``temperature`` and ``top_p``: false only when a price-map row says so. - - The GPT 5.6 and newer rows carry ``supports_sampling_params: false`` because AWS answers either param with - a 400 on Converse and on native Chat Completions alike, so both routes drop them under ``drop_params`` and - refuse them otherwise. - """ - return not any( - entry is not None and entry.get("supports_sampling_params") is False - for entry in _bedrock_price_map_entries(model) - ) +def bedrock_model_is_openai_gpt(model: str) -> bool: + """A GPT-5.x or GPT-6.x id, never GPT-OSS: the families whose sampling params AWS ties to reasoning being off.""" + return _openai_gpt_version(model) is not None BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset( diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 37d96c45d6d..5ea3c8c31bd 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -599,13 +599,13 @@ def test_supported_params_include_reasoning_effort_for_gpt56(local_cost_map): [ ( "bedrock/global.openai.gpt-5.6-sol", - ("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"), - ("logit_bias", "reasoning_effort", "tools", "functions", "stop"), + ("n",), + ("temperature", "top_p", "frequency_penalty", "logprobs", "logit_bias", "reasoning_effort", "stop"), ), ( "bedrock/us.openai.gpt-6.1-sol", - ("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"), - ("logit_bias", "reasoning_effort", "tools", "functions", "stop"), + ("n",), + ("temperature", "top_p", "presence_penalty", "top_logprobs", "reasoning_effort", "tools", "functions"), ), ( "us.xai.grok-4.6", @@ -628,11 +628,6 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod @pytest.mark.parametrize( "model, param", [ - ("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"frequency_penalty": 0.5}), - ("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"logprobs": True, "top_logprobs": 2}), - ("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"temperature": 0.2}), - ("bedrock/chat_completions/global.openai.gpt-6-sol", {"top_p": 0.9}), - ("bedrock/chat_completions/global.openai.gpt-6-sol", {"presence_penalty": 0.5}), ("bedrock/chat_completions/us.xai.grok-4.6", {"presence_penalty": 0.5}), ("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {"logit_bias": {"1": 1}}), ], @@ -650,6 +645,51 @@ def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_ma assert param.keys().isdisjoint(json.loads(requests[0].content)) +GPT_PARAMS_TIED_TO_REASONING_OFF = { + "temperature": 0.2, + "top_p": 0.9, + "frequency_penalty": 0.5, + "presence_penalty": 0.5, + "logprobs": True, + "top_logprobs": 2, +} + + +@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"]) +@pytest.mark.parametrize("reasoning", [{}, {"reasoning_effort": "low"}], ids=["effort_unset", "effort_low"]) +@pytest.mark.parametrize("param", list(GPT_PARAMS_TIED_TO_REASONING_OFF)) +def test_gpt_sampling_params_are_refused_or_dropped_while_reasoning( + local_cost_map, fake_aws_env, model, reasoning, param +): + requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/"))) + request = {"model": model, "messages": [{"role": "user", "content": "hello"}], "client": client, **reasoning} + with pytest.raises(litellm.UnsupportedParamsError, match=param): + litellm.completion(**request, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]}) + litellm.completion(**request, drop_params=True, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]}) + + body = json.loads(requests[0].content) + assert str(requests[0].url).endswith("/openai/v1/chat/completions") + assert param not in body + assert body.get("reasoning_effort") == reasoning.get("reasoning_effort") + + +@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"]) +def test_gpt_sampling_params_reach_aws_with_reasoning_effort_none(local_cost_map, fake_aws_env, model): + requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/"))) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + reasoning_effort="none", + client=client, + **GPT_PARAMS_TIED_TO_REASONING_OFF, + ) + + body = json.loads(requests[0].content) + assert str(requests[0].url).endswith("/openai/v1/chat/completions") + assert body["reasoning_effort"] == "none" + assert {key: body[key] for key in GPT_PARAMS_TIED_TO_REASONING_OFF} == GPT_PARAMS_TIED_TO_REASONING_OFF + + def test_split_reasoning_tag_splits_leading_tag(): assert split_reasoning_tag("plan it\n\n\nHello") == ("plan it\n", "Hello")