fix(bedrock): refuse GPT sampling and logprob params natively only while reasoning is on

On bedrock-runtime's native chat completions endpoint, GPT-5.x and GPT-6.x
accept temperature, top_p, frequency_penalty, presence_penalty, logprobs,
and top_logprobs once reasoning_effort is "none", and refuse them with any
other effort or when the effort is unset. The previous commit refused the
sampling params unconditionally from the cost map's supports_sampling_params
flag, which lost the reasoning-off case and never covered the penalties or
logprobs. The refusal now keys on the model being a GPT id and reasoning
being active, raises a 400 UnsupportedParamsError naming the params unless
drop_params drops them, and lets everything through under "none". Grok and
gpt-oss keep their unconditional family refusals.
This commit is contained in:
mateo-berri 2026-09-30 17:04:54 -07:00
parent 22e35c6b21
commit 952adfc025
3 changed files with 88 additions and 39 deletions

View file

@ -34,7 +34,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import (
BedrockError,
bedrock_model_supports_sampling_params,
bedrock_model_is_openai_gpt,
split_bedrock_region_path,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
@ -50,37 +50,42 @@ if TYPE_CHECKING:
REASONING_OPEN_TAG: Final = "<reasoning>"
REASONING_CLOSE_TAG: Final = "</reasoning>"
GPT_CHAT_COMPLETIONS_REFUSED_PARAMS: Final = frozenset(
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
)
CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY: Final = MappingProxyType(
{
"openai.gpt-5": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS,
"openai.gpt-6": GPT_CHAT_COMPLETIONS_REFUSED_PARAMS,
"openai.gpt-oss": frozenset(("logit_bias",)),
"xai.": frozenset(("frequency_penalty", "presence_penalty")),
}
)
CHAT_COMPLETIONS_SAMPLING_PARAMS: Final = frozenset(("temperature", "top_p"))
GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING: Final = frozenset(
("temperature", "top_p", "frequency_penalty", "presence_penalty", "logprobs", "top_logprobs")
)
def chat_completions_params_refused_for(model: str) -> frozenset[str]:
"""The OpenAI params AWS's Chat Completions endpoint rejects for this model whatever else the request says.
Each family answers them with a 400 (GPT 5.6 and newer, gpt-oss) or a 503 (Grok), and a model whose price-map row
says ``supports_sampling_params: false`` answers ``temperature`` and ``top_p`` with a 400 too, where
Converse dropped the same params under ``drop_params``, so the native config leaves them out of its
supported list and the usual drop-or-raise handling applies before the request reaches AWS.
GPT-OSS answers ``logit_bias`` with a 400 and Grok answers the penalties with a 503, so the native config leaves
them out of its supported params and litellm refuses them, or drops them under ``drop_params``, before sending.
"""
model_id: Final = split_bedrock_region_path(model)[1]
family_refused: Final = frozenset().union(
return frozenset().union(
*(refused for family, refused in CHAT_COMPLETIONS_REFUSED_PARAMS_BY_FAMILY.items() if family in model_id)
)
if bedrock_model_supports_sampling_params(model):
return family_refused
return family_refused | CHAT_COMPLETIONS_SAMPLING_PARAMS
def chat_completions_params_refused_while_reasoning(model: str, params: Mapping[str, object]) -> frozenset[str]:
"""The params of this request that AWS ties to ``reasoning_effort: "none"`` on the GPT-5.x and GPT-6.x families.
AWS answers ``temperature``, ``top_p``, the penalties, and logprobs with a 400 while the model reasons, which
is every effort but ``"none"`` and the default when none is set, and accepts all of them under ``"none"``.
"""
if params.get("reasoning_effort") == "none" or not bedrock_model_is_openai_gpt(model):
return frozenset()
return GPT_CHAT_COMPLETIONS_PARAMS_REFUSED_WHILE_REASONING & frozenset(params)
def _without_params(params: Mapping[str, object], dropped: frozenset[str]) -> Mapping[str, object]:
return MappingProxyType({key: value for key, value in params.items() if key not in dropped})
CHAT_COMPLETIONS_REFUSED_REASONING_EFFORTS_BY_FAMILY: Final = MappingProxyType({"xai.": frozenset(("none",))})
@ -105,7 +110,7 @@ def chat_completions_reasoning_efforts_refused_for(model: str) -> frozenset[str]
def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) -> Mapping[str, object]:
if params.get("reasoning_effort") not in chat_completions_reasoning_efforts_refused_for(model):
return params
return MappingProxyType({key: value for key, value in params.items() if key != "reasoning_effort"})
return _without_params(params, frozenset(("reasoning_effort",)))
def _held_close_tag_prefix(text: str) -> int:
@ -329,8 +334,20 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig):
drop_params=drop_params,
replace_max_completion_tokens_with_max_tokens=replace_max_completion_tokens_with_max_tokens,
)
refused_while_reasoning: Final = chat_completions_params_refused_while_reasoning(model, non_default_params)
if refused_while_reasoning and not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} doesn't support {sorted(refused_while_reasoning)} while reasoning is active on "
"Bedrock's Chat Completions endpoint. Set reasoning_effort to 'none' to send them, or set "
"`litellm.drop_params = True` to drop them"
),
status_code=400,
)
return dict( # mutable-ok: get_optional_params keeps filling this dict
without_refused_reasoning_effort(model, with_max_completion_tokens(mapped))
without_refused_reasoning_effort(
model, with_max_completion_tokens(_without_params(mapped, refused_while_reasoning))
)
)
def _inference_params(

View file

@ -882,17 +882,9 @@ def bedrock_runtime_chat_completions_enforces_response_format(model: str) -> boo
return _bedrock_price_map_flag(model, "supports_bedrock_runtime_chat_completions_response_format")
def bedrock_model_supports_sampling_params(model: str) -> bool:
"""Whether the model takes ``temperature`` and ``top_p``: false only when a price-map row says so.
The GPT 5.6 and newer rows carry ``supports_sampling_params: false`` because AWS answers either param with
a 400 on Converse and on native Chat Completions alike, so both routes drop them under ``drop_params`` and
refuse them otherwise.
"""
return not any(
entry is not None and entry.get("supports_sampling_params") is False
for entry in _bedrock_price_map_entries(model)
)
def bedrock_model_is_openai_gpt(model: str) -> bool:
"""A GPT-5.x or GPT-6.x id, never GPT-OSS: the families whose sampling params AWS ties to reasoning being off."""
return _openai_gpt_version(model) is not None
BEDROCK_CONVERSE_ONLY_REQUEST_KEYS: Final = frozenset(

View file

@ -599,13 +599,13 @@ def test_supported_params_include_reasoning_effort_for_gpt56(local_cost_map):
[
(
"bedrock/global.openai.gpt-5.6-sol",
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"),
("logit_bias", "reasoning_effort", "tools", "functions", "stop"),
("n",),
("temperature", "top_p", "frequency_penalty", "logprobs", "logit_bias", "reasoning_effort", "stop"),
),
(
"bedrock/us.openai.gpt-6.1-sol",
("frequency_penalty", "presence_penalty", "logprobs", "top_logprobs", "temperature", "top_p", "n"),
("logit_bias", "reasoning_effort", "tools", "functions", "stop"),
("n",),
("temperature", "top_p", "presence_penalty", "top_logprobs", "reasoning_effort", "tools", "functions"),
),
(
"us.xai.grok-4.6",
@ -628,11 +628,6 @@ def test_supported_params_leave_out_what_each_family_refuses(local_cost_map, mod
@pytest.mark.parametrize(
"model, param",
[
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"frequency_penalty": 0.5}),
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"logprobs": True, "top_logprobs": 2}),
("bedrock/chat_completions/global.openai.gpt-5.6-sol", {"temperature": 0.2}),
("bedrock/chat_completions/global.openai.gpt-6-sol", {"top_p": 0.9}),
("bedrock/chat_completions/global.openai.gpt-6-sol", {"presence_penalty": 0.5}),
("bedrock/chat_completions/us.xai.grok-4.6", {"presence_penalty": 0.5}),
("bedrock/chat_completions/openai.gpt-oss-20b-1:0", {"logit_bias": {"1": 1}}),
],
@ -650,6 +645,51 @@ def test_refused_params_are_dropped_or_refused_before_reaching_aws(local_cost_ma
assert param.keys().isdisjoint(json.loads(requests[0].content))
GPT_PARAMS_TIED_TO_REASONING_OFF = {
"temperature": 0.2,
"top_p": 0.9,
"frequency_penalty": 0.5,
"presence_penalty": 0.5,
"logprobs": True,
"top_logprobs": 2,
}
@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"])
@pytest.mark.parametrize("reasoning", [{}, {"reasoning_effort": "low"}], ids=["effort_unset", "effort_low"])
@pytest.mark.parametrize("param", list(GPT_PARAMS_TIED_TO_REASONING_OFF))
def test_gpt_sampling_params_are_refused_or_dropped_while_reasoning(
local_cost_map, fake_aws_env, model, reasoning, param
):
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
request = {"model": model, "messages": [{"role": "user", "content": "hello"}], "client": client, **reasoning}
with pytest.raises(litellm.UnsupportedParamsError, match=param):
litellm.completion(**request, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]})
litellm.completion(**request, drop_params=True, **{param: GPT_PARAMS_TIED_TO_REASONING_OFF[param]})
body = json.loads(requests[0].content)
assert str(requests[0].url).endswith("/openai/v1/chat/completions")
assert param not in body
assert body.get("reasoning_effort") == reasoning.get("reasoning_effort")
@pytest.mark.parametrize("model", ["bedrock/global.openai.gpt-5.6-sol", "bedrock/us.openai.gpt-6-sol"])
def test_gpt_sampling_params_reach_aws_with_reasoning_effort_none(local_cost_map, fake_aws_env, model):
requests, client = _recording_client(json=_chat_completion_json("ok", model.removeprefix("bedrock/")))
litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
reasoning_effort="none",
client=client,
**GPT_PARAMS_TIED_TO_REASONING_OFF,
)
body = json.loads(requests[0].content)
assert str(requests[0].url).endswith("/openai/v1/chat/completions")
assert body["reasoning_effort"] == "none"
assert {key: body[key] for key in GPT_PARAMS_TIED_TO_REASONING_OFF} == GPT_PARAMS_TIED_TO_REASONING_OFF
def test_split_reasoning_tag_splits_leading_tag():
assert split_reasoning_tag("<reasoning>plan it\n</reasoning>\n\nHello") == ("plan it\n", "Hello")