Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_bedrock_v1_messages_stream_passthrough

This commit is contained in:
mateo-berri 2026-09-22 19:16:18 -07:00
commit 0d2546e74e
5 changed files with 174 additions and 37 deletions

View file

@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
"""
return self._is_deepseek_r1_model(model=model, base_model=base_model)
@classmethod
def _supports_sampling_params(cls, model: str) -> bool:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
base_model: Final = BedrockModelInfo.get_base_model(model)
if base_model.startswith("anthropic"):
return True
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
for candidate in candidates:
if (
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
candidate, "supports_sampling_params"
)
) is not None:
return flag
return True
def get_supported_openai_params(self, model: str) -> list[str]:
from litellm.utils import supports_function_calling
supports_sampling: Final = self._supports_sampling_params(model)
supported_params: Final = [
"max_tokens",
"max_completion_tokens",
"stream",
"stream_options",
"stop",
"temperature",
"top_p",
*(("temperature", "top_p") if supports_sampling else ()),
"extra_headers",
"response_format",
"requestMetadata",
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
value = [value]
optional_params["stopSequences"] = value
if param == "temperature" or param == "top_p":
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key="topP" if param == "top_p" else param,
)
if base_model.startswith("anthropic"):
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key="topP" if param == "top_p" else param,
)
elif not self._supports_sampling_params(model):
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} does not support {param}={value}. "
"To drop unsupported params, set `litellm.drop_params = True`."
),
status_code=400,
)
else:
optional_params["topP" if param == "top_p" else param] = value
if param == "tools" and isinstance(value, list):
self._apply_tool_call_transformation(
tools=cast(list[OpenAIChatCompletionToolParam], value),

View file

@ -55087,7 +55087,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 4e-06,
@ -55116,7 +55117,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
@ -55145,7 +55147,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
@ -55174,7 +55177,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
@ -55203,7 +55207,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
@ -55232,7 +55237,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"bedrock_mantle/openai.gpt-6-astra": {
"input_cost_per_token": 1.1e-05,

View file

@ -55087,7 +55087,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 4e-06,
@ -55116,7 +55117,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
@ -55145,7 +55147,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
@ -55174,7 +55177,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
@ -55203,7 +55207,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
@ -55232,7 +55237,8 @@
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_xhigh_reasoning_effort": true,
"supports_vision": true
"supports_vision": true,
"supports_sampling_params": false
},
"bedrock_mantle/openai.gpt-6-astra": {
"input_cost_per_token": 1.1e-05,

View file

@ -1,12 +1,11 @@
import json
import os
from typing import Final
from unittest.mock import MagicMock, patch
import httpx
import pytest
from typing import Final
from unittest.mock import MagicMock, patch
import litellm
from litellm import ModelResponse
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
@ -7500,3 +7499,92 @@ def test_eager_input_streaming_non_boolean_is_a_bad_request():
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
[_eager_openai_tool(eager_input_streaming="true")],
)
@pytest.mark.parametrize("model", ("anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7"))
def test_converse_accepts_anthropic_default_temperature(model: str) -> None:
result: Final = litellm.utils.get_optional_params(
model=model,
custom_llm_provider="bedrock",
temperature=1,
drop_params=False,
)
assert result["temperature"] == 1
def test_get_supported_openai_params_drops_sampling_params_for_gpt5_models():
config = AmazonConverseConfig()
for model in [
"bedrock/converse/global.openai.gpt-5.6-luna",
"global.openai.gpt-5.6-luna",
"global.openai.gpt-5.6-sol",
"us.openai.gpt-5.6-terra",
"eu.openai.gpt-5.6-luna",
"openai.gpt-5.6-luna",
"bedrock/openai.gpt-5.6-luna",
]:
supported = config.get_supported_openai_params(model=model)
assert "temperature" not in supported
assert "top_p" not in supported
supported_oss = config.get_supported_openai_params(model="openai.gpt-oss-120b-1:0")
assert "temperature" in supported_oss
assert "top_p" in supported_oss
def test_map_openai_params_drops_temperature_and_top_p_when_drop_params_true():
config = AmazonConverseConfig()
for model in [
"bedrock/converse/global.openai.gpt-5.6-luna",
"openai.gpt-5.6-luna",
"eu.openai.gpt-5.6-luna",
]:
result = config.map_openai_params(
non_default_params={"temperature": 1.0, "top_p": 0.9, "max_tokens": 50},
optional_params={},
model=model,
drop_params=True,
)
assert "temperature" not in result
assert "topP" not in result
assert result.get("maxTokens") == 50
def test_map_openai_params_raises_unsupported_params_when_drop_params_false(monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
config = AmazonConverseConfig()
for model in [
"bedrock/converse/global.openai.gpt-5.6-luna",
"openai.gpt-5.6-luna",
]:
with pytest.raises(litellm.utils.UnsupportedParamsError) as exc_info:
config.map_openai_params(
non_default_params={"temperature": 1.0},
optional_params={},
model=model,
drop_params=False,
)
assert "does not support temperature=1.0" in str(exc_info.value)
def test_map_openai_params_retains_sampling_params_for_supported_models():
config = AmazonConverseConfig()
result = config.map_openai_params(
non_default_params={"temperature": 0.7, "top_p": 0.8},
optional_params={},
model="openai.gpt-oss-120b-1:0",
drop_params=False,
)
assert result.get("temperature") == 0.7
assert result.get("topP") == 0.8
def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
litellm.model_cost,
"global.custom-test-reasoning-model",
{"supports_sampling_params": False},
)
assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False
assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True

View file

@ -8,7 +8,7 @@ import logging
import os
import queue
import threading
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from pathlib import PurePath
@ -5283,23 +5283,31 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
) -> None:
"""Regression: an error raised after the deployment call already succeeded (e.g. inside
async_post_call_success_deployment_hook or post_call_processing) is not a deployment
attempt failure and must not reach async_post_call_failure_deployment_hook."""
attempt failure and must not reach async_post_call_failure_deployment_hook. The raising
callback is a guardrail because a plain logger's success hook error is isolated and
logged instead of propagating out of the call."""
class ExplodingSuccessLogger(CustomLogger):
class ExplodingSuccessGuardrail(CustomGuardrail):
def __init__(self) -> None:
super().__init__()
self.failure_calls: list[Exception] = []
super().__init__(guardrail_name="exploding")
self.failure_calls: tuple[Exception, ...] = ()
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
async def async_post_call_success_deployment_hook(
self, request_data: Mapping[str, object], response: LLMResponseTypes, call_type: CallTypes | None
) -> LLMResponseTypes | None:
raise RuntimeError("boom in success hook, model call itself succeeded")
async def async_post_call_failure_deployment_hook(
self, request_data, exception, call_type, fallback_depth=None
):
self.failure_calls.append(exception)
self,
request_data: Mapping[str, object],
exception: Exception,
call_type: CallTypes | None,
fallback_depth: int | None = None,
) -> None:
self.failure_calls = (*self.failure_calls, exception)
exploding_logger = ExplodingSuccessLogger()
monkeypatch.setattr(litellm, "callbacks", [exploding_logger])
exploding_guardrail: Final = ExplodingSuccessGuardrail()
monkeypatch.setattr(litellm, "callbacks", [exploding_guardrail])
with pytest.raises(RuntimeError, match="boom in success hook"):
await litellm.acompletion(
@ -5308,7 +5316,7 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
mock_response="this call succeeds",
)
assert exploding_logger.failure_calls == []
assert exploding_guardrail.failure_calls == ()
@pytest.mark.asyncio