mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_bedrock_v1_messages_stream_passthrough
This commit is contained in:
commit
0d2546e74e
5 changed files with 174 additions and 37 deletions
|
|
@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
return self._is_deepseek_r1_model(model=model, base_model=base_model)
|
||||
|
||||
@classmethod
|
||||
def _supports_sampling_params(cls, model: str) -> bool:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if base_model.startswith("anthropic"):
|
||||
return True
|
||||
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
|
||||
for candidate in candidates:
|
||||
if (
|
||||
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
|
||||
candidate, "supports_sampling_params"
|
||||
)
|
||||
) is not None:
|
||||
return flag
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
supports_sampling: Final = self._supports_sampling_params(model)
|
||||
supported_params: Final = [
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
"stream_options",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
*(("temperature", "top_p") if supports_sampling else ()),
|
||||
"extra_headers",
|
||||
"response_format",
|
||||
"requestMetadata",
|
||||
|
|
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
|
|||
value = [value]
|
||||
optional_params["stopSequences"] = value
|
||||
if param == "temperature" or param == "top_p":
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
if base_model.startswith("anthropic"):
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
elif not self._supports_sampling_params(model):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}={value}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
else:
|
||||
optional_params["topP" if param == "top_p" else param] = value
|
||||
if param == "tools" and isinstance(value, list):
|
||||
self._apply_tool_call_transformation(
|
||||
tools=cast(list[OpenAIChatCompletionToolParam], value),
|
||||
|
|
|
|||
|
|
@ -55087,7 +55087,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 4e-06,
|
||||
|
|
@ -55116,7 +55117,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
|
|
@ -55145,7 +55147,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -55174,7 +55177,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
|
|
@ -55203,7 +55207,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -55232,7 +55237,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-6-astra": {
|
||||
"input_cost_per_token": 1.1e-05,
|
||||
|
|
|
|||
|
|
@ -55087,7 +55087,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 4e-06,
|
||||
|
|
@ -55116,7 +55117,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
|
|
@ -55145,7 +55147,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -55174,7 +55177,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
|
|
@ -55203,7 +55207,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -55232,7 +55237,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-6-astra": {
|
||||
"input_cost_per_token": 1.1e-05,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
|
@ -7500,3 +7499,92 @@ def test_eager_input_streaming_non_boolean_is_a_bad_request():
|
|||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming="true")],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7"))
|
||||
def test_converse_accepts_anthropic_default_temperature(model: str) -> None:
|
||||
result: Final = litellm.utils.get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
temperature=1,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["temperature"] == 1
|
||||
|
||||
|
||||
def test_get_supported_openai_params_drops_sampling_params_for_gpt5_models():
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"global.openai.gpt-5.6-luna",
|
||||
"global.openai.gpt-5.6-sol",
|
||||
"us.openai.gpt-5.6-terra",
|
||||
"eu.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
"bedrock/openai.gpt-5.6-luna",
|
||||
]:
|
||||
supported = config.get_supported_openai_params(model=model)
|
||||
assert "temperature" not in supported
|
||||
assert "top_p" not in supported
|
||||
|
||||
supported_oss = config.get_supported_openai_params(model="openai.gpt-oss-120b-1:0")
|
||||
assert "temperature" in supported_oss
|
||||
assert "top_p" in supported_oss
|
||||
|
||||
|
||||
def test_map_openai_params_drops_temperature_and_top_p_when_drop_params_true():
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
"eu.openai.gpt-5.6-luna",
|
||||
]:
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"temperature": 1.0, "top_p": 0.9, "max_tokens": 50},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=True,
|
||||
)
|
||||
assert "temperature" not in result
|
||||
assert "topP" not in result
|
||||
assert result.get("maxTokens") == 50
|
||||
|
||||
|
||||
def test_map_openai_params_raises_unsupported_params_when_drop_params_false(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
]:
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError) as exc_info:
|
||||
config.map_openai_params(
|
||||
non_default_params={"temperature": 1.0},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
assert "does not support temperature=1.0" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_map_openai_params_retains_sampling_params_for_supported_models():
|
||||
config = AmazonConverseConfig()
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.7, "top_p": 0.8},
|
||||
optional_params={},
|
||||
model="openai.gpt-oss-120b-1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
assert result.get("temperature") == 0.7
|
||||
assert result.get("topP") == 0.8
|
||||
|
||||
|
||||
def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"global.custom-test-reasoning-model",
|
||||
{"supports_sampling_params": False},
|
||||
)
|
||||
assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False
|
||||
assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import logging
|
|||
import os
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import PurePath
|
||||
|
|
@ -5283,23 +5283,31 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
|
|||
) -> None:
|
||||
"""Regression: an error raised after the deployment call already succeeded (e.g. inside
|
||||
async_post_call_success_deployment_hook or post_call_processing) is not a deployment
|
||||
attempt failure and must not reach async_post_call_failure_deployment_hook."""
|
||||
attempt failure and must not reach async_post_call_failure_deployment_hook. The raising
|
||||
callback is a guardrail because a plain logger's success hook error is isolated and
|
||||
logged instead of propagating out of the call."""
|
||||
|
||||
class ExplodingSuccessLogger(CustomLogger):
|
||||
class ExplodingSuccessGuardrail(CustomGuardrail):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.failure_calls: list[Exception] = []
|
||||
super().__init__(guardrail_name="exploding")
|
||||
self.failure_calls: tuple[Exception, ...] = ()
|
||||
|
||||
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
|
||||
async def async_post_call_success_deployment_hook(
|
||||
self, request_data: Mapping[str, object], response: LLMResponseTypes, call_type: CallTypes | None
|
||||
) -> LLMResponseTypes | None:
|
||||
raise RuntimeError("boom in success hook, model call itself succeeded")
|
||||
|
||||
async def async_post_call_failure_deployment_hook(
|
||||
self, request_data, exception, call_type, fallback_depth=None
|
||||
):
|
||||
self.failure_calls.append(exception)
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
exception: Exception,
|
||||
call_type: CallTypes | None,
|
||||
fallback_depth: int | None = None,
|
||||
) -> None:
|
||||
self.failure_calls = (*self.failure_calls, exception)
|
||||
|
||||
exploding_logger = ExplodingSuccessLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [exploding_logger])
|
||||
exploding_guardrail: Final = ExplodingSuccessGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [exploding_guardrail])
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom in success hook"):
|
||||
await litellm.acompletion(
|
||||
|
|
@ -5308,7 +5316,7 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
|
|||
mock_response="this call succeeds",
|
||||
)
|
||||
|
||||
assert exploding_logger.failure_calls == []
|
||||
assert exploding_guardrail.failure_calls == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue