scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge pull request #21024 from BerriAI/litellm_oss_staging_02_12_2026

Browse source 
Litellm oss staging 02 12 2026
This commit is contained in:
Sameer Kankute 2026-02-13 18:37:44 +05:30 • committed by GitHub
commit f343159911
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 1839 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -41,6 +41,10 @@ class EnterpriseRouteChecks:
return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
@staticmethod
def should_call_route(route: str):
"""
@ -58,6 +62,7 @@ class EnterpriseRouteChecks:
)
elif (
RouteChecks.is_llm_api_route(route=route)
and route not in EnterpriseRouteChecks.LLM_API_EXEMPT_ROUTES
and EnterpriseRouteChecks.is_llm_api_route_disabled()
):
raise HTTPException(

66
litellm/caching/dual_cache.py
View file

@ -12,7 +12,8 @@ import asyncio
import time
import traceback
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, List, Optional, Union
from threading import Lock
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
if TYPE_CHECKING:
from litellm.types.caching import RedisPipelineIncrementOperation
@ -71,6 +72,7 @@ class DualCache(BaseCache):
self.last_redis_batch_access_time = LimitedSizeOrderedDict(
max_size=default_max_redis_batch_cache_size
)
self._last_redis_batch_access_time_lock = Lock()
self.redis_batch_cache_expiry = (
default_redis_batch_cache_expiry
or litellm.default_redis_batch_cache_expiry
@ -236,22 +238,46 @@ class DualCache(BaseCache):
except Exception:
verbose_logger.error(traceback.format_exc())
def get_redis_batch_keys(
def _reserve_redis_batch_keys(
self,
current_time: float,
keys: List[str],
result: List[Any],
) -> List[str]:
sublist_keys = []
for key, value in zip(keys, result):
if value is None:
) -> Tuple[List[str], Dict[str, Optional[float]]]:
"""
Atomically choose keys to fetch from Redis and reserve their access time.
This prevents check-then-act races under concurrent async callers.
"""
sublist_keys: List[str] = []
previous_access_times: Dict[str, Optional[float]] = {}
with self._last_redis_batch_access_time_lock:
for key, value in zip(keys, result):
if value is not None:
continue
if (
key not in self.last_redis_batch_access_time
or current_time - self.last_redis_batch_access_time[key]
>= self.redis_batch_cache_expiry
):
sublist_keys.append(key)
return sublist_keys
previous_access_times[key] = self.last_redis_batch_access_time.get(
key
)
self.last_redis_batch_access_time[key] = current_time
return sublist_keys, previous_access_times
def _rollback_redis_batch_key_reservations(
self, previous_access_times: Dict[str, Optional[float]]
) -> None:
with self._last_redis_batch_access_time_lock:
for key, previous_time in previous_access_times.items():
if previous_time is None:
self.last_redis_batch_access_time.pop(key, None)
else:
self.last_redis_batch_access_time[key] = previous_time
async def async_batch_get_cache(
self,
@ -276,19 +302,23 @@ class DualCache(BaseCache):
- check the redis cache
"""
current_time = time.time()
sublist_keys = self.get_redis_batch_keys(current_time, keys, result)
sublist_keys, previous_access_times = self._reserve_redis_batch_keys(
current_time, keys, result
)
# Only hit Redis if the last access time was more than 5 seconds ago
# Only hit Redis if enough time has passed since last access.
if len(sublist_keys) > 0:
# If not found in in-memory cache, try fetching from Redis
redis_result = await self.redis_cache.async_batch_get_cache(
sublist_keys, parent_otel_span=parent_otel_span
)
# Update the last access time for ALL queried keys
# This includes keys with None values to throttle repeated Redis queries
for key in sublist_keys:
self.last_redis_batch_access_time[key] = current_time
try:
# If not found in in-memory cache, try fetching from Redis
redis_result = await self.redis_cache.async_batch_get_cache(
sublist_keys, parent_otel_span=parent_otel_span
)
except Exception:
# Do not throttle subsequent callers if the Redis read fails.
self._rollback_redis_batch_key_reservations(
previous_access_times
)
raise
# Short-circuit if redis_result is None or contains only None values
if redis_result is None or all(v is None for v in redis_result.values()):

4
litellm/constants.py
View file

@ -1016,10 +1016,12 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
BEDROCK_CONVERSE_MODELS = [
"qwen.qwen3-coder-480b-a35b-v1:0",
"qwen.qwen3-coder-next",
"qwen.qwen3-235b-a22b-2507-v1:0",
"qwen.qwen3-coder-30b-a3b-v1:0",
"qwen.qwen3-32b-v1:0",
"deepseek.v3-v1:0",
"deepseek.v3.2",
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
@ -1062,6 +1064,8 @@ BEDROCK_CONVERSE_MODELS = [
"amazon.nova-pro-v1:0",
"writer.palmyra-x4-v1:0",
"writer.palmyra-x5-v1:0",
"minimax.minimax-m2.1",
"moonshotai.kimi-k2.5",
]

2
litellm/litellm_core_utils/litellm_logging.py
View file

@ -3127,7 +3127,7 @@ class Logging(LiteLLMLoggingBaseClass):
self, dynamic_success_callbacks: Optional[List], global_callbacks: List
) -> List:
if dynamic_success_callbacks is None:
return global_callbacks
return list(global_callbacks)
return list(set(dynamic_success_callbacks + global_callbacks))
def _remove_internal_litellm_callbacks(self, callbacks: List) -> List:

17
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -772,12 +772,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
try:
return ModelResponseStream(
id=chunk["id"],
object="chat.completion.chunk",
created=chunk.get("created"),
model=chunk.get("model"),
choices=chunk.get("choices", []),
)
kwargs = {
"id": chunk["id"],
"object": "chat.completion.chunk",
"created": chunk.get("created"),
"model": chunk.get("model"),
"choices": chunk.get("choices", []),
}
if "usage" in chunk and chunk["usage"] is not None:
kwargs["usage"] = chunk["usage"]
return ModelResponseStream(**kwargs)
except Exception as e:
raise e

12
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -529,6 +529,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
raise e
def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
"""Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values."""
extra_body: Optional[dict] = optional_params.pop("extra_body", None)
if extra_body is not None:
for k, v in extra_body.items():
if k in data and isinstance(data[k], dict) and isinstance(v, dict):
data[k].update(v)
else:
data[k] = v
def _transform_request_body(
messages: List[AllMessageValues],
model: str,
@ -619,6 +630,7 @@ def _transform_request_body(
# Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty
if labels and custom_llm_provider != LlmProviders.GEMINI:
data["labels"] = labels
_pop_and_merge_extra_body(data, optional_params)
except Exception as e:
raise e

632
litellm/model_prices_and_context_window_backup.json
View file

@ -6105,6 +6105,32 @@
"output_cost_per_token": 2.4e-05,
"supports_tool_choice": true
},
"bedrock/ap-northeast-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6116,6 +6142,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6128,7 +6181,7 @@
"supports_reasoning": true
},
"bedrock/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.3e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -6159,6 +6212,32 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07
},
"bedrock/ap-south-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6170,6 +6249,86 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ap-south-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.05e-06,
"litellm_provider": "bedrock",
@ -6188,6 +6347,46 @@
"mode": "chat",
"output_cost_per_token": 6.9e-07
},
"bedrock/eu-north-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": {
"input_cost_per_second": 0.01635,
"litellm_provider": "bedrock",
@ -6275,6 +6474,32 @@
"output_cost_per_token": 2.4e-05,
"supports_tool_choice": true
},
"bedrock/eu-central-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.86e-06,
"litellm_provider": "bedrock",
@ -6293,6 +6518,32 @@
"mode": "chat",
"output_cost_per_token": 6.5e-07
},
"bedrock/eu-west-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.45e-06,
"litellm_provider": "bedrock",
@ -6311,6 +6562,32 @@
"mode": "chat",
"output_cost_per_token": 7.8e-07
},
"bedrock/eu-west-2/minimax.minimax-m2.1": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.86e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.86e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
"input_cost_per_token": 2e-07,
"litellm_provider": "bedrock",
@ -6341,6 +6618,32 @@
"output_cost_per_token": 9.1e-07,
"supports_tool_choice": true
},
"bedrock/eu-south-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock",
@ -6375,6 +6678,32 @@
"mode": "chat",
"output_cost_per_token": 1.01e-06
},
"bedrock/sa-east-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6386,6 +6715,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/sa-east-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
"input_cost_per_second": 0.011,
"litellm_provider": "bedrock",
@ -6522,6 +6878,32 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-east-1/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6533,6 +6915,59 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6544,6 +6979,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-2/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
"input_cost_per_token": 9.6e-07,
"litellm_provider": "bedrock",
@ -6950,6 +7412,32 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-west-2/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6961,6 +7449,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-west-2/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
"cache_creation_input_token_cost": 1e-06,
"cache_read_input_token_cost": 8e-08,
@ -10874,6 +11389,19 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"dolphin": {
"input_cost_per_token": 5e-07,
"litellm_provider": "nlp_cloud",
@ -21344,6 +21872,19 @@
"output_cost_per_token": 1.2e-06,
"supports_system_messages": true
},
"minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -22100,6 +22641,20 @@
"supports_reasoning": true,
"supports_system_messages": true
},
"moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"moonshot/kimi-k2-0711-preview": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 6e-07,
@ -25534,6 +26089,19 @@
"supports_system_messages": true,
"supports_vision": true
},
"qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"recraft/recraftv2": {
"litellm_provider": "recraft",
"mode": "image_generation",
@ -27970,6 +28538,30 @@
"supports_reasoning": true,
"supports_tool_choice": false
},
"us.deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"eu.deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"us.meta.llama3-1-405b-instruct-v1:0": {
"input_cost_per_token": 5.32e-06,
"litellm_provider": "bedrock",
@ -32080,6 +32672,23 @@
"1280x720"
]
},
"openai/sora-2-pro-high-res": {
"litellm_provider": "openai",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.5,
"source": "https://platform.openai.com/docs/api-reference/videos",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
],
"supported_resolutions": [
"1024x1792",
"1792x1024"
]
},
"azure/sora-2": {
"litellm_provider": "azure",
"mode": "video_generation",
@ -35719,7 +36328,9 @@
"gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image": 8e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "openai",
"max_input_tokens": 128000,
@ -35750,7 +36361,9 @@
"gpt-realtime-mini-2025-12-15": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image": 8e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "openai",
"max_input_tokens": 128000,
@ -35812,6 +36425,23 @@
"1280x720"
]
},
"sora-2-pro-high-res": {
"litellm_provider": "openai",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.5,
"source": "https://platform.openai.com/docs/api-reference/videos",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
],
"supported_resolutions": [
"1024x1792",
"1792x1024"
]
},
"chatgpt-image-latest": {
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,

4
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -71,9 +71,7 @@ try:
from mcp.shared.tool_name_validation import (
validate_tool_name, # pyright: ignore[reportAssignmentType]
)
from mcp.shared.tool_name_validation import (
SEP_986_URL,
)
from mcp.shared.tool_name_validation import SEP_986_URL
except ImportError:
from pydantic import BaseModel

2
litellm/proxy/utils.py
View file

@ -740,7 +740,7 @@ class ProxyLogging:
self, dynamic_success_callbacks: Optional[List], global_callbacks: List
) -> List:
if dynamic_success_callbacks is None:
return global_callbacks
return list(global_callbacks)
return list(set(dynamic_success_callbacks + global_callbacks))
def _parse_pre_mcp_call_hook_response(

1
litellm/responses/main.py
View file

@ -682,6 +682,7 @@ def responses(
_is_async=_is_async,
stream=stream,
extra_headers=extra_headers,
extra_body=extra_body,
**kwargs,
)

594
model_prices_and_context_window.json
View file

@ -6105,6 +6105,32 @@
"output_cost_per_token": 2.4e-05,
"supports_tool_choice": true
},
"bedrock/ap-northeast-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6116,6 +6142,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6128,7 +6181,7 @@
"supports_reasoning": true
},
"bedrock/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.3e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
@ -6159,6 +6212,32 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07
},
"bedrock/ap-south-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6170,6 +6249,86 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ap-south-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.05e-06,
"litellm_provider": "bedrock",
@ -6188,6 +6347,46 @@
"mode": "chat",
"output_cost_per_token": 6.9e-07
},
"bedrock/eu-north-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": {
"input_cost_per_second": 0.01635,
"litellm_provider": "bedrock",
@ -6275,6 +6474,32 @@
"output_cost_per_token": 2.4e-05,
"supports_tool_choice": true
},
"bedrock/eu-central-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.86e-06,
"litellm_provider": "bedrock",
@ -6293,6 +6518,32 @@
"mode": "chat",
"output_cost_per_token": 6.5e-07
},
"bedrock/eu-west-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.45e-06,
"litellm_provider": "bedrock",
@ -6311,6 +6562,32 @@
"mode": "chat",
"output_cost_per_token": 7.8e-07
},
"bedrock/eu-west-2/minimax.minimax-m2.1": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.86e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.86e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": {
"input_cost_per_token": 2e-07,
"litellm_provider": "bedrock",
@ -6341,6 +6618,32 @@
"output_cost_per_token": 9.1e-07,
"supports_tool_choice": true
},
"bedrock/eu-south-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock",
@ -6375,6 +6678,32 @@
"mode": "chat",
"output_cost_per_token": 1.01e-06
},
"bedrock/sa-east-1/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6386,6 +6715,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/sa-east-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.44e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
"input_cost_per_second": 0.011,
"litellm_provider": "bedrock",
@ -6522,6 +6878,32 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-east-1/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6533,6 +6915,59 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6544,6 +6979,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-2/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
"input_cost_per_token": 9.6e-07,
"litellm_provider": "bedrock",
@ -6950,6 +7412,32 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-west-2/deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -6961,6 +7449,33 @@
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-west-2/moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
"cache_creation_input_token_cost": 1e-06,
"cache_read_input_token_cost": 8e-08,
@ -10874,6 +11389,19 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"dolphin": {
"input_cost_per_token": 5e-07,
"litellm_provider": "nlp_cloud",
@ -21344,6 +21872,19 @@
"output_cost_per_token": 1.2e-06,
"supports_system_messages": true
},
"minimax.minimax-m2.1": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 196000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -22100,6 +22641,20 @@
"supports_reasoning": true,
"supports_system_messages": true
},
"moonshotai.kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"moonshot/kimi-k2-0711-preview": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 6e-07,
@ -25534,6 +26089,19 @@
"supports_system_messages": true,
"supports_vision": true
},
"qwen.qwen3-coder-next": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 262144,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"recraft/recraftv2": {
"litellm_provider": "recraft",
"mode": "image_generation",
@ -27970,6 +28538,30 @@
"supports_reasoning": true,
"supports_tool_choice": false
},
"us.deepseek.v3.2": {
"input_cost_per_token": 6.2e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.85e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"eu.deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 2.22e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"us.meta.llama3-1-405b-instruct-v1:0": {
"input_cost_per_token": 5.32e-06,
"litellm_provider": "bedrock",

70
tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py
View file

@ -182,6 +182,76 @@ class TestEnterpriseRouteChecks:
EnterpriseRouteChecks.should_call_route("/config/update")
@patch("litellm.proxy.proxy_server.premium_user", True)
class TestEnterpriseRouteChecksModelListExemption:
"""Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS"""
@patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled")
@patch.object(EnterpriseRouteChecks, "is_management_routes_disabled")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route")
def test_models_route_allowed_when_llm_api_disabled(
self,
mock_is_management_route,
mock_is_llm_api_route,
mock_is_management_disabled,
mock_is_llm_api_disabled,
):
"""Test that /models is allowed even when LLM API routes are disabled"""
mock_is_management_route.return_value = False
mock_is_llm_api_route.return_value = True
mock_is_management_disabled.return_value = False
mock_is_llm_api_disabled.return_value = True
# Should not raise exception for /models
EnterpriseRouteChecks.should_call_route("/models")
@patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled")
@patch.object(EnterpriseRouteChecks, "is_management_routes_disabled")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route")
def test_v1_models_route_allowed_when_llm_api_disabled(
self,
mock_is_management_route,
mock_is_llm_api_route,
mock_is_management_disabled,
mock_is_llm_api_disabled,
):
"""Test that /v1/models is allowed even when LLM API routes are disabled"""
mock_is_management_route.return_value = False
mock_is_llm_api_route.return_value = True
mock_is_management_disabled.return_value = False
mock_is_llm_api_disabled.return_value = True
# Should not raise exception for /v1/models
EnterpriseRouteChecks.should_call_route("/v1/models")
@patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled")
@patch.object(EnterpriseRouteChecks, "is_management_routes_disabled")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route")
@patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route")
def test_chat_completions_still_blocked_when_llm_api_disabled(
self,
mock_is_management_route,
mock_is_llm_api_route,
mock_is_management_disabled,
mock_is_llm_api_disabled,
):
"""Test that non-exempt LLM routes like /v1/chat/completions are still blocked"""
mock_is_management_route.return_value = False
mock_is_llm_api_route.return_value = True
mock_is_management_disabled.return_value = False
mock_is_llm_api_disabled.return_value = True
with pytest.raises(HTTPException) as exc_info:
EnterpriseRouteChecks.should_call_route("/v1/chat/completions")
assert exc_info.value.status_code == 403
assert "LLM API routes are disabled for this instance." in str(
exc_info.value.detail
)
class TestEnterpriseRouteChecksErrorMessages:
"""Test that error messages correctly identify which feature requires Enterprise license"""

170
tests/litellm/test_bedrock_extended_beta_models.py Normal file
View file

@ -0,0 +1,170 @@
"""
Test suite for AWS Bedrock extended beta model support
Tests model configuration, pricing, and regional availability for:
- DeepSeek V3.2
- Minimax M2.1
- Moonshot AI Kimi K2.5
- Qwen3 Coder Next
"""
import os
# Set env var to use local model cost map instead of fetching from remote
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "true"
import pytest
from litellm import get_model_info
# Model configurations: (model_name, regions, max_input, max_output)
MODEL_CONFIGS = [
(
"deepseek.v3.2",
[
"ap-northeast-1",
"ap-south-1",
"ap-southeast-3",
"eu-north-1",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-2",
],
163840,
163840,
),
(
"minimax.minimax-m2.1",
[
"ap-northeast-1",
"ap-south-1",
"ap-southeast-3",
"eu-central-1",
"eu-north-1",
"eu-south-1",
"eu-west-1",
"eu-west-2",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-2",
],
196000,
8192,
),
(
"moonshotai.kimi-k2.5",
[
"ap-northeast-1",
"ap-south-1",
"ap-southeast-3",
"eu-north-1",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-2",
],
262144,
262144,
),
(
"qwen.qwen3-coder-next",
[
"ap-northeast-1",
"ap-south-1",
"ap-southeast-3",
"eu-central-1",
"eu-south-1",
"eu-west-1",
"eu-west-2",
"sa-east-1",
"us-east-1",
"us-east-2",
"us-west-2",
],
262144,
8192,
),
]
class TestBedrockNewModels:
"""Unified test suite for all new Bedrock models"""
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_model_info_primary_region(
self, model_name, regions, max_input, max_output
):
"""Test model configuration in primary region (us-east-1)"""
model = f"bedrock/us-east-1/{model_name}"
model_info = get_model_info(model)
assert model_info is not None, f"Model {model_name} not found"
assert model_info["max_input_tokens"] == max_input
assert model_info["max_output_tokens"] == max_output
assert model_info["litellm_provider"] == "bedrock"
assert model_info["mode"] == "chat"
assert model_info["supports_function_calling"] is True
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_pricing_configured(self, model_name, regions, max_input, max_output):
"""Verify pricing is set for all models"""
model = f"bedrock/us-east-1/{model_name}"
model_info = get_model_info(model)
assert (
model_info["input_cost_per_token"] > 0
), f"Missing input cost for {model_name}"
assert (
model_info["output_cost_per_token"] > 0
), f"Missing output cost for {model_name}"
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_region_count(self, model_name, regions, max_input, max_output):
"""Verify each bedrock/{region}/{model_name} resolves via get_model_info"""
for region in regions:
model = f"bedrock/{region}/{model_name}"
model_info = get_model_info(model)
assert model_info is not None, f"Model {model_name} not found in {region}"
assert model_info["max_input_tokens"] == max_input
assert model_info["max_output_tokens"] == max_output
@pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS)
def test_sample_regional_variants(self, model_name, regions, max_input, max_output):
"""Test sample regional variants (us-east-1, eu-west-1, ap-northeast-1)"""
for region in ["us-east-1", "ap-northeast-1"]:
if region in regions:
model = f"bedrock/{region}/{model_name}"
model_info = get_model_info(model)
assert (
model_info is not None
), f"Model {model_name} not found in {region}"
assert model_info["max_input_tokens"] == max_input
assert model_info["litellm_provider"] == "bedrock"
class TestModelSpecificFeatures:
"""Model-specific capability tests"""
def test_deepseek_v3_2_context_window(self):
"""DeepSeek V3.2 has 163K context window"""
model_info = get_model_info("bedrock/us-east-1/deepseek.v3.2")
assert model_info["max_input_tokens"] == 163840
def test_minimax_m2_1_context_window(self):
"""Minimax M2.1 has 196K input, 8K output"""
model_info = get_model_info("bedrock/us-east-1/minimax.minimax-m2.1")
assert model_info["max_input_tokens"] == 196000
assert model_info["max_output_tokens"] == 8192
def test_moonshotai_kimi_k2_5_context_window(self):
"""Moonshot AI Kimi K2.5 has 256K context window"""
model_info = get_model_info("bedrock/us-east-1/moonshotai.kimi-k2.5")
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 262144
def test_qwen3_coder_next_context_window(self):
"""Qwen3 Coder Next has 256K input, 8K output"""
model_info = get_model_info("bedrock/us-east-1/qwen.qwen3-coder-next")
assert model_info["max_input_tokens"] == 262144
assert model_info["max_output_tokens"] == 8192

26
tests/logging_callback_tests/test_unit_tests_init_callbacks.py
View file

@ -293,3 +293,29 @@ def test_get_combined_callback_list():
assert "lago" in _logging.get_combined_callback_list(
dynamic_success_callbacks=["langfuse"], global_callbacks=["lago"]
)
def test_get_combined_callback_list_returns_copy_when_dynamic_is_none():
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_logging = LiteLLMLoggingObj(
model="claude-3-opus-20240229",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="123",
function_id="456",
)
global_callbacks = ["langfuse"]
combined_callbacks = _logging.get_combined_callback_list(
dynamic_success_callbacks=None, global_callbacks=global_callbacks
)
assert combined_callbacks == ["langfuse"]
assert combined_callbacks is not global_callbacks
combined_callbacks.append("new_callback")
assert global_callbacks == ["langfuse"]

58
tests/test_litellm/caching/test_dual_cache.py Normal file
View file

@ -0,0 +1,58 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cache import RedisCache
@pytest.mark.asyncio
async def test_dual_cache_async_batch_get_cache_coalesces_concurrent_redis_reads():
dual_cache = DualCache(
redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10
)
keys = ["shared_a", "shared_b"]
start_gate = asyncio.Event()
async def _mock_async_batch_get_cache(key_list, parent_otel_span=None):
await asyncio.sleep(0.05)
return {k: None for k in key_list}
with patch.object(
dual_cache.redis_cache,
"async_batch_get_cache",
new=AsyncMock(side_effect=_mock_async_batch_get_cache),
) as mock_async_batch_get_cache:
async def worker():
await start_gate.wait()
return await dual_cache.async_batch_get_cache(keys=keys)
tasks = [asyncio.create_task(worker()) for _ in range(50)]
start_gate.set()
await asyncio.gather(*tasks)
assert mock_async_batch_get_cache.call_count == 1
@pytest.mark.asyncio
async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_error():
dual_cache = DualCache(
redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10
)
keys = ["shared_a", "shared_b"]
with patch.object(
dual_cache.redis_cache,
"async_batch_get_cache",
new=AsyncMock(side_effect=RuntimeError("redis unavailable")),
) as mock_async_batch_get_cache:
first_result = await dual_cache.async_batch_get_cache(keys=keys)
second_result = await dual_cache.async_batch_get_cache(keys=keys)
assert first_result is None
assert second_result is None
assert mock_async_batch_get_cache.call_count == 2
assert "shared_a" not in dual_cache.last_redis_batch_access_time
assert "shared_b" not in dual_cache.last_redis_batch_access_time

74
tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py
View file

@ -9,7 +9,10 @@ import pytest
sys.path.insert(0, os.path.abspath("../../../../.."))
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIGPTConfig,
OpenAIChatCompletionStreamingHandler,
)
class TestOpenAIGPTConfig:
@ -136,6 +139,75 @@ class TestGetOptionalParamsIntegration:
assert regular_params.get("user") == "my-end-user"
assert responses_params.get("user") == "my-end-user"
class TestOpenAIChatCompletionStreamingHandler:
"""Tests for OpenAIChatCompletionStreamingHandler.chunk_parser()"""
def test_chunk_parser_preserves_usage(self):
"""
Test that chunk_parser preserves the usage field from streaming chunks.
"""
handler = OpenAIChatCompletionStreamingHandler(
streaming_response=None, sync_stream=True
)
usage_chunk = {
"id": "gen-123",
"created": 1234567890,
"model": "openai/gpt-4o-mini",
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": ""},
"finish_reason": None,
}
],
"usage": {
"prompt_tokens": 13797,
"completion_tokens": 350,
"total_tokens": 14147,
},
}
result = handler.chunk_parser(usage_chunk)
assert result.usage is not None
assert result.usage.prompt_tokens == 13797
assert result.usage.completion_tokens == 350
assert result.usage.total_tokens == 14147
def test_chunk_parser_without_usage(self):
"""Test that chunk_parser works normally for chunks without usage."""
handler = OpenAIChatCompletionStreamingHandler(
streaming_response=None, sync_stream=True
)
chunk = {
"id": "gen-123",
"created": 1234567890,
"model": "openai/gpt-4o-mini",
"object": "chat.completion.chunk",
"choices": [
{
"index": 0,
"delta": {"role": "assistant", "content": "Hello"},
"finish_reason": None,
}
],
}
result = handler.chunk_parser(chunk)
assert result.id == "gen-123"
assert result.choices[0].delta.content == "Hello"
assert not hasattr(result, "usage") or result.usage is None
class TestPromptCacheKeyIntegration:
"""Tests for prompt_cache_key support"""
def test_prompt_cache_key_in_optional_params(self):
"""Test that 'prompt_cache_key' flows through get_optional_params for OpenAI models."""
from litellm.utils import get_optional_params

103
tests/test_litellm/proxy/auth/test_route_checks.py
View file

@ -936,6 +936,109 @@ def test_proxy_admin_viewer_can_access_global_spend_tags():
)
class TestModelsRouteExemptFromDisableLLMEndpoints:
"""
Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS.
When DISABLE_LLM_API_ENDPOINTS is set, inference routes like /v1/chat/completions
should be blocked, but /models and /v1/models should remain accessible because
they are read-only model listing routes needed by the Admin UI.
Relevant issue: https://github.com/BerriAI/litellm/issues/new (UI breaks with DISABLE_LLM_ENDPOINTS)
"""
def _get_enterprise_route_checks(self):
"""Import EnterpriseRouteChecks from the local enterprise source file."""
import importlib.util
local_file = os.path.join(
os.path.dirname(__file__),
"..", "..", "..", "..", "enterprise",
"litellm_enterprise", "proxy", "auth", "route_checks.py",
)
local_file = os.path.abspath(local_file)
spec = importlib.util.spec_from_file_location(
"local_enterprise_route_checks", local_file
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.EnterpriseRouteChecks
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_models_route_allowed_when_llm_api_disabled(self):
"""Test that /models is allowed even when LLM API routes are disabled"""
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
), patch.object(
EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False
):
# /models should NOT raise - it's exempt
EnterpriseRouteChecks.should_call_route("/models")
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_v1_models_route_allowed_when_llm_api_disabled(self):
"""Test that /v1/models is allowed even when LLM API routes are disabled"""
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
), patch.object(
EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False
):
# /v1/models should NOT raise - it's exempt
EnterpriseRouteChecks.should_call_route("/v1/models")
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_chat_completions_still_blocked_when_llm_api_disabled(self):
"""Test that non-exempt LLM routes like /v1/chat/completions are still blocked"""
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
), patch.object(
EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False
):
with pytest.raises(HTTPException) as exc_info:
EnterpriseRouteChecks.should_call_route("/v1/chat/completions")
assert exc_info.value.status_code == 403
assert "LLM API routes are disabled for this instance." in str(
exc_info.value.detail
)
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_embeddings_still_blocked_when_llm_api_disabled(self):
"""Test that /v1/embeddings is still blocked when LLM API routes are disabled"""
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True
), patch.object(
EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False
):
with pytest.raises(HTTPException) as exc_info:
EnterpriseRouteChecks.should_call_route("/v1/embeddings")
assert exc_info.value.status_code == 403
@patch("litellm.proxy.proxy_server.premium_user", True)
def test_should_models_route_allowed_when_llm_api_not_disabled(self):
"""Test that /models works normally when LLM API routes are not disabled"""
EnterpriseRouteChecks = self._get_enterprise_route_checks()
with patch.object(
EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False
), patch.object(
EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False
):
# Should not raise
EnterpriseRouteChecks.should_call_route("/models")
EnterpriseRouteChecks.should_call_route("/v1/models")
def test_route_in_additional_public_routes_wildcard_match():
"""
Test that route_in_additonal_public_routes supports wildcard patterns.

32
tests/test_litellm/responses/test_responses_utils.py
View file

@ -2,6 +2,7 @@ import base64
import json
import os
import sys
from unittest.mock import MagicMock, patch
import pytest
from fastapi.testclient import TestClient
@ -352,3 +353,34 @@ class TestResponsesAPIProviderSpecificParams:
# Should not raise any exception
result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params)
assert "temperature" in result
def test_responses_extra_body_forwarded_to_completion_transformation_handler():
"""
Regression test: extra_body must be forwarded to response_api_handler
when responses_api_provider_config is None (completion transformation path).
Before the fix, extra_body was a named parameter of responses() but was
not passed to litellm_completion_transformation_handler.response_api_handler(),
so it was silently dropped.
"""
with patch(
"litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config",
return_value=None,
), patch(
"litellm.responses.main.litellm_completion_transformation_handler.response_api_handler",
) as mock_handler:
mock_handler.return_value = MagicMock()
litellm.responses(
model="openai/gpt-4o",
input="Hello",
extra_body={"custom_key": "custom_value"},
)
mock_handler.assert_called_once()
call_kwargs = mock_handler.call_args
# extra_body can be a positional or keyword arg; check both
assert call_kwargs.kwargs.get("extra_body") == {
"custom_key": "custom_value"
}