diff --git a/enterprise/litellm_enterprise/proxy/auth/route_checks.py b/enterprise/litellm_enterprise/proxy/auth/route_checks.py index 6f7cf9143f4..fc57292a8d2 100644 --- a/enterprise/litellm_enterprise/proxy/auth/route_checks.py +++ b/enterprise/litellm_enterprise/proxy/auth/route_checks.py @@ -41,6 +41,10 @@ class EnterpriseRouteChecks: return get_secret_bool("DISABLE_ADMIN_ENDPOINTS") is True + # Routes that should remain accessible even when LLM API endpoints are disabled. + # These are read-only model listing routes needed by the Admin UI. + LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"] + @staticmethod def should_call_route(route: str): """ @@ -58,6 +62,7 @@ class EnterpriseRouteChecks: ) elif ( RouteChecks.is_llm_api_route(route=route) + and route not in EnterpriseRouteChecks.LLM_API_EXEMPT_ROUTES and EnterpriseRouteChecks.is_llm_api_route_disabled() ): raise HTTPException( diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 3edc3f42820..6df570c72b9 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -12,7 +12,8 @@ import asyncio import time import traceback from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, List, Optional, Union +from threading import Lock +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union if TYPE_CHECKING: from litellm.types.caching import RedisPipelineIncrementOperation @@ -71,6 +72,7 @@ class DualCache(BaseCache): self.last_redis_batch_access_time = LimitedSizeOrderedDict( max_size=default_max_redis_batch_cache_size ) + self._last_redis_batch_access_time_lock = Lock() self.redis_batch_cache_expiry = ( default_redis_batch_cache_expiry or litellm.default_redis_batch_cache_expiry @@ -236,22 +238,46 @@ class DualCache(BaseCache): except Exception: verbose_logger.error(traceback.format_exc()) - def get_redis_batch_keys( + def _reserve_redis_batch_keys( self, current_time: float, keys: List[str], result: List[Any], - ) -> List[str]: - sublist_keys = [] - for key, value in zip(keys, result): - if value is None: + ) -> Tuple[List[str], Dict[str, Optional[float]]]: + """ + Atomically choose keys to fetch from Redis and reserve their access time. + This prevents check-then-act races under concurrent async callers. + """ + sublist_keys: List[str] = [] + previous_access_times: Dict[str, Optional[float]] = {} + + with self._last_redis_batch_access_time_lock: + for key, value in zip(keys, result): + if value is not None: + continue + if ( key not in self.last_redis_batch_access_time or current_time - self.last_redis_batch_access_time[key] >= self.redis_batch_cache_expiry ): sublist_keys.append(key) - return sublist_keys + previous_access_times[key] = self.last_redis_batch_access_time.get( + key + ) + self.last_redis_batch_access_time[key] = current_time + + return sublist_keys, previous_access_times + + def _rollback_redis_batch_key_reservations( + self, previous_access_times: Dict[str, Optional[float]] + ) -> None: + with self._last_redis_batch_access_time_lock: + for key, previous_time in previous_access_times.items(): + if previous_time is None: + self.last_redis_batch_access_time.pop(key, None) + else: + self.last_redis_batch_access_time[key] = previous_time async def async_batch_get_cache( self, @@ -276,19 +302,23 @@ class DualCache(BaseCache): - check the redis cache """ current_time = time.time() - sublist_keys = self.get_redis_batch_keys(current_time, keys, result) + sublist_keys, previous_access_times = self._reserve_redis_batch_keys( + current_time, keys, result + ) - # Only hit Redis if the last access time was more than 5 seconds ago + # Only hit Redis if enough time has passed since last access. if len(sublist_keys) > 0: - # If not found in in-memory cache, try fetching from Redis - redis_result = await self.redis_cache.async_batch_get_cache( - sublist_keys, parent_otel_span=parent_otel_span - ) - - # Update the last access time for ALL queried keys - # This includes keys with None values to throttle repeated Redis queries - for key in sublist_keys: - self.last_redis_batch_access_time[key] = current_time + try: + # If not found in in-memory cache, try fetching from Redis + redis_result = await self.redis_cache.async_batch_get_cache( + sublist_keys, parent_otel_span=parent_otel_span + ) + except Exception: + # Do not throttle subsequent callers if the Redis read fails. + self._rollback_redis_batch_key_reservations( + previous_access_times + ) + raise # Short-circuit if redis_result is None or contains only None values if redis_result is None or all(v is None for v in redis_result.values()): diff --git a/litellm/constants.py b/litellm/constants.py index 4e89ddd7bd9..addd659be73 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1016,10 +1016,12 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ BEDROCK_CONVERSE_MODELS = [ "qwen.qwen3-coder-480b-a35b-v1:0", + "qwen.qwen3-coder-next", "qwen.qwen3-235b-a22b-2507-v1:0", "qwen.qwen3-coder-30b-a3b-v1:0", "qwen.qwen3-32b-v1:0", "deepseek.v3-v1:0", + "deepseek.v3.2", "openai.gpt-oss-20b-1:0", "openai.gpt-oss-120b-1:0", "anthropic.claude-haiku-4-5-20251001-v1:0", @@ -1062,6 +1064,8 @@ BEDROCK_CONVERSE_MODELS = [ "amazon.nova-pro-v1:0", "writer.palmyra-x4-v1:0", "writer.palmyra-x5-v1:0", + "minimax.minimax-m2.1", + "moonshotai.kimi-k2.5", ] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 82a7af64f97..a765d30ddab 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3127,7 +3127,7 @@ class Logging(LiteLLMLoggingBaseClass): self, dynamic_success_callbacks: Optional[List], global_callbacks: List ) -> List: if dynamic_success_callbacks is None: - return global_callbacks + return list(global_callbacks) return list(set(dynamic_success_callbacks + global_callbacks)) def _remove_internal_litellm_callbacks(self, callbacks: List) -> List: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 16368907070..5b9840d95b0 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -772,12 +772,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator): def chunk_parser(self, chunk: dict) -> ModelResponseStream: try: - return ModelResponseStream( - id=chunk["id"], - object="chat.completion.chunk", - created=chunk.get("created"), - model=chunk.get("model"), - choices=chunk.get("choices", []), - ) + kwargs = { + "id": chunk["id"], + "object": "chat.completion.chunk", + "created": chunk.get("created"), + "model": chunk.get("model"), + "choices": chunk.get("choices", []), + } + if "usage" in chunk and chunk["usage"] is not None: + kwargs["usage"] = chunk["usage"] + return ModelResponseStream(**kwargs) except Exception as e: raise e diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 00b461dcda0..6a5ac92816b 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -529,6 +529,17 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 raise e +def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: + """Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values.""" + extra_body: Optional[dict] = optional_params.pop("extra_body", None) + if extra_body is not None: + for k, v in extra_body.items(): + if k in data and isinstance(data[k], dict) and isinstance(v, dict): + data[k].update(v) + else: + data[k] = v + + def _transform_request_body( messages: List[AllMessageValues], model: str, @@ -619,6 +630,7 @@ def _transform_request_body( # Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels + _pop_and_merge_extra_body(data, optional_params) except Exception as e: raise e diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7c82d47dcdc..e6b7cf17297 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6105,6 +6105,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6116,6 +6142,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6128,7 +6181,7 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { - "input_cost_per_token": 7.3e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -6159,6 +6212,32 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6170,6 +6249,86 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6188,6 +6347,46 @@ "mode": "chat", "output_cost_per_token": 6.9e-07 }, + "bedrock/eu-north-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.01635, "litellm_provider": "bedrock", @@ -6275,6 +6474,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/eu-central-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", @@ -6293,6 +6518,32 @@ "mode": "chat", "output_cost_per_token": 6.5e-07 }, + "bedrock/eu-west-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", @@ -6311,6 +6562,32 @@ "mode": "chat", "output_cost_per_token": 7.8e-07 }, + "bedrock/eu-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 7.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", @@ -6341,6 +6618,32 @@ "output_cost_per_token": 9.1e-07, "supports_tool_choice": true }, + "bedrock/eu-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -6375,6 +6678,32 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6386,6 +6715,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6522,6 +6878,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6533,6 +6915,59 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6544,6 +6979,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6950,6 +7412,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6961,6 +7449,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, @@ -10874,6 +11389,19 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -21344,6 +21872,19 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -22100,6 +22641,20 @@ "supports_reasoning": true, "supports_system_messages": true }, + "moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 6e-07, @@ -25534,6 +26089,19 @@ "supports_system_messages": true, "supports_vision": true }, + "qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", @@ -27970,6 +28538,30 @@ "supports_reasoning": true, "supports_tool_choice": false }, + "us.deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "eu.deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, "litellm_provider": "bedrock", @@ -32080,6 +32672,23 @@ "1280x720" ] }, + "openai/sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, "azure/sora-2": { "litellm_provider": "azure", "mode": "video_generation", @@ -35719,7 +36328,9 @@ "gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -35750,7 +36361,9 @@ "gpt-realtime-mini-2025-12-15": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, + "input_cost_per_image": 8e-07, "input_cost_per_token": 6e-07, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -35812,6 +36425,23 @@ "1280x720" ] }, + "sora-2-pro-high-res": { + "litellm_provider": "openai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.5, + "source": "https://platform.openai.com/docs/api-reference/videos", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ], + "supported_resolutions": [ + "1024x1792", + "1792x1024" + ] + }, "chatgpt-image-latest": { "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 62a06468e9a..e003841ac09 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -71,9 +71,7 @@ try: from mcp.shared.tool_name_validation import ( validate_tool_name, # pyright: ignore[reportAssignmentType] ) - from mcp.shared.tool_name_validation import ( - SEP_986_URL, - ) + from mcp.shared.tool_name_validation import SEP_986_URL except ImportError: from pydantic import BaseModel diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0aace65ff6b..d977751004c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -740,7 +740,7 @@ class ProxyLogging: self, dynamic_success_callbacks: Optional[List], global_callbacks: List ) -> List: if dynamic_success_callbacks is None: - return global_callbacks + return list(global_callbacks) return list(set(dynamic_success_callbacks + global_callbacks)) def _parse_pre_mcp_call_hook_response( diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 7105f1ae6fb..e943789a1cd 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -682,6 +682,7 @@ def responses( _is_async=_is_async, stream=stream, extra_headers=extra_headers, + extra_body=extra_body, **kwargs, ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 09de81b38fa..e6b7cf17297 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6105,6 +6105,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/ap-northeast-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6116,6 +6142,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6128,7 +6181,7 @@ "supports_reasoning": true }, "bedrock/moonshotai.kimi-k2.5": { - "input_cost_per_token": 7.3e-07, + "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 262144, @@ -6159,6 +6212,32 @@ "mode": "chat", "output_cost_per_token": 7.2e-07 }, + "bedrock/ap-south-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6170,6 +6249,86 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/ap-south-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.05e-06, "litellm_provider": "bedrock", @@ -6188,6 +6347,46 @@ "mode": "chat", "output_cost_per_token": 6.9e-07 }, + "bedrock/eu-north-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-north-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.01635, "litellm_provider": "bedrock", @@ -6275,6 +6474,32 @@ "output_cost_per_token": 2.4e-05, "supports_tool_choice": true }, + "bedrock/eu-central-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-central-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.86e-06, "litellm_provider": "bedrock", @@ -6293,6 +6518,32 @@ "mode": "chat", "output_cost_per_token": 6.5e-07 }, + "bedrock/eu-west-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 3.45e-06, "litellm_provider": "bedrock", @@ -6311,6 +6562,32 @@ "mode": "chat", "output_cost_per_token": 7.8e-07 }, + "bedrock/eu-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 7.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.86e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", @@ -6341,6 +6618,32 @@ "output_cost_per_token": 9.1e-07, "supports_tool_choice": true }, + "bedrock/eu-south-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/eu-south-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -6375,6 +6678,32 @@ "mode": "chat", "output_cost_per_token": 1.01e-06 }, + "bedrock/sa-east-1/deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6386,6 +6715,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/sa-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/sa-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.44e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "input_cost_per_second": 0.011, "litellm_provider": "bedrock", @@ -6522,6 +6878,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-east-1/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6533,6 +6915,59 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-1/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-1/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6544,6 +6979,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-east-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-east-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "input_cost_per_token": 9.6e-07, "litellm_provider": "bedrock", @@ -6950,6 +7412,32 @@ "output_cost_per_token": 7e-07, "supports_tool_choice": true }, + "bedrock/us-west-2/deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -6961,6 +7449,33 @@ "supports_function_calling": true, "supports_reasoning": true }, + "bedrock/us-west-2/moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, + "bedrock/us-west-2/qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, @@ -10874,6 +11389,19 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -21344,6 +21872,19 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax.minimax-m2.1": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 196000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -22100,6 +22641,20 @@ "supports_reasoning": true, "supports_system_messages": true }, + "moonshotai.kimi-k2.5": { + "input_cost_per_token": 6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 6e-07, @@ -25534,6 +26089,19 @@ "supports_system_messages": true, "supports_vision": true }, + "qwen.qwen3-coder-next": { + "input_cost_per_token": 5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 262144, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", @@ -27970,6 +28538,30 @@ "supports_reasoning": true, "supports_tool_choice": false }, + "us.deepseek.v3.2": { + "input_cost_per_token": 6.2e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.85e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "eu.deepseek.v3.2": { + "input_cost_per_token": 7.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 2.22e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "us.meta.llama3-1-405b-instruct-v1:0": { "input_cost_per_token": 5.32e-06, "litellm_provider": "bedrock", diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py index 706e3b71870..24e27977963 100644 --- a/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py +++ b/tests/enterprise/litellm_enterprise/proxy/auth/test_route_checks.py @@ -182,6 +182,76 @@ class TestEnterpriseRouteChecks: EnterpriseRouteChecks.should_call_route("/config/update") +@patch("litellm.proxy.proxy_server.premium_user", True) +class TestEnterpriseRouteChecksModelListExemption: + """Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS""" + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_models_route_allowed_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that /models is allowed even when LLM API routes are disabled""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + # Should not raise exception for /models + EnterpriseRouteChecks.should_call_route("/models") + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_v1_models_route_allowed_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that /v1/models is allowed even when LLM API routes are disabled""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + # Should not raise exception for /v1/models + EnterpriseRouteChecks.should_call_route("/v1/models") + + @patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled") + @patch.object(EnterpriseRouteChecks, "is_management_routes_disabled") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_llm_api_route") + @patch("litellm.proxy.auth.route_checks.RouteChecks.is_management_route") + def test_chat_completions_still_blocked_when_llm_api_disabled( + self, + mock_is_management_route, + mock_is_llm_api_route, + mock_is_management_disabled, + mock_is_llm_api_disabled, + ): + """Test that non-exempt LLM routes like /v1/chat/completions are still blocked""" + mock_is_management_route.return_value = False + mock_is_llm_api_route.return_value = True + mock_is_management_disabled.return_value = False + mock_is_llm_api_disabled.return_value = True + + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/chat/completions") + + assert exc_info.value.status_code == 403 + assert "LLM API routes are disabled for this instance." in str( + exc_info.value.detail + ) + + class TestEnterpriseRouteChecksErrorMessages: """Test that error messages correctly identify which feature requires Enterprise license""" diff --git a/tests/litellm/test_bedrock_extended_beta_models.py b/tests/litellm/test_bedrock_extended_beta_models.py new file mode 100644 index 00000000000..ebbbd6cab5c --- /dev/null +++ b/tests/litellm/test_bedrock_extended_beta_models.py @@ -0,0 +1,170 @@ +""" +Test suite for AWS Bedrock extended beta model support +Tests model configuration, pricing, and regional availability for: +- DeepSeek V3.2 +- Minimax M2.1 +- Moonshot AI Kimi K2.5 +- Qwen3 Coder Next +""" + +import os + +# Set env var to use local model cost map instead of fetching from remote +os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "true" + +import pytest + +from litellm import get_model_info + +# Model configurations: (model_name, regions, max_input, max_output) +MODEL_CONFIGS = [ + ( + "deepseek.v3.2", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-north-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 163840, + 163840, + ), + ( + "minimax.minimax-m2.1", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 196000, + 8192, + ), + ( + "moonshotai.kimi-k2.5", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-north-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 262144, + 262144, + ), + ( + "qwen.qwen3-coder-next", + [ + "ap-northeast-1", + "ap-south-1", + "ap-southeast-3", + "eu-central-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-2", + ], + 262144, + 8192, + ), +] + + +class TestBedrockNewModels: + """Unified test suite for all new Bedrock models""" + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_model_info_primary_region( + self, model_name, regions, max_input, max_output + ): + """Test model configuration in primary region (us-east-1)""" + model = f"bedrock/us-east-1/{model_name}" + model_info = get_model_info(model) + + assert model_info is not None, f"Model {model_name} not found" + assert model_info["max_input_tokens"] == max_input + assert model_info["max_output_tokens"] == max_output + assert model_info["litellm_provider"] == "bedrock" + assert model_info["mode"] == "chat" + assert model_info["supports_function_calling"] is True + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_pricing_configured(self, model_name, regions, max_input, max_output): + """Verify pricing is set for all models""" + model = f"bedrock/us-east-1/{model_name}" + model_info = get_model_info(model) + + assert ( + model_info["input_cost_per_token"] > 0 + ), f"Missing input cost for {model_name}" + assert ( + model_info["output_cost_per_token"] > 0 + ), f"Missing output cost for {model_name}" + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_region_count(self, model_name, regions, max_input, max_output): + """Verify each bedrock/{region}/{model_name} resolves via get_model_info""" + for region in regions: + model = f"bedrock/{region}/{model_name}" + model_info = get_model_info(model) + assert model_info is not None, f"Model {model_name} not found in {region}" + assert model_info["max_input_tokens"] == max_input + assert model_info["max_output_tokens"] == max_output + + @pytest.mark.parametrize("model_name,regions,max_input,max_output", MODEL_CONFIGS) + def test_sample_regional_variants(self, model_name, regions, max_input, max_output): + """Test sample regional variants (us-east-1, eu-west-1, ap-northeast-1)""" + for region in ["us-east-1", "ap-northeast-1"]: + if region in regions: + model = f"bedrock/{region}/{model_name}" + model_info = get_model_info(model) + assert ( + model_info is not None + ), f"Model {model_name} not found in {region}" + assert model_info["max_input_tokens"] == max_input + assert model_info["litellm_provider"] == "bedrock" + + +class TestModelSpecificFeatures: + """Model-specific capability tests""" + + def test_deepseek_v3_2_context_window(self): + """DeepSeek V3.2 has 163K context window""" + model_info = get_model_info("bedrock/us-east-1/deepseek.v3.2") + assert model_info["max_input_tokens"] == 163840 + + def test_minimax_m2_1_context_window(self): + """Minimax M2.1 has 196K input, 8K output""" + model_info = get_model_info("bedrock/us-east-1/minimax.minimax-m2.1") + assert model_info["max_input_tokens"] == 196000 + assert model_info["max_output_tokens"] == 8192 + + def test_moonshotai_kimi_k2_5_context_window(self): + """Moonshot AI Kimi K2.5 has 256K context window""" + model_info = get_model_info("bedrock/us-east-1/moonshotai.kimi-k2.5") + assert model_info["max_input_tokens"] == 262144 + assert model_info["max_output_tokens"] == 262144 + + def test_qwen3_coder_next_context_window(self): + """Qwen3 Coder Next has 256K input, 8K output""" + model_info = get_model_info("bedrock/us-east-1/qwen.qwen3-coder-next") + assert model_info["max_input_tokens"] == 262144 + assert model_info["max_output_tokens"] == 8192 diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 345c5b2f2ac..f8dba78798d 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -293,3 +293,29 @@ def test_get_combined_callback_list(): assert "lago" in _logging.get_combined_callback_list( dynamic_success_callbacks=["langfuse"], global_callbacks=["lago"] ) + + +def test_get_combined_callback_list_returns_copy_when_dynamic_is_none(): + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + _logging = LiteLLMLoggingObj( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="123", + function_id="456", + ) + + global_callbacks = ["langfuse"] + combined_callbacks = _logging.get_combined_callback_list( + dynamic_success_callbacks=None, global_callbacks=global_callbacks + ) + + assert combined_callbacks == ["langfuse"] + assert combined_callbacks is not global_callbacks + + combined_callbacks.append("new_callback") + + assert global_callbacks == ["langfuse"] diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py new file mode 100644 index 00000000000..9974c23e4b4 --- /dev/null +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -0,0 +1,58 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCache + + +@pytest.mark.asyncio +async def test_dual_cache_async_batch_get_cache_coalesces_concurrent_redis_reads(): + dual_cache = DualCache( + redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10 + ) + keys = ["shared_a", "shared_b"] + start_gate = asyncio.Event() + + async def _mock_async_batch_get_cache(key_list, parent_otel_span=None): + await asyncio.sleep(0.05) + return {k: None for k in key_list} + + with patch.object( + dual_cache.redis_cache, + "async_batch_get_cache", + new=AsyncMock(side_effect=_mock_async_batch_get_cache), + ) as mock_async_batch_get_cache: + + async def worker(): + await start_gate.wait() + return await dual_cache.async_batch_get_cache(keys=keys) + + tasks = [asyncio.create_task(worker()) for _ in range(50)] + start_gate.set() + await asyncio.gather(*tasks) + + assert mock_async_batch_get_cache.call_count == 1 + + +@pytest.mark.asyncio +async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_error(): + dual_cache = DualCache( + redis_cache=MagicMock(spec=RedisCache), default_redis_batch_cache_expiry=10 + ) + keys = ["shared_a", "shared_b"] + + with patch.object( + dual_cache.redis_cache, + "async_batch_get_cache", + new=AsyncMock(side_effect=RuntimeError("redis unavailable")), + ) as mock_async_batch_get_cache: + first_result = await dual_cache.async_batch_get_cache(keys=keys) + second_result = await dual_cache.async_batch_get_cache(keys=keys) + + assert first_result is None + assert second_result is None + assert mock_async_batch_get_cache.call_count == 2 + assert "shared_a" not in dual_cache.last_redis_batch_access_time + assert "shared_b" not in dual_cache.last_redis_batch_access_time diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index c0695bf3588..e6ab199168d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -9,7 +9,10 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) -from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIChatCompletionStreamingHandler, +) class TestOpenAIGPTConfig: @@ -136,6 +139,75 @@ class TestGetOptionalParamsIntegration: assert regular_params.get("user") == "my-end-user" assert responses_params.get("user") == "my-end-user" + +class TestOpenAIChatCompletionStreamingHandler: + """Tests for OpenAIChatCompletionStreamingHandler.chunk_parser()""" + + def test_chunk_parser_preserves_usage(self): + """ + Test that chunk_parser preserves the usage field from streaming chunks. + + """ + handler = OpenAIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + usage_chunk = { + "id": "gen-123", + "created": 1234567890, + "model": "openai/gpt-4o-mini", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "finish_reason": None, + } + ], + "usage": { + "prompt_tokens": 13797, + "completion_tokens": 350, + "total_tokens": 14147, + }, + } + + result = handler.chunk_parser(usage_chunk) + + assert result.usage is not None + assert result.usage.prompt_tokens == 13797 + assert result.usage.completion_tokens == 350 + assert result.usage.total_tokens == 14147 + + def test_chunk_parser_without_usage(self): + """Test that chunk_parser works normally for chunks without usage.""" + handler = OpenAIChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + chunk = { + "id": "gen-123", + "created": 1234567890, + "model": "openai/gpt-4o-mini", + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello"}, + "finish_reason": None, + } + ], + } + + result = handler.chunk_parser(chunk) + + assert result.id == "gen-123" + assert result.choices[0].delta.content == "Hello" + assert not hasattr(result, "usage") or result.usage is None + + +class TestPromptCacheKeyIntegration: + """Tests for prompt_cache_key support""" + def test_prompt_cache_key_in_optional_params(self): """Test that 'prompt_cache_key' flows through get_optional_params for OpenAI models.""" from litellm.utils import get_optional_params diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 65bb329e7ea..c339ed2b762 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -936,6 +936,109 @@ def test_proxy_admin_viewer_can_access_global_spend_tags(): ) +class TestModelsRouteExemptFromDisableLLMEndpoints: + """ + Test that /models and /v1/models are exempt from DISABLE_LLM_API_ENDPOINTS. + + When DISABLE_LLM_API_ENDPOINTS is set, inference routes like /v1/chat/completions + should be blocked, but /models and /v1/models should remain accessible because + they are read-only model listing routes needed by the Admin UI. + + Relevant issue: https://github.com/BerriAI/litellm/issues/new (UI breaks with DISABLE_LLM_ENDPOINTS) + """ + + def _get_enterprise_route_checks(self): + """Import EnterpriseRouteChecks from the local enterprise source file.""" + import importlib.util + + local_file = os.path.join( + os.path.dirname(__file__), + "..", "..", "..", "..", "enterprise", + "litellm_enterprise", "proxy", "auth", "route_checks.py", + ) + local_file = os.path.abspath(local_file) + + spec = importlib.util.spec_from_file_location( + "local_enterprise_route_checks", local_file + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.EnterpriseRouteChecks + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_models_route_allowed_when_llm_api_disabled(self): + """Test that /models is allowed even when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # /models should NOT raise - it's exempt + EnterpriseRouteChecks.should_call_route("/models") + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_v1_models_route_allowed_when_llm_api_disabled(self): + """Test that /v1/models is allowed even when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # /v1/models should NOT raise - it's exempt + EnterpriseRouteChecks.should_call_route("/v1/models") + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_chat_completions_still_blocked_when_llm_api_disabled(self): + """Test that non-exempt LLM routes like /v1/chat/completions are still blocked""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/chat/completions") + + assert exc_info.value.status_code == 403 + assert "LLM API routes are disabled for this instance." in str( + exc_info.value.detail + ) + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_embeddings_still_blocked_when_llm_api_disabled(self): + """Test that /v1/embeddings is still blocked when LLM API routes are disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + with pytest.raises(HTTPException) as exc_info: + EnterpriseRouteChecks.should_call_route("/v1/embeddings") + + assert exc_info.value.status_code == 403 + + @patch("litellm.proxy.proxy_server.premium_user", True) + def test_should_models_route_allowed_when_llm_api_not_disabled(self): + """Test that /models works normally when LLM API routes are not disabled""" + EnterpriseRouteChecks = self._get_enterprise_route_checks() + + with patch.object( + EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False + ), patch.object( + EnterpriseRouteChecks, "is_management_routes_disabled", return_value=False + ): + # Should not raise + EnterpriseRouteChecks.should_call_route("/models") + EnterpriseRouteChecks.should_call_route("/v1/models") + + def test_route_in_additional_public_routes_wildcard_match(): """ Test that route_in_additonal_public_routes supports wildcard patterns. diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 8f7acb6c120..7feab9c6035 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -2,6 +2,7 @@ import base64 import json import os import sys +from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -352,3 +353,34 @@ class TestResponsesAPIProviderSpecificParams: # Should not raise any exception result = ResponsesAPIRequestUtils.get_requested_response_api_optional_param(params) assert "temperature" in result + + +def test_responses_extra_body_forwarded_to_completion_transformation_handler(): + """ + Regression test: extra_body must be forwarded to response_api_handler + when responses_api_provider_config is None (completion transformation path). + + Before the fix, extra_body was a named parameter of responses() but was + not passed to litellm_completion_transformation_handler.response_api_handler(), + so it was silently dropped. + """ + with patch( + "litellm.responses.main.ProviderConfigManager.get_provider_responses_api_config", + return_value=None, + ), patch( + "litellm.responses.main.litellm_completion_transformation_handler.response_api_handler", + ) as mock_handler: + mock_handler.return_value = MagicMock() + + litellm.responses( + model="openai/gpt-4o", + input="Hello", + extra_body={"custom_key": "custom_value"}, + ) + + mock_handler.assert_called_once() + call_kwargs = mock_handler.call_args + # extra_body can be a positional or keyword arg; check both + assert call_kwargs.kwargs.get("extra_body") == { + "custom_key": "custom_value" + }