diff --git a/litellm/__init__.py b/litellm/__init__.py index 0d6a788e368..cffdbacf597 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -213,6 +213,15 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = ( log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False +# When True (default — preserves historical behavior), the Router appends +# internal config names (model_group, fallback model groups, deployment +# timeouts, fallback failure details) onto exception messages and surfaces +# them to clients via ProxyException.message. Set to False if you do NOT +# want the proxy's internal model_name / fallback wiring visible to clients. +# Deprecation: planned to flip to False (redact by default) in a future +# major release; opt in early with `litellm.expose_router_debug_in_errors +# = False`. +expose_router_debug_in_errors: bool = True filter_invalid_headers: Optional[bool] = False add_user_information_to_llm_headers: Optional[bool] = ( None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers @@ -235,6 +244,17 @@ modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) use_chat_completions_url_for_anthropic_messages: bool = bool( os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) ) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API +# When True, strip the OpenAI-flavored `usage.total_tokens` field that +# LiteLLM injects into non-streaming /v1/messages responses, bringing the +# wire response into line with the Anthropic spec (matches the streaming +# SSE path, which already omits total_tokens). Default False to preserve +# backward compatibility for clients that read the LiteLLM-shaped +# `usage.total_tokens` today. Planned to flip to True in a future major +# release; opt in early via Python: +# `litellm.strip_anthropic_total_tokens = True` +# Or via `litellm_settings.strip_anthropic_total_tokens: true` in +# config.yaml. +strip_anthropic_total_tokens: bool = False route_all_chat_openai_to_responses: bool = ( os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true" ) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge @@ -413,7 +433,7 @@ anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", ) -suppress_debug_info = False +suppress_debug_info: bool = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None s3_audit_callback_params: Optional[Dict] = None diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 6087e55b136..edb97b310d7 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -170,6 +170,16 @@ def get_error_message(error_obj) -> Optional[str]: ####### EXCEPTION MAPPING ################ +def _get_body_error_code(error_str: str) -> int | None: + """Return error.code from a JSON error body, or None if not parseable.""" + try: + body = json.loads(error_str) + code = body.get("error", {}).get("code") + return int(code) if code is not None else None + except Exception: + return None + + def _get_response_headers(original_exception: Exception) -> Optional[httpx.Headers]: """ Extract and return the response headers from an exception, if present. @@ -1415,6 +1425,29 @@ def exception_type( # type: ignore ), ), ) + elif ( + isinstance(getattr(original_exception, "status_code", None), int) + and 500 <= original_exception.status_code < 600 + and _get_body_error_code(error_str) == 429 + ): + # upstream gateway wraps a 429 inside a 5xx envelope + # e.g. HTTP 500/503 with {"error":{"code":429,...}}. + # Scoped to 5xx so HTTP 400/401 with body code:429 + # still maps to BadRequestError / AuthenticationError. + exception_mapping_worked = True + raise RateLimitError( + message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}", + model=model, + llm_provider=custom_llm_provider, + litellm_debug_info=extra_information, + response=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) elif ( "500 Internal Server Error" in error_str or "The model is overloaded." in error_str diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d241c501797..afd96029995 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5416,19 +5416,20 @@ class StandardLoggingPayloadSetup: tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG] ) # Limit to first 100 lines + # Prefer the `.message` attribute (set by ProxyException and every + # litellm.exceptions.* class) over str(exc); ProxyException does not + # call super().__init__() nor define __str__, so str() on it returns + # an empty string, which used to silently strip the human-readable + # message from spend_logs.metadata.error_information. + # Use isinstance, not truthiness: an explicit empty string on + # `.message` is a deliberate value and must not be replaced by + # `str(exc)`. explicit_message = getattr(original_exception, "message", None) - error_message = ( - explicit_message - if isinstance(explicit_message, str) and explicit_message - else str(original_exception) - ) + if isinstance(explicit_message, str): + error_message = explicit_message + else: + error_message = str(original_exception) if original_exception else "" - # Duck-typed read so bare-Exception subclasses like - # `litellm.BudgetExceededError` can participate without joining the - # RateLimitError hierarchy (which would break `except BudgetExceededError`). - # Validated against the enum value sets so a third-party exception that - # happens to declare a `.category` or `.rate_limit_type` string attribute - # can't leak garbage into the payload or Prometheus label cardinality. rate_limit_category = validate_rate_limit_category( getattr(original_exception, "category", None) ) @@ -5441,7 +5442,7 @@ class StandardLoggingPayloadSetup: error_class=error_class, llm_provider=_llm_provider_in_exception, traceback=traceback_info, - error_message=error_message if original_exception else "", + error_message=error_message, error_rate_limit_category=rate_limit_category, error_rate_limit_type=rate_limit_type, ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 6c6b8611da6..7a7fde3087e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -191,6 +191,11 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st return base_key +def _parse_above_token_threshold(key: str) -> float: + threshold_str = key.split("_above_")[1].split("_tokens")[0] + return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) + + def _get_token_base_cost( model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None ) -> Tuple[float, float, float, float, float]: @@ -256,15 +261,13 @@ def _get_token_base_cost( # Only sort the threshold keys (typically 1-2 keys instead of 66+) threshold: Optional[float] = None - for key in sorted(threshold_keys, reverse=True): + for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True): value = model_info.get(key) if value is not None: try: # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] - threshold = float(threshold_str.replace("k", "")) * ( - 1000 if "k" in threshold_str else 1 - ) + threshold = _parse_above_token_threshold(key) if usage.prompt_tokens > threshold: # Prefer a service_tier-specific above-threshold key when available, # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 44ba1ce3c86..3cd3a249c33 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -218,8 +218,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): - Qualifier goes as query parameter - Only the payload goes in the request body + Payload shape: + - ``prompt`` is always present and contains the text-only flatten of the + last message's content (existing behavior). + - ``content`` is added ONLY when the ``forward_multimodal_content`` litellm + param is truthy AND the last message's ``content`` is a list containing a + non-text block (e.g. ``image_url``, ``file``, ``input_audio``). The list is + forwarded verbatim so the agent's ``@app.entrypoint`` handler can parse the + OpenAI-shaped multimodal blocks. This is opt-in because an AgentCore agent + must be explicitly written to read ``payload["content"]``; by default the + payload stays byte-identical to the legacy ``{"prompt": "..."}`` shape. + Returns: - dict: Payload dict containing the prompt + dict: Payload dict containing the prompt and (optionally) the OpenAI + content list. """ verbose_logger.debug( f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}" @@ -231,6 +243,20 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Create the payload - this is what goes in the body (raw JSON) payload: dict = {"prompt": prompt} + # Opt-in: when forward_multimodal_content is set, forward the OpenAI content + # list verbatim under "content" so an attachment-aware agent can read the raw + # blocks (image_url, file, etc.). Default off keeps the payload byte-identical + # to the legacy {"prompt": "..."} shape for agents that only read the prompt. + if self._should_forward_multimodal_content(optional_params, litellm_params): + last_content = messages[-1].get("content") + if isinstance(last_content, list) and any( + isinstance(block, dict) and block.get("type") not in (None, "text") + for block in last_content + ): + # Copy so the payload never aliases messages[-1]["content"]; shallow, + # not deep, to avoid cloning large base64 media on the request path. + payload["content"] = list(last_content) + # Get or generate session ID - this goes in the header runtime_session_id = self._get_runtime_session_id(optional_params) headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id @@ -246,6 +272,29 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): verbose_logger.debug(f"PAYLOAD: {payload}") return payload + @staticmethod + def _should_forward_multimodal_content( + optional_params: dict, litellm_params: dict + ) -> bool: + """Whether to forward raw OpenAI content blocks under ``payload["content"]``. + + Opt-in via the ``forward_multimodal_content`` litellm param (default ``False``) + because AgentCore agents must be explicitly written to read the field. The + value may arrive as a bool or a config/env string ("true", "1", ...). Checks + ``optional_params`` first (where other AgentCore params land), then + ``litellm_params``. + """ + for source in (optional_params, litellm_params): + if not isinstance(source, dict): + continue + value = source.get("forward_multimodal_content") + if value is None: + continue + if isinstance(value, str): + return value.strip().lower() in ("1", "true", "yes", "on") + return bool(value) + return False + def _extract_sse_json(self, line: str) -> Optional[Dict]: """Extract and parse JSON from an SSE data line.""" if not line.startswith("data:"): diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 62f707b3622..b97a59a93a6 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -116,6 +116,16 @@ class AiohttpResponseStream(httpx.AsyncByteStream): # For other exceptions, use the normal mapping with map_aiohttp_exceptions(): raise + finally: + # Release the aiohttp connection when iteration ends for any + # reason (read timeout, cancellation from a client disconnect, + # GeneratorExit). Without this, abnormally terminated streams + # permanently hold a slot in the TCPConnector pool; once the + # pool is exhausted every request to that host times out (408) + # until the proxy is restarted, even after the backend recovers. + # On a fully-read response the connection was already released + # at EOF and close() is a no-op. + self._aiohttp_response.close() async def aclose(self) -> None: with map_aiohttp_exceptions(): diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index cca3b3da37a..341c2fc7350 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -392,7 +392,10 @@ class FireworksAIConfig(OpenAIGPTConfig): headers: dict, ) -> dict: if not model.startswith("accounts/") and "#" not in model: - model = f"accounts/fireworks/models/{model}" + if model.endswith("-fast"): + model = f"accounts/fireworks/routers/{model}" + else: + model = f"accounts/fireworks/models/{model}" messages = self._transform_messages_helper( messages=messages, model=model, litellm_params=litellm_params ) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 5464b5bb7ee..b8b750b8c12 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -18,6 +18,9 @@ from typing import ( overload, ) +import os +from urllib.parse import urlparse + import httpx import litellm @@ -426,6 +429,32 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def _should_preserve_cache_control_for_endpoint( + self, + custom_llm_provider: str | None, + api_base: str | None, + ) -> bool: + """ + The generic `openai` provider also reaches OpenAI-compatible endpoints + (a LiteLLM proxy, vLLM, an Anthropic-compatible gateway) via a custom + api_base. Those can understand cache_control, so it must survive there. + Real OpenAI cannot, so it is still stripped for an openai.com host. + """ + if custom_llm_provider != "openai": + return False + resolved_api_base = ( + api_base + or litellm.api_base + or os.getenv("OPENAI_BASE_URL") + or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return False + hostname = urlparse(resolved_api_base).hostname + if hostname is None: + return False + return hostname != "openai.com" and not hostname.endswith(".openai.com") + def transform_request( self, model: str, @@ -441,11 +470,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): dict: The transformed request. Sent as the body of the API call. """ messages = self._transform_messages(messages=messages, model=model) - messages, tools = self.remove_cache_control_flag_from_messages_and_tools( - model=model, messages=messages, tools=optional_params.get("tools", []) - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools + if not self._should_preserve_cache_control_for_endpoint( + litellm_params.get("custom_llm_provider"), litellm_params.get("api_base") + ): + messages, tools = self.remove_cache_control_flag_from_messages_and_tools( + model=model, messages=messages, tools=optional_params.get("tools", []) + ) + if tools is not None and len(tools) > 0: + optional_params["tools"] = tools optional_params.pop("max_retries", None) @@ -466,16 +498,19 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): transformed_messages = await self._transform_messages( messages=messages, model=model, is_async=True ) - ( - transformed_messages, - tools, - ) = self.remove_cache_control_flag_from_messages_and_tools( - model=model, - messages=transformed_messages, - tools=optional_params.get("tools", []), - ) - if tools is not None and len(tools) > 0: - optional_params["tools"] = tools + if not self._should_preserve_cache_control_for_endpoint( + litellm_params.get("custom_llm_provider"), litellm_params.get("api_base") + ): + ( + transformed_messages, + tools, + ) = self.remove_cache_control_flag_from_messages_and_tools( + model=model, + messages=transformed_messages, + tools=optional_params.get("tools", []), + ) + if tools is not None and len(tools) > 0: + optional_params["tools"] = tools if self.__class__._is_base_class: return { "model": model, diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 0f9c3cad841..bf055f91aa0 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -58,11 +58,8 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: ## CALCULATE OUTPUT COST output_cost_per_token = _safe_float_cast(model_info.get("output_cost_per_token")) - completion_cost: float = (usage.completion_tokens or 0) * output_cost_per_token - ## ADD REASONING TOKENS COST (if present) reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0 - # Also check completion_tokens_details if reasoning_tokens is not directly available if ( reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") @@ -73,9 +70,19 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: ) reasoning_cost_value = model_info.get("output_cost_per_reasoning_token") + + # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage + # convention (codified for the central path in PR #18607). When a reasoning rate is + # configured we subtract before the output-rate multiplication so the reasoning + # tokens are not billed twice. if reasoning_tokens > 0 and reasoning_cost_value is not None: - reasoning_cost_per_token = _safe_float_cast(reasoning_cost_value) - completion_cost += reasoning_tokens * reasoning_cost_per_token + non_reasoning_completion_tokens = max( + 0, (usage.completion_tokens or 0) - reasoning_tokens + ) + completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token + completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value) + else: + completion_cost = (usage.completion_tokens or 0) * output_cost_per_token ## ADD SEARCH QUERIES COST (if present) num_search_queries = 0 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0ee4a33c4ca..39d612f252d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10912,13 +10912,13 @@ "supports_tool_choice": true }, "command-r7b-12-2024": { - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3.75e-08, "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 3.75e-08, + "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", "supports_function_calling": true, "supports_tool_choice": true @@ -14612,6 +14612,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -14687,43 +14719,64 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, - "max_output_tokens": 202800, - "max_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://fireworks.ai/models/fireworks/glm-5p1", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/glm-5p2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://fireworks.ai/pricing", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/gpt-oss-20b": { - "input_cost_per_token": 5e-08, + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 7e-08, "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://fireworks.ai/pricing", + "output_cost_per_token": 3e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { "input_cost_per_token": 6e-07, @@ -14779,6 +14832,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2p6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/kimi-k2p7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "input_cost_per_token": 3e-06, "litellm_provider": "fireworks_ai", @@ -14896,6 +14981,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -14948,6 +15065,38 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "fireworks_ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -14968,15 +15117,80 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, - "max_output_tokens": 202800, - "max_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://fireworks.ai/models/fireworks/glm-5p1", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/glm-5p1-fast": { + "cache_read_input_token_cost": 5.2e-07, + "input_cost_per_token": 2.8e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/glm-5p2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/gpt-oss-120b": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/gpt-oss-20b": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 7e-08, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/kimi-k2p5": { "cache_read_input_token_cost": 1e-07, @@ -14992,6 +15206,70 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/kimi-k2p6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p6-fast": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p7-code-fast": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/minimax-m2p1": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -15006,6 +15284,54 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/minimax-m2p7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/qwen3p7-plus": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/nomic-ai/nomic-embed-text-v1": { "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", @@ -39467,6 +39793,22 @@ "litellm_provider": "fireworks_ai", "mode": "chat" }, + "fireworks_ai/accounts/fireworks/models/qwen3p7-plus": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -39629,6 +39971,54 @@ "litellm_provider": "fireworks_ai", "mode": "chat" }, + "fireworks_ai/accounts/fireworks/routers/glm-5p1-fast": { + "cache_read_input_token_cost": 5.2e-07, + "input_cost_per_token": 2.8e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "novita/deepseek/deepseek-v3.2": { "litellm_provider": "novita", "mode": "chat", diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 97cfa74ea45..b0141d3207c 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -23,9 +23,20 @@ import os from urllib.parse import quote # Constants -LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" +# +# NOTE: The environment-backed values below are read once, when this module is +# first imported, and cached for the lifetime of the process. Changing the +# corresponding environment variables after import has no effect unless the +# module is reloaded (e.g. ``importlib.reload``). Tests that override these +# variables must reload this module — see +# ``tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py``. +LITELLM_MCP_SERVER_NAME = os.environ.get( + "LITELLM_MCP_SERVER_NAME", "litellm-mcp-server" +) LITELLM_MCP_SERVER_VERSION = "1.0.0" -LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" +LITELLM_MCP_SERVER_DESCRIPTION = os.environ.get( + "LITELLM_MCP_SERVER_DESCRIPTION", "MCP Server for LiteLLM" +) MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-") MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 765e90bc896..8e2ec423cde 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -361,6 +361,16 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + # realtime (GA WebRTC HTTP routes) + "/realtime/client_secrets", + "/v1/realtime/client_secrets", + "/openai/v1/realtime/client_secrets", + "/realtime/calls", + "/v1/realtime/calls", + "/openai/v1/realtime/calls", + "/realtime/transcription_sessions", + "/v1/realtime/transcription_sessions", + "/openai/v1/realtime/transcription_sessions", # responses API "/responses", "/v1/responses", diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1995ff275c9..856b788b54b 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -5,6 +5,7 @@ Unified /v1/messages endpoint - (Anthropic Spec) from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse +import litellm from litellm._logging import verbose_proxy_logger from litellm.anthropic_interface.exceptions import AnthropicExceptionMapping from litellm.integrations.custom_guardrail import ModifyResponseException @@ -23,6 +24,40 @@ from litellm.types.utils import TokenCountResponse router = APIRouter() +def _strip_total_tokens_from_anthropic_response(response: Any) -> None: + """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM + injects into Anthropic /v1/messages responses. + + The Anthropic /v1/messages spec only defines: + input_tokens, output_tokens, cache_creation_input_tokens, + cache_read_input_tokens, cache_creation.{ephemeral_5m,ephemeral_1h} + The streaming SSE path (message_delta.usage) already does not include + total_tokens; this brings the non-streaming path into the same shape. + + Handles both shapes returned by `base_process_llm_request`: + - plain `dict` (most common — `AnthropicMessagesResponse` is a TypedDict + and is `dict` at runtime) + - Pydantic model whose `usage` attribute is dict-shaped (e.g. a + BaseModel that holds raw Anthropic usage as a `dict[str, int]`) + + Streaming results (StreamingResponse, AsyncIterator, etc.) and Pydantic + models with strongly-typed Usage sub-models are left untouched — + those paths either have separate serialization handling or impose + type constraints the helper does not try to subvert. + """ + if response is None: + return + if isinstance(response, dict): + usage = response.get("usage") + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + return + # Pydantic-model fallback: only mutate if `usage` is a dict. + usage = getattr(response, "usage", None) + if isinstance(usage, dict) and "total_tokens" in usage: + usage.pop("total_tokens", None) + + @router.post( "/v1/messages", tags=["[beta] Anthropic `/v1/messages`"], @@ -72,6 +107,18 @@ async def anthropic_response( user_api_base=user_api_base, version=version, ) + # Optionally strip the non-Anthropic `usage.total_tokens` field + # LiteLLM adds internally. Anthropic's official /v1/messages spec + # only defines input_tokens / output_tokens / cache_*_input_tokens; + # total_tokens is an OpenAI convention. Default off + # (`litellm.strip_anthropic_total_tokens = False`) to preserve + # backward compatibility for clients that currently read it; set + # to True to align the wire response with the spec (and with the + # streaming SSE path, which already omits total_tokens). + # spend_logs / Prometheus still compute total internally — this + # only affects the wire response. + if litellm.strip_anthropic_total_tokens: + _strip_total_tokens_from_anthropic_response(result) return result except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 3fa500bbafe..94b2ed84f20 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1267,6 +1267,14 @@ _MODEL_ROUTING_BODY_TARGET_MODEL_ROUTE_MARKERS = ( "/vector_stores", ) _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS = ("/evals",) +# Realtime WebRTC routes carry the effective model inside the nested +# ``session.model`` field (see realtime_endpoints.endpoints), so the model the +# request will actually use is not present at the top level. Extract it here so +# can_key_call_model() validates the real target model. +_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS = ( + "/realtime/client_secrets", + "/realtime/calls", +) _MODEL_ROUTING_ID_FIELDS = ( "file_id", "input_file_id", @@ -1449,6 +1457,12 @@ def _extract_model_candidates_from_request( _append_model_candidates(candidates, body_model) if uses_body_target_model_sources or not body_model: _append_model_candidates(candidates, request_data.get("target_model_names")) + if _route_matches_any_marker( + route=route, markers=_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS + ): + session = request_data.get("session") + if isinstance(session, dict): + _append_model_candidates(candidates, session.get("model")) if uses_completion_model_sources and isinstance( request_data.get("completion"), dict ): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 4b7b20d75d0..aab92a54577 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1613,199 +1613,215 @@ class DBSpendUpdateWriter: start_time = time.time() try: - for i in range(n_retry_times + 1): - try: - # Sort the transactions to minimize the probability of deadlocks by reducing the chance of concurrent - # trasactions locking the same rows/ranges in different orders. - transactions_to_process = dict( - sorted( - daily_spend_transactions.items(), - # Normally to avoid deadlocks we would sort by the index, but since we have sprinkled indexes - # on our schema like we're discount Salt Bae, we just sort by all fields that have an index, - # in an ad-hoc (but hopefully sensible) order of indexes. The actual ordering matters less than - # ensuring that all concurrent transactions sort in the same order. - # We could in theory use the dict key, as it contains basically the same fields, but this is more - # robust to future changes in the key format. - # If _update_daily_spend ever gets the ability to write to multiple tables at once, the sorting - # should sort by the table first. - key=lambda x: ( - x[1].get("date") or "", - x[1].get(entity_id_field) or "", - x[1].get("api_key") or "", - x[1].get("model") or "", - x[1].get("custom_llm_provider") or "", - ), - )[:BATCH_SIZE] - ) - - if len(transactions_to_process) == 0: - verbose_proxy_logger.debug( - f"No new transactions to process for daily {entity_type} spend update" - ) - break - + while daily_spend_transactions: + for i in range(n_retry_times + 1): try: - async with prisma_client.db.batch_() as batcher: - for _, transaction in transactions_to_process.items(): - entity_id = transaction.get(entity_id_field) + # Sort the transactions to minimize the probability of deadlocks by reducing the chance of concurrent + # trasactions locking the same rows/ranges in different orders. + transactions_to_process = dict( + sorted( + daily_spend_transactions.items(), + # Normally to avoid deadlocks we would sort by the index, but since we have sprinkled indexes + # on our schema like we're discount Salt Bae, we just sort by all fields that have an index, + # in an ad-hoc (but hopefully sensible) order of indexes. The actual ordering matters less than + # ensuring that all concurrent transactions sort in the same order. + # We could in theory use the dict key, as it contains basically the same fields, but this is more + # robust to future changes in the key format. + # If _update_daily_spend ever gets the ability to write to multiple tables at once, the sorting + # should sort by the table first. + key=lambda x: ( + x[1].get("date") or "", + x[1].get(entity_id_field) or "", + x[1].get("api_key") or "", + x[1].get("model") or "", + x[1].get("custom_llm_provider") or "", + ), + )[:BATCH_SIZE] + ) - # Construct the where clause dynamically - where_clause = { - unique_constraint_name: { + if len(transactions_to_process) == 0: + verbose_proxy_logger.debug( + f"No new transactions to process for daily {entity_type} spend update" + ) + return + + try: + async with prisma_client.db.batch_() as batcher: + for _, transaction in transactions_to_process.items(): + entity_id = transaction.get(entity_id_field) + + # Construct the where clause dynamically + where_clause = { + unique_constraint_name: { + entity_id_field: entity_id, + "date": transaction["date"], + "api_key": transaction["api_key"], + "model": transaction["model"], + "custom_llm_provider": transaction.get( + "custom_llm_provider" + ) + or "", + "mcp_namespaced_tool_name": transaction.get( + "mcp_namespaced_tool_name" + ) + or "", + "endpoint": transaction.get("endpoint") + or "", + } + } + + # Get the table dynamically + table = getattr(batcher, table_name) + + # Common data structure for both create and update + common_data = { entity_id_field: entity_id, "date": transaction["date"], "api_key": transaction["api_key"], - "model": transaction["model"], - "custom_llm_provider": transaction.get( - "custom_llm_provider" - ) - or "", + "model": transaction.get("model"), + "model_group": transaction.get("model_group"), "mcp_namespaced_tool_name": transaction.get( "mcp_namespaced_tool_name" ) or "", + "custom_llm_provider": transaction.get( + "custom_llm_provider" + ), "endpoint": transaction.get("endpoint") or "", - } - } - - # Get the table dynamically - table = getattr(batcher, table_name) - - # Common data structure for both create and update - common_data = { - entity_id_field: entity_id, - "date": transaction["date"], - "api_key": transaction["api_key"], - "model": transaction.get("model"), - "model_group": transaction.get("model_group"), - "mcp_namespaced_tool_name": transaction.get( - "mcp_namespaced_tool_name" - ) - or "", - "custom_llm_provider": transaction.get( - "custom_llm_provider" - ), - "endpoint": transaction.get("endpoint") or "", - "prompt_tokens": transaction["prompt_tokens"], - "completion_tokens": transaction[ - "completion_tokens" - ], - "spend": transaction["spend"], - "api_requests": transaction["api_requests"], - "successful_requests": transaction[ - "successful_requests" - ], - "failed_requests": transaction["failed_requests"], - } - - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = ( - transaction.get("cache_read_input_tokens", 0) - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = ( - transaction.get( - "cache_creation_input_tokens", 0 - ) - ) - - if entity_type == "tag" and "request_id" in transaction: - common_data["request_id"] = transaction.get( - "request_id" - ) - - # Create update data structure - update_data = { - "prompt_tokens": { - "increment": transaction["prompt_tokens"] - }, - "completion_tokens": { - "increment": transaction["completion_tokens"] - }, - "spend": {"increment": transaction["spend"]}, - "api_requests": { - "increment": transaction["api_requests"] - }, - "successful_requests": { - "increment": transaction["successful_requests"] - }, - "failed_requests": { - "increment": transaction["failed_requests"] - }, - } - - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get( - "cache_read_input_tokens", 0 - ) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get( - "cache_creation_input_tokens", 0 - ) + "prompt_tokens": transaction["prompt_tokens"], + "completion_tokens": transaction[ + "completion_tokens" + ], + "spend": transaction["spend"], + "api_requests": transaction["api_requests"], + "successful_requests": transaction[ + "successful_requests" + ], + "failed_requests": transaction[ + "failed_requests" + ], } - if entity_type == "tag" and "request_id" in transaction: - update_data["request_id"] = transaction.get( - "request_id" + # Add cache-related fields if they exist + if "cache_read_input_tokens" in transaction: + common_data["cache_read_input_tokens"] = ( + transaction.get( + "cache_read_input_tokens", 0 + ) + ) + if "cache_creation_input_tokens" in transaction: + common_data["cache_creation_input_tokens"] = ( + transaction.get( + "cache_creation_input_tokens", 0 + ) + ) + + if ( + entity_type == "tag" + and "request_id" in transaction + ): + common_data["request_id"] = transaction.get( + "request_id" + ) + + # Create update data structure + update_data = { + "prompt_tokens": { + "increment": transaction["prompt_tokens"] + }, + "completion_tokens": { + "increment": transaction[ + "completion_tokens" + ] + }, + "spend": {"increment": transaction["spend"]}, + "api_requests": { + "increment": transaction["api_requests"] + }, + "successful_requests": { + "increment": transaction[ + "successful_requests" + ] + }, + "failed_requests": { + "increment": transaction["failed_requests"] + }, + } + + # Add cache-related fields to update if they exist + if "cache_read_input_tokens" in transaction: + update_data["cache_read_input_tokens"] = { + "increment": transaction.get( + "cache_read_input_tokens", 0 + ) + } + if "cache_creation_input_tokens" in transaction: + update_data["cache_creation_input_tokens"] = { + "increment": transaction.get( + "cache_creation_input_tokens", 0 + ) + } + + if ( + entity_type == "tag" + and "request_id" in transaction + ): + update_data["request_id"] = transaction.get( + "request_id" + ) + + # Add endpoint to update_data so existing rows get their endpoint field updated + update_data["endpoint"] = ( + transaction.get("endpoint") or "" ) - # Add endpoint to update_data so existing rows get their endpoint field updated - update_data["endpoint"] = ( - transaction.get("endpoint") or "" - ) + table.upsert( + where=where_clause, + data={ + "create": common_data, + "update": update_data, + }, + ) + except Exception as batch_error: + # Log detailed error information for debugging batch upsert failures + # This helps diagnose issues like unique constraint violations + spend_log_error( + "Daily %s spend batch upsert failed. " + "Table: %s, Constraint: %s, Batch size: %d, Error: %s", + entity_type, + table_name, + unique_constraint_name, + len(transactions_to_process), + str(batch_error), + exc=batch_error, + ) + raise - table.upsert( - where=where_clause, - data={ - "create": common_data, - "update": update_data, - }, - ) - except Exception as batch_error: - # Log detailed error information for debugging batch upsert failures - # This helps diagnose issues like unique constraint violations - spend_log_error( - "Daily %s spend batch upsert failed. " - "Table: %s, Constraint: %s, Batch size: %d, Error: %s", - entity_type, - table_name, - unique_constraint_name, - len(transactions_to_process), - str(batch_error), - exc=batch_error, + verbose_proxy_logger.debug( + f"Processed {len(transactions_to_process)} daily {entity_type} transactions in {time.time() - start_time:.2f}s" ) - raise - verbose_proxy_logger.debug( - f"Processed {len(transactions_to_process)} daily {entity_type} transactions in {time.time() - start_time:.2f}s" - ) + # Remove processed transactions + for key in transactions_to_process.keys(): + daily_spend_transactions.pop(key, None) - # Remove processed transactions - for key in transactions_to_process.keys(): - daily_spend_transactions.pop(key, None) + break - break - - except DB_CONNECTION_ERROR_TYPES as e: - if i >= n_retry_times: - _raise_failed_update_spend_exception( - e=e, - start_time=start_time, - proxy_logging_obj=proxy_logging_obj, + except DB_CONNECTION_ERROR_TYPES as e: + if i >= n_retry_times: + _raise_failed_update_spend_exception( + e=e, + start_time=start_time, + proxy_logging_obj=proxy_logging_obj, + ) + await asyncio.sleep( + # Sleep a random amount to avoid retrying and deadlocking again: when two transactions deadlock they are + # cancelled basically at the same time, so if they wait the same time they will also retry at the same time + # and thus they are more likely to deadlock again. + # Instead, we sleep a random amount so that they retry at slightly different times, lowering the chance of + # repeated deadlocks, and therefore of exceeding the retry limit. + random.uniform(2**i, 2 ** (i + 1)) ) - await asyncio.sleep( - # Sleep a random amount to avoid retrying and deadlocking again: when two transactions deadlock they are - # cancelled basically at the same time, so if they wait the same time they will also retry at the same time - # and thus they are more likely to deadlock again. - # Instead, we sleep a random amount so that they retry at slightly different times, lowering the chance of - # repeated deadlocks, and therefore of exceeding the retry limit. - random.uniform(2**i, 2 ** (i + 1)) - ) except Exception as e: if "transactions_to_process" in locals(): diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index e6b040ef2ee..341a8767db0 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -35,16 +35,25 @@ _PRISMA_TO_PG_TABLE: Dict[str, str] = { def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics: - """Update metrics with new record data.""" - existing_metrics.spend += record.spend - existing_metrics.prompt_tokens += record.prompt_tokens - existing_metrics.completion_tokens += record.completion_tokens - existing_metrics.total_tokens += record.prompt_tokens + record.completion_tokens - existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens - existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens - existing_metrics.api_requests += record.api_requests - existing_metrics.successful_requests += record.successful_requests - existing_metrics.failed_requests += record.failed_requests + """Update metrics with new record data. + + Rollup rows can carry None for numeric fields when SUM() spans zero rows + (e.g. a key with no spend), so coalesce to 0 before accumulating to avoid + a TypeError. Mirrors the handling in ``_record_to_spend_metrics``. + """ + prompt_tokens = record.prompt_tokens or 0 + completion_tokens = record.completion_tokens or 0 + existing_metrics.spend += record.spend or 0.0 + existing_metrics.prompt_tokens += prompt_tokens + existing_metrics.completion_tokens += completion_tokens + existing_metrics.total_tokens += prompt_tokens + completion_tokens + existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0 + existing_metrics.cache_creation_input_tokens += ( + record.cache_creation_input_tokens or 0 + ) + existing_metrics.api_requests += record.api_requests or 0 + existing_metrics.successful_requests += record.successful_requests or 0 + existing_metrics.failed_requests += record.failed_requests or 0 return existing_metrics diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index 458cba686e6..f28bcc2bcd4 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -397,7 +397,7 @@ def _set_object_metadata_field( field_name: Name of the metadata field to set value: Value to set for the field """ - if field_name in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if field_name in LiteLLM_ManagementEndpoint_MetadataFields_Premium and value: _premium_user_check(field_name) object_data.metadata = object_data.metadata or {} @@ -563,13 +563,11 @@ def _update_metadata_field(updated_kv: dict, field_name: str) -> None: field_name: Name of the metadata field being updated """ if field_name in LiteLLM_ManagementEndpoint_MetadataFields_Premium: - value = updated_kv.get(field_name) - # Skip the premium check for empty collections ([] or {}). - # The UI sends these as defaults even when the user hasn't configured - # any enterprise features (see issue #20304). However, we still - # proceed with the update so that users can intentionally clear a - # previously-set field by sending an empty list/dict. - if value is not None and value != [] and value != {}: + # The UI sends falsy defaults (False, [], {}) even when the user has not + # enabled any enterprise feature (see #20304, #30285); require a license + # only for a truthy value. The falsy value is still persisted below so a + # previously-set field can be cleared. + if updated_kv.get(field_name): _premium_user_check() if field_name in updated_kv and updated_kv[field_name] is not None: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d6ecc59f263..143d61a0b3a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1793,7 +1793,8 @@ def prepare_metadata_fields( if k in LiteLLM_ManagementEndpoint_MetadataFields_Premium: from litellm.proxy.utils import _premium_user_check - _premium_user_check(k) + if v: + _premium_user_check(k) casted_metadata[k] = v except Exception as e: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 0df4675b67f..e86982307e7 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -49,6 +49,8 @@ from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.proxy._experimental.mcp_server.utils import ( build_env_var_setup_url, collect_env_var_references, + LITELLM_MCP_SERVER_DESCRIPTION, + LITELLM_MCP_SERVER_NAME, get_server_prefix, parse_admin_env_vars, ) @@ -89,8 +91,6 @@ def does_mcp_server_exist( DEFAULT_MCP_REGISTRY_VERSION = "1.0.0" -LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" -LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM" try: importlib.import_module("mcp") diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 4d4d1ef2774..3d90e7b5ab9 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2961,6 +2961,7 @@ async def team_member_update( returned_team_info: TeamInfoResponseObject = await team_info( http_request=http_request, team_id=data.team_id, + key_limit=None, user_api_key_dict=user_api_key_dict, ) @@ -3577,6 +3578,9 @@ async def team_info( team_id: str = fastapi.Query( default=None, description="Team ID in the request parameters" ), + key_limit: int | None = fastapi.Query( + default=None, description="Limit the number of keys returned", gt=0 + ), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -3632,6 +3636,7 @@ async def team_info( table_name="key", query_type="find_all", expires=datetime.now(), + limit=key_limit, ) if keys is None: diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index a953dbec6b7..409ce6f50f2 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -130,6 +130,12 @@ async def _prepare_client_secret_session( session_model = req.session.model if req.session else None model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL if session_type != "transcription": + await can_key_call_resolved_model( + model=model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates = _transcription_model_candidates_from_session( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 451c32b334d..a7bc94f7430 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3432,6 +3432,7 @@ class PrismaClient: r.expires = r.expires.isoformat() elif query_type == "find_all" and team_id is not None: response = await VerificationTokenRepository(self).table.find_many( + take=limit, where={"team_id": team_id}, include={"litellm_budget_table": True}, ) @@ -6328,15 +6329,37 @@ def create_model_info_response( "created": DEFAULT_MODEL_CREATED_AT_TIME, "owned_by": provider, } + + # Surface context-window limits for OpenAI-compatible discovery clients. + # Only emitted when known, so wildcard routes and limitless backends stay clean. + # Limits are best-effort enrichment, so a single malformed deployment degrades + # to the base response rather than 500-ing the whole listing. + if llm_router is not None: + try: + model_group_info = llm_router.get_model_group_info(model_id) + except Exception as e: + verbose_proxy_logger.debug( + "create_model_info_response: get_model_group_info failed for %s: %s", + model_id, + e, + ) + model_group_info = None + if model_group_info is not None: + if model_group_info.max_input_tokens is not None: + base["max_input_tokens"] = int(model_group_info.max_input_tokens) + if model_group_info.max_output_tokens is not None: + base["max_output_tokens"] = int(model_group_info.max_output_tokens) + if not include_metadata: return base effective_fallback_type = fallback_type if fallback_type is not None else "general" - valid_fallback_types = ("general", "context_window", "content_policy") + + valid_fallback_types = ["general", "context_window", "content_policy"] if effective_fallback_type not in valid_fallback_types: raise HTTPException( status_code=400, - detail=f"Invalid fallback_type. Must be one of: {list(valid_fallback_types)}", + detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}", ) fallbacks = get_all_fallbacks( diff --git a/litellm/router.py b/litellm/router.py index 5f26097443f..e54eadfb872 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3045,7 +3045,8 @@ class Router: deployment_timeout_param = _timeout_debug_deployment_dict.get( "litellm_params", {} ).get("timeout", None) - e.message += f"\n\nDeployment Info: request_timeout: {deployment_request_timeout_param}\ntimeout: {deployment_timeout_param}" + if litellm.expose_router_debug_in_errors: + e.message += f"\n\nDeployment Info: request_timeout: {deployment_request_timeout_param}\ntimeout: {deployment_timeout_param}" # Set per-deployment num_retries on exception for retry logic if deployment is not None: self._set_deployment_num_retries_on_exception(e, deployment) @@ -6644,7 +6645,8 @@ class Router: ) ) - e.message += "\n{}".format(error_message) + if litellm.expose_router_debug_in_errors: + e.message += "\n{}".format(error_message) elif isinstance(e, litellm.ContentPolicyViolationError): if content_policy_fallbacks is not None: content_policy_fallback_model_group: Optional[List[str]] = ( @@ -6679,7 +6681,8 @@ class Router: ) ) - e.message += "\n{}".format(error_message) + if litellm.expose_router_debug_in_errors: + e.message += "\n{}".format(error_message) if fallbacks is not None and model_group is not None: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") ( @@ -6697,7 +6700,10 @@ class Router: verbose_router_logger.info( f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" ) - if hasattr(original_exception, "message"): + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + ): original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" # type: ignore raise original_exception @@ -6728,7 +6734,10 @@ class Router: ) fallback_failure_exception_str = str(new_exception) - if hasattr(original_exception, "message"): + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + ): # add the available fallbacks to the exception original_exception.message += ". Received Model Group={}\nAvailable Model Group Fallbacks={}".format( # type: ignore model_group, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 55216caa941..6eb65d7be02 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -211,6 +211,9 @@ class PiiEntityType(str, Enum): # UK UK_NHS = "UK_NHS" UK_NINO = "UK_NINO" + UK_PASSPORT = "UK_PASSPORT" + UK_POSTCODE = "UK_POSTCODE" + UK_VEHICLE_REGISTRATION = "UK_VEHICLE_REGISTRATION" # Spain ES_NIF = "ES_NIF" ES_NIE = "ES_NIE" @@ -265,7 +268,13 @@ PII_ENTITY_CATEGORIES_MAP = { PiiEntityType.US_PASSPORT, PiiEntityType.US_SSN, ], - PiiEntityCategory.UK: [PiiEntityType.UK_NHS, PiiEntityType.UK_NINO], + PiiEntityCategory.UK: [ + PiiEntityType.UK_NHS, + PiiEntityType.UK_NINO, + PiiEntityType.UK_PASSPORT, + PiiEntityType.UK_POSTCODE, + PiiEntityType.UK_VEHICLE_REGISTRATION, + ], PiiEntityCategory.SPAIN: [PiiEntityType.ES_NIF, PiiEntityType.ES_NIE], PiiEntityCategory.ITALY: [ PiiEntityType.IT_FISCAL_CODE, @@ -319,8 +328,7 @@ class PresidioPresidioConfigModelUserInterface(BaseModel): presidio_filter_scope: Optional[Literal["input", "output", "both"]] = Field( default=None, description=( - "Where to apply Presidio checks: 'input' (user -> model), " - "'output' (model -> user), or 'both' (default)." + "Where to apply Presidio checks: 'input' (user -> model), 'output' (model -> user), or 'both' (default)." ), ) output_parse_pii: Optional[bool] = Field( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0c925bb276b..80034e50393 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3246,6 +3246,11 @@ all_litellm_params = ( "order", "enable_json_schema_validation", "use_xai_oauth", + "_litellm_rate_limit_descriptors", + "_litellm_tpm_reserved_tokens", + "_litellm_tpm_reserved_model", + "_litellm_tpm_reserved_scopes", + "_litellm_tpm_reservation_released", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) + list(CustomPricingLiteLLMParams.model_fields.keys()) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d6ab0e10657..ba8b09498e8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10912,13 +10912,13 @@ "supports_tool_choice": true }, "command-r7b-12-2024": { - "input_cost_per_token": 1.5e-07, + "input_cost_per_token": 3.75e-08, "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 3.75e-08, + "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", "supports_function_calling": true, "supports_tool_choice": true @@ -14612,6 +14612,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/firefunction-v2": { "input_cost_per_token": 9e-07, "litellm_provider": "fireworks_ai", @@ -14687,43 +14719,64 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, - "max_output_tokens": 202800, - "max_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://fireworks.ai/models/fireworks/glm-5p1", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/glm-5p2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/gpt-oss-120b": { + "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://fireworks.ai/pricing", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/gpt-oss-20b": { - "input_cost_per_token": 5e-08, + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 7e-08, "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://fireworks.ai/pricing", + "output_cost_per_token": 3e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { "input_cost_per_token": 6e-07, @@ -14779,6 +14832,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2p6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/models/kimi-k2p7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "input_cost_per_token": 3e-06, "litellm_provider": "fireworks_ai", @@ -14896,6 +14981,38 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -14948,6 +15065,38 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "fireworks_ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -14968,15 +15117,80 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, - "max_output_tokens": 202800, - "max_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://fireworks.ai/models/fireworks/glm-5p1", + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/glm-5p1-fast": { + "cache_read_input_token_cost": 5.2e-07, + "input_cost_per_token": 2.8e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/glm-5p2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/gpt-oss-120b": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/gpt-oss-20b": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_token": 7e-08, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "fireworks_ai/kimi-k2p5": { "cache_read_input_token_cost": 1e-07, @@ -14992,6 +15206,70 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/kimi-k2p6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p6-fast": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/kimi-k2p7-code-fast": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/minimax-m2p1": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -15006,6 +15284,54 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "fireworks_ai/minimax-m2p7": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 196608, + "max_output_tokens": 196608, + "max_tokens": 196608, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/minimax-m3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 512000, + "max_output_tokens": 512000, + "max_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/qwen3p7-plus": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/nomic-ai/nomic-embed-text-v1": { "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", @@ -39497,6 +39823,22 @@ "litellm_provider": "fireworks_ai", "mode": "chat" }, + "fireworks_ai/accounts/fireworks/models/qwen3p7-plus": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 4e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.6e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -39659,6 +40001,54 @@ "litellm_provider": "fireworks_ai", "mode": "chat" }, + "fireworks_ai/accounts/fireworks/routers/glm-5p1-fast": { + "cache_read_input_token_cost": 5.2e-07, + "input_cost_per_token": 2.8e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 202800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "fireworks_ai/accounts/fireworks/routers/kimi-k2p6-fast": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 2e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "fireworks_ai/accounts/fireworks/routers/kimi-k2p7-code-fast": { + "cache_read_input_token_cost": 3.8e-07, + "input_cost_per_token": 1.9e-06, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 8e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "scaleway/qwen/qwen3.5-397b-a17b": { "input_cost_per_token": 6e-07, "litellm_provider": "scaleway", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index ed3e96803f9..9b3152fae07 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -35,6 +35,7 @@ sys.path.insert( from litellm.litellm_core_utils.llm_cost_calc.utils import ( PromptTokensDetailsResult, _calculate_input_cost, + _get_token_base_cost, calculate_cache_writing_cost, generic_cost_per_token, ) @@ -298,6 +299,26 @@ def test_generic_cost_per_token_above_200k_tokens(): ) +def test_get_token_base_cost_picks_highest_crossed_tier(): + """Regression test for #30345. + + With graduated tiers at 90k and 128k whose keys have different digit lengths, a request + crossing both must be billed at the highest tier it crosses (128k), not the lower one that + happens to sort first lexicographically. + """ + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_90k_tokens": 5e-6, + "input_cost_per_token_above_128k_tokens": 9e-6, + } + usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) + + prompt_base_cost = _get_token_base_cost(model_info, usage)[0] + + assert prompt_base_cost == 9e-6 + + def test_generic_cost_per_token_gpt54_above_272k_tokens(): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 7dab0e02623..35c02184a51 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -11,6 +11,7 @@ sys.path.insert( from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, + _get_body_error_code, exception_type, extract_and_raise_litellm_exception, ) @@ -359,6 +360,122 @@ def test_vertex_ai_rate_limit_error_mapping(error_message, should_raise_rate_lim ) +class TestGetBodyErrorCode: + """Unit tests for _get_body_error_code helper.""" + + def test_parses_int_code(self): + body = ( + '{"error":{"message":"high demand","type":"upstream_error",' + '"param":"","code":429}}' + ) + assert _get_body_error_code(body) == 429 + + def test_parses_string_code(self): + # some gateways serialize code as a string + body = '{"error":{"message":"x","code":"503"}}' + assert _get_body_error_code(body) == 503 + + def test_returns_none_on_non_json(self): + assert _get_body_error_code("not json") is None + + def test_returns_none_when_no_error_key(self): + assert _get_body_error_code('{"ok":true}') is None + + def test_returns_none_when_no_code_key(self): + assert _get_body_error_code('{"error":{"message":"x"}}') is None + + +# Test cases for Gemini upstream-error body-code mapping. +# +# Body code 429 wrapped in a 5xx HTTP envelope (e.g. new-api gateways) +# must map to RateLimitError so Router retries kick in. A 4xx HTTP +# envelope with body code:429 must NOT — it falls through to whatever +# the HTTP status code maps to (BadRequestError, AuthenticationError, +# etc.), matching upstream's existing semantics. +gemini_body_code_429_test_cases = [ + # (status_code, error_body, expected_exception_type, description) + ( + 500, + '{"error":{"message":" This model is currently experiencing high demand.' + " Spikes in demand are usually temporary. Please try again later." + ' (request id: x)","type":"upstream_error","param":"","code":429}}', + litellm.RateLimitError, + "HTTP 500 envelope with body code:429 -> RateLimitError", + ), + ( + 503, + '{"error":{"message":"upstream unavailable","type":"upstream_error",' + '"param":"","code":429}}', + litellm.RateLimitError, + "HTTP 503 envelope with body code:429 -> RateLimitError", + ), + ( + 502, + '{"error":{"message":"bad gateway","code":429}}', + litellm.RateLimitError, + "HTTP 502 envelope with body code:429 -> RateLimitError", + ), + ( + 500, + '{"error":{"message":"server boom","code":500}}', + litellm.InternalServerError, + "HTTP 500 with body code:500 stays InternalServerError", + ), + ( + 500, + "plain text 500 error", + litellm.InternalServerError, + "HTTP 500 with non-JSON body falls through to status_code mapping", + ), + ( + 400, + '{"error":{"message":"malformed","code":429}}', + litellm.BadRequestError, + "HTTP 400 with body code:429 must NOT be promoted to RateLimitError", + ), + ( + 401, + '{"error":{"message":"bad key","code":429}}', + litellm.AuthenticationError, + "HTTP 401 with body code:429 must NOT be promoted to RateLimitError", + ), +] + + +@pytest.mark.parametrize( + "status_code, error_body, expected_exception, description", + gemini_body_code_429_test_cases, +) +def test_gemini_upstream_error_body_code_429_maps_to_rate_limit( + status_code, error_body, expected_exception, description +): + """ + Body code 429 inside a 5xx envelope -> RateLimitError so Router + retries kick in. Body code 429 inside a 4xx envelope must fall + through to the HTTP-status-code branch (P1 from greptile review). + """ + model = "gemini/gemini-2.5-flash" + custom_llm_provider = "gemini" + + # Build an exception that looks like what _handle_error produces: + # a BaseLLMException-style object with .status_code and .message + class _FakeGeminiError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + super().__init__(message) + + original_exception = _FakeGeminiError(status_code=status_code, message=error_body) + + with pytest.raises(expected_exception) as excinfo: + exception_type( + model=model, + original_exception=original_exception, + custom_llm_provider=custom_llm_provider, + ) + assert isinstance(excinfo.value, expected_exception), description + + class TestExtractAndRaiseLitellmException: """Tests for extract_and_raise_litellm_exception function""" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index b3c19a09388..e0d7f22f817 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2116,6 +2116,88 @@ def test_get_error_information_error_code_priority(): assert result["error_class"] == "NoCodeException" +def test_get_error_information_prefers_message_attribute_over_str(): + """ + Regression for empty-error_message-in-spend-logs. + + ProxyException sets `self.message` but does NOT call + `super().__init__(message)` nor define `__str__`, so `str(exc)` + returns the empty string. Before the fix, get_error_information + used `str(original_exception)` and silently stripped the + human-readable message from spend_logs.metadata.error_information, + making dashboard "LLM Failure" rows un-triagable. + + Asserts the `.message` attribute is consulted first. + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + # Simulate a ProxyException-shaped exception: .message set, but + # super().__init__() NOT called and no __str__ override. + class ProxyExceptionLike(Exception): + def __init__(self, message, code): + self.message = str(message) + self.code = str(code) + # NOTE: deliberately NOT calling super().__init__(message) + + msg = "Authentication Error, Invalid proxy server token passed. key=..." + exc = ProxyExceptionLike(message=msg, code=401) + + # Sanity check: this exception type's str() really is empty + assert str(exc) == "", ( + "Test premise broken — bare-base Exception now returns message; " + "review whether ProxyException fix landed at the class level instead" + ) + + result = StandardLoggingPayloadSetup.get_error_information(exc) + assert ( + result["error_message"] == msg + ), f"expected message from .message attribute, got {result['error_message']!r}" + assert result["error_code"] == "401" + assert result["error_class"] == "ProxyExceptionLike" + + +def test_get_error_information_preserves_explicit_empty_message(): + """ + An exception that deliberately sets `.message = ""` must surface + the empty string verbatim, not fall through to `str(exc)`. + + Regression for greptile P2 finding on PR #30381: a truthiness + check (`if message_attr:`) would silently mask an explicit empty + message and substitute `str(original_exception)` — which for + ProxyException-shaped objects is also empty, but for plain + `Exception("boom")` would inject the wrong string and corrupt + the error_information signal. + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + class ProxyExceptionLike(Exception): + def __init__(self, message, code): + self.message = message + self.code = str(code) + super().__init__("unrelated-args-summary") + + exc = ProxyExceptionLike(message="", code=500) + result = StandardLoggingPayloadSetup.get_error_information(exc) + assert result["error_message"] == "", ( + "explicit empty .message must survive verbatim; got " + f"{result['error_message']!r}" + ) + + +def test_get_error_information_falls_back_to_str_when_no_message_attr(): + """ + Plain Exception (no `.message` attr) must still produce a useful + error_message via str(exc), preserving prior behavior for + non-litellm exception types. + """ + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + exc = ValueError("boom") + result = StandardLoggingPayloadSetup.get_error_information(exc) + assert result["error_message"] == "boom" + assert result["error_class"] == "ValueError" + + # ────────────────────────────────────────────────────────────────────── # Tests for _get_assembled_streaming_response non-streaming early return # ────────────────────────────────────────────────────────────────────── diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index 64b43b15dcd..448afd5f3a5 100644 --- a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -5,6 +5,7 @@ Tests: - Accept header fix (sign_request sets Accept: application/json, text/event-stream) - JSON response parsing fallback chain (_parse_json_response supports multiple schemas) - Streaming Content-Type fallback (JSON responses converted to single-chunk streams) +- Multimodal content preservation (transform_request forwards OpenAI content blocks) """ import json @@ -389,3 +390,249 @@ class TestAgentCoreStreamingJsonFallback: client=client, api_key="test-jwt-token", ) + + +class TestAgentCoreMultimodalContent: + """Tests for transform_request forwarding OpenAI multimodal content blocks. + + AgentCore Runtime is schemaless on the agent side — the agent author's + @app.entrypoint handler parses whatever JSON arrives. transform_request + only emits {"prompt": ""} by default and drops image_url, file, and + other non-text blocks. + + When the ``forward_multimodal_content`` litellm param is set, the OpenAI + content list is forwarded verbatim under a "content" field whenever the last + message contains a non-text block. This is opt-in: an agent must be written + to read payload["content"]. Without the flag, the payload is byte-identical + to the legacy {"prompt": "..."} shape. + """ + + @pytest.fixture + def config(self): + return AmazonAgentCoreConfig() + + @pytest.fixture + def transform_kwargs(self): + """Default kwargs — forwarding is OFF (no opt-in flag).""" + return { + "model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:111111111111:runtime/test_agent", + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + } + + @pytest.fixture + def opted_in_kwargs(self, transform_kwargs): + """Kwargs with the opt-in flag set in optional_params.""" + return { + **transform_kwargs, + "optional_params": {"forward_multimodal_content": True}, + } + + def test_string_content_payload_byte_identical_to_legacy( + self, config, transform_kwargs + ): + """String content → exactly {"prompt": ""}, no extra fields.""" + messages = [{"role": "user", "content": "hello agent"}] + payload = config.transform_request(messages=messages, **transform_kwargs) + assert payload == {"prompt": "hello agent"} + + def test_file_block_not_forwarded_by_default(self, config, transform_kwargs): + """Default (no opt-in flag): file blocks are NOT forwarded — backward compat.""" + content = [ + {"type": "text", "text": "summarize this report"}, + { + "type": "file", + "file": { + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0xLjQK", + }, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **transform_kwargs) + assert payload == {"prompt": "summarize this report"} + assert "content" not in payload + + def test_text_only_list_content_no_content_field(self, config, opted_in_kwargs): + """All-text content list → no "content" field even when opted in.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "hello agent"}], + } + ] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + assert payload == {"prompt": "hello agent"} + assert "content" not in payload + + def test_file_data_block_passthrough(self, config, opted_in_kwargs): + """Opted in: a file block → "content" carries the original list verbatim.""" + content = [ + {"type": "text", "text": "summarize this report"}, + { + "type": "file", + "file": { + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0xLjQK", + }, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + assert payload["prompt"] == "summarize this report" + # Contents forwarded verbatim, but as a distinct list (no aliasing). + assert payload["content"] == content + assert payload["content"] is not content + + def test_image_url_block_passthrough(self, config, opted_in_kwargs): + """Opted in: an image_url block → "content" carries it verbatim.""" + content = [ + {"type": "text", "text": "what is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + assert payload["prompt"] == "what is in this image?" + assert payload["content"] == content + assert payload["content"] is not content + + def test_mixed_text_and_files_payload_shape(self, config, opted_in_kwargs): + """Opted in: text + file + image → both "prompt" (text-only) and "content".""" + content = [ + {"type": "text", "text": "first sentence."}, + { + "type": "file", + "file": { + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0xLjQK", + }, + }, + {"type": "text", "text": "second sentence."}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + # prompt is the text-only flatten produced by convert_content_list_to_str. + assert "first sentence." in payload["prompt"] + assert "second sentence." in payload["prompt"] + assert "JVBERi0xLjQK" not in payload["prompt"] + assert "iVBORw0KGgo=" not in payload["prompt"] + # content carries every block in original order. + assert payload["content"] == content + + def test_forwarded_content_does_not_alias_message(self, config, opted_in_kwargs): + """Regression: the forwarded list is a shallow copy, so mutating the + returned payload before serialization must not leak back into the caller's + messages[-1]["content"].""" + content = [ + {"type": "text", "text": "describe this"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + + payload["content"].append({"type": "text", "text": "injected"}) + + assert len(messages[-1]["content"]) == 2 + assert {"type": "text", "text": "injected"} not in messages[-1]["content"] + + def test_only_last_message_content_preserved(self, config, opted_in_kwargs): + """Opted in: file blocks in earlier messages don't trigger "content" — last only.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "context"}, + { + "type": "file", + "file": { + "filename": "old.pdf", + "file_data": "data:application/pdf;base64,Zm9v", + }, + }, + ], + }, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "follow-up question with no files"}, + ] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + assert payload == {"prompt": "follow-up question with no files"} + assert "content" not in payload + + def test_unknown_non_text_block_type_passthrough(self, config, opted_in_kwargs): + """Opted in: unknown block types (e.g. input_audio) flow through.""" + content = [ + {"type": "text", "text": "transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": "U29tZUF1ZGlvQnl0ZXM=", "format": "wav"}, + }, + ] + messages = [{"role": "user", "content": content}] + payload = config.transform_request(messages=messages, **opted_in_kwargs) + assert payload["prompt"] == "transcribe this" + assert payload["content"] == content + assert payload["content"] is not content + + def test_forward_flag_as_string_true(self, config, transform_kwargs): + """The opt-in flag accepts config/env string values like "true".""" + content = [ + {"type": "text", "text": "hi"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + kwargs = { + **transform_kwargs, + "optional_params": {"forward_multimodal_content": "true"}, + } + payload = config.transform_request(messages=messages, **kwargs) + assert payload["content"] == content + assert payload["content"] is not content + + def test_forward_flag_false_explicit(self, config, transform_kwargs): + """Explicit falsy flag → no content field.""" + content = [ + {"type": "text", "text": "hi"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + kwargs = { + **transform_kwargs, + "optional_params": {"forward_multimodal_content": False}, + } + payload = config.transform_request(messages=messages, **kwargs) + assert "content" not in payload + + def test_forward_flag_via_litellm_params(self, config, transform_kwargs): + """The opt-in flag is also honored when set in litellm_params.""" + content = [ + {"type": "text", "text": "hi"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + messages = [{"role": "user", "content": content}] + kwargs = { + **transform_kwargs, + "litellm_params": {"forward_multimodal_content": True}, + } + payload = config.transform_request(messages=messages, **kwargs) + assert payload["content"] == content + assert payload["content"] is not content diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 474ffee3304..0c5e386c438 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -63,10 +63,14 @@ class MockAiohttpResponse: ): self.status = status self.headers = headers or {} + self.closed = False self.content = MockContent( content_chunks, exception_to_raise, exception_at_chunk ) + def close(self): + self.closed = True + async def __aexit__(self, exc_type, exc_val, exc_tb): pass @@ -613,3 +617,64 @@ async def test_handle_session_closed_during_request(): assert counts["requests"] == 2 # First request failed, second succeeded assert counts["sessions"] == 2 # Created 2 sessions for retry assert response.status_code == 200 + + +@pytest.mark.asyncio +async def test_response_stream_closes_response_on_error(): + """ + Regression test for #30192: when body iteration ends with an error, the + underlying aiohttp response must be closed so its connector slot is + released. Leaked slots exhaust the pool and every later request times + out (408) until the proxy restarts, even after the backend recovers. + """ + mock_response = MockAiohttpResponse( + content_chunks=[b"chunk1", b"chunk2"], + exception_to_raise=aiohttp.ServerTimeoutError("read timeout"), + exception_at_chunk=1, + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + with pytest.raises(httpx.TimeoutException): + async for _ in stream: + pass + + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_response_stream_closes_response_on_cancellation(): + """ + Regression test for #30192: a task cancelled mid-stream (e.g. the caller + disconnects during a traffic spike) must not leak its aiohttp connection. + """ + mock_response = MockAiohttpResponse( + content_chunks=[b"chunk1", b"chunk2", b"chunk3"], + exception_to_raise=asyncio.CancelledError(), + exception_at_chunk=1, + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + with pytest.raises(asyncio.CancelledError): + async for _ in stream: + pass + + assert mock_response.closed is True + + +@pytest.mark.asyncio +async def test_response_stream_closes_response_on_generator_exit(): + """ + Regression test for #30192: when the consumer stops iterating early and the + stream generator is closed (GeneratorExit), the underlying aiohttp response + must still be closed so its connector slot is released. + """ + mock_response = MockAiohttpResponse( + content_chunks=[b"chunk1", b"chunk2", b"chunk3"], + ) + + stream = AiohttpResponseStream(mock_response) # type: ignore + iterator = stream.__aiter__() + assert await iterator.__anext__() == b"chunk1" + await iterator.aclose() + + assert mock_response.closed is True diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 0221db1b23d..683ec158f44 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -554,3 +554,31 @@ def test_map_response_format_json_object_unchanged(): drop_params=False, ) assert result == {"response_format": {"type": "json_object"}} + + +def test_transform_request_routes_short_form_router_to_routers_path(): + """A bare router model name ending in -fast must be rewritten to the + ``accounts/fireworks/routers/`` path, not the default ``models/`` path.""" + config = FireworksAIConfig() + result = config.transform_request( + model="glm-5p1-fast", + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert result["model"] == "accounts/fireworks/routers/glm-5p1-fast" + + +def test_transform_request_routes_short_form_model_to_models_path(): + """A bare direct-model name must still be rewritten to the + ``accounts/fireworks/models/`` path.""" + config = FireworksAIConfig() + result = config.transform_request( + model="glm-5p2", + messages=[{"role": "user", "content": "Hi"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert result["model"] == "accounts/fireworks/models/glm-5p2" diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index bb9cda2584c..20a1bf85751 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -9,6 +9,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) +import litellm from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, @@ -571,3 +572,162 @@ class TestGPT5ReasoningEffortPreservation: assert optional_params.get("temperature") == 0.5 assert non_default_params.get("reasoning_effort") == "none" + + +class TestCacheControlPreservationForCustomEndpoint: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/30319 + + The AnthropicCacheControlHook injects cache_control when a user passes + cache_control_injection_points, but the base OpenAIGPTConfig used to strip + it unconditionally, making the feature a guaranteed no-op for the generic + openai provider pointed at a cache_control-aware endpoint (a LiteLLM proxy, + vLLM, an Anthropic-compatible gateway). cache_control must survive there + while still being stripped for real api.openai.com. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _cache_controlled_messages(): + return [ + { + "role": "system", + "content": "You are helpful.", + "cache_control": {"type": "ephemeral"}, + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + def _transform(self, custom_llm_provider, api_base, optional_params=None): + return self.config.transform_request( + model="claude-sonnet-4", + messages=self._cache_controlled_messages(), + optional_params=optional_params or {}, + litellm_params={ + "custom_llm_provider": custom_llm_provider, + "api_base": api_base, + }, + headers={}, + ) + + def test_predicate_openai_provider_custom_api_base_preserves(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint( + "openai", "http://localhost:4000/v1" + ) + is True + ) + + def test_predicate_real_openai_no_api_base_strips(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint("openai", None) + is False + ) + + def test_predicate_explicit_openai_host_strips(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint( + "openai", "https://api.openai.com/v1" + ) + is False + ) + + def test_predicate_non_openai_provider_strips(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint( + "deepseek", "https://api.deepseek.com" + ) + is False + ) + + def test_predicate_resolves_openai_base_url_env(self, monkeypatch): + monkeypatch.setenv("OPENAI_BASE_URL", "http://localhost:4000/v1") + assert ( + self.config._should_preserve_cache_control_for_endpoint("openai", None) + is True + ) + + def test_predicate_resolves_openai_api_base_env(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_BASE", "http://localhost:4000/v1") + assert ( + self.config._should_preserve_cache_control_for_endpoint("openai", None) + is True + ) + + def test_predicate_lookalike_host_is_not_treated_as_openai(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint( + "openai", "https://api.openai.com.evil.example/v1" + ) + is True + ) + + def test_predicate_openai_subdomain_strips(self): + assert ( + self.config._should_preserve_cache_control_for_endpoint( + "openai", "https://eu.api.openai.com/v1" + ) + is False + ) + + def test_transform_request_preserves_for_custom_api_base(self): + body = self._transform("openai", "http://localhost:4000/v1") + assert all("cache_control" in m for m in body["messages"]) + + def test_transform_request_strips_for_real_openai(self): + body = self._transform("openai", None) + assert all("cache_control" not in m for m in body["messages"]) + + def test_transform_request_strips_for_non_openai_provider(self): + body = self._transform("fireworks_ai", "https://api.fireworks.ai/inference/v1") + assert all("cache_control" not in m for m in body["messages"]) + + def test_transform_request_preserves_tool_cache_control(self): + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {}}, + "cache_control": {"type": "ephemeral"}, + } + ] + body = self._transform( + "openai", "http://localhost:4000/v1", optional_params={"tools": tools} + ) + assert "cache_control" in body["tools"][0] + + @pytest.mark.asyncio + async def test_async_transform_request_preserves_for_custom_api_base(self): + body = await self.config.async_transform_request( + model="claude-sonnet-4", + messages=self._cache_controlled_messages(), + optional_params={}, + litellm_params={ + "custom_llm_provider": "openai", + "api_base": "http://localhost:4000/v1", + }, + headers={}, + ) + assert all("cache_control" in m for m in body["messages"]) + + @pytest.mark.asyncio + async def test_async_transform_request_strips_for_real_openai(self): + body = await self.config.async_transform_request( + model="gpt-4o", + messages=self._cache_controlled_messages(), + optional_params={}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + assert all("cache_control" not in m for m in body["messages"]) diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index d408f55c004..e2d1ab72c5e 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -1,7 +1,7 @@ """ Test file for Perplexity cost calculator functionality. -Tests the cost calculation for Perplexity models including citation tokens, +Tests the cost calculation for Perplexity models including citation tokens, search queries, and reasoning tokens. """ @@ -21,7 +21,11 @@ from litellm.cost_calculator import completion_cost, cost_per_token from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) -from litellm.types.utils import Usage, PromptTokensDetailsWrapper +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + Usage, + PromptTokensDetailsWrapper, +) from litellm.utils import get_model_info @@ -135,13 +139,14 @@ class TestPerplexityCostCalculator: model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 20 tokens * $3e-6 = $0.00006 - # Total completion cost: $0.00046 + # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity + # convention codified in PR #18607. Non-reasoning portion = 50 - 20 = 30. + # Input: 100 tokens * $2e-6 = $0.0002 + # Output (text): 30 tokens * $8e-6 = $0.00024 + # Reasoning: 20 tokens * $3e-6 = $0.00006 + # Total completion cost = $0.0003 expected_prompt_cost = 100 * 2e-6 - expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -159,13 +164,10 @@ class TestPerplexityCostCalculator: model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 20 tokens * $3e-6 = $0.00006 - # Total completion cost: $0.00046 + # Same convention as the direct-attribute case above; reasoning is a subset of + # completion_tokens, so non-reasoning portion = 50 - 20 = 30. expected_prompt_cost = 100 * 2e-6 - expected_completion_cost = (50 * 8e-6) + (20 * 3e-6) + expected_completion_cost = ((50 - 20) * 8e-6) + (20 * 3e-6) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -187,16 +189,16 @@ class TestPerplexityCostCalculator: model="sonar-deep-research", usage=usage ) - # Expected costs: - # Input: 100 tokens * $2e-6 = $0.0002 - # Citation: 30 tokens * $2e-6 = $0.00006 - # Total prompt cost: $0.00026 - # Output: 50 tokens * $8e-6 = $0.0004 - # Reasoning: 15 tokens * $3e-6 = $0.000045 - # Search: 2 queries * ($0.005 / 1000) = $0.00001 - # Total completion cost: $0.000455 + # Expected costs (reasoning is a subset of completion_tokens): + # Input: 100 tokens * $2e-6 = $0.0002 + # Citation: 30 tokens * $2e-6 = $0.00006 + # Total prompt cost = $0.00026 + # Output (text): (50 - 15) tokens * $8e-6 = $0.00028 + # Reasoning: 15 tokens * $3e-6 = $0.000045 + # Search: 2 queries * ($0.005 / 1000) = $0.00001 + # Total completion cost = $0.000335 expected_prompt_cost = (100 * 2e-6) + (30 * 2e-6) - expected_completion_cost = (50 * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005) + expected_completion_cost = ((50 - 15) * 8e-6) + (15 * 3e-6) + (2 / 1000 * 0.005) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-6) @@ -306,11 +308,11 @@ class TestPerplexityCostCalculator: completion_response=response, custom_llm_provider="perplexity" ) - # Calculate expected total cost + # Calculate expected total cost (reasoning is a subset of completion_tokens) expected_prompt_cost = (100 * 2e-6) + (15 * 2e-6) # Input + citation expected_completion_cost = ( - (50 * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005) - ) # Output + reasoning + search + ((50 - 10) * 8e-6) + (10 * 3e-6) + (1 / 1000 * 0.005) + ) # Output (text) + reasoning + search expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) @@ -353,10 +355,13 @@ class TestPerplexityCostCalculator: model="sonar-deep-research", usage=usage ) - # Calculate expected costs + # Calculate expected costs. `completion_tokens` includes `reasoning_tokens`, + # so non-reasoning portion = 50 - reasoning_tokens. expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) expected_completion_cost = ( - (50 * 8e-6) + (reasoning_tokens * 3e-6) + (search_queries / 1000 * 0.005) + ((50 - reasoning_tokens) * 8e-6) + + (reasoning_tokens * 3e-6) + + (search_queries / 1000 * 0.005) ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) @@ -413,3 +418,36 @@ class TestPerplexityCostCalculator: assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-6) assert math.isclose(completion_cost, expected_completion, rel_tol=1e-6) + + def test_reasoning_tokens_not_double_billed(self): + """ + Regression: `completion_tokens` includes `reasoning_tokens` per the + OpenAI/Perplexity usage convention (codified for the central path in PR #18607). + When `output_cost_per_reasoning_token` is configured the manual fallback must + subtract reasoning from completion before applying the output rate so the + reasoning tokens are not billed at BOTH the output rate and the reasoning rate. + + Uses the exact usage shape produced by the live response fixture in + `tests/llm_translation/test_perplexity_reasoning.py`. + """ + usage = Usage( + prompt_tokens=9, + completion_tokens=20, + total_tokens=29, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=15 + ), + ) + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-deep-research", usage=usage + ) + + # sonar-deep-research rates: input 2e-6, output 8e-6, reasoning 3e-6. + # Non-reasoning portion of the 20 completion tokens = 20 - 15 = 5. + # Pre-fix this asserted 20 * 8e-6 + 15 * 3e-6 = 2.05e-4 (a 2.16x overcharge). + expected_prompt = 9 * 2e-6 + expected_completion = (20 - 15) * 8e-6 + 15 * 3e-6 + + assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9) + assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9) diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py index 1b03fd7df88..e59fbc9f272 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_integration.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_integration.py @@ -104,12 +104,10 @@ class TestPerplexityIntegration: ) citation_tokens = citation_chars // 4 - expected_prompt_cost = (100 * 2e-6) + ( - citation_tokens * 2e-6 - ) # Input + citation + expected_prompt_cost = (100 * 2e-6) + (citation_tokens * 2e-6) expected_completion_cost = ( - (50 * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005) - ) # Output + reasoning + search + ((50 - 10) * 8e-6) + (10 * 3e-6) + (2 / 1000 * 0.005) + ) expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) @@ -152,11 +150,10 @@ class TestPerplexityIntegration: usage_object=usage, ) - # Calculate expected costs - expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) # Input + citation + expected_prompt_cost = (200 * 2e-6) + (40 * 2e-6) expected_completion_cost = ( - (100 * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005) - ) # Output + reasoning + search + ((100 - 25) * 8e-6) + (25 * 3e-6) + (3 / 1000 * 0.005) + ) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-6) assert math.isclose(completion_cost_val, expected_completion_cost, rel_tol=1e-6) @@ -263,15 +260,14 @@ class TestPerplexityIntegration: custom_llm_provider="perplexity", ) - # Calculate expected cost - expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) # $0.11 + expected_prompt_cost = (50000 * 2e-6) + (5000 * 2e-6) expected_completion_cost = ( - (25000 * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005) - ) # $0.23 - expected_total = expected_prompt_cost + expected_completion_cost # $0.34 + ((25000 - 10000) * 8e-6) + (10000 * 3e-6) + (100 / 1000 * 0.005) + ) + expected_total = expected_prompt_cost + expected_completion_cost assert math.isclose(total_cost, expected_total, rel_tol=1e-6) - assert total_cost > 0.3 # Sanity check for high-volume scenario + assert total_cost > 0.25 def test_transformation_preserves_existing_usage_fields(self): """Test that transformation doesn't overwrite existing standard usage fields.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py new file mode 100644 index 00000000000..ac7082c2668 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_identity_env.py @@ -0,0 +1,73 @@ +"""Regression tests for the configurable MCP gateway identity. + +``LITELLM_MCP_SERVER_NAME`` and ``LITELLM_MCP_SERVER_DESCRIPTION`` are read from +the environment at import time in +``litellm.proxy._experimental.mcp_server.utils`` and must flow through to every +consumer, including the well-known registry entry built in +``mcp_management_endpoints``. The env values are reloaded into the modules and +restored afterwards so the override does not leak into other tests. +""" + +import contextlib +import importlib +import os + +import pytest + +pytest.importorskip("mcp") + +UTILS_MODULE = "litellm.proxy._experimental.mcp_server.utils" +MGMT_MODULE = "litellm.proxy.management_endpoints.mcp_management_endpoints" + + +@contextlib.contextmanager +def _env_and_reload(**env): + saved = {key: os.environ.get(key) for key in env} + + def _apply_env(values): + for key, value in values.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def _reload(): + utils = importlib.reload(importlib.import_module(UTILS_MODULE)) + mgmt = importlib.reload(importlib.import_module(MGMT_MODULE)) + return utils, mgmt + + try: + _apply_env(env) + yield _reload() + finally: + _apply_env(saved) + _reload() + + +def test_defaults_used_when_env_unset(): + with _env_and_reload( + LITELLM_MCP_SERVER_NAME=None, LITELLM_MCP_SERVER_DESCRIPTION=None + ) as (utils, _mgmt): + assert utils.LITELLM_MCP_SERVER_NAME == "litellm-mcp-server" + assert utils.LITELLM_MCP_SERVER_DESCRIPTION == "MCP Server for LiteLLM" + + +def test_env_overrides_server_identity(): + with _env_and_reload( + LITELLM_MCP_SERVER_NAME="acme-gateway", + LITELLM_MCP_SERVER_DESCRIPTION="Acme internal MCP gateway", + ) as (utils, _mgmt): + assert utils.LITELLM_MCP_SERVER_NAME == "acme-gateway" + assert utils.LITELLM_MCP_SERVER_DESCRIPTION == "Acme internal MCP gateway" + + +def test_env_override_propagates_to_registry_entry(): + with _env_and_reload( + LITELLM_MCP_SERVER_NAME="acme-gateway", + LITELLM_MCP_SERVER_DESCRIPTION="Acme internal MCP gateway", + ) as (_utils, mgmt): + entry = mgmt._build_builtin_registry_entry("http://localhost:4000") + + assert entry["name"] == "acme-gateway" + assert entry["title"] == "acme-gateway" + assert entry["description"] == "Acme internal MCP gateway" diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index f6189382d74..a4da4587b7f 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -86,3 +86,96 @@ class TestEventLoggingBatchEndpoint: assert response.status_code == 200 assert response.json() == {"status": "ok"} + + +class TestStripTotalTokens(unittest.TestCase): + """Cover ``_strip_total_tokens_from_anthropic_response``. + + The Anthropic /v1/messages spec does not define ``usage.total_tokens``. + LiteLLM injects it internally; the helper must remove it from the wire + response so the non-streaming path matches the streaming SSE shape and + direct Anthropic API responses. + """ + + def test_strips_total_tokens_when_present(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = { + "id": "msg_123", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + }, + } + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response["usage"] + assert response["usage"]["input_tokens"] == 100 + assert response["usage"]["output_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 0 + + def test_no_op_when_total_tokens_absent(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"usage": {"input_tokens": 100, "output_tokens": 50}} + _strip_total_tokens_from_anthropic_response(response) + assert response["usage"] == {"input_tokens": 100, "output_tokens": 50} + + def test_no_op_when_usage_missing(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + response = {"id": "msg_123"} + _strip_total_tokens_from_anthropic_response(response) + assert response == {"id": "msg_123"} + + def test_no_op_on_non_dict_response(self): + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # Streaming responses (StreamingResponse, async iterators) are not dicts. + # The helper must not raise or attempt to mutate them. + for value in (None, "stream", 42, [{"usage": {"total_tokens": 1}}]): + _strip_total_tokens_from_anthropic_response(value) # no raise + + def test_strips_total_tokens_on_pydantic_model_with_dict_usage(self): + """Greptile P1 on #30382: helper must not silently no-op when the + response is a Pydantic-shaped object whose `usage` attribute is a + plain dict (the common case for objects wrapping raw upstream JSON). + """ + from types import SimpleNamespace + + from litellm.proxy.anthropic_endpoints.endpoints import ( + _strip_total_tokens_from_anthropic_response, + ) + + # SimpleNamespace mimics the .usage attribute access pattern; the + # helper's contract: if .usage is dict-shaped, strip total_tokens. + response = SimpleNamespace( + usage={"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + ) + _strip_total_tokens_from_anthropic_response(response) + assert "total_tokens" not in response.usage + assert response.usage == {"input_tokens": 100, "output_tokens": 50} + + +class TestStripTotalTokensFeatureFlag(unittest.TestCase): + """The strip is gated behind `litellm.strip_anthropic_total_tokens`. + + Default off (backward compat). Greptile P1 on #30382 required a + user-controlled flag so existing clients reading the LiteLLM-shaped + `usage.total_tokens` continue to work after this PR lands. + """ + + def test_flag_defaults_off(self): + import litellm + + assert litellm.strip_anthropic_total_tokens is False diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 4bc007f6878..e652c109987 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -604,6 +604,59 @@ def test_get_model_from_request_handles_managed_id_decoder_failures(): ) +@pytest.mark.parametrize( + "route", + [ + "/realtime/client_secrets", + "/v1/realtime/client_secrets", + "/openai/v1/realtime/client_secrets", + "/realtime/calls", + "/v1/realtime/calls", + "/openai/v1/realtime/calls", + ], +) +def test_get_model_from_request_extracts_realtime_session_model(route): + """The effective realtime model lives in ``session.model`` (not the + top-level ``model``). It must be surfaced so can_key_call_model() can + validate the model a restricted key is actually requesting. + + Regression test for the model-access bypass on the GA Realtime WebRTC + HTTP routes (https://github.com/BerriAI/litellm/issues/29923). + """ + assert ( + get_model_from_request( + request_data={"session": {"type": "realtime", "model": "gpt-realtime"}}, + route=route, + ) + == "gpt-realtime" + ) + + +def test_get_model_from_request_realtime_includes_top_level_and_session_model(): + """When both top-level and session model are present, both are returned so + neither path can smuggle a disallowed model past the model-access check.""" + models = get_model_from_request( + request_data={ + "model": "gpt-4o-realtime-preview", + "session": {"type": "realtime", "model": "gpt-realtime"}, + }, + route="/v1/realtime/client_secrets", + ) + assert models == ["gpt-4o-realtime-preview", "gpt-realtime"] + + +def test_get_model_from_request_ignores_session_model_on_non_realtime_routes(): + """A nested ``session.model`` must not leak into model resolution for + unrelated routes.""" + assert ( + get_model_from_request( + request_data={"session": {"type": "realtime", "model": "gpt-realtime"}}, + route="/v1/chat/completions", + ) + is None + ) + + def test_abbreviate_api_key(): assert abbreviate_api_key("sk-test-1234") == "sk-...1234" diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 07b04961205..7a4597c4e02 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -460,6 +460,32 @@ def test_mcp_inference_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/client_secrets", + "/v1/realtime/client_secrets", + "/openai/v1/realtime/client_secrets", + "/realtime/calls", + "/v1/realtime/calls", + "/openai/v1/realtime/calls", + "/realtime/transcription_sessions", + "/v1/realtime/transcription_sessions", + "/openai/v1/realtime/transcription_sessions", + ], +) +def test_realtime_webrtc_http_routes_classified_as_llm_api(route): + """GA Realtime WebRTC HTTP routes must be classified as LLM API routes so + non-admin virtual keys can call them instead of hitting the admin-only + 401 branch in non_proxy_admin_allowed_routes_check. + + Regression test for https://github.com/BerriAI/litellm/issues/29923 + """ + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 79e6494eab0..04c93f48ca9 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -239,6 +239,54 @@ async def test_update_daily_spend_sorting(): mock_table.upsert.assert_has_calls(upsert_calls) +@pytest.mark.asyncio +async def test_update_daily_spend_drains_all_batches_over_batch_size(): + """ + Regression for #30281: >BATCH_SIZE (100) unique entities in one flush must all + be written and the in-memory dict fully drained within a single call. Pre-fix, + only the first 100 sorted items were upserted then the method returned, silently + dropping the remaining entities. + """ + mock_prisma_client = MagicMock() + mock_batcher = MagicMock() + mock_table = MagicMock() + mock_prisma_client.db.batch_.return_value.__aenter__.return_value = mock_batcher + mock_batcher.litellm_dailyuserspend = mock_table + + num_entities = 250 + daily_spend_transactions = { + f"test_key_{i}": { + "user_id": f"user{i:04d}", + "date": "2024-01-01", + "api_key": "test-api-key", + "model": "gpt-4", + "custom_llm_provider": "openai", + "prompt_tokens": 10, + "completion_tokens": 20, + "spend": 0.1, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + for i in range(num_entities) + } + + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=1, + prisma_client=mock_prisma_client, + proxy_logging_obj=MagicMock(), + daily_spend_transactions=daily_spend_transactions, + entity_type="user", + entity_id_field="user_id", + table_name="litellm_dailyuserspend", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name_endpoint", + ) + + assert mock_table.upsert.call_count == num_entities + assert mock_prisma_client.db.batch_.call_count == 3 + assert daily_spend_transactions == {} + + @pytest.mark.asyncio async def test_update_daily_spend_tag_with_request_id(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index bf507cb065d..2d2c18bb46e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -1,5 +1,6 @@ import os import sys +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest @@ -12,10 +13,13 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( _adjust_dates_for_timezone, _build_aggregated_sql_query, _is_user_agent_tag, + _record_to_spend_metrics, get_api_key_metadata, get_daily_activity, get_daily_activity_aggregated, + update_metrics, ) +from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics @pytest.mark.asyncio @@ -810,3 +814,45 @@ async def test_get_daily_activity_aggregated_empty_result_set(): assert result.metadata.total_failed_requests == 0 assert result.metadata.total_cache_read_input_tokens == 0 assert result.metadata.total_cache_creation_input_tokens == 0 + + +def _no_spend_record(): + """A rollup row for a key with no spend, where SUM() returns NULL (None).""" + return SimpleNamespace( + spend=None, + prompt_tokens=None, + completion_tokens=None, + cache_read_input_tokens=None, + cache_creation_input_tokens=None, + api_requests=None, + successful_requests=None, + failed_requests=None, + ) + + +def test_record_to_spend_metrics_handles_none_values(): + """Keys with no spend produce NULL aggregates; treat them as zero, not a crash.""" + metrics = _record_to_spend_metrics(_no_spend_record()) + assert metrics.spend == 0 + assert metrics.prompt_tokens == 0 + assert metrics.completion_tokens == 0 + assert metrics.total_tokens == 0 + assert metrics.api_requests == 0 + assert metrics.successful_requests == 0 + assert metrics.failed_requests == 0 + assert metrics.cache_read_input_tokens == 0 + assert metrics.cache_creation_input_tokens == 0 + + +def test_update_metrics_handles_none_values(): + """update_metrics should coalesce NULL aggregates instead of raising TypeError.""" + metrics = update_metrics(SpendMetrics(), _no_spend_record()) + assert metrics.spend == 0 + assert metrics.prompt_tokens == 0 + assert metrics.completion_tokens == 0 + assert metrics.total_tokens == 0 + assert metrics.api_requests == 0 + assert metrics.successful_requests == 0 + assert metrics.failed_requests == 0 + assert metrics.cache_read_input_tokens == 0 + assert metrics.cache_creation_input_tokens == 0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_utils.py b/tests/test_litellm/proxy/management_endpoints/test_common_utils.py index d53ea6fa34d..7a8d04507dc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_utils.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_utils.py @@ -157,6 +157,32 @@ class TestUpdateMetadataFieldsEmptyCollections: assert "guardrails" not in updated_kv assert updated_kv["metadata"]["guardrails"] == ["my-guardrail"] + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_false_boolean_does_not_trigger_premium_check(self, mock_premium_check): + """ + Regression #30285: /team/update sends disable_global_guardrails=False + (the UI's unchanged default). A falsy boolean must not trigger the + premium check, so non-premium users are not wrongly 403'd. + """ + updated_kv = {"team_id": "test-team", "disable_global_guardrails": False} + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_not_called() + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_false_boolean_still_updates_metadata(self, mock_premium_check): + """A falsy boolean must still be moved into metadata so it persists.""" + updated_kv = {"team_id": "test-team", "disable_global_guardrails": False} + _update_metadata_fields(updated_kv=updated_kv) + assert "disable_global_guardrails" not in updated_kv + assert updated_kv["metadata"]["disable_global_guardrails"] is False + + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") + def test_true_boolean_triggers_premium_check(self, mock_premium_check): + """Control: enabling the premium feature (True) still requires a license.""" + updated_kv = {"team_id": "test-team", "disable_global_guardrails": True} + _update_metadata_fields(updated_kv=updated_kv) + mock_premium_check.assert_called() + @patch("litellm.proxy.management_endpoints.common_utils._premium_user_check") def test_ui_typical_payload_does_not_trigger_premium_check( self, mock_premium_check diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 9c5206722aa..cc8b4c7f5cc 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1547,6 +1547,51 @@ async def test_prepare_key_update_data_budget_limits_serializes_windows(): assert windows[0]["reset_at"] is not None +@pytest.mark.asyncio +async def test_prepare_key_update_data_disable_global_guardrails_false_no_premium( + monkeypatch, +): + """ + Regression #30285: editing a key via the UI sends disable_global_guardrails=False + (unchanged default). A non-premium user must NOT get a 403, and False must persist. + """ + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + data = UpdateKeyRequest(key="sk-1", disable_global_guardrails=False) + existing_key = LiteLLM_VerificationToken(token="hashed") + + result = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + assert result["metadata"]["disable_global_guardrails"] is False + + +@pytest.mark.asyncio +async def test_prepare_key_update_data_disable_global_guardrails_true_requires_premium( + monkeypatch, +): + """Control: enabling the premium feature (True) without a license still 403s.""" + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + data = UpdateKeyRequest(key="sk-1", disable_global_guardrails=True) + existing_key = LiteLLM_VerificationToken(token="hashed") + + with pytest.raises(HTTPException) as exc_info: + await prepare_key_update_data(data=data, existing_key_row=existing_key) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_prepare_key_update_data_disable_global_guardrails_true_premium_persists( + monkeypatch, +): + """A premium user enabling the feature (True) succeeds and the value persists.""" + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + data = UpdateKeyRequest(key="sk-1", disable_global_guardrails=True) + existing_key = LiteLLM_VerificationToken(token="hashed") + + result = await prepare_key_update_data(data=data, existing_key_row=existing_key) + + assert result["metadata"]["disable_global_guardrails"] is True + + @pytest.mark.asyncio async def test_validate_team_id_used_in_service_account_request_requires_team_id(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index b81807ee19e..a649bc7225e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -9313,3 +9313,34 @@ async def test_clear_team_member_budget_fields_no_budget_row_skips_update(): mock_update_budget.assert_not_awaited() assert "team_member_budget" not in result assert "team_member_rpm_limit" not in result + + +@pytest.mark.asyncio +async def test_team_info_forwards_key_limit_to_get_data(): + """/team/info must thread its ``key_limit`` query param into the key + lookup so the database caps how many keys are returned for the team. + """ + from fastapi import Request + + from litellm.proxy.management_endpoints import team_endpoints + + mock_prisma = MagicMock() + mock_prisma.db.litellm_teamtable.find_unique = AsyncMock( + return_value=LiteLLM_TeamTable(team_id="team-1") + ) + mock_prisma.get_data = AsyncMock(return_value=[]) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch.object( + team_endpoints, "get_all_team_memberships", AsyncMock(return_value=[]) + ), + ): + await team_endpoints.team_info( + http_request=MagicMock(spec=Request), + team_id="team-1", + key_limit=7, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert mock_prisma.get_data.await_args.kwargs["limit"] == 7 diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index 0f87fcda588..40d590132aa 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -696,6 +696,7 @@ async def test_v1_models_translates_team_model_with_metadata(monkeypatch): router.get_fully_blocked_model_names.return_value = set() router.model_list = [team_dep] router.get_model_list.return_value = [team_dep] + router.get_model_group_info.return_value = None monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "user_model", None) @@ -742,6 +743,7 @@ async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch router.get_model_list.return_value = [team_dep] # Fallbacks are keyed on the internal routing name, as the router stores them. router.fallbacks = [{"model_name_teamX_uuid9": ["gpt-4o-backup"]}] + router.get_model_group_info.return_value = None monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "user_model", None) @@ -799,6 +801,7 @@ async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch {"model_name_teamX_uuid9": ["teamX-backup"]}, {"model_name_teamY_uuidZ": ["teamY-backup"]}, ] + router.get_model_group_info.return_value = None monkeypatch.setattr(ps, "llm_router", router) monkeypatch.setattr(ps, "user_model", None) diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 65853df392f..e0e51e7b966 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -901,6 +901,109 @@ def test_session_type_coerced_for_unknown_value(): assert session_type == "realtime" +@pytest.mark.asyncio +async def test_client_secrets_realtime_default_model_blocked_when_not_in_key_scope( + proxy_app, +): + """ + Regression: omitting both model and session.model must NOT bypass the authz + check. The endpoint defaults to gpt-4o-realtime-preview; a key that cannot + reach that model must receive 403. + """ + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["some-other-model"], + ) + try: + client = TestClient(proxy_app, raise_server_exceptions=False) + with ( + patch("litellm.proxy.proxy_server.route_request") as mock_route_request, + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={}, + ) + + assert response.status_code == 403 + assert "gpt-4o-realtime-preview" in response.text + mock_route_request.assert_not_called() + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_client_secrets_realtime_explicit_model_blocked_when_not_in_key_scope( + proxy_app, +): + """An explicit model not in the key's allowed list must also be rejected.""" + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-4o-realtime-preview"], + ) + try: + client = TestClient(proxy_app, raise_server_exceptions=False) + with ( + patch("litellm.proxy.proxy_server.route_request") as mock_route_request, + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging, + ): + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-4o-realtime-mini"}, + ) + + assert response.status_code == 403 + assert "gpt-4o-realtime-mini" in response.text + mock_route_request.assert_not_called() + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_client_secrets_realtime_default_model_allowed_when_in_key_scope( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, +): + """Omitting model should succeed when the default (gpt-4o-realtime-preview) is in scope.""" + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-4o-realtime-preview"], + ) + try: + client = TestClient(proxy_app) + with ( + patch( + "litellm.proxy.proxy_server.route_request", + side_effect=mock_route_request_client_secrets, + ), + patch( + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging, + ): + mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={}, + ) + + assert response.status_code == 200 + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + @pytest.mark.asyncio async def test_transcription_sessions_returns_upstream_error_verbatim( proxy_app, diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index d6b4c48a80c..3e86f0e8f3c 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -427,6 +427,134 @@ class TestPostCallFailureHookLiftsFirstApiCallStartTime: assert "litellm_logging_obj" not in request_data +from litellm.proxy.utils import create_model_info_response +from litellm.types.router import ModelGroupInfo + + +def _router_returning(model_group_info): + router = MagicMock() + router.get_model_group_info = MagicMock(return_value=model_group_info) + return router + + +def test_create_model_info_response_includes_max_tokens_when_available(): + router = _router_returning( + ModelGroupInfo( + model_group="qwen-vllm", + providers=["hosted_vllm"], + max_input_tokens=32768, + max_output_tokens=8192, + ) + ) + + response = create_model_info_response( + model_id="qwen-vllm", provider="openai", llm_router=router + ) + + router.get_model_group_info.assert_called_once_with("qwen-vllm") + assert response["id"] == "qwen-vllm" + assert response["object"] == "model" + assert response["max_input_tokens"] == 32768 + assert response["max_output_tokens"] == 8192 + + +def test_create_model_info_response_emits_integer_token_counts(): + # ModelGroupInfo types the limits as float; OpenAI-compatible clients expect + # plain integers, so the response must not leak 128000.0. + router = _router_returning( + ModelGroupInfo( + model_group="gpt-4o", + providers=["openai"], + max_input_tokens=128000.0, + max_output_tokens=16384.0, + ) + ) + + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=router + ) + + assert response["max_input_tokens"] == 128000 + assert isinstance(response["max_input_tokens"], int) + assert response["max_output_tokens"] == 16384 + assert isinstance(response["max_output_tokens"], int) + + +def test_create_model_info_response_omits_unknown_individual_limit(): + router = _router_returning( + ModelGroupInfo( + model_group="partial", + providers=["openai"], + max_input_tokens=4096, + max_output_tokens=None, + ) + ) + + response = create_model_info_response( + model_id="partial", provider="openai", llm_router=router + ) + + assert response["max_input_tokens"] == 4096 + assert "max_output_tokens" not in response + + +def test_create_model_info_response_omits_limits_when_both_none(): + router = _router_returning( + ModelGroupInfo( + model_group="no-limits", + providers=["openai"], + max_input_tokens=None, + max_output_tokens=None, + ) + ) + + response = create_model_info_response( + model_id="no-limits", provider="openai", llm_router=router + ) + + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_omits_limits_when_group_unknown(): + # Wildcard routes / access groups have no ModelGroupInfo. + router = _router_returning(None) + + response = create_model_info_response( + model_id="openai/*", provider="openai", llm_router=router + ) + + assert response["id"] == "openai/*" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_degrades_when_group_info_raises(): + # A malformed deployment must not turn the listing into a 500; the entry + # falls back to the base fields without limits. + router = MagicMock() + router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) + + response = create_model_info_response( + model_id="broken", provider="openai", llm_router=router + ) + + assert response["id"] == "broken" + assert "max_input_tokens" not in response + assert "max_output_tokens" not in response + + +def test_create_model_info_response_no_router_keeps_base_fields(): + response = create_model_info_response( + model_id="some-model", provider="openai", llm_router=None + ) + + assert response == { + "id": "some-model", + "object": "model", + "created": response["created"], + "owned_by": "openai", + } class TestPostCallFailureHookLLMExceptionAlerting: """The llm_exceptions alert is for infra / LLM-API failures, not user errors (https://github.com/BerriAI/litellm/issues/3395). Already-normalized diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py index 08d1ef619a7..d517c1c346f 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -514,3 +514,26 @@ async def test_get_data_combined_view_returns_view_for_deprecated_key( assert isinstance(response, LiteLLM_VerificationTokenView) assert response.token == active_hash + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", [5, None]) +async def test_get_data_team_keys_forward_limit_as_take( + prisma_client: PrismaClient, limit: Any +) -> None: + """The /team/info ``key_limit`` must reach Prisma as ``take`` so the + database caps how many of a team's keys come back. + ``limit=None`` leaves ``take`` unset so every key is returned. + """ + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + await prisma_client.get_data( + team_id="team-1", + table_name="key", + query_type="find_all", + limit=limit, + ) + assert prisma_client.db.litellm_verificationtoken.find_many.await_args.kwargs == { + "take": limit, + "where": {"team_id": "team-1"}, + "include": {"litellm_budget_table": True}, + } diff --git a/tests/test_litellm/test_command_r7b_pricing.py b/tests/test_litellm/test_command_r7b_pricing.py new file mode 100644 index 00000000000..b952c365910 --- /dev/null +++ b/tests/test_litellm/test_command_r7b_pricing.py @@ -0,0 +1,83 @@ +""" +Regression test: ``command-r7b-12-2024`` had its input/output per-token +costs transposed in the model-cost maps (input=1.5e-07 / output=3.75e-08), +even though Cohere publishes $0.0375/1M input and $0.15/1M output, i.e. +output is ~4x input like every other ``command-r`` entry. + +These tests pin the corrected values in both the primary price map and the +``litellm/`` backup, and verify ``get_model_info`` surfaces them, so the +swap cannot silently regress. +""" + +import json +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm + +MODEL = "command-r7b-12-2024" +EXPECTED_INPUT_COST = 3.75e-08 +EXPECTED_OUTPUT_COST = 1.5e-07 + + +def _load_json(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _backup_path() -> str: + return os.path.join( + os.path.dirname(litellm.__file__), + "model_prices_and_context_window_backup.json", + ) + + +def _main_path() -> str: + # This test lives at ``tests/test_litellm/``; the primary price map sits at + # the repo root, two directories up. Resolve it relative to this file so the + # test works regardless of where ``litellm`` itself is installed (e.g. a pip + # install into site-packages). + return os.path.join( + os.path.dirname(__file__), + "..", + "..", + "model_prices_and_context_window.json", + ) + + +class TestCommandR7bPricingData: + """The JSON price maps must carry Cohere's published costs, with output + more expensive than input.""" + + def test_backup_costs_not_swapped(self): + entry = _load_json(_backup_path())[MODEL] + assert entry["input_cost_per_token"] == EXPECTED_INPUT_COST + assert entry["output_cost_per_token"] == EXPECTED_OUTPUT_COST + assert entry["output_cost_per_token"] > entry["input_cost_per_token"] + + def test_main_costs_not_swapped(self): + entry = _load_json(_main_path())[MODEL] + assert entry["input_cost_per_token"] == EXPECTED_INPUT_COST + assert entry["output_cost_per_token"] == EXPECTED_OUTPUT_COST + assert entry["output_cost_per_token"] > entry["input_cost_per_token"] + + +class TestCommandR7bPricingModelInfo: + """``get_model_info`` must report the corrected, un-swapped costs.""" + + def test_get_model_info_costs(self): + # Patch litellm.model_cost with the local backup so the test is not + # dependent on the remote fetch hitting a not-yet-merged main branch. + original = litellm.model_cost + try: + litellm.model_cost = _load_json(_backup_path()) + info = litellm.get_model_info(MODEL) + assert info["input_cost_per_token"] == EXPECTED_INPUT_COST + assert info["output_cost_per_token"] == EXPECTED_OUTPUT_COST + assert info["output_cost_per_token"] > info["input_cost_per_token"] + finally: + litellm.model_cost = original diff --git a/tests/test_litellm/test_router_exception_redaction.py b/tests/test_litellm/test_router_exception_redaction.py new file mode 100644 index 00000000000..e40bf661da4 --- /dev/null +++ b/tests/test_litellm/test_router_exception_redaction.py @@ -0,0 +1,311 @@ +""" +Tests for `litellm.expose_router_debug_in_errors`. + +The Router historically appended internal config names (model_group, +fallback_model_group, fallback failure detail, deployment timeouts, +context_window_fallbacks dict, etc.) onto the message of the exception +it re-raises. That message is then surfaced to clients by +ProxyException, leaking the proxy's internal wiring. + +The flag defaults to True to preserve historical behavior (no +breaking change for existing deployments). Set it to False to redact +those strings from the raised exception's message. + +These tests verify that with the flag ON (default) the historical +leak strings appear in the raised exception's message, and with the +flag OFF the proxy's internal wiring is redacted. + +Five leak sites are gated in `litellm/router.py`: + +1. Deployment timeout debug after `litellm.Timeout` +2. ContextWindowExceededError fallback hint +3. ContentPolicyViolationError fallback hint +4. "No fallback model group found for..." when fallbacks dict misses +5. "Received Model Group=...\\nAvailable Model Group Fallbacks=..." + (always fires on terminal raise from the fallback orchestrator) + +Site 5 is the broadest — it fires for every failing call that goes +through the fallback orchestrator with any non-context-window / +non-content-policy error, regardless of whether `fallbacks` is set. +""" + +from __future__ import annotations + +import pytest + +import litellm +from litellm import Router + +_RECEIVED_MODEL_GROUP_PHRASE = "Received Model Group=" +_AVAILABLE_FALLBACKS_PHRASE = "Available Model Group Fallbacks=" +_CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" +_INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" + + +def _router_with_rate_limit_failure() -> Router: + return Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.RateLimitError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + num_retries=0, + ) + + +def _router_with_context_window_failure() -> Router: + return Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.ContextWindowExceededError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + num_retries=0, + ) + + +@pytest.fixture(autouse=True) +def _reset_expose_flag(): + """Each test starts with the flag in its default (on) state.""" + original = litellm.expose_router_debug_in_errors + litellm.expose_router_debug_in_errors = True + try: + yield + finally: + litellm.expose_router_debug_in_errors = original + + +def test_flag_defaults_on(): + assert litellm.expose_router_debug_in_errors is True + + +# --- Site 5: "Received Model Group=..." on terminal raise -------------------- + + +@pytest.mark.asyncio +async def test_flag_off_does_not_leak_received_model_group(): + litellm.expose_router_debug_in_errors = False + router = _router_with_rate_limit_failure() + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert _RECEIVED_MODEL_GROUP_PHRASE not in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg + + +@pytest.mark.asyncio +async def test_default_leaks_received_model_group(): + router = _router_with_rate_limit_failure() + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert _RECEIVED_MODEL_GROUP_PHRASE in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + + +# --- Site 2: ContextWindowExceededError fallback hint ------------------------ + + +@pytest.mark.asyncio +async def test_flag_off_does_not_leak_context_window_fallback_hint(): + litellm.expose_router_debug_in_errors = False + router = _router_with_context_window_failure() + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert _CONTEXT_WINDOW_HINT_PHRASE not in msg, msg + assert _RECEIVED_MODEL_GROUP_PHRASE not in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg + + +@pytest.mark.asyncio +async def test_default_leaks_context_window_fallback_hint(): + router = _router_with_context_window_failure() + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert _CONTEXT_WINDOW_HINT_PHRASE in msg, msg + # Site 5 also fires for ContextWindow errors that exit the + # orchestrator without fallback resolution, so the model_group + # name leaks under the default behavior. + assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + + +# --- Site 4: "No fallback model group found..." when fallbacks miss --------- + + +@pytest.mark.asyncio +async def test_flag_off_does_not_leak_when_no_fallback_group_found(): + litellm.expose_router_debug_in_errors = False + router = Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.RateLimitError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + # Fallbacks defined for a different model_group, so resolution + # ends with fallback_model_group=None and hits site 4. + fallbacks=[{"some-other-group": ["some-other-target"]}], + num_retries=0, + ) + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert "No fallback model group found" not in msg, msg + assert "some-other-group" not in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg + + +@pytest.mark.asyncio +async def test_default_leaks_when_no_fallback_group_found(): + router = Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": { + "model": "gpt-4o", + "api_key": "key", + "mock_response": "litellm.RateLimitError", + }, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + fallbacks=[{"some-other-group": ["some-other-target"]}], + num_retries=0, + ) + with pytest.raises(litellm.RateLimitError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + ) + msg = excinfo.value.message + assert "No fallback model group found" in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + + +# --- Site 1: Deployment timeout debug on litellm.Timeout -------------------- + + +def _router_with_plain_deployment() -> Router: + """Plain deployment, no preconfigured mock_response — caller supplies via kwargs. + + Exception instances cannot live in `model_list[*].litellm_params` because + `Router.__init__` deep-copies model_list and several LiteLLM exceptions + (Timeout, ContentPolicyViolationError) require positional args that + `__reduce__` cannot reconstruct. Passing the trigger at call-site bypasses + the deepcopy entirely. + """ + return Router( + model_list=[ + { + "model_name": _INTERNAL_MODEL_GROUP_NAME, + "litellm_params": {"model": "gpt-4o", "api_key": "key"}, + "model_info": {"id": "secret-deployment-id"}, + }, + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +async def test_flag_off_does_not_leak_deployment_timeout_debug(): + litellm.expose_router_debug_in_errors = False + router = _router_with_plain_deployment() + with pytest.raises(litellm.Timeout) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_timeout=True, + timeout=0.001, + ) + msg = excinfo.value.message + assert "Deployment Info: request_timeout:" not in msg, msg + + +@pytest.mark.asyncio +async def test_default_leaks_deployment_timeout_debug(): + router = _router_with_plain_deployment() + with pytest.raises(litellm.Timeout) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_timeout=True, + timeout=0.001, + ) + msg = excinfo.value.message + assert "Deployment Info: request_timeout:" in msg, msg + + +# --- Site 3: ContentPolicyViolationError fallback hint (no fallback set) ---- + + +def _content_policy_error() -> litellm.ContentPolicyViolationError: + return litellm.ContentPolicyViolationError( + message="mocked policy violation", + model="gpt-4o", + llm_provider="openai", + ) + + +@pytest.mark.asyncio +async def test_flag_off_does_not_leak_content_policy_fallback_hint(): + litellm.expose_router_debug_in_errors = False + router = _router_with_plain_deployment() + with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_response=_content_policy_error(), + ) + msg = excinfo.value.message + assert "content_policy_fallback=" not in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg + + +@pytest.mark.asyncio +async def test_default_leaks_content_policy_fallback_hint(): + router = _router_with_plain_deployment() + with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_response=_content_policy_error(), + ) + msg = excinfo.value.message + assert "content_policy_fallback=" in msg, msg + assert _INTERNAL_MODEL_GROUP_NAME in msg, msg diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 400c693abf1..44e0b55ee3b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4244,6 +4244,254 @@ def test_deepseek_v4_models_in_backup_cost_map(): assert info["cache_read_input_token_cost"] == expected_cache +_FIREWORKS_MODELS = [ + ( + "accounts/fireworks/models/glm-5p2", + 1.4e-06, + 4.4e-06, + 2.6e-07, + 1048576, + 131072, + False, + True, + ), + ( + "accounts/fireworks/models/glm-5p1", + 1.4e-06, + 4.4e-06, + 2.6e-07, + 202800, + 131072, + False, + True, + ), + ( + "accounts/fireworks/routers/glm-5p1-fast", + 2.8e-06, + 8.8e-06, + 5.2e-07, + 202800, + 131072, + False, + True, + ), + ( + "accounts/fireworks/models/qwen3p7-plus", + 4e-07, + 1.6e-06, + 8e-08, + 262144, + 65536, + True, + True, + ), + ( + "accounts/fireworks/models/minimax-m3", + 3e-07, + 1.2e-06, + 6e-08, + 512000, + 512000, + False, + True, + ), + ( + "accounts/fireworks/models/minimax-m2p7", + 3e-07, + 1.2e-06, + 6e-08, + 196608, + 196608, + False, + True, + ), + ( + "accounts/fireworks/models/kimi-k2p7-code", + 9.5e-07, + 4e-06, + 1.9e-07, + 262144, + 262144, + True, + True, + ), + ( + "accounts/fireworks/routers/kimi-k2p7-code-fast", + 1.9e-06, + 8e-06, + 3.8e-07, + 262144, + 262144, + True, + True, + ), + ( + "accounts/fireworks/models/kimi-k2p6", + 9.5e-07, + 4e-06, + 1.6e-07, + 262144, + 262144, + True, + True, + ), + ( + "accounts/fireworks/routers/kimi-k2p6-fast", + 2e-06, + 8e-06, + 3e-07, + 262144, + 262144, + True, + True, + ), + ( + "accounts/fireworks/models/gpt-oss-120b", + 1.5e-07, + 6e-07, + 1.5e-08, + 131072, + 32768, + False, + True, + ), + ( + "accounts/fireworks/models/gpt-oss-20b", + 7e-08, + 3e-07, + 3.5e-08, + 131072, + 32768, + False, + True, + ), + ( + "accounts/fireworks/models/deepseek-v4-pro", + 1.74e-06, + 3.48e-06, + 1.45e-07, + 1048576, + 384000, + False, + True, + ), + ( + "accounts/fireworks/models/deepseek-v4-flash", + 1.4e-07, + 2.8e-07, + 2.8e-08, + 1048576, + 384000, + False, + True, + ), +] + +_FIREWORKS_SHORT_FORMS = [ + "glm-5p2", + "glm-5p1", + "qwen3p7-plus", + "minimax-m3", + "minimax-m2p7", + "kimi-k2p7-code", + "kimi-k2p6", + "gpt-oss-120b", + "gpt-oss-20b", + "deepseek-v4-pro", + "deepseek-v4-flash", +] + +_FIREWORKS_ROUTER_SHORT_FORMS = [ + "glm-5p1-fast", + "kimi-k2p6-fast", + "kimi-k2p7-code-fast", +] + + +def _assert_fireworks_entry( + model_cost, + model_path, + expected_input, + expected_output, + expected_cache, + expected_max_input, + expected_max_output, + expected_vision, + expected_reasoning, +): + info = model_cost.get(f"fireworks_ai/{model_path}") + assert info is not None, f"fireworks_ai/{model_path} missing from model cost map" + assert info["litellm_provider"] == "fireworks_ai" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] == expected_input + assert info["output_cost_per_token"] == expected_output + assert info["cache_read_input_token_cost"] == expected_cache + assert info["max_input_tokens"] == expected_max_input + assert info["max_output_tokens"] == expected_max_output + assert info["max_tokens"] == expected_max_output + assert info["supports_function_calling"] is True + assert info["supports_tool_choice"] is True + assert info["supports_reasoning"] is expected_reasoning + assert info["supports_response_schema"] is True + assert info["supports_vision"] is expected_vision + + +def test_fireworks_models_in_cost_map(): + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + for entry in _FIREWORKS_MODELS: + _assert_fireworks_entry(model_cost, *entry) + + for short in _FIREWORKS_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/models/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get( + long_key + ), f"short-form {short_key} does not match long-form {long_key}" + + for short in _FIREWORKS_ROUTER_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get( + long_key + ), f"short-form {short_key} does not match long-form {long_key}" + + +def test_fireworks_models_in_backup_cost_map(): + import json + from pathlib import Path + + json_path = ( + Path(__file__).parents[2] + / "litellm" + / "model_prices_and_context_window_backup.json" + ) + with open(json_path) as f: + model_cost = json.load(f) + + for entry in _FIREWORKS_MODELS: + _assert_fireworks_entry(model_cost, *entry) + + for short in _FIREWORKS_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/models/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get( + long_key + ), f"short-form {short_key} does not match long-form {long_key}" + + for short in _FIREWORKS_ROUTER_SHORT_FORMS: + long_key = f"fireworks_ai/accounts/fireworks/routers/{short}" + short_key = f"fireworks_ai/{short}" + assert model_cost.get(short_key) == model_cost.get( + long_key + ), f"short-form {short_key} does not match long-form {long_key}" + + class TestBedrockBaseModelLabelKeepsTools: """Regression for #29618: a Bedrock deployment whose ``base_model`` is a friendly label must not silently drop ``tools``/``tool_choice`` under ``drop_params``.""" diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index a4074ccdaaa..fde71ae65f2 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -321,6 +321,29 @@ class TestNativeFinishReason: assert choice.provider_specific_fields["native_finish_reason"] == "MAX_TOKENS" +def test_parallel_request_limiter_internal_fields_in_all_litellm_params(): + """ + Regression test: internal fields written by parallel_request_limiter_v3 must + be in all_litellm_params so they are stripped before forwarding to upstream + providers. If missing, they are sent as extra body parameters and providers + like OpenAI reject the request with a 400 invalid_request_error. + """ + from litellm.types.utils import all_litellm_params + + internal_fields = [ + "_litellm_rate_limit_descriptors", + "_litellm_tpm_reserved_tokens", + "_litellm_tpm_reserved_model", + "_litellm_tpm_reserved_scopes", + "_litellm_tpm_reservation_released", + ] + for field in internal_fields: + assert field in all_litellm_params, ( + f"{field!r} is not in all_litellm_params. " + "It will be forwarded to upstream providers and cause 400 errors." + ) + + def test_delta_maps_reasoning_to_reasoning_content(): """ Test that Delta maps 'reasoning' field to 'reasoning_content'. diff --git a/tests/test_litellm/types/test_uk_pii_entities.py b/tests/test_litellm/types/test_uk_pii_entities.py new file mode 100644 index 00000000000..378970adf9b --- /dev/null +++ b/tests/test_litellm/types/test_uk_pii_entities.py @@ -0,0 +1,54 @@ +""" +Test UK PII entity types in guardrails module +""" + +from litellm.types.guardrails import PiiEntityType, PiiEntityCategory, PII_ENTITY_CATEGORIES_MAP + + +class TestUKPiiEntities: + """Test UK PII entity type definitions and mappings""" + + def test_uk_pii_entity_types_exist(self): + """Test all UK PII entity types are defined""" + assert hasattr(PiiEntityType, "UK_NHS") + assert hasattr(PiiEntityType, "UK_NINO") + assert hasattr(PiiEntityType, "UK_PASSPORT") + assert hasattr(PiiEntityType, "UK_POSTCODE") + assert hasattr(PiiEntityType, "UK_VEHICLE_REGISTRATION") + + def test_uk_pii_entity_values(self): + """Test UK PII entity types have correct string values""" + assert PiiEntityType.UK_NHS == "UK_NHS" + assert PiiEntityType.UK_NINO == "UK_NINO" + assert PiiEntityType.UK_PASSPORT == "UK_PASSPORT" + assert PiiEntityType.UK_POSTCODE == "UK_POSTCODE" + assert PiiEntityType.UK_VEHICLE_REGISTRATION == "UK_VEHICLE_REGISTRATION" + + def test_uk_category_exists(self): + """Test UK category exists in PII_ENTITY_CATEGORIES_MAP""" + assert PiiEntityCategory.UK in PII_ENTITY_CATEGORIES_MAP + + def test_uk_category_contains_all_entities(self): + """Test UK category contains all UK PII entity types""" + uk_entities = PII_ENTITY_CATEGORIES_MAP[PiiEntityCategory.UK] + + assert PiiEntityType.UK_NHS in uk_entities + assert PiiEntityType.UK_NINO in uk_entities + assert PiiEntityType.UK_PASSPORT in uk_entities + assert PiiEntityType.UK_POSTCODE in uk_entities + assert PiiEntityType.UK_VEHICLE_REGISTRATION in uk_entities + + def test_uk_entities_match_presidio_recognizers(self): + """Test UK entity type names match Presidio recognizer names""" + expected_entities = { + "UK_NHS", + "UK_NINO", + "UK_PASSPORT", + "UK_POSTCODE", + "UK_VEHICLE_REGISTRATION", + } + + uk_entities = PII_ENTITY_CATEGORIES_MAP[PiiEntityCategory.UK] + actual_entities = set(uk_entities) + + assert actual_entities == expected_entities diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 5299ec0fce8..62d8f2644ec 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -11,6 +11,7 @@ import Navbar from "./navbar"; import { agentHubPublicModelsCall, skillHubPublicCall, + getProxyBaseUrl, getPublicModelHubInfo, getUiConfig, mcpHubPublicServersCall, @@ -1929,7 +1930,7 @@ import asyncio config = { "mcpServers": { "${selectedMcpServer.server_name}": { - "url": "http://localhost:4000/${selectedMcpServer.server_name}/mcp", + "url": "${getProxyBaseUrl()}/${selectedMcpServer.server_name}/mcp", "headers": { "x-litellm-api-key": "Bearer sk-1234" } @@ -1969,7 +1970,7 @@ import asyncio config = { "mcpServers": { "${selectedMcpServer.server_name}": { - "url": "http://localhost:4000/${selectedMcpServer.server_name}/mcp", + "url": "${getProxyBaseUrl()}/${selectedMcpServer.server_name}/mcp", "headers": { "x-litellm-api-key": "Bearer sk-1234" } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13b735ddf7c..01b7f58d696 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -48849,6 +48849,8 @@ export interface operations { query?: { /** @description Team ID in the request parameters */ team_id?: string; + /** @description Limit the number of keys returned */ + key_limit?: number | null; }; header?: never; path?: never;