diff --git a/docs/my-website/blog/security_hardening_april_2026/index.md b/docs/my-website/blog/security_hardening_april_2026/index.md new file mode 100644 index 00000000000..1af4caa3e1f --- /dev/null +++ b/docs/my-website/blog/security_hardening_april_2026/index.md @@ -0,0 +1,66 @@ +--- +slug: security-hardening-april-2026 +title: "Security Update: Vulnerability Disclosures and Ongoing Hardening" +date: 2026-04-03T12:00:00 +authors: + - krrish + - ishaan-alt +description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program." +tags: [security] +hide_table_of_contents: false +--- + +After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading. + +We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions. + +The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users. + +The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.** + +{/* truncate */} + +## Vulnerabilities + +### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical) + +Found by Veria Labs. + +When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead. + +**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround. + +Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6) + +### CVE-2026-35029: Privilege escalation via `/config/update` (High) + +Found by Lakera. + +`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint. + +Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) + +### Password hash exposure and pass-the-hash login (High) + +Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/). + +Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses. + +Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8) + +## Bug bounty program + +After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues. + +Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities: + +| Severity | Bounty | Example | +|----------|--------|---------| +| Critical | $1,500 – $3,000 | Supply chain compromise | +| High | $500 – $1,500 | Unauthenticated access to protected data | + +We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security). + +## What's next + +Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed. diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1077731779d..67bba28e4c5 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM): endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + if acompletion and provider == "anthropic" and self.is_claude_messages_api_model( + model + ): + if isinstance(client, HTTPHandler): + client = None + return self._async_anthropic_messages_completion( + model=model, + messages=messages, + endpoint_url=endpoint_url, + proxy_endpoint_url=proxy_endpoint_url, + credentials=credentials, + aws_region_name=aws_region_name, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + litellm_params=litellm_params, + logger_fn=logger_fn, + extra_headers=extra_headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) # type: ignore[return-value] + prompt, chat_history = self.convert_messages_to_prompt( model, messages, provider, custom_prompt_dict ) @@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM): encoding=encoding, ) + async def _async_anthropic_messages_completion( + self, + model: str, + messages: list, + endpoint_url: str, + proxy_endpoint_url: str, + credentials, + aws_region_name: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj: Logging, + optional_params: dict, + stream, + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, + ) -> Union[ModelResponse, CustomStreamWrapper]: + transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params or {}, + headers=extra_headers or {}, + ) + data = json.dumps(transformed_request) + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=data, + headers=headers, + ) + + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + + if stream is True: + return await self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) + return await self.async_completion( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) + async def async_completion( self, model: str, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 7936b6ea644..2b28473ad3e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, +) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, + convert_url_to_base64, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: - # Filter out AWS authentication parameters before passing to Anthropic transformation - # AWS params should only be used for signing requests, not included in request body + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + self._convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + async def async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + await self._async_convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + def _build_bedrock_anthropic_request_base( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: filtered_params = { k: v for k, v in optional_params.items() @@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): } filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) - _anthropic_request = AnthropicConfig.transform_request( + anthropic_request = AnthropicConfig.transform_request( self, model=model, messages=messages, @@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): headers=headers, ) - _anthropic_request.pop("model", None) - _anthropic_request.pop("stream", None) - # Bedrock Invoke doesn't support output_format parameter - _anthropic_request.pop("output_format", None) - # Bedrock Invoke doesn't support output_config parameter - # Fixes: https://github.com/BerriAI/litellm/issues/22797 - _anthropic_request.pop("output_config", None) - if "anthropic_version" not in _anthropic_request: - _anthropic_request["anthropic_version"] = self.anthropic_version + anthropic_request.pop("model", None) + anthropic_request.pop("stream", None) + anthropic_request.pop("output_format", None) + anthropic_request.pop("output_config", None) + if "anthropic_version" not in anthropic_request: + anthropic_request["anthropic_version"] = self.anthropic_version # Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(_anthropic_request) + remove_custom_field_from_tools(anthropic_request) + return anthropic_request + def _compute_bedrock_invoke_beta_headers( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + headers: dict, + ) -> List[str]: tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools) input_examples_used = self.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = self.get_anthropic_beta_list( model=model, optional_params=optional_params, @@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") - # Filter out beta headers that Bedrock Invoke doesn't support - # Uses centralized configuration from anthropic_beta_headers_config.json - beta_list = list(beta_set) - _anthropic_request["anthropic_beta"] = beta_list + auto_beta_list = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + return sorted(user_beta_set.union(set(auto_beta_list))) - return _anthropic_request + def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None: + """ + Bedrock Invoke does not accept document URL sources. Convert to base64 payloads. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } + + async def _async_convert_document_url_sources_to_base64( + self, anthropic_request: dict + ) -> None: + """ + Async version of document URL conversion for async completion paths. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = await async_convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: """ diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index a890cf2a04d..784c6a50cfd 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -12,6 +12,7 @@ from typing import ( import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -436,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig( ) input_examples_used = anthropic_model_info.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = anthropic_model_info.get_anthropic_beta_list( model=model, optional_params=anthropic_messages_optional_request_params, @@ -460,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - if beta_set: - anthropic_messages_request["anthropic_beta"] = list(beta_set) + filtered_auto_betas = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas))) + if filtered_betas: + anthropic_messages_request["anthropic_beta"] = filtered_betas return anthropic_messages_request diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 51b7b56a285..4072aab6544 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6672,6 +6672,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6795,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6847,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6887,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6972,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7100,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7158,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7216,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7286,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7375,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7589,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7667,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7995,6 +8163,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -21292,6 +21474,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23307,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -32726,6 +32936,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 386b9d27723..09200c96841 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -221,16 +221,21 @@ async def create_response( f"Error consuming first chunk from generator: {e}" ) - # Fallback to a generic error stream + # Preserve status code from HTTPException (e.g., guardrail blocks) + error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + error_detail = getattr(e, "detail", "Error processing stream start") + if not isinstance(error_detail, str): + error_detail = str(error_detail) + async def error_gen_message() -> AsyncGenerator[str, None]: - yield f"data: {json.dumps({'error': {'message': 'Error processing stream start', 'code': status.HTTP_500_INTERNAL_SERVER_ERROR}})}\n\n" + yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n" yield "data: [DONE]\n\n" return StreamingResponse( error_gen_message(), media_type=media_type, headers=headers, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + status_code=error_status, ) async def combined_generator() -> AsyncGenerator[str, None]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 2d3f048f81b..3ee8e6a27bd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -14,6 +14,8 @@ from fastapi import HTTPException if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +import json + import litellm from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -203,8 +205,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): response.text, ) raise HTTPException( - status_code=response.status_code, - detail=f"Model Armor API error: {response.text}", + status_code=400, + detail=f"Model Armor API error (upstream {response.status_code}): {response.text}", ) json_response = response.json() @@ -746,8 +748,21 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): yield chunk return - except HTTPException: - raise + except HTTPException as e: + # Yield error as SSE event so create_response() detects it and + # returns a proper JSON error response with the correct status code. + # (Raising from a generator hits create_response's generic except → 500.) + detail = ( + e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)} + ) + error_value = detail.get("error", detail) + if isinstance(error_value, dict): + error_obj = dict(error_value) + else: + error_obj = {"message": str(error_value)} + error_obj["code"] = e.status_code + yield f"data: {json.dumps({'error': error_obj})}\n\n" + return except Exception as e: verbose_proxy_logger.error( "Model Armor streaming error: %s", str(e), exc_info=True diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c140de98191..e105d9b8586 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -942,9 +942,9 @@ async def _check_team_key_limits( where={"team_id": team_table.team_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_team_key_model_specific_limits( keys=keys, @@ -1101,9 +1101,9 @@ async def _check_org_key_limits( where={"organization_id": org_table.organization_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_org_key_model_specific_limits( keys=keys, @@ -2157,7 +2157,7 @@ async def update_key_fn( # Delete - key from cache, since it's been updated! # key updated - a new model could have been added to this key. it should not block requests after this is done await _delete_cache_key_object( - hashed_token=hash_token(key), + hashed_token=_hash_token_if_needed(key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 51b7b56a285..71918ed049b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6672,6 +6672,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6795,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6847,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6887,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6972,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7100,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7158,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7216,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7286,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7375,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7589,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7667,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7995,6 +8163,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -18665,13 +18847,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18681,8 +18861,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18715,13 +18894,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18731,8 +18908,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18760,14 +18936,10 @@ "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18777,8 +18949,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -18809,14 +18979,10 @@ "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18826,8 +18992,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -18857,11 +19021,13 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 1e-08, - "cache_read_input_token_cost_batches": 3.8e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -18870,6 +19036,7 @@ "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 9e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21292,6 +21459,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23292,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -32726,6 +32921,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, diff --git a/pyproject.toml b/pyproject.toml index c2b6a31bf4d..c9d7148735c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.83.0" +version = "1.83.1" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.9" @@ -238,7 +238,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.83.0" +version = "1.83.1" version_files = [ "pyproject.toml:^version", ] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 8080491f662..97ca55073df 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -380,8 +380,9 @@ async def test_model_armor_api_error_handling(): call_type="completion" ) - assert exc_info.value.status_code == 500 + assert exc_info.value.status_code == 400 assert "Model Armor API error" in str(exc_info.value.detail) + assert "upstream 500" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -485,6 +486,128 @@ async def test_model_armor_streaming_response(): assert len(result_chunks) > 0 mock_post.assert_called() +@pytest.mark.asyncio +async def test_model_armor_streaming_block_yields_sse_error(): + """Test that streaming content block yields SSE error event instead of raising HTTPException.""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock Model Armor API response that triggers a block (SDP MATCH_FOUND) + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock( + return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [ + { + "infoType": "PASSWORD", + "likelihood": "VERY_LIKELY", + } + ], + } + } + } + }, + } + } + ) + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + + async def mock_stream(): + chunks = [ + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta( + content="My password is " + ) + ) + ] + ), + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta(content="hunter2") + ) + ] + ), + ] + for chunk in chunks: + yield chunk + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "What's your password?"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + + result_chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_stream(), + request_data=request_data, + ): + result_chunks.append(chunk) + + # Should yield exactly one SSE error event (not raise HTTPException) + assert len(result_chunks) == 1 + error_data = json.loads(result_chunks[0].removeprefix("data: ")) + assert "error" in error_data + assert error_data["error"]["code"] == 400 + + +@pytest.mark.asyncio +async def test_model_armor_api_failure_returns_400(): + """Test that Model Armor API failures raise HTTP 400, not the upstream status code.""" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock a 500 response from the Model Armor GCP API + mock_response = AsyncMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_model_armor_request( + content="test content", + source="user_prompt", + ) + + # Should be 400, NOT the upstream 500 + assert exc_info.value.status_code == 400 + assert "upstream 500" in str(exc_info.value.detail) + + def test_model_armor_ui_friendly_name(): """Test the UI-friendly name of the Model Armor guardrail""" from litellm.types.proxy.guardrails.guardrail_hooks.model_armor import ( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 45ec4b726a8..781b559651f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,7 +4,7 @@ from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request, status +from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, StreamingResponse import litellm @@ -899,6 +899,33 @@ class TestCommonRequestProcessingHelpers: assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_generator_raises_http_exception( + self, + ): + """ + Test that when a generator raises HTTPException, the response preserves + the original status code instead of hardcoding 500. + """ + mock_gen = AsyncMock() + mock_gen.__anext__.side_effect = HTTPException( + status_code=400, detail="Content blocked by guardrail" + ) + + response = await create_response(mock_gen, "text/event-stream", {}) + assert response.status_code == 400 + content = await self.consume_stream(response) + import json + + expected_error_data = { + "error": { + "message": "Content blocked by guardrail", + "code": 400, + } + } + assert len(content) == 2 + assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" + assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_first_chunk_error_string_code(self): """ Test that when the first chunk contains a string error code, a JSON error response is returned