diff --git a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md index 2722a4a024c..9c654cd1560 100644 --- a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md +++ b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md @@ -37,57 +37,7 @@ Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization). `my_guardrail.py`: -```python -import os -from typing import Optional, List -from fastapi import HTTPException - -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.types.guardrails import PiiEntityType -from litellm._logging import verbose_proxy_logger -from litellm.llms.custom_httpx.http_handler import ( - get_async_httpx_client, - httpxSpecialProvider, -) - -class MyGuardrail(CustomGuardrail): - def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs): - self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY") - self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com") - super().__init__(default_on=True) - - async def apply_guardrail( - self, - text: str, - language: Optional[str] = None, - entities: Optional[List[PiiEntityType]] = None, - request_data: Optional[dict] = None, - ) -> str: - result = await self._check_with_api(text, request_data) - - if result.get("action") == "BLOCK": - raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}") - - return text - - async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict: - async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}", - } - - response = await async_client.post( - f"{self.api_base}/check", - headers=headers, - json={"text": text}, - timeout=5, - ) - - response.raise_for_status() - return response.json() -``` +Follow from [Custom Guardrail](../proxy/guardrails/custom_guardrail#custom-guardrail) tutorial. ### Create the Init File diff --git a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md index b8ba64d333a..de008f79624 100644 --- a/docs/my-website/docs/proxy/guardrails/custom_guardrail.md +++ b/docs/my-website/docs/proxy/guardrails/custom_guardrail.md @@ -4,151 +4,86 @@ import TabItem from '@theme/TabItem'; # Custom Guardrail -Use this is you want to write code to run a custom guardrail +Use this if you want to write code to run a custom guardrail ## Quick Start ### 1. Write a `CustomGuardrail` Class -A CustomGuardrail has 4 methods to enforce guardrails -- `async_pre_call_hook` - (Optional) modify input or reject request before making LLM API call -- `async_moderation_hook` - (Optional) reject request, runs while making LLM API call (help to lower latency) -- `async_post_call_success_hook`- (Optional) apply guardrail on input/output, runs after making LLM API call -- `async_post_call_streaming_iterator_hook` - (Optional) pass the entire stream to the guardrail - - -**[See detailed spec of methods here](#customguardrail-methods)** +The simplest way to create a custom guardrail is by implementing the `apply_guardrail` method. This method is called to check text content and can block requests by raising an exception. **Example `CustomGuardrail` Class** -Create a new file called `custom_guardrail.py` and add this code to it +Create a new file called `custom_guardrail.py` and add this code to it: + ```python -from typing import Any, AsyncGenerator, Literal, Optional, Union - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm.caching.caching import DualCache +import os +from typing import Optional, List from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import ModelResponseStream - +from litellm.types.guardrails import PiiEntityType +from litellm._logging import verbose_proxy_logger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) class myCustomGuardrail(CustomGuardrail): - def __init__( - self, - **kwargs, - ): - # store kwargs as optional_params - self.optional_params = kwargs - + def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs): + self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY") + self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com") super().__init__(**kwargs) - async def async_pre_call_hook( + async def apply_guardrail( self, - user_api_key_dict: UserAPIKeyAuth, - cache: DualCache, - data: dict, - call_type: Literal[ - "completion", - "text_completion", - "embeddings", - "image_generation", - "moderation", - "audio_transcription", - "pass_through_endpoint", - "rerank" - ], - ) -> Optional[Union[Exception, str, dict]]: + text: str, # IMPORTANT: This is the text to check against your guardrail rules. It's extracted from the request or response across all LLM call types. + language: Optional[str] = None, # ignore + entities: Optional[List[PiiEntityType]] = None, # ignore + request_data: Optional[dict] = None, # ignore + ) -> str: """ - Runs before the LLM API call - Runs on only Input - Use this if you want to MODIFY the input + Check text content against your guardrail rules. + Raise an exception to block the request. + Return the text (optionally modified) to allow it through. """ + result = await self._check_with_api(text, request_data) + + if result.get("action") == "BLOCK": + raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}") + + return text - # In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM - _messages = data.get("messages") - if _messages: - for message in _messages: - _content = message.get("content") - if isinstance(_content, str): - if "litellm" in _content.lower(): - _content = _content.replace("litellm", "********") - message["content"] = _content - - verbose_proxy_logger.debug( - "async_pre_call_hook: Message after masking %s", _messages + async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict: + async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + response = await async_client.post( + f"{self.api_base}/check", + headers=headers, + json={"text": text}, + timeout=5, ) - - return data - - async def async_moderation_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"], - ): - """ - Runs in parallel to LLM API call - Runs on only Input - - This can NOT modify the input, only used to reject or accept a call before going to LLM API - """ - - # this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call - # In this guardrail, if a user inputs `litellm` we will mask it. - _messages = data.get("messages") - if _messages: - for message in _messages: - _content = message.get("content") - if isinstance(_content, str): - if "litellm" in _content.lower(): - raise ValueError("Guardrail failed words - `litellm` detected") - - async def async_post_call_success_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - response, - ): - """ - Runs on response from LLM API call - - It can be used to reject a response - - If a response contains the word "coffee" -> we will raise an exception - """ - verbose_proxy_logger.debug("async_pre_call_hook response: %s", response) - if isinstance(response, litellm.ModelResponse): - for choice in response.choices: - if isinstance(choice, litellm.Choices): - verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice) - if ( - choice.message.content - and isinstance(choice.message.content, str) - and "coffee" in choice.message.content - ): - raise ValueError("Guardrail failed Coffee Detected") - - async def async_post_call_streaming_iterator_hook( - self, - user_api_key_dict: UserAPIKeyAuth, - response: Any, - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream, None]: - """ - Passes the entire stream to the guardrail - - This is useful for guardrails that need to see the entire response, such as PII masking. - - See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168 - - Triggered by mode: 'post_call' - """ - async for item in response: - yield item - + + response.raise_for_status() + return response.json() ``` +:::tip Advanced: Using Individual Event Hooks + +If you need more fine-grained control, you can implement individual event hooks instead of (or in addition to) `apply_guardrail`: + +- `async_pre_call_hook` - Modify input or reject request before making LLM API call +- `async_moderation_hook` - Reject request, runs in parallel with LLM API call (helps lower latency) +- `async_post_call_success_hook` - Apply guardrail on input/output, runs after making LLM API call +- `async_post_call_streaming_iterator_hook` - Pass the entire stream to the guardrail + +**[See examples of individual event hooks here](#advanced-individual-event-hooks)** | **[See detailed spec of methods here](#customguardrail-methods)** + +::: + ### 2. Pass your custom guardrail class in LiteLLM `config.yaml` In the config below, we point the guardrail to our custom guardrail by setting `guardrail: custom_guardrail.myCustomGuardrail` @@ -166,9 +101,32 @@ model_list: api_key: os.environ/OPENAI_API_KEY guardrails: - - guardrail_name: "custom-pre-guard" + - guardrail_name: "my-custom-guardrail" litellm_params: guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change + mode: "during_call" # runs apply_guardrail method + api_key: os.environ/MY_GUARDRAIL_API_KEY + api_base: https://api.myguardrail.com +``` + +:::info Mode Options + +- `during_call` - Default mode, runs `apply_guardrail` method (or `async_moderation_hook` if using individual hooks) +- `pre_call` - Runs `async_pre_call_hook` for input modification +- `post_call` - Runs `async_post_call_success_hook` for output validation + +::: + +
+Advanced: Multiple modes with individual event hooks + +If you're using individual event hooks, you can configure multiple guardrails with different modes: + +```yaml +guardrails: + - guardrail_name: "custom-pre-guard" + litellm_params: + guardrail: custom_guardrail.myCustomGuardrail mode: "pre_call" # runs async_pre_call_hook - guardrail_name: "custom-during-guard" litellm_params: @@ -180,6 +138,8 @@ guardrails: mode: "post_call" # runs async_post_call_success_hook ``` +
+ ### 3. Start LiteLLM Gateway @@ -218,15 +178,76 @@ litellm --config config.yaml --detailed_debug ### 4. Test it -#### Test `"custom-pre-guard"` - - **[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + + + +This request will be blocked if it violates your guardrail policy: + +```shell +curl -i -X POST http://localhost:4000/v1/chat/completions \ +-H "Content-Type: application/json" \ +-H "Authorization: Bearer sk-1234" \ +-d '{ + "model": "gpt-4", + "messages": [ + { + "role": "user", + "content": "Content that violates policy" + } + ], + "guardrails": ["my-custom-guardrail"] +}' +``` + +Expected response when blocked: + +```json +{ + "error": { + "message": "Content blocked: Policy violation", + "type": "None", + "param": "None", + "code": "500" + } +} +``` + + + + + +This request passes the guardrail: + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "What is the weather like today?"} + ], + "guardrails": ["my-custom-guardrail"] + }' +``` + + + + + +
+Advanced: Testing individual event hooks + +If you're using individual event hooks, you can test each mode separately: + +#### Test `"custom-pre-guard"` + -Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#1-write-a-customguardrail-class) +Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -244,37 +265,6 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after pre-guard - -```json -{ - "id": "chatcmpl-9zREDkBIG20RJB4pMlyutmi1hXQWc", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "It looks like you've chosen a string of asterisks. This could be a way to censor or hide certain text. However, without more context, I can't provide a specific word or phrase. If there's something specific you'd like me to say or if you need help with a topic, feel free to let me know!", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "created": 1724429701, - "model": "gpt-4o-2024-05-13", - "object": "chat.completion", - "system_fingerprint": "fp_3aa7262c27", - "usage": { - "completion_tokens": 65, - "prompt_tokens": 14, - "total_tokens": 79 - }, - "service_tier": null -} - -``` - @@ -282,7 +272,7 @@ Expected response after pre-guard ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -H "Authorization: Bearer sk-1234" \ -d '{ "model": "gpt-3.5-turbo", "messages": [ @@ -294,20 +284,14 @@ curl -i http://localhost:4000/v1/chat/completions \ - - #### Test `"custom-during-guard"` - -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** - -Expect this to fail since since `litellm` is in the message content. [This runs the `async_moderation_hook`](#1-write-a-customguardrail-class) - +Expect this to fail since `litellm` is in the message content. [This runs the `async_moderation_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -325,7 +309,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after running during-guard +Expected response: ```json { @@ -345,7 +329,7 @@ Expected response after running during-guard ```shell curl -i http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -H "Authorization: Bearer sk-1234" \ -d '{ "model": "gpt-3.5-turbo", "messages": [ @@ -357,21 +341,14 @@ curl -i http://localhost:4000/v1/chat/completions \ - - #### Test `"custom-post-guard"` - - -**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** - -Expect this to fail since since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#1-write-a-customguardrail-class) - +Expect this to fail since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#advanced-individual-event-hooks) ```shell curl -i -X POST http://localhost:4000/v1/chat/completions \ @@ -389,7 +366,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \ }' ``` -Expected response after running during-guard +Expected response: ```json { @@ -407,7 +384,7 @@ Expected response after running during-guard ```shell - curl -i -X POST http://localhost:4000/v1/chat/completions \ +curl -i -X POST http://localhost:4000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ -d '{ @@ -424,9 +401,10 @@ Expected response after running during-guard - +
+ ## ✨ Pass additional parameters to guardrail :::info @@ -539,10 +517,143 @@ The `get_guardrail_dynamic_request_body_params` method will return: } ``` +## Advanced: Individual Event Hooks + +Pro: More flexibility +Con: You need to implement this for each LLM call type (chat completions, text completions, embeddings, image generation, moderation, audio transcription, pass through endpoint, rerank, etc. ) + +For more fine-grained control over when and how your guardrail runs, you can implement individual event hooks. This gives you flexibility to: +- Modify inputs before the LLM call +- Run checks in parallel with the LLM call (lower latency) +- Validate or modify outputs after the LLM call +- Process streaming responses + +### Example with Individual Event Hooks + +```python +from typing import Any, AsyncGenerator, Literal, Optional, Union + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.caching.caching import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import ModelResponseStream, CallTypes + + +class myCustomGuardrail(CustomGuardrail): + def __init__( + self, + **kwargs, + ): + # store kwargs as optional_params + self.optional_params = kwargs + + super().__init__(**kwargs) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Optional[CallTypes], + ) -> Optional[Union[Exception, str, dict]]: + """ + Runs before the LLM API call + Runs on only Input + Use this if you want to MODIFY the input + """ + + # In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM + _messages = data.get("messages") + if _messages: + for message in _messages: + _content = message.get("content") + if isinstance(_content, str): + if "litellm" in _content.lower(): + _content = _content.replace("litellm", "********") + message["content"] = _content + + verbose_proxy_logger.debug( + "async_pre_call_hook: Message after masking %s", _messages + ) + + return data + + async def async_moderation_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"], + ): + """ + Runs in parallel to LLM API call + Runs on only Input + + This can NOT modify the input, only used to reject or accept a call before going to LLM API + """ + + # this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call + # In this guardrail, if a user inputs `litellm` we will mask it. + _messages = data.get("messages") + if _messages: + for message in _messages: + _content = message.get("content") + if isinstance(_content, str): + if "litellm" in _content.lower(): + raise ValueError("Guardrail failed words - `litellm` detected") + + async def async_post_call_success_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response, + ): + """ + Runs on response from LLM API call + + It can be used to reject a response + + If a response contains the word "coffee" -> we will raise an exception + """ + verbose_proxy_logger.debug("async_pre_call_hook response: %s", response) + if isinstance(response, litellm.ModelResponse): + for choice in response.choices: + if isinstance(choice, litellm.Choices): + verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice) + if ( + choice.message.content + and isinstance(choice.message.content, str) + and "coffee" in choice.message.content + ): + raise ValueError("Guardrail failed Coffee Detected") + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: Any, + request_data: dict, + ) -> AsyncGenerator[ModelResponseStream, None]: + """ + Passes the entire stream to the guardrail + + This is useful for guardrails that need to see the entire response, such as PII masking. + + See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168 + + Triggered by mode: 'post_call' + """ + async for item in response: + yield item + +``` + ## **CustomGuardrail methods** | Component | Description | Optional | Checked Data | Can Modify Input | Can Modify Output | Can Fail Call | |-----------|-------------|----------|--------------|------------------|-------------------|----------------| +| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ | | `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ | | `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ | | `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ | +| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ | diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 367f788dd34..1ba1e73cb33 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1224,6 +1224,102 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/eu/gpt-5-2025-08-07": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5-mini-2025-08-07": { + "cache_read_input_token_cost": 2.75e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/eu/gpt-5-nano-2025-08-07": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "input_cost_per_token": 1.65e-05, @@ -2738,14 +2834,14 @@ }, "azure/o3-2025-04-16": { "deprecation_date": "2026-04-16", - "cache_read_input_token_cost": 2.5e-06, - "input_cost_per_token": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "max_tokens": 100000, "mode": "chat", - "output_cost_per_token": 4e-05, + "output_cost_per_token": 8e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -3004,6 +3100,107 @@ "litellm_provider": "azure", "mode": "audio_speech" }, + "azure/us/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "input_cost_per_token_batches": 1.1e-06, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "output_cost_per_token_batches": 4.4e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "azure/us/gpt-4.1-mini-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_batches": 2.2e-07, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.76e-06, + "output_cost_per_token_batches": 8.8e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "azure/us/gpt-4.1-nano-2025-04-14": { + "deprecation_date": "2026-11-04", + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 1.1e-07, + "input_cost_per_token_batches": 6e-08, + "litellm_provider": "azure", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "output_cost_per_token_batches": 2.2e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/gpt-4o-2024-08-06": { "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, @@ -3118,6 +3315,102 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "azure/us/gpt-5-2025-08-07": { + "cache_read_input_token_cost": 1.375e-07, + "input_cost_per_token": 1.375e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.1e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5-mini-2025-08-07": { + "cache_read_input_token_cost": 2.75e-08, + "input_cost_per_token": 2.75e-07, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/us/gpt-5-nano-2025-08-07": { + "cache_read_input_token_cost": 5.5e-09, + "input_cost_per_token": 5.5e-08, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-07, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, "input_cost_per_token": 1.65e-05, @@ -3163,6 +3456,36 @@ "supports_prompt_caching": true, "supports_vision": false }, + "azure/us/o3-2025-04-16": { + "deprecation_date": "2026-04-16", + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 2.2e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 8.8e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/us/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "input_cost_per_token": 1.21e-06, @@ -3179,6 +3502,23 @@ "supports_tool_choice": true, "supports_vision": false }, + "azure/us/o4-mini-2025-04-16": { + "cache_read_input_token_cost": 3.1e-07, + "input_cost_per_token": 1.21e-06, + "litellm_provider": "azure", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 4.84e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/whisper-1": { "input_cost_per_second": 0.0001, "litellm_provider": "azure",