scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

docs: simplify docs

Browse source
This commit is contained in:
Krrish Dholakia 2025-11-17 09:39:32 -08:00
parent 0292b84dc4
commit 3c6f81e6ce
3 changed files with 636 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
View file

@ -37,57 +37,7 @@ Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization).
`my_guardrail.py`:
```python
import os
from typing import Optional, List
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class MyGuardrail(CustomGuardrail):
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(default_on=True)
async def apply_guardrail(
self,
text: str,
language: Optional[str] = None,
entities: Optional[List[PiiEntityType]] = None,
request_data: Optional[dict] = None,
) -> str:
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
response.raise_for_status()
return response.json()
```
Follow from [Custom Guardrail](../proxy/guardrails/custom_guardrail#custom-guardrail) tutorial.
### Create the Init File

473
docs/my-website/docs/proxy/guardrails/custom_guardrail.md
View file

@ -4,151 +4,86 @@ import TabItem from '@theme/TabItem';
# Custom Guardrail
Use this is you want to write code to run a custom guardrail
Use this if you want to write code to run a custom guardrail
## Quick Start
### 1. Write a `CustomGuardrail` Class
A CustomGuardrail has 4 methods to enforce guardrails
- `async_pre_call_hook` - (Optional) modify input or reject request before making LLM API call
- `async_moderation_hook` - (Optional) reject request, runs while making LLM API call (help to lower latency)
- `async_post_call_success_hook`- (Optional) apply guardrail on input/output, runs after making LLM API call
- `async_post_call_streaming_iterator_hook` - (Optional) pass the entire stream to the guardrail
**[See detailed spec of methods here](#customguardrail-methods)**
The simplest way to create a custom guardrail is by implementing the `apply_guardrail` method. This method is called to check text content and can block requests by raising an exception.
**Example `CustomGuardrail` Class**
Create a new file called `custom_guardrail.py` and add this code to it
Create a new file called `custom_guardrail.py` and add this code to it:
```python
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
import os
from typing import Optional, List
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import ModelResponseStream
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class myCustomGuardrail(CustomGuardrail):
def __init__(
self,
**kwargs,
):
# store kwargs as optional_params
self.optional_params = kwargs
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(**kwargs)
async def async_pre_call_hook(
async def apply_guardrail(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: Literal[
"completion",
"text_completion",
"embeddings",
"image_generation",
"moderation",
"audio_transcription",
"pass_through_endpoint",
"rerank"
],
) -> Optional[Union[Exception, str, dict]]:
text: str, # IMPORTANT: This is the text to check against your guardrail rules. It's extracted from the request or response across all LLM call types.
language: Optional[str] = None, # ignore
entities: Optional[List[PiiEntityType]] = None, # ignore
request_data: Optional[dict] = None, # ignore
) -> str:
"""
Runs before the LLM API call
Runs on only Input
Use this if you want to MODIFY the input
Check text content against your guardrail rules.
Raise an exception to block the request.
Return the text (optionally modified) to allow it through.
"""
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
# In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
_content = _content.replace("litellm", "********")
message["content"] = _content
verbose_proxy_logger.debug(
"async_pre_call_hook: Message after masking %s", _messages
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
return data
async def async_moderation_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"],
):
"""
Runs in parallel to LLM API call
Runs on only Input
This can NOT modify the input, only used to reject or accept a call before going to LLM API
"""
# this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call
# In this guardrail, if a user inputs `litellm` we will mask it.
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
raise ValueError("Guardrail failed words - `litellm` detected")
async def async_post_call_success_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response,
):
"""
Runs on response from LLM API call
It can be used to reject a response
If a response contains the word "coffee" -> we will raise an exception
"""
verbose_proxy_logger.debug("async_pre_call_hook response: %s", response)
if isinstance(response, litellm.ModelResponse):
for choice in response.choices:
if isinstance(choice, litellm.Choices):
verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice)
if (
choice.message.content
and isinstance(choice.message.content, str)
and "coffee" in choice.message.content
):
raise ValueError("Guardrail failed Coffee Detected")
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_data: dict,
) -> AsyncGenerator[ModelResponseStream, None]:
"""
Passes the entire stream to the guardrail
This is useful for guardrails that need to see the entire response, such as PII masking.
See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168
Triggered by mode: 'post_call'
"""
async for item in response:
yield item
response.raise_for_status()
return response.json()
```
:::tip Advanced: Using Individual Event Hooks
If you need more fine-grained control, you can implement individual event hooks instead of (or in addition to) `apply_guardrail`:
- `async_pre_call_hook` - Modify input or reject request before making LLM API call
- `async_moderation_hook` - Reject request, runs in parallel with LLM API call (helps lower latency)
- `async_post_call_success_hook` - Apply guardrail on input/output, runs after making LLM API call
- `async_post_call_streaming_iterator_hook` - Pass the entire stream to the guardrail
**[See examples of individual event hooks here](#advanced-individual-event-hooks)** | **[See detailed spec of methods here](#customguardrail-methods)**
:::
### 2. Pass your custom guardrail class in LiteLLM `config.yaml`
In the config below, we point the guardrail to our custom guardrail by setting `guardrail: custom_guardrail.myCustomGuardrail`
@ -166,9 +101,32 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "custom-pre-guard"
- guardrail_name: "my-custom-guardrail"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change
mode: "during_call" # runs apply_guardrail method
api_key: os.environ/MY_GUARDRAIL_API_KEY
api_base: https://api.myguardrail.com
```
:::info Mode Options
- `during_call` - Default mode, runs `apply_guardrail` method (or `async_moderation_hook` if using individual hooks)
- `pre_call` - Runs `async_pre_call_hook` for input modification
- `post_call` - Runs `async_post_call_success_hook` for output validation
:::
<details>
<summary>Advanced: Multiple modes with individual event hooks</summary>
If you're using individual event hooks, you can configure multiple guardrails with different modes:
```yaml
guardrails:
- guardrail_name: "custom-pre-guard"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail
mode: "pre_call" # runs async_pre_call_hook
- guardrail_name: "custom-during-guard"
litellm_params:
@ -180,6 +138,8 @@ guardrails:
mode: "post_call" # runs async_post_call_success_hook
```
</details>
### 3. Start LiteLLM Gateway
<Tabs>
@ -218,15 +178,76 @@ litellm --config config.yaml --detailed_debug
### 4. Test it
#### Test `"custom-pre-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Blocked Request" value = "blocked">
This request will be blocked if it violates your guardrail policy:
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"messages": [
{
"role": "user",
"content": "Content that violates policy"
}
],
"guardrails": ["my-custom-guardrail"]
}'
```
Expected response when blocked:
```json
{
"error": {
"message": "Content blocked: Policy violation",
"type": "None",
"param": "None",
"code": "500"
}
}
```
</TabItem>
<TabItem label="Successful Call" value = "allowed">
This request passes the guardrail:
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "What is the weather like today?"}
],
"guardrails": ["my-custom-guardrail"]
}'
```
</TabItem>
</Tabs>
<details>
<summary>Advanced: Testing individual event hooks</summary>
If you're using individual event hooks, you can test each mode separately:
#### Test `"custom-pre-guard"`
<Tabs>
<TabItem label="Modify input" value = "not-allowed">
Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#1-write-a-customguardrail-class)
Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -244,37 +265,6 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after pre-guard
```json
{
"id": "chatcmpl-9zREDkBIG20RJB4pMlyutmi1hXQWc",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "It looks like you've chosen a string of asterisks. This could be a way to censor or hide certain text. However, without more context, I can't provide a specific word or phrase. If there's something specific you'd like me to say or if you need help with a topic, feel free to let me know!",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"created": 1724429701,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion",
"system_fingerprint": "fp_3aa7262c27",
"usage": {
"completion_tokens": 65,
"prompt_tokens": 14,
"total_tokens": 79
},
"service_tier": null
}
```
</TabItem>
<TabItem label="Successful Call " value = "allowed">
@ -282,7 +272,7 @@ Expected response after pre-guard
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
@ -294,20 +284,14 @@ curl -i http://localhost:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Test `"custom-during-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Unsuccessful call" value = "not-allowed">
Expect this to fail since since `litellm` is in the message content. [This runs the `async_moderation_hook`](#1-write-a-customguardrail-class)
Expect this to fail since `litellm` is in the message content. [This runs the `async_moderation_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -325,7 +309,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after running during-guard
Expected response:
```json
{
@ -345,7 +329,7 @@ Expected response after running during-guard
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
@ -357,21 +341,14 @@ curl -i http://localhost:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Test `"custom-post-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Unsuccessful call" value = "not-allowed">
Expect this to fail since since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#1-write-a-customguardrail-class)
Expect this to fail since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -389,7 +366,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after running during-guard
Expected response:
```json
{
@ -407,7 +384,7 @@ Expected response after running during-guard
<TabItem label="Successful Call " value = "allowed">
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
curl -i -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
@ -424,9 +401,10 @@ Expected response after running during-guard
</TabItem>
</Tabs>
</details>
## ✨ Pass additional parameters to guardrail
:::info
@ -539,10 +517,143 @@ The `get_guardrail_dynamic_request_body_params` method will return:
}
```
## Advanced: Individual Event Hooks
Pro: More flexibility
Con: You need to implement this for each LLM call type (chat completions, text completions, embeddings, image generation, moderation, audio transcription, pass through endpoint, rerank, etc. )
For more fine-grained control over when and how your guardrail runs, you can implement individual event hooks. This gives you flexibility to:
- Modify inputs before the LLM call
- Run checks in parallel with the LLM call (lower latency)
- Validate or modify outputs after the LLM call
- Process streaming responses
### Example with Individual Event Hooks
```python
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import ModelResponseStream, CallTypes
class myCustomGuardrail(CustomGuardrail):
def __init__(
self,
**kwargs,
):
# store kwargs as optional_params
self.optional_params = kwargs
super().__init__(**kwargs)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: Optional[CallTypes],
) -> Optional[Union[Exception, str, dict]]:
"""
Runs before the LLM API call
Runs on only Input
Use this if you want to MODIFY the input
"""
# In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
_content = _content.replace("litellm", "********")
message["content"] = _content
verbose_proxy_logger.debug(
"async_pre_call_hook: Message after masking %s", _messages
)
return data
async def async_moderation_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"],
):
"""
Runs in parallel to LLM API call
Runs on only Input
This can NOT modify the input, only used to reject or accept a call before going to LLM API
"""
# this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call
# In this guardrail, if a user inputs `litellm` we will mask it.
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
raise ValueError("Guardrail failed words - `litellm` detected")
async def async_post_call_success_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response,
):
"""
Runs on response from LLM API call
It can be used to reject a response
If a response contains the word "coffee" -> we will raise an exception
"""
verbose_proxy_logger.debug("async_pre_call_hook response: %s", response)
if isinstance(response, litellm.ModelResponse):
for choice in response.choices:
if isinstance(choice, litellm.Choices):
verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice)
if (
choice.message.content
and isinstance(choice.message.content, str)
and "coffee" in choice.message.content
):
raise ValueError("Guardrail failed Coffee Detected")
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_data: dict,
) -> AsyncGenerator[ModelResponseStream, None]:
"""
Passes the entire stream to the guardrail
This is useful for guardrails that need to see the entire response, such as PII masking.
See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168
Triggered by mode: 'post_call'
"""
async for item in response:
yield item
```
## **CustomGuardrail methods**
| Component | Description | Optional | Checked Data | Can Modify Input | Can Modify Output | Can Fail Call |
|-----------|-------------|----------|--------------|------------------|-------------------|----------------|
| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ |
| `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ |
| `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ |
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ |
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ |

346
litellm/model_prices_and_context_window_backup.json
View file

@ -1224,6 +1224,102 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/eu/gpt-5-2025-08-07": {
"cache_read_input_token_cost": 1.375e-07,
"input_cost_per_token": 1.375e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.1e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/eu/gpt-5-mini-2025-08-07": {
"cache_read_input_token_cost": 2.75e-08,
"input_cost_per_token": 2.75e-07,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.2e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/eu/gpt-5-nano-2025-08-07": {
"cache_read_input_token_cost": 5.5e-09,
"input_cost_per_token": 5.5e-08,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/eu/o1-2024-12-17": {
"cache_read_input_token_cost": 8.25e-06,
"input_cost_per_token": 1.65e-05,
@ -2738,14 +2834,14 @@
},
"azure/o3-2025-04-16": {
"deprecation_date": "2026-04-16",
"cache_read_input_token_cost": 2.5e-06,
"input_cost_per_token": 1e-05,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4e-05,
"output_cost_per_token": 8e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -3004,6 +3100,107 @@
"litellm_provider": "azure",
"mode": "audio_speech"
},
"azure/us/gpt-4.1-2025-04-14": {
"deprecation_date": "2026-11-04",
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_batches": 1.1e-06,
"litellm_provider": "azure",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 8.8e-06,
"output_cost_per_token_batches": 4.4e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": false
},
"azure/us/gpt-4.1-mini-2025-04-14": {
"deprecation_date": "2026-11-04",
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 4.4e-07,
"input_cost_per_token_batches": 2.2e-07,
"litellm_provider": "azure",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.76e-06,
"output_cost_per_token_batches": 8.8e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": false
},
"azure/us/gpt-4.1-nano-2025-04-14": {
"deprecation_date": "2026-11-04",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 1.1e-07,
"input_cost_per_token_batches": 6e-08,
"litellm_provider": "azure",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4.4e-07,
"output_cost_per_token_batches": 2.2e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/us/gpt-4o-2024-08-06": {
"deprecation_date": "2026-02-27",
"cache_read_input_token_cost": 1.375e-06,
@ -3118,6 +3315,102 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/us/gpt-5-2025-08-07": {
"cache_read_input_token_cost": 1.375e-07,
"input_cost_per_token": 1.375e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.1e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/us/gpt-5-mini-2025-08-07": {
"cache_read_input_token_cost": 2.75e-08,
"input_cost_per_token": 2.75e-07,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.2e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/us/gpt-5-nano-2025-08-07": {
"cache_read_input_token_cost": 5.5e-09,
"input_cost_per_token": 5.5e-08,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/us/o1-2024-12-17": {
"cache_read_input_token_cost": 8.25e-06,
"input_cost_per_token": 1.65e-05,
@ -3163,6 +3456,36 @@
"supports_prompt_caching": true,
"supports_vision": false
},
"azure/us/o3-2025-04-16": {
"deprecation_date": "2026-04-16",
"cache_read_input_token_cost": 5.5e-07,
"input_cost_per_token": 2.2e-06,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 8.8e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/us/o3-mini-2025-01-31": {
"cache_read_input_token_cost": 6.05e-07,
"input_cost_per_token": 1.21e-06,
@ -3179,6 +3502,23 @@
"supports_tool_choice": true,
"supports_vision": false
},
"azure/us/o4-mini-2025-04-16": {
"cache_read_input_token_cost": 3.1e-07,
"input_cost_per_token": 1.21e-06,
"litellm_provider": "azure",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.84e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/whisper-1": {
"input_cost_per_second": 0.0001,
"litellm_provider": "azure",