From 29057ba6add3dd17288e06d90424f9f2d71f4a2c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 19 Nov 2025 11:55:38 +0530 Subject: [PATCH 01/68] Added support for azure anthopic models via chat completion --- docs/my-website/docs/providers/anthropic.md | 26 +- docs/my-website/docs/providers/azure/azure.md | 14 +- .../docs/providers/azure/azure_anthropic.md | 378 ++++++++++++++++++ litellm/__init__.py | 1 + .../get_llm_provider_logic.py | 18 + litellm/llms/azure/anthropic/__init__.py | 8 + litellm/llms/azure/anthropic/handler.py | 236 +++++++++++ .../llms/azure/anthropic/transformation.py | 96 +++++ litellm/main.py | 58 +++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + .../anthropic/test_azure_anthropic_handler.py | 216 ++++++++++ .../test_azure_anthropic_provider_routing.py | 82 ++++ .../test_azure_anthropic_transformation.py | 191 +++++++++ 14 files changed, 1318 insertions(+), 9 deletions(-) create mode 100644 docs/my-website/docs/providers/azure/azure_anthropic.md create mode 100644 litellm/llms/azure/anthropic/__init__.py create mode 100644 litellm/llms/azure/anthropic/handler.py create mode 100644 litellm/llms/azure/anthropic/transformation.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index afcb6a34d9c..08b52066e16 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -17,11 +17,11 @@ LiteLLM supports all anthropic models. | Property | Details | |-------|-------| -| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. | -| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) | -| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) | -| API Endpoint for Provider | https://api.anthropic.com | -| Supported Endpoints | `/chat/completions` | +| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. Also available via Azure Foundry. | +| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`). For Azure Foundry deployments, use `azure/claude-*` (see [Azure Anthropic documentation](../providers/azure/azure_anthropic)) | +| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint for Provider | https://api.anthropic.com (or Azure Foundry endpoint: `https://.services.ai.azure.com/anthropic`) | +| Supported Endpoints | `/chat/completions`, `/v1/messages` (passthrough) | ## Supported OpenAI Parameters @@ -59,6 +59,22 @@ os.environ["ANTHROPIC_API_KEY"] = "your-api-key" # os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending ``` +:::tip Azure Foundry Support + +Claude models are also available via Microsoft Azure Foundry. Use the `azure/` prefix instead of `anthropic/` and configure Azure authentication. See the [Azure Anthropic documentation](../providers/azure/azure_anthropic) for details. + +Example: +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +::: + ### Custom API Base When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL. diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 2f845357328..d338a0287e0 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -9,10 +9,10 @@ import TabItem from '@theme/TabItem'; | Property | Details | |-------|-------| -| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series | -| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | -| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) +| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. | +| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) (Claude passthrough) | +| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) ## API Keys, Params api_key, api_base, api_version etc can be passed directly to `litellm.completion` - see here or set as `litellm.api_key` params see here @@ -27,6 +27,12 @@ os.environ["AZURE_AD_TOKEN"] = "" os.environ["AZURE_API_TYPE"] = "" ``` +:::info Azure Foundry Claude Models + +Azure also supports Claude models via Azure Foundry. Use `azure/claude-*` model names (e.g., `azure/claude-sonnet-4-5`) with Azure authentication. See the [Azure Anthropic documentation](./azure_anthropic) for details. + +::: + ## **Usage - LiteLLM Python SDK** Open In Colab diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md new file mode 100644 index 00000000000..9a45e1db599 --- /dev/null +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -0,0 +1,378 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure Anthropic (Claude via Azure Foundry) + +LiteLLM supports Claude models deployed via Microsoft Azure Foundry, including Claude Sonnet 4.5, Claude Haiku 4.5, and Claude Opus 4.1. + +## Available Models + +Azure Foundry supports the following Claude models: + +- `claude-sonnet-4-5` - Anthropic's most capable model for building real-world agents and handling complex, long-horizon tasks +- `claude-haiku-4-5` - Near-frontier performance with the right speed and cost for high-volume use cases +- `claude-opus-4-1` - Industry leader for coding, delivering sustained performance on long-running tasks + +| Property | Details | +|-------|-------| +| Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. | +| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) | +| Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint | `https://.services.ai.azure.com/anthropic/v1/messages` | +| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages` (passthrough) | + +## Key Features + +- **Extended thinking**: Enhanced reasoning capabilities for complex tasks +- **Image and text input**: Strong vision capabilities for analyzing charts, graphs, technical diagrams, and reports +- **Code generation**: Advanced thinking with code generation, analysis, and debugging (Claude Sonnet 4.5 and Claude Opus 4.1) +- **Same API as Anthropic**: All request/response transformations are identical to the main Anthropic provider + +## Authentication + +Azure Anthropic supports two authentication methods: + +1. **API Key**: Use the `api-key` header +2. **Azure AD Token**: Use `Authorization: Bearer ` header (Microsoft Entra ID) + +## API Keys and Configuration + +```python +import os + +# Option 1: API Key authentication +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Option 2: Azure AD Token authentication +os.environ["AZURE_AD_TOKEN"] = "your-azure-ad-token" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Optional: Azure AD Token Provider (for automatic token refresh) +os.environ["AZURE_TENANT_ID"] = "your-tenant-id" +os.environ["AZURE_CLIENT_ID"] = "your-client-id" +os.environ["AZURE_CLIENT_SECRET"] = "your-client-secret" +os.environ["AZURE_SCOPE"] = "https://cognitiveservices.azure.com/.default" +``` + +## Usage - LiteLLM Python SDK + +### Basic Completion + +```python +from litellm import completion + +# Set environment variables +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Make a completion request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What are 3 things to visit in Seattle?"} + ], + max_tokens=1000, + temperature=0.7, +) + +print(response) +``` + +### Completion with API Key Parameter + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Completion with Azure AD Token + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + azure_ad_token="your-azure-ad-token", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Streaming + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Write a short story"} + ], + stream=True, + max_tokens=1000, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + +### Tool Calling + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What's the weather in Seattle?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + } + ], + tool_choice="auto", + max_tokens=1000, +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export AZURE_API_KEY="your-azure-api-key" +export AZURE_API_BASE="https://.services.ai.azure.com/anthropic" +``` + +### 2. Configure the proxy + +```yaml +model_list: + - model_name: claude-sonnet-4-5 + litellm_params: + model: azure/claude-sonnet-4-5 + api_base: https://.services.ai.azure.com/anthropic + api_key: os.environ/AZURE_API_KEY +``` + +### 3. Test it + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "max_tokens": 1000 +}' +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000 +) + +print(response) +``` + + + + +## Messages API Passthrough + +Azure Anthropic also supports the native Anthropic Messages API via passthrough. The endpoint structure is the same as Anthropic's `/v1/messages` API. + +### Using Anthropic SDK + +```python +from anthropic import Anthropic + +client = Anthropic( + api_key="your-azure-api-key", + base_url="https://.services.ai.azure.com/anthropic" +) + +response = client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1000, + messages=[ + {"role": "user", "content": "Hello, world"} + ] +) + +print(response) +``` + +### Using LiteLLM Proxy Passthrough + +```bash +curl --request POST \ + --url http://0.0.0.0:4000/anthropic/v1/messages \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --header "Authorization: bearer sk-anything" \ + --data '{ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Hello, world"} + ] +}' +``` + +## Supported OpenAI Parameters + +Azure Anthropic supports the same parameters as the main Anthropic provider: + +``` +"stream", +"stop", +"temperature", +"top_p", +"max_tokens", +"max_completion_tokens", +"tools", +"tool_choice", +"extra_headers", +"parallel_tool_calls", +"response_format", +"user", +"thinking", +"reasoning_effort" +``` + +:::info + +Azure Anthropic API requires `max_tokens` to be passed. LiteLLM automatically passes `max_tokens=4096` when no `max_tokens` are provided. + +::: + +## Differences from Standard Anthropic Provider + +The only difference between Azure Anthropic and the standard Anthropic provider is authentication: + +- **Standard Anthropic**: Uses `x-api-key` header +- **Azure Anthropic**: Uses `api-key` header or `Authorization: Bearer ` for Azure AD authentication + +All other request/response transformations, tool calling, streaming, and feature support are identical. + +## API Base URL Format + +The API base URL should follow this format: + +``` +https://.services.ai.azure.com/anthropic +``` + +LiteLLM will automatically append `/v1/messages` if not already present in the URL. + +## Example: Full Configuration + +```python +import os +from litellm import completion + +# Configure Azure Anthropic +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic" + +# Make a request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Explain quantum computing in simple terms."} + ], + max_tokens=1000, + temperature=0.7, + stream=False, +) + +print(response.choices[0].message.content) +``` + +## Troubleshooting + +### Missing API Base Error + +If you see an error about missing API base, ensure you've set: + +```python +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" +``` + +Or pass it directly: + +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + # ... +) +``` + +### Authentication Errors + +- **API Key**: Ensure `AZURE_API_KEY` is set or passed as `api_key` parameter +- **Azure AD Token**: Ensure `AZURE_AD_TOKEN` is set or passed as `azure_ad_token` parameter +- **Token Provider**: For automatic token refresh, configure `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` + +## Related Documentation + +- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models +- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup + diff --git a/litellm/__init__.py b/litellm/__init__.py index c86768490f3..929835c547e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1108,6 +1108,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo +from .llms.azure.anthropic.transformation import AzureAnthropicConfig from .llms.groq.stt.transformation import GroqSTTConfig from .llms.anthropic.completion.transformation import AnthropicTextConfig from .llms.triton.completion.transformation import TritonConfig diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ef0ebe074d7..de616a332d7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -22,6 +22,19 @@ def _is_non_openai_azure_model(model: str) -> bool: return False +def _is_azure_anthropic_model(model: str) -> Optional[str]: + try: + model_parts = model.split("/", 1) + if len(model_parts) > 1: + model_name = model_parts[1].lower() + # Check if model name contains claude + if "claude" in model_name or model_name.startswith("claude"): + return model_parts[1] # Return model name without "azure/" prefix + except Exception: + pass + return None + + def handle_cohere_chat_model_custom_llm_provider( model: str, custom_llm_provider: Optional[str] = None ) -> Tuple[str, Optional[str]]: @@ -123,6 +136,11 @@ def get_llm_provider( # noqa: PLR0915 # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus if model.split("/", 1)[0] == "azure": + # Check if it's an Azure Anthropic model (claude models) + azure_anthropic_model = _is_azure_anthropic_model(model) + if azure_anthropic_model: + custom_llm_provider = "azure_anthropic" + return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base if _is_non_openai_azure_model(model): custom_llm_provider = "openai" return model, custom_llm_provider, dynamic_api_key, api_base diff --git a/litellm/llms/azure/anthropic/__init__.py b/litellm/llms/azure/anthropic/__init__.py new file mode 100644 index 00000000000..b40c4dfa9b3 --- /dev/null +++ b/litellm/llms/azure/anthropic/__init__.py @@ -0,0 +1,8 @@ +""" +Azure Anthropic provider - supports Claude models via Azure Foundry +""" +from .handler import AzureAnthropicChatCompletion +from .transformation import AzureAnthropicConfig + +__all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"] + diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure/anthropic/handler.py new file mode 100644 index 00000000000..cfa5eeddaa9 --- /dev/null +++ b/litellm/llms/azure/anthropic/handler.py @@ -0,0 +1,236 @@ +""" +Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authentication +""" +import copy +import json +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +import httpx + +import litellm +from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +from .transformation import AzureAnthropicConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as CustomStreamWrapperType + from litellm.llms.base_llm.chat.transformation import BaseConfig + + +class AzureAnthropicChatCompletion(AnthropicChatCompletion): + """ + Azure Anthropic chat completion handler. + Reuses all Anthropic logic but with Azure authentication. + """ + + def __init__(self) -> None: + super().__init__() + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_llm_provider: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + optional_params: dict, + timeout: Union[float, httpx.Timeout], + litellm_params: dict, + acompletion=None, + logger_fn=None, + headers={}, + client=None, + ): + """ + Completion method that uses Azure authentication instead of Anthropic's x-api-key. + All other logic is the same as AnthropicChatCompletion. + """ + from litellm.utils import ProviderConfigManager + + optional_params = copy.deepcopy(optional_params) + stream = optional_params.pop("stream", None) + json_mode: bool = optional_params.pop("json_mode", False) + is_vertex_request: bool = optional_params.pop("is_vertex_request", False) + _is_function_call = False + messages = copy.deepcopy(messages) + + # Use AzureAnthropicConfig instead of AnthropicConfig + headers = AzureAnthropicConfig().validate_environment( + api_key=api_key, + headers=headers, + model=model, + messages=messages, + optional_params={**optional_params, "is_vertex_request": is_vertex_request}, + litellm_params=litellm_params, + ) + + config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=litellm.types.utils.LlmProviders(custom_llm_provider), + ) + if config is None: + raise ValueError( + f"Provider config not found for model: {model} and provider: {custom_llm_provider}" + ) + + data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + print_verbose(f"_is_function_call: {_is_function_call}") + if acompletion is True: + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + print_verbose("makes async azure anthropic streaming POST request") + data["stream"] = stream + return self.acompletion_stream_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + json_mode=json_mode, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=( + client + if client is not None and isinstance(client, AsyncHTTPHandler) + else None + ), + ) + else: + return self.acompletion_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + provider_config=config, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + client=client, + json_mode=json_mode, + timeout=timeout, + ) + else: + ## COMPLETION CALL + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + data["stream"] = stream + # Import the make_sync_call from parent + from litellm.llms.anthropic.chat.handler import make_sync_call + + completion_stream, response_headers = make_sync_call( + client=client, + api_base=api_base, + headers=headers, # type: ignore + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + timeout=timeout, + json_mode=json_mode, + ) + from litellm.llms.anthropic.common_utils import process_anthropic_headers + + return CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="azure_anthropic", + logging_obj=logging_obj, + _response_headers=process_anthropic_headers(response_headers), + ) + + else: + if client is None or not isinstance(client, HTTPHandler): + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client(params={"timeout": timeout}) + else: + client = client + + try: + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout, + ) + except Exception as e: + from litellm.llms.anthropic.common_utils import AnthropicError + + status_code = getattr(e, "status_code", 500) + error_headers = getattr(e, "headers", None) + error_text = getattr(e, "text", str(e)) + error_response = getattr(e, "response", None) + if error_headers is None and error_response: + error_headers = getattr(error_response, "headers", None) + if error_response and hasattr(error_response, "text"): + error_text = getattr(error_response, "text", error_text) + raise AnthropicError( + message=error_text, + status_code=status_code, + headers=error_headers, + ) + + return config.transform_response( + model=model, + raw_response=response, + model_response=model_response, + logging_obj=logging_obj, + api_key=api_key, + request_data=data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + json_mode=json_mode, + ) + diff --git a/litellm/llms/azure/anthropic/transformation.py b/litellm/llms/azure/anthropic/transformation.py new file mode 100644 index 00000000000..81beeb74ae1 --- /dev/null +++ b/litellm/llms/azure/anthropic/transformation.py @@ -0,0 +1,96 @@ +""" +Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication +""" +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import litellm +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + pass + + +class AzureAnthropicConfig(AnthropicConfig): + """ + Azure Anthropic configuration that extends AnthropicConfig. + The only difference is authentication - Azure uses api-key header or Azure AD token + instead of x-api-key header. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "azure_anthropic" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: Union[dict, GenericLiteLLMParams], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Dict: + """ + Validate environment and set up Azure authentication headers. + Azure supports: + 1. API key via 'api-key' header + 2. Azure AD token via 'Authorization: Bearer ' header + """ + # Convert dict to GenericLiteLLMParams if needed + if isinstance(litellm_params, dict): + # Ensure api_key is included if provided + if api_key and "api_key" not in litellm_params: + litellm_params = {**litellm_params, "api_key": api_key} + litellm_params_obj = GenericLiteLLMParams(**litellm_params) + else: + litellm_params_obj = litellm_params or GenericLiteLLMParams() + # Set api_key if provided and not already set + if api_key and not litellm_params_obj.api_key: + litellm_params_obj.api_key = api_key + + # Use Azure authentication logic + headers = BaseAzureLLM._base_validate_azure_environment( + headers=headers, litellm_params=litellm_params_obj + ) + + # Get tools and other anthropic-specific setup + tools = optional_params.get("tools") + prompt_caching_set = self.is_cache_control_set(messages=messages) + computer_tool_used = self.is_computer_tool_used(tools=tools) + mcp_server_used = self.is_mcp_server_used( + mcp_servers=optional_params.get("mcp_servers") + ) + pdf_used = self.is_pdf_used(messages=messages) + file_id_used = self.is_file_id_used(messages=messages) + user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ) + + # Get anthropic headers (but we'll replace x-api-key with Azure auth) + anthropic_headers = self.get_anthropic_headers( + computer_tool_used=computer_tool_used, + prompt_caching_set=prompt_caching_set, + pdf_used=pdf_used, + api_key=api_key or "", # Azure auth is already in headers + file_id_used=file_id_used, + is_vertex_request=optional_params.get("is_vertex_request", False), + user_anthropic_beta_headers=user_anthropic_beta_headers, + mcp_server_used=mcp_server_used, + ) + + # Remove x-api-key from anthropic headers since Azure uses different auth + anthropic_headers.pop("x-api-key", None) + + # Merge headers - Azure auth (api-key or Authorization) takes precedence + headers = {**anthropic_headers, **headers} + + # Ensure anthropic-version header is set + if "anthropic-version" not in headers: + headers["anthropic-version"] = "2023-06-01" + + return headers + diff --git a/litellm/main.py b/litellm/main.py index 88c3f7bc55b..4857f1b9754 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -152,6 +152,7 @@ from .litellm_core_utils.prompt_templates.factory import ( ) from .litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.azure.anthropic.handler import AzureAnthropicChatCompletion from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, _check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -253,6 +254,7 @@ openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions = GroqChatCompletion() azure_ai_embedding = AzureAIEmbedding() anthropic_chat_completions = AnthropicChatCompletion() +azure_anthropic_chat_completions = AzureAnthropicChatCompletion() azure_chat_completions = AzureChatCompletion() azure_o1_chat_completions = AzureOpenAIO1ChatCompletion() azure_text_completions = AzureTextCompletion() @@ -2353,6 +2355,62 @@ def completion( # type: ignore # noqa: PLR0915 original_response=response, ) response = response + elif custom_llm_provider == "azure_anthropic": + # Azure Anthropic uses same API as Anthropic but with Azure authentication + api_key = ( + api_key + or litellm.azure_key + or litellm.api_key + or get_secret("AZURE_API_KEY") + or get_secret("AZURE_OPENAI_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # Azure Foundry endpoint format: https://.services.ai.azure.com/anthropic/v1/messages + api_base = ( + api_base + or litellm.api_base + or get_secret("AZURE_API_BASE") + ) + + if api_base is None: + raise ValueError( + "Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. " + "Expected format: https://.services.ai.azure.com/anthropic" + ) + + # Ensure the URL ends with /v1/messages + if not api_base.endswith("/v1/messages"): + if not api_base.endswith("/anthropic"): + api_base = api_base.rstrip("/") + "/anthropic" + api_base = api_base.rstrip("/") + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response elif custom_llm_provider == "nlp_cloud": nlp_cloud_key = ( api_key diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9bef6ebef7d..3c8fa885f67 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2545,6 +2545,7 @@ class LlmProviders(str, Enum): AZURE = "azure" AZURE_TEXT = "azure_text" AZURE_AI = "azure_ai" + AZURE_ANTHROPIC = "azure_anthropic" SAGEMAKER = "sagemaker" SAGEMAKER_CHAT = "sagemaker_chat" BEDROCK = "bedrock" diff --git a/litellm/utils.py b/litellm/utils.py index 47fd06a9f27..e114e4cc051 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7148,6 +7148,8 @@ class ProviderConfigManager: return litellm.AzureAIStudioConfig() elif litellm.LlmProviders.AZURE_TEXT == provider: return litellm.AzureOpenAITextConfig() + elif litellm.LlmProviders.AZURE_ANTHROPIC == provider: + return litellm.AzureAnthropicConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMChatConfig() elif litellm.LlmProviders.NLP_CLOUD == provider: diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py new file mode 100644 index 00000000000..34aed42478d --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py @@ -0,0 +1,216 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.handler import AzureAnthropicChatCompletion +from litellm.types.utils import ModelResponse + + +class TestAzureAnthropicChatCompletion: + def test_inherits_from_anthropic_chat_completion(self): + """Test that AzureAnthropicChatCompletion inherits from AnthropicChatCompletion""" + handler = AzureAnthropicChatCompletion() + assert isinstance(handler, AzureAnthropicChatCompletion) + # Check that it has methods from parent class + assert hasattr(handler, "acompletion_function") + assert hasattr(handler, "acompletion_stream_function") + + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_uses_azure_anthropic_config(self, mock_azure_config, mock_provider_manager): + """Test that completion method uses AzureAnthropicConfig""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = {"model": "claude-sonnet-4-5", "messages": []} + mock_config.transform_response.return_value = ModelResponse() + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + with patch.object( + handler, "acompletion_function", return_value=ModelResponse() + ) as mock_acompletion: + handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=True, + ) + + # Verify AzureAnthropicConfig was used + mock_azure_config.assert_called_once() + mock_config_instance.validate_environment.assert_called_once() + + @patch("litellm.llms.anthropic.chat.handler.make_sync_call") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_streaming(self, mock_azure_config, mock_provider_manager, mock_make_sync_call): + # Note: decorators are applied in reverse order + """Test completion with streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + "stream": True, + } + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + # Mock streaming response + mock_stream = MagicMock() + mock_headers = MagicMock() + mock_make_sync_call.return_value = (mock_stream, mock_headers) + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {"stream": True} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=False, + ) + + # Verify streaming was handled + mock_make_sync_call.assert_called_once() + assert result is not None + + @patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_non_streaming(self, mock_azure_config, mock_provider_manager, mock_get_client): + # Note: decorators are applied in reverse order + """Test completion without streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + } + mock_response = ModelResponse() + mock_config.transform_response.return_value = mock_response + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + # Mock HTTP client + mock_client = MagicMock() + mock_response_obj = MagicMock() + mock_response_obj.status_code = 200 + mock_response_obj.text = json.dumps({ + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + mock_response_obj.json.return_value = { + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + mock_client.post.return_value = mock_response_obj + mock_get_client.return_value = mock_client + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + client=None, # Let it create the client + acompletion=False, + ) + + # Verify non-streaming was handled + mock_client.post.assert_called_once() + assert result is not None + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py new file mode 100644 index 00000000000..a7aa2983175 --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py @@ -0,0 +1,82 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import pytest + +from litellm.litellm_core_utils.get_llm_provider_logic import _is_azure_anthropic_model, get_llm_provider + + +class TestAzureAnthropicProviderRouting: + def test_is_azure_anthropic_model_with_claude(self): + """Test _is_azure_anthropic_model detects Claude models""" + # Test various Claude model names + assert _is_azure_anthropic_model("azure/claude-sonnet-4-5") == "claude-sonnet-4-5" + assert _is_azure_anthropic_model("azure/claude-opus-4-1") == "claude-opus-4-1" + assert _is_azure_anthropic_model("azure/claude-haiku-4-5") == "claude-haiku-4-5" + assert _is_azure_anthropic_model("azure/claude-3-5-sonnet") == "claude-3-5-sonnet" + assert _is_azure_anthropic_model("azure/claude-3-opus") == "claude-3-opus" + + def test_is_azure_anthropic_model_case_insensitive(self): + """Test _is_azure_anthropic_model is case insensitive""" + assert _is_azure_anthropic_model("azure/CLAUDE-sonnet-4-5") == "CLAUDE-sonnet-4-5" + assert _is_azure_anthropic_model("azure/Claude-Sonnet-4-5") == "Claude-Sonnet-4-5" + + def test_is_azure_anthropic_model_with_non_claude(self): + """Test _is_azure_anthropic_model returns None for non-Claude models""" + assert _is_azure_anthropic_model("azure/gpt-4") is None + assert _is_azure_anthropic_model("azure/gpt-35-turbo") is None + assert _is_azure_anthropic_model("azure/command-r-plus") is None + + def test_is_azure_anthropic_model_with_invalid_format(self): + """Test _is_azure_anthropic_model handles invalid formats""" + assert _is_azure_anthropic_model("azure") is None + assert _is_azure_anthropic_model("claude-sonnet-4-5") is None + assert _is_azure_anthropic_model("") is None + + def test_get_llm_provider_routes_azure_claude_to_azure_anthropic(self): + """Test that get_llm_provider routes azure/claude-* models to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-sonnet-4-5" # Should strip "azure/" prefix + + def test_get_llm_provider_routes_azure_claude_opus(self): + """Test routing for Claude Opus models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-opus-4-1" + ) + assert provider == "azure_anthropic" + assert model == "claude-opus-4-1" + + def test_get_llm_provider_routes_azure_claude_haiku(self): + """Test routing for Claude Haiku models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-haiku-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-haiku-4-5" + + def test_get_llm_provider_does_not_route_non_claude_azure_models(self): + """Test that non-Claude Azure models are not routed to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/gpt-4" + ) + assert provider != "azure_anthropic" + # Should be routed to regular azure provider + assert provider == "azure" or provider == "openai" + + def test_get_llm_provider_with_custom_llm_provider_override(self): + """Test that custom_llm_provider parameter can override routing""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5", custom_llm_provider="azure" + ) + # When custom_llm_provider is explicitly set, it should be respected + # But the routing logic should still detect it as azure_anthropic + # This depends on the order of checks in get_llm_provider + assert provider in ["azure_anthropic", "azure"] + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py new file mode 100644 index 00000000000..43f0b5c439d --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py @@ -0,0 +1,191 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import pytest +from unittest.mock import MagicMock, patch + +from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig +from litellm.types.router import GenericLiteLLMParams + + +class TestAzureAnthropicConfig: + def test_custom_llm_provider(self): + """Test that custom_llm_provider returns 'azure_anthropic'""" + config = AzureAnthropicConfig() + assert config.custom_llm_provider == "azure_anthropic" + + def test_validate_environment_with_dict_litellm_params(self): + """Test validate_environment with dict litellm_params""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that dict was converted to GenericLiteLLMParams + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert call_args[1]["litellm_params"].api_key == "test-api-key" + assert "anthropic-version" in result + + def test_validate_environment_with_generic_litellm_params(self): + """Test validate_environment with GenericLiteLLMParams object""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = GenericLiteLLMParams(api_key="test-api-key") + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that GenericLiteLLMParams was passed through + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert "anthropic-version" in result + + def test_validate_environment_sets_api_key_in_litellm_params(self): + """Test that api_key parameter is set in litellm_params if provided""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {} # Empty dict, no api_key + api_key = "provided-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "provided-api-key"} + config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that api_key was set in litellm_params + call_args = mock_validate.call_args + assert call_args[1]["litellm_params"].api_key == "provided-api-key" + + def test_validate_environment_removes_x_api_key(self): + """Test that x-api-key header is removed (Azure uses api-key instead)""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object( + config, "get_anthropic_headers", return_value={"x-api-key": "should-be-removed"} + ): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Verify x-api-key was removed + assert "x-api-key" not in result + assert "api-key" in result + + def test_validate_environment_sets_anthropic_version(self): + """Test that anthropic-version header is set""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object(config, "get_anthropic_headers", return_value={}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2023-06-01" + + def test_validate_environment_preserves_existing_anthropic_version(self): + """Test that existing anthropic-version header is preserved""" + config = AzureAnthropicConfig() + headers = {"anthropic-version": "2024-01-01"} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key", "anthropic-version": "2024-01-01"} + with patch.object(config, "get_anthropic_headers", return_value={"anthropic-version": "2024-01-01"}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2024-01-01" + + def test_inherits_anthropic_config_methods(self): + """Test that AzureAnthropicConfig inherits methods from AnthropicConfig""" + config = AzureAnthropicConfig() + + # Test that it has AnthropicConfig methods + assert hasattr(config, "get_anthropic_headers") + assert hasattr(config, "is_cache_control_set") + assert hasattr(config, "is_computer_tool_used") + assert hasattr(config, "transform_request") + assert hasattr(config, "transform_response") + From 3e58fe42b7a4e9ddf334525e18677714d68d3af2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 19 Nov 2025 17:07:46 -0300 Subject: [PATCH 02/68] fix: Support response_format parameter in completion -> responses bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #16810 ## Problem When using completion() with models that have mode: "responses" (like o3-pro, gpt-5-codex), the response_format parameter with JSON schemas was being ignored or incorrectly handled, causing: - Large schemas (>512 chars) to fail with "metadata.schema_dict_json: string too long" error - Structured outputs to be silently dropped - Users' code to break unexpectedly ## Root Cause The completion -> responses bridge in litellm/completion_extras/litellm_responses_transformation/transformation.py was missing the conversion of response_format (Chat Completion format) to text.format (Responses API format). The inverse bridge (responses -> completion) already had this conversion implemented in commit 29f0ed223a, but the completion -> responses direction was incomplete. ## Solution Added _transform_response_format_to_text_format() method that converts: - response_format with json_schema → text.format with json_schema - response_format with json_object → text.format with json_object - response_format with text → text.format with text Updated transform_request() to detect and convert response_format parameter before sending to litellm.responses(). ## Changes - Added _transform_response_format_to_text_format() method (lines 592-647) - Modified transform_request() to handle response_format (lines 199-203) - Added comprehensive tests to validate the conversion ## Testing - 5 new unit tests covering all conversion scenarios - Real API test with OpenAI confirming large schemas (>512 chars) work - No more metadata.schema_dict_json errors ## Impact Users can now use completion() with models that have mode: "responses" and: - Use large JSON schemas without hitting metadata 512 char limit - Get proper structured outputs - Have their existing code continue working --- .../transformation.py | 62 ++++++++ ...responses_transformation_transformation.py | 136 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 54d32fe3466..a6dbd5d0dcd 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -196,6 +196,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(List[Dict[str, Any]], value) ) ) + elif key == "response_format": + # Convert response_format to text.format + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key in ("metadata"): @@ -589,6 +594,63 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return Reasoning(effort="minimal") return None + def _transform_response_format_to_text_format( + self, response_format: Union[Dict[str, Any], Any] + ) -> Optional[Dict[str, Any]]: + """ + Transform Chat Completion response_format parameter to Responses API text.format parameter. + + Chat Completion response_format structure: + { + "type": "json_schema", + "json_schema": { + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + + Responses API text parameter structure: + { + "format": { + "type": "json_schema", + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + """ + if not response_format: + return None + + if isinstance(response_format, dict): + format_type = response_format.get("type") + + if format_type == "json_schema": + json_schema = response_format.get("json_schema", {}) + return { + "format": { + "type": "json_schema", + "name": json_schema.get("name", "response_schema"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", False), + } + } + elif format_type == "json_object": + return { + "format": { + "type": "json_object" + } + } + elif format_type == "text": + return { + "format": { + "type": "text" + } + } + + return None + def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str: """Map responses API status to chat completion finish_reason""" if not status: diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py new file mode 100644 index 00000000000..adbaf219079 --- /dev/null +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -0,0 +1,136 @@ +""" +Test for response_format to text.format conversion in completion -> responses bridge +""" +import pytest +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) + + +def test_transform_response_format_to_text_format_json_schema(): + """Test conversion of response_format with json_schema to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + # Chat Completion format + response_format = { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + + # Convert to Responses API format + result = handler._transform_response_format_to_text_format(response_format) + + # Verify conversion + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_schema" + assert result["format"]["name"] == "person_schema" + assert result["format"]["strict"] is True + assert "schema" in result["format"] + assert result["format"]["schema"]["type"] == "object" + assert "properties" in result["format"]["schema"] + + +def test_transform_response_format_to_text_format_json_object(): + """Test conversion of response_format with json_object to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "json_object" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_object" + + +def test_transform_response_format_to_text_format_text(): + """Test conversion of response_format with text to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "text" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "text" + + +def test_transform_response_format_to_text_format_none(): + """Test that None input returns None""" + handler = LiteLLMResponsesTransformationHandler() + + result = handler._transform_response_format_to_text_format(None) + + assert result is None + + +def test_transform_request_with_response_format(): + """Test that transform_request correctly handles response_format parameter""" + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + {"role": "user", "content": "Extract person info: John Doe, 30 years old"} + ] + + optional_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + } + + litellm_params = {} + headers = {} + + # Mock logging object + class MockLoggingObj: + pass + + litellm_logging_obj = MockLoggingObj() + + result = handler.transform_request( + model="o3-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + litellm_logging_obj=litellm_logging_obj, + ) + + # Verify that text parameter was set with converted format + assert "text" in result + assert result["text"] is not None + assert "format" in result["text"] + assert result["text"]["format"]["type"] == "json_schema" + assert result["text"]["format"]["name"] == "person_schema" + assert "schema" in result["text"]["format"] From f5a3349fefe2e943e4ba063268a8089b9c92a247 Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Tue, 25 Nov 2025 10:13:07 +0900 Subject: [PATCH 03/68] feat: add UI support for configuring tool permission guardrails (#17050) * feat: add UI support for configuring tool permission guardrails * chore: rename UI/Docs references to "LiteLLM Tool Permission Guardrail" --- .../docs/proxy/guardrails/tool_permission.md | 4 +- .../proxy/guardrails/guardrail_endpoints.py | 99 +++--- .../guardrail_hooks/tool_permission.py | 15 +- .../guardrails/guardrail_initializers.py | 13 +- litellm/types/guardrails.py | 15 +- .../guardrail_hooks/tool_permission.py | 24 +- .../guardrails/add_guardrail_form.tsx | 75 +++- .../components/guardrails/guardrail_info.tsx | 140 ++++++-- .../guardrails/guardrail_info_helpers.tsx | 1 + .../ToolPermissionRulesEditor.test.tsx | 65 ++++ .../ToolPermissionRulesEditor.tsx | 322 ++++++++++++++++++ 11 files changed, 675 insertions(+), 98 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.tsx diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 2e0b72a8a8a..19b674c9e55 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -2,9 +2,9 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Tool Permission Guardrail +# LiteLLM Tool Permission Guardrail -LiteLLM provides a Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools). +LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools). ## Quick Start ### 1. Define Guardrails on your LiteLLM config.yaml diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index e64fbe9084e..a1cfead9bb2 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -31,6 +31,7 @@ from litellm.types.guardrails import ( PiiEntityType, PresidioPresidioConfigModelUserInterface, SupportedGuardrailIntegrations, + ToolPermissionGuardrailConfigModel, ) #### GUARDRAILS ENDPOINTS #### @@ -635,7 +636,9 @@ async def get_guardrail_info(guardrail_id: str): raise HTTPException(status_code=500, detail="Prisma client not initialized") try: - guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = GUARDRAIL_DEFINITION_LOCATION.DB + guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = ( + GUARDRAIL_DEFINITION_LOCATION.DB + ) result = await GUARDRAIL_REGISTRY.get_guardrail_by_id_from_db( guardrail_id=guardrail_id, prisma_client=prisma_client ) @@ -702,10 +705,12 @@ async def get_guardrail_ui_settings(): # Convert the PII_ENTITY_CATEGORIES_MAP to the format expected by the UI category_maps = [] for category, entities in PII_ENTITY_CATEGORIES_MAP.items(): - category_maps.append({ - "category": category.value, - "entities": [entity.value for entity in entities] - }) + category_maps.append( + { + "category": category.value, + "entities": [entity.value for entity in entities], + } + ) return GuardrailUIAddGuardrailSettings( supported_entities=[entity.value for entity in PiiEntityType], @@ -728,20 +733,20 @@ async def get_guardrail_ui_settings(): async def validate_blocked_words_file(request: Dict[str, str]): """ Validate a blocked_words YAML file content. - + Args: request: Dictionary with 'file_content' key containing the YAML string - + Returns: Dictionary with 'valid' boolean and either 'message'/'errors' depending on result - + Example Request: ```json { "file_content": "blocked_words:\\n - keyword: \\"test\\"\\n action: \\"BLOCK\\"" } ``` - + Example Success Response: ```json { @@ -749,7 +754,7 @@ async def validate_blocked_words_file(request: Dict[str, str]): "message": "Valid YAML file with 2 blocked words" } ``` - + Example Error Response: ```json { @@ -759,56 +764,54 @@ async def validate_blocked_words_file(request: Dict[str, str]): ``` """ import yaml - + try: file_content = request.get("file_content", "") if not file_content: - return { - "valid": False, - "error": "No file content provided" - } - + return {"valid": False, "error": "No file content provided"} + data = yaml.safe_load(file_content) - + if not isinstance(data, dict) or "blocked_words" not in data: return { "valid": False, - "error": "Invalid format: file must contain 'blocked_words' key with a list" + "error": "Invalid format: file must contain 'blocked_words' key with a list", } - + blocked_words_list = data["blocked_words"] if not isinstance(blocked_words_list, list): - return { - "valid": False, - "error": "'blocked_words' must be a list" - } - + return {"valid": False, "error": "'blocked_words' must be a list"} + # Validate each entry errors = [] for idx, word_data in enumerate(blocked_words_list): if not isinstance(word_data, dict): errors.append(f"Entry {idx}: must be an object") continue - + if "keyword" not in word_data: errors.append(f"Entry {idx}: missing 'keyword' field") elif not isinstance(word_data["keyword"], str): errors.append(f"Entry {idx}: 'keyword' must be a string") - + if "action" not in word_data: errors.append(f"Entry {idx}: missing 'action' field") elif word_data["action"] not in ["BLOCK", "MASK"]: - errors.append(f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'") - - if "description" in word_data and not isinstance(word_data["description"], str): + errors.append( + f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'" + ) + + if "description" in word_data and not isinstance( + word_data["description"], str + ): errors.append(f"Entry {idx}: 'description' must be a string") - + if errors: return {"valid": False, "errors": errors} - + return { "valid": True, - "message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)" + "message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)", } except yaml.YAMLError as e: return {"valid": False, "error": f"Invalid YAML syntax: {str(e)}"} @@ -931,30 +934,32 @@ def _should_skip_optional_params(field_name: str, field_annotation: Any) -> bool """Check if optional_params field should be skipped (not meaningfully overridden).""" if field_name != "optional_params": return False - + if field_annotation is None: return True - + # Check if the annotation is still a generic TypeVar (not specialized) if isinstance(field_annotation, TypeVar) or ( hasattr(field_annotation, "__origin__") and field_annotation.__origin__ is TypeVar ): return True - + # Also skip if it's a generic type that wasn't specialized if hasattr(field_annotation, "__name__") and field_annotation.__name__ in ( "T", "TypeVar", ): return True - + # Handle Optional[T] where T is still a TypeVar if hasattr(field_annotation, "__args__"): - non_none_args = [arg for arg in field_annotation.__args__ if arg is not type(None)] + non_none_args = [ + arg for arg in field_annotation.__args__ if arg is not type(None) + ] if non_none_args and isinstance(non_none_args[0], TypeVar): return True - + return False @@ -1041,9 +1046,11 @@ def _extract_fields_recursive( for field_name, field in model.model_fields.items(): field_annotation = field.annotation - + # Skip optional_params if it's not meaningfully overridden - if _should_skip_optional_params(field_name=field_name, field_annotation=field_annotation): + if _should_skip_optional_params( + field_name=field_name, field_annotation=field_annotation + ): continue # Handle Optional types and get the actual type @@ -1153,12 +1160,18 @@ async def get_provider_specific_params(): bedrock_fields = _get_fields_from_model(BedrockGuardrailConfigModel) presidio_fields = _get_fields_from_model(PresidioPresidioConfigModelUserInterface) lakera_v2_fields = _get_fields_from_model(LakeraV2GuardrailConfigModel) + tool_permission_fields = _get_fields_from_model(ToolPermissionGuardrailConfigModel) + + tool_permission_fields[ + "ui_friendly_name" + ] = ToolPermissionGuardrailConfigModel.ui_friendly_name() # Return the provider-specific parameters provider_params = { SupportedGuardrailIntegrations.BEDROCK.value: bedrock_fields, SupportedGuardrailIntegrations.PRESIDIO.value: presidio_fields, SupportedGuardrailIntegrations.LAKERA_V2.value: lakera_v2_fields, + SupportedGuardrailIntegrations.TOOL_PERMISSION.value: tool_permission_fields, } ### get the config model for the guardrail - go through the registry and get the config model for the guardrail @@ -1175,6 +1188,7 @@ async def get_provider_specific_params(): return provider_params + @router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse) @router.post("/apply_guardrail", response_model=ApplyGuardrailResponse) async def apply_guardrail( @@ -1183,11 +1197,11 @@ async def apply_guardrail( ): """ Apply a guardrail to text input and return the processed result. - + This endpoint allows testing guardrails by applying them to custom text inputs. """ from litellm.proxy.utils import handle_exception_on_proxy - + try: active_guardrail: Optional[ CustomGuardrail @@ -1207,4 +1221,3 @@ async def apply_guardrail( return ApplyGuardrailResponse(response_text=response_text) except Exception as e: raise handle_exception_on_proxy(e) - diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 95d8f894dc2..eef8043b237 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -62,8 +62,11 @@ class ToolPermissionGuardrail(CustomGuardrail): self.rules: List[ToolPermissionRule] = [] self._compiled_rule_patterns: Dict[str, Dict[str, re.Pattern]] = {} if rules: - for rule_dict in rules: - rule = ToolPermissionRule(**rule_dict) + for rule_item in rules: + if isinstance(rule_item, ToolPermissionRule): + rule = rule_item + else: + rule = ToolPermissionRule(**rule_item) self.rules.append(rule) if rule.allowed_param_patterns: @@ -88,6 +91,14 @@ class ToolPermissionGuardrail(CustomGuardrail): self.default_action, ) + @staticmethod + def get_config_model(): + from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( + ToolPermissionGuardrailConfigModel, + ) + + return ToolPermissionGuardrailConfigModel + def _matches_pattern(self, tool_name: str, pattern: str) -> bool: """ Check if a tool name matches a pattern diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index f2083e9c67e..9bb965ef14e 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -1,4 +1,6 @@ # litellm/proxy/guardrails/guardrail_initializers.py +from typing import Any, Dict, List, Optional + import litellm from litellm.proxy._types import CommonProxyErrors from litellm.types.guardrails import * @@ -128,10 +130,19 @@ def initialize_tool_permission(litellm_params: LitellmParams, guardrail: Guardra ToolPermissionGuardrail, ) + rules: Optional[List[Dict[str, Any]]] = None + if litellm_params.rules: + rules = [] + for rule in litellm_params.rules: + if hasattr(rule, "model_dump"): + rules.append(rule.model_dump()) + else: + rules.append(dict(rule)) + _tool_permission_callback = ToolPermissionGuardrail( guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, - rules=litellm_params.rules, + rules=rules, default_action=getattr(litellm_params, "default_action", "deny"), on_disallowed_action=getattr(litellm_params, "on_disallowed_action", "block"), default_on=litellm_params.default_on, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 5f6295e151f..24a235def59 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -14,6 +14,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.grayswan import ( from litellm.types.proxy.guardrails.guardrail_hooks.ibm import ( IBMGuardrailsBaseConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( + ToolPermissionGuardrailConfigModel, +) """ @@ -415,18 +418,6 @@ class NomaGuardrailConfigModel(BaseModel): ) -class ToolPermissionGuardrailConfigModel(BaseModel): - """Configuration parameters for the Tool Permission guardrail""" - - rules: Optional[List[Dict]] = Field( - default=None, description="List of permission rules for tool usage" - ) - default_action: Optional[str] = Field( - default="Deny", - description="Default action when no rule matches (Allow or Deny)", - ) - - class ZscalerAIGuardConfigModel(BaseModel): """Configuration parameters for the Zscaler AI Guard guardrail""" diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py index b2248c51930..e78cfad8bdb 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -1,8 +1,10 @@ # Tool Permission Guardrail Type Definitions -from typing import Dict, Literal, Optional +from typing import Dict, List, Literal, Optional from pydantic import BaseModel, Field +from .base import GuardrailConfigModel + class ToolPermissionRule(BaseModel): """ @@ -43,3 +45,23 @@ class PermissionError(BaseModel): tool_name: str = Field(description="Name of the denied tool") rule_id: Optional[str] = Field(description="ID of the rule that caused denial") message: str = Field(description="Error message") + + +class ToolPermissionGuardrailConfigModel(GuardrailConfigModel): + """Configuration parameters exposed to the UI for the Tool Permission guardrail.""" + + rules: Optional[List[ToolPermissionRule]] = Field( + default=None, + description="Ordered allow/deny rules. Patterns support * wildcards and optional regex constraints on tool arguments.", + ) + default_action: Literal["allow", "deny"] = Field( + default="deny", description="Fallback decision when no rule matches" + ) + on_disallowed_action: Literal["block", "rewrite"] = Field( + default="block", + description="Choose whether disallowed tools block the request or get rewritten out of the payload", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "LiteLLM Tool Permission Guardrail" diff --git a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx index 55fe800a431..20ca36f6d16 100644 --- a/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useMemo } from "react"; import { Form, Typography, Select, Modal, Tag, Steps } from "antd"; import { Button, TextInput } from "@tremor/react"; import { @@ -16,6 +16,9 @@ import GuardrailProviderFields from "./guardrail_provider_fields"; import GuardrailOptionalParams from "./guardrail_optional_params"; import NotificationsManager from "../molecules/notifications_manager"; import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; +import ToolPermissionRulesEditor, { + ToolPermissionConfig, +} from "./tool_permission/ToolPermissionRulesEditor"; const { Title, Text, Link } = Typography; const { Option } = Select; @@ -100,6 +103,20 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a // Content Filter state const [selectedPatterns, setSelectedPatterns] = useState([]); const [blockedWords, setBlockedWords] = useState([]); + const [toolPermissionConfig, setToolPermissionConfig] = useState({ + rules: [], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", + }); + + const isToolPermissionProvider = useMemo(() => { + if (!selectedProvider) { + return false; + } + const providerValue = guardrail_provider_map[selectedProvider]; + return (providerValue || "").toLowerCase() === "tool_permission"; + }, [selectedProvider]); // Fetch guardrail UI settings + provider params on mount / accessToken change useEffect(() => { @@ -145,6 +162,13 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a setSelectedCategories([]); setGlobalSeverityThreshold(2); setCategorySpecificThresholds({}); + + setToolPermissionConfig({ + rules: [], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", + }); }; const handleEntitySelect = (entity: string) => { @@ -225,6 +249,14 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a setSelectedCategories([]); setGlobalSeverityThreshold(2); setCategorySpecificThresholds({}); + setSelectedPatterns([]); + setBlockedWords([]); + setToolPermissionConfig({ + rules: [], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", + }); setCurrentStep(0); }; @@ -315,6 +347,20 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a } } + if (guardrailProvider === "tool_permission") { + if (toolPermissionConfig.rules.length === 0) { + NotificationsManager.fromBackend("Add at least one tool permission rule"); + setLoading(false); + return; + } + guardrailData.litellm_params.rules = toolPermissionConfig.rules; + guardrailData.litellm_params.default_action = toolPermissionConfig.default_action; + guardrailData.litellm_params.on_disallowed_action = toolPermissionConfig.on_disallowed_action; + if (toolPermissionConfig.violation_message_template) { + guardrailData.litellm_params.violation_message_template = toolPermissionConfig.violation_message_template; + } + } + /****************************** * Add provider-specific params * ---------------------------------- @@ -535,11 +581,13 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a {/* Use the GuardrailProviderFields component to render provider-specific fields */} - + {!isToolPermissionProvider && ( + + )} ); }; @@ -593,7 +641,20 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a }; const renderOptionalParams = () => { - if (!selectedProvider || !providerParams) return null; + if (!selectedProvider) return null; + + if (isToolPermissionProvider) { + return ( + + ); + } + + if (!providerParams) { + return null; + } console.log("guardrail_provider_map: ", guardrail_provider_map); console.log("selectedProvider: ", selectedProvider); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx index fa34cf55636..0d1a205f66f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx @@ -26,6 +26,9 @@ import PiiConfiguration from "./pii_configuration"; import GuardrailProviderFields from "./guardrail_provider_fields"; import GuardrailOptionalParams from "./guardrail_optional_params"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; +import ToolPermissionRulesEditor, { + ToolPermissionConfig, +} from "./tool_permission/ToolPermissionRulesEditor"; import { ArrowLeftIcon } from "@heroicons/react/outline"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; import { CheckIcon, CopyIcon } from "lucide-react"; @@ -83,6 +86,14 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } | null>(null); const [copiedStates, setCopiedStates] = useState>({}); const [hasUnsavedContentFilterChanges, setHasUnsavedContentFilterChanges] = useState(false); + const emptyToolPermissionConfig: ToolPermissionConfig = { + rules: [], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", + }; + const [toolPermissionConfig, setToolPermissionConfig] = useState(emptyToolPermissionConfig); + const [toolPermissionDirty, setToolPermissionDirty] = useState(false); // Content Filter data ref (managed by ContentFilterManager) const contentFilterDataRef = React.useRef<{ patterns: any[]; blockedWords: any[] }>({ @@ -180,6 +191,29 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } }, [guardrailData, guardrailProviderSpecificParams, form]); + const resetToolPermissionEditor = useCallback(() => { + if (guardrailData?.litellm_params?.guardrail === "tool_permission") { + setToolPermissionConfig({ + rules: (guardrailData.litellm_params?.rules as ToolPermissionConfig["rules"]) || [], + default_action: ((guardrailData.litellm_params?.default_action || "deny") as ToolPermissionConfig["default_action"]).toLowerCase() as ToolPermissionConfig["default_action"], + on_disallowed_action: ((guardrailData.litellm_params?.on_disallowed_action || "block") as ToolPermissionConfig["on_disallowed_action"]).toLowerCase() as ToolPermissionConfig["on_disallowed_action"], + violation_message_template: guardrailData.litellm_params?.violation_message_template || "", + }); + } else { + setToolPermissionConfig(emptyToolPermissionConfig); + } + setToolPermissionDirty(false); + }, [guardrailData]); + + useEffect(() => { + resetToolPermissionEditor(); + }, [resetToolPermissionEditor]); + + const handleToolPermissionConfigChange = (config: ToolPermissionConfig) => { + setToolPermissionConfig(config); + setToolPermissionDirty(true); + }; + const handlePiiEntitySelect = (entity: string) => { setSelectedPiiEntities((prev) => { if (prev.includes(entity)) { @@ -255,6 +289,31 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, } } + if (guardrailData.litellm_params?.guardrail === "tool_permission") { + const originalRules = guardrailData.litellm_params?.rules || []; + const currentRules = toolPermissionConfig.rules || []; + const rulesChanged = JSON.stringify(originalRules) !== JSON.stringify(currentRules); + + const originalDefault = (guardrailData.litellm_params?.default_action || "deny").toLowerCase(); + const currentDefault = (toolPermissionConfig.default_action || "deny").toLowerCase(); + const defaultChanged = originalDefault !== currentDefault; + + const originalOnDisallowed = (guardrailData.litellm_params?.on_disallowed_action || "block").toLowerCase(); + const currentOnDisallowed = (toolPermissionConfig.on_disallowed_action || "block").toLowerCase(); + const onDisallowedChanged = originalOnDisallowed !== currentOnDisallowed; + + const originalMessage = guardrailData.litellm_params?.violation_message_template || ""; + const currentMessage = toolPermissionConfig.violation_message_template || ""; + const messageChanged = originalMessage !== currentMessage; + + if (toolPermissionDirty || rulesChanged || defaultChanged || onDisallowedChanged || messageChanged) { + updateData.litellm_params.rules = currentRules; + updateData.litellm_params.default_action = currentDefault; + updateData.litellm_params.on_disallowed_action = currentOnDisallowed; + updateData.litellm_params.violation_message_template = currentMessage || null; + } + } + /****************************** * Add provider-specific params (reusing logic from add_guardrail_form.tsx) * ---------------------------------- @@ -273,7 +332,8 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, console.log("currentProvider: ", currentProvider); // Use pre-fetched provider params to copy recognised params - if (guardrailProviderSpecificParams && currentProvider) { + const isToolPermissionGuardrail = guardrailData.litellm_params?.guardrail === "tool_permission"; + if (guardrailProviderSpecificParams && currentProvider && !isToolPermissionGuardrail) { const providerKey = guardrail_provider_map[currentProvider]?.toLowerCase(); const providerSpecificParams = guardrailProviderSpecificParams[providerKey] || {}; @@ -488,6 +548,12 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, )} + {guardrailData.litellm_params?.guardrail === "tool_permission" && ( + + + + )} + {/* Content Filter Configuration Display */} = ({ guardrailId, onClose, Provider Settings - {/* Provider-specific fields */} - guardrail_provider_map[key] === guardrailData.litellm_params?.guardrail, - ) || null - } - accessToken={accessToken} - providerParams={guardrailProviderSpecificParams} - value={guardrailData.litellm_params} - /> + {guardrailData.litellm_params?.guardrail === "tool_permission" ? ( + + ) : ( + <> + {/* Provider-specific fields */} + guardrail_provider_map[key] === guardrailData.litellm_params?.guardrail, + ) || null + } + accessToken={accessToken} + providerParams={guardrailProviderSpecificParams} + value={guardrailData.litellm_params} + /> - {/* Optional parameters */} - {guardrailProviderSpecificParams && - (() => { - const currentProvider = Object.keys(guardrail_provider_map).find( - (key) => guardrail_provider_map[key] === guardrailData.litellm_params?.guardrail, - ); - if (!currentProvider) return null; + {/* Optional parameters */} + {guardrailProviderSpecificParams && + (() => { + const currentProvider = Object.keys(guardrail_provider_map).find( + (key) => guardrail_provider_map[key] === guardrailData.litellm_params?.guardrail, + ); + if (!currentProvider) return null; - const providerKey = guardrail_provider_map[currentProvider]?.toLowerCase(); - const providerFields = guardrailProviderSpecificParams[providerKey]; + const providerKey = guardrail_provider_map[currentProvider]?.toLowerCase(); + const providerFields = guardrailProviderSpecificParams[providerKey]; - if (!providerFields || !providerFields.optional_params) return null; + if (!providerFields || !providerFields.optional_params) return null; - return ( - - ); - })()} + return ( + + ); + })()} + + )} Advanced Settings @@ -619,6 +694,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, onClick={() => { setIsEditing(false); setHasUnsavedContentFilterChanges(false); + resetToolPermissionEditor(); }} > Cancel @@ -672,6 +748,10 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, Last Updated
{formatDate(guardrailData.updated_at)}
+ + {guardrailData.litellm_params?.guardrail === "tool_permission" && ( + + )} )} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx index fa665d4911c..c6314c95bef 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx @@ -46,6 +46,7 @@ export const guardrail_provider_map: Record = { Bedrock: "bedrock", Lakera: "lakera_v2", LitellmContentFilter: "litellm_content_filter", + ToolPermission: "tool_permission", }; // Function to populate provider map from API response - updates the original map diff --git a/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.test.tsx b/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.test.tsx new file mode 100644 index 00000000000..59736694888 --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.test.tsx @@ -0,0 +1,65 @@ +import React from "react"; +import { describe, it, expect, vi } from "vitest"; +import { render, screen, waitFor, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import ToolPermissionRulesEditor, { + ToolPermissionConfig, +} from "./ToolPermissionRulesEditor"; + +describe("ToolPermissionRulesEditor", () => { + it("renders empty state and lets users add a new rule", async () => { + const onChange = vi.fn(); + render(); + + expect(screen.getByText(/No tool rules added yet/i)).toBeInTheDocument(); + + await userEvent.click(screen.getByRole("button", { name: /add rule/i })); + + expect(onChange).toHaveBeenCalled(); + const payload = onChange.mock.calls[0][0] as ToolPermissionConfig; + expect(payload.rules).toHaveLength(1); + expect(payload.rules[0].decision).toBe("allow"); + }); + + it("captures violation message and argument constraints", async () => { + let latestConfig: ToolPermissionConfig | null = null; + const initialConfig: ToolPermissionConfig = { + rules: [ + { + id: "allow_bash", + tool_name: "Bash", + decision: "allow", + }, + ], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", + }; + + const Wrapper = () => { + const [state, setState] = React.useState(initialConfig); + const handleChange = (next: ToolPermissionConfig) => { + latestConfig = next; + setState(next); + }; + return ; + }; + + render(); + + await userEvent.click(screen.getByRole("button", { name: /restrict tool arguments/i })); + const initialInput = await screen.findByPlaceholderText(/messages\[0\].content/i); + await userEvent.clear(initialInput); + fireEvent.change(initialInput, { target: { value: "input.location" } }); + + const violationArea = await screen.findByPlaceholderText(/violates our org policy/i); + await userEvent.clear(violationArea); + fireEvent.change(violationArea, { target: { value: "Do not run bash" } }); + + await waitFor(() => { + expect(latestConfig).not.toBeNull(); + expect(latestConfig?.rules[0].allowed_param_patterns).toEqual({ "input.location": "" }); + expect(latestConfig?.violation_message_template).toBe("Do not run bash"); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.tsx b/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.tsx new file mode 100644 index 00000000000..790876ed3f0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/tool_permission/ToolPermissionRulesEditor.tsx @@ -0,0 +1,322 @@ +import React from "react"; +import { Card, Text } from "@tremor/react"; +import { Button, Divider, Empty, Input, Select, Space, Tooltip } from "antd"; +import { InfoCircleOutlined, PlusOutlined, DeleteOutlined } from "@ant-design/icons"; + +export type ToolPermissionDecision = "allow" | "deny"; +export type ToolPermissionDefaultAction = "allow" | "deny"; +export type ToolPermissionOnDisallowedAction = "block" | "rewrite"; + +export interface ToolPermissionRuleConfig { + id: string; + tool_name: string; + decision: ToolPermissionDecision; + allowed_param_patterns?: Record; +} + +export interface ToolPermissionConfig { + rules: ToolPermissionRuleConfig[]; + default_action: ToolPermissionDefaultAction; + on_disallowed_action: ToolPermissionOnDisallowedAction; + violation_message_template?: string; +} + +interface ToolPermissionRulesEditorProps { + value?: ToolPermissionConfig; + onChange?: (config: ToolPermissionConfig) => void; + disabled?: boolean; +} + +const DEFAULT_CONFIG: ToolPermissionConfig = { + rules: [], + default_action: "deny", + on_disallowed_action: "block", + violation_message_template: "", +}; + +const ensureConfig = (config?: ToolPermissionConfig): ToolPermissionConfig => ({ + ...DEFAULT_CONFIG, + ...(config || {}), + rules: config?.rules ? [...config.rules] : [], +}); + +const ToolPermissionRulesEditor: React.FC = ({ + value, + onChange, + disabled = false, +}) => { + const config = ensureConfig(value); + + const updateConfig = (partial: Partial) => { + const nextConfig: ToolPermissionConfig = { + ...config, + ...partial, + }; + onChange?.(nextConfig); + }; + + const updateRule = (ruleIndex: number, updates: Partial) => { + const nextRules = config.rules.map((rule, index) => + index === ruleIndex ? { ...rule, ...updates } : rule, + ); + updateConfig({ rules: nextRules }); + }; + + const addRule = () => { + const nextRules = [ + ...config.rules, + { + id: `rule_${Math.random().toString(36).slice(2, 8)}`, + tool_name: "", + decision: "allow" as ToolPermissionDecision, + allowed_param_patterns: undefined, + }, + ]; + updateConfig({ rules: nextRules }); + }; + + const removeRule = (ruleIndex: number) => { + const nextRules = config.rules.filter((_, index) => index !== ruleIndex); + updateConfig({ rules: nextRules }); + }; + + const updateAllowedParamEntries = ( + ruleIndex: number, + mutate: (entries: [string, string][]) => void, + ) => { + const targetRule = config.rules[ruleIndex]; + if (!targetRule) { + return; + } + const entries = Object.entries(targetRule.allowed_param_patterns || {}); + mutate(entries); + const updatedObject: Record = {}; + entries.forEach(([key, value]) => { + updatedObject[key] = value; + }); + updateRule(ruleIndex, { + allowed_param_patterns: + Object.keys(updatedObject).length > 0 ? updatedObject : undefined, + }); + }; + + const updateAllowedParamPath = ( + ruleIndex: number, + entryIndex: number, + nextPath: string, + ) => { + updateAllowedParamEntries(ruleIndex, (entries) => { + if (!entries[entryIndex]) { + return; + } + const [, value] = entries[entryIndex]; + entries[entryIndex] = [nextPath, value]; + }); + }; + + const updateAllowedParamPattern = ( + ruleIndex: number, + entryIndex: number, + pattern: string, + ) => { + updateAllowedParamEntries(ruleIndex, (entries) => { + if (!entries[entryIndex]) { + return; + } + const [path] = entries[entryIndex]; + entries[entryIndex] = [path, pattern]; + }); + }; + + const renderAllowedParamPatterns = (rule: ToolPermissionRuleConfig, index: number) => { + const entries = Object.entries(rule.allowed_param_patterns || {}); + if (entries.length === 0) { + return ( + + ); + } + + return ( +
+ Argument constraints (dot or array paths) + {entries.map(([path, pattern], patternIndex) => ( + + updateAllowedParamPath(index, patternIndex, e.target.value)} + /> + updateAllowedParamPattern(index, patternIndex, e.target.value)} + /> + +
+ ); + }; + + return ( + +
+
+ LiteLLM Tool Permission Guardrail + + Use wildcards (e.g., mcp__github_*) to scope which tools can run and optionally constrain + payload fields. + +
+ {!disabled && ( + + )} +
+ + + + {config.rules.length === 0 ? ( + + ) : ( +
+ {config.rules.map((rule, index) => ( + +
+ Rule {index + 1} + +
+
+
+ Rule ID + updateRule(index, { id: e.target.value })} + /> +
+
+ Tool Name / Pattern + updateRule(index, { tool_name: e.target.value })} + /> +
+
+ +
+ Decision + +
+ +
{renderAllowedParamPatterns(rule, index)}
+ + ))} +
+ )} + + + +
+
+ Default action + +
+
+ + On disallowed action + + + + + +
+
+ +
+ Violation message (optional) + updateConfig({ violation_message_template: e.target.value })} + /> +
+ + ); +}; + +export default ToolPermissionRulesEditor; From aec65904862ad77da05138aa7be305359ece482c Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Mon, 24 Nov 2025 20:31:59 -0500 Subject: [PATCH 04/68] add strands tutorial (#17039) * add strands tutorial * configgg --- docs/my-website/docs/mcp.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 887c5278144..a9f7e249133 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -248,6 +248,41 @@ mcp_servers: X-Custom-Header: "some-value" ``` +### MCP Walkthroughs + +- **Strands (STDIO)** – [watch tutorial](https://screen.studio/share/ruv4D73F) + +> Add it from the UI + +```json title="strands-mcp" showLineNumbers +{ + "mcpServers": { + "strands-agents": { + "command": "uvx", + "args": ["strands-agents-mcp-server"], + "env": { + "FASTMCP_LOG_LEVEL": "INFO" + }, + "disabled": false, + "autoApprove": ["search_docs", "fetch_doc"] + } + } +} +``` + +> config.yml + +```yaml title="config.yml – strands MCP" showLineNumbers +mcp_servers: + strands_mcp: + transport: "stdio" + command: "uvx" + args: ["strands-agents-mcp-server"] + env: + FASTMCP_LOG_LEVEL: "INFO" +``` + + ### MCP Aliases You can define aliases for your MCP servers in the `litellm_settings` section. This allows you to: From 629404a10034c1642e9dd76403c85c0dad90576a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 07:09:26 +0530 Subject: [PATCH 05/68] Add cost tracking for cohere embed passthrough endpoint (#17029) * Add cost tracking for cohere embed passthrough endpoint * update passthrough code * update passthrough code * fixed lint and mypy errors --- .../cohere_passthrough_logging_handler.py | 138 +++++++++++++++- .../pass_through_endpoints/success_handler.py | 4 +- ...test_cohere_passthrough_logging_handler.py | 154 ++++++++++++++++++ 3 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py index a8228de6e01..743f4e4f96a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py @@ -1,14 +1,30 @@ +from datetime import datetime from typing import List, Optional, Union +import httpx + +import litellm from litellm import stream_chunk_builder from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig from litellm.llms.cohere.common_utils import ( ModelResponseIterator as CohereModelResponseIterator, ) -from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse +from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) +from litellm.types.utils import ( + LlmProviders, + ModelResponse, + TextCompletionResponse, +) from .base_passthrough_logging_handler import BasePassthroughLoggingHandler @@ -54,3 +70,123 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler): break complete_streaming_response = stream_chunk_builder(chunks=all_openai_chunks) return complete_streaming_response + + def cohere_passthrough_handler( # noqa: PLR0915 + self, + httpx_response: httpx.Response, + response_body: dict, + logging_obj: LiteLLMLoggingObj, + url_route: str, + result: str, + start_time: datetime, + end_time: datetime, + cache_hit: bool, + request_body: dict, + **kwargs, + ) -> PassThroughEndpointLoggingTypedDict: + """ + Handle Cohere passthrough logging with route detection and cost tracking. + """ + # Check if this is an embed endpoint + if "/v1/embed" in url_route: + model = request_body.get("model", response_body.get("model", "")) + try: + cohere_embed_config = CohereEmbeddingConfig() + litellm_model_response = litellm.EmbeddingResponse() + handler_instance = CoherePassthroughLoggingHandler() + + input_texts = request_body.get("texts", []) + if not input_texts: + input_texts = request_body.get("input", []) + + # Transform the response + litellm_model_response = cohere_embed_config._transform_response( + response=httpx_response, + api_key="", + logging_obj=logging_obj, + data=request_body, + model_response=litellm_model_response, + model=model, + encoding=litellm.encoding, + input=input_texts, + ) + + # Calculate cost using LiteLLM's cost calculator + response_cost = litellm.completion_cost( + completion_response=litellm_model_response, + model=model, + custom_llm_provider="cohere", + call_type="aembedding", + ) + + # Set the calculated cost in _hidden_params to prevent recalculation + if not hasattr(litellm_model_response, "_hidden_params"): + litellm_model_response._hidden_params = {} + litellm_model_response._hidden_params["response_cost"] = response_cost + + kwargs["response_cost"] = response_cost + kwargs["model"] = model + kwargs["custom_llm_provider"] = "cohere" + + # Extract user information for tracking + passthrough_logging_payload: Optional[ + PassthroughStandardLoggingPayload + ] = kwargs.get("passthrough_logging_payload") + if passthrough_logging_payload: + user = handler_instance._get_user_from_metadata( + passthrough_logging_payload=passthrough_logging_payload, + ) + if user: + kwargs.setdefault("litellm_params", {}) + kwargs["litellm_params"].update( + {"proxy_server_request": {"body": {"user": user}}} + ) + + # Create standard logging object + if litellm_model_response is not None: + get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=litellm_model_response, + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + status="success", + ) + + # Update logging object with cost information + logging_obj.model_call_details["model"] = model + logging_obj.model_call_details["custom_llm_provider"] = "cohere" + logging_obj.model_call_details["response_cost"] = response_cost + + return { + "result": litellm_model_response, + "kwargs": kwargs, + } + except Exception: + # For other routes (e.g., /v2/chat), fall back to chat handler + return super().passthrough_chat_handler( + httpx_response=httpx_response, + response_body=response_body, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) + + # For non-embed routes (e.g., /v2/chat), fall back to chat handler + return super().passthrough_chat_handler( + httpx_response=httpx_response, + response_body=response_body, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 6a0cfd44438..cc50d2c2d8e 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -48,7 +48,7 @@ class PassThroughEndpointLogging: self.TRACKED_ANTHROPIC_ROUTES = ["/messages"] # Cohere - self.TRACKED_COHERE_ROUTES = ["/v2/chat"] + self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"] self.assemblyai_passthrough_logging_handler = ( AssemblyAIPassthroughLoggingHandler() ) @@ -177,7 +177,7 @@ class PassThroughEndpointLogging: kwargs = anthropic_passthrough_logging_handler_result["kwargs"] elif self.is_cohere_route(url_route): cohere_passthrough_logging_handler_result = ( - cohere_passthrough_logging_handler.passthrough_chat_handler( + cohere_passthrough_logging_handler.cohere_passthrough_handler( httpx_response=httpx_response, response_body=response_body or {}, logging_obj=logging_obj, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py new file mode 100644 index 00000000000..0b6d3fdeced --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_cohere_passthrough_logging_handler.py @@ -0,0 +1,154 @@ +import json +import os +import sys +from datetime import datetime +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.cohere_passthrough_logging_handler import ( + CoherePassthroughLoggingHandler, +) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) + + +class TestCoherePassthroughLoggingHandler: + """Test the Cohere passthrough logging handler for embed cost tracking.""" + + def setup_method(self): + """Set up test fixtures""" + self.start_time = datetime.now() + self.end_time = datetime.now() + self.handler = CoherePassthroughLoggingHandler() + + # Mock Cohere embed response + self.mock_cohere_embed_response = { + "embeddings": [ + [0.1, 0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9, 1.0], + ], + "meta": { + "billed_units": { + "input_tokens": 3, + } + }, + } + + def _create_mock_logging_obj(self) -> LiteLLMLoggingObj: + """Create a mock logging object""" + mock_logging_obj = MagicMock() + mock_logging_obj.model_call_details = {} + return mock_logging_obj + + def _create_mock_httpx_response(self, response_data: dict = None) -> httpx.Response: + """Create a mock httpx response""" + if response_data is None: + response_data = self.mock_cohere_embed_response + + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.text = json.dumps(response_data) + mock_response.json.return_value = response_data + mock_response.headers = {"content-type": "application/json"} + return mock_response + + def _create_passthrough_logging_payload(self) -> PassthroughStandardLoggingPayload: + """Create a mock passthrough logging payload""" + return PassthroughStandardLoggingPayload( + url="https://api.cohere.com/v1/embed", + request_body={"model": "embed-english-v3.0", "texts": ["test passthrough"]}, + request_method="POST", + ) + + @patch("litellm.completion_cost") + @patch( + "litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload" + ) + @patch("litellm.llms.cohere.embed.v1_transformation.CohereEmbeddingConfig._transform_response") + def test_cohere_embed_passthrough_cost_tracking( + self, mock_transform_response, mock_get_standard_logging, mock_completion_cost + ): + """Test successful cost tracking for Cohere embed passthrough""" + # Arrange + from litellm.types.utils import EmbeddingResponse + + # Create a mock embedding response + mock_embedding_response = EmbeddingResponse() + mock_embedding_response.data = [ + {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}, + {"object": "embedding", "index": 1, "embedding": [0.4, 0.5, 0.6]}, + ] + mock_embedding_response.model = "embed-english-v3.0" + mock_embedding_response.object = "list" + from litellm.types.utils import Usage + mock_embedding_response.usage = Usage( + prompt_tokens=3, completion_tokens=0, total_tokens=3 + ) + + mock_transform_response.return_value = mock_embedding_response + mock_completion_cost.return_value = 3.6e-07 # Expected cost for embed-v4.0 + mock_get_standard_logging.return_value = {"test": "logging_payload"} + + mock_httpx_response = self._create_mock_httpx_response() + mock_logging_obj = self._create_mock_logging_obj() + passthrough_payload = self._create_passthrough_logging_payload() + + kwargs = { + "passthrough_logging_payload": passthrough_payload, + } + + request_body = { + "model": "embed-english-v3.0", + "texts": ["test passthrough"], + } + + # Act + result = self.handler.cohere_passthrough_handler( + httpx_response=mock_httpx_response, + response_body=self.mock_cohere_embed_response, + logging_obj=mock_logging_obj, + url_route="https://api.cohere.com/v1/embed", + result="", + start_time=self.start_time, + end_time=self.end_time, + cache_hit=False, + request_body=request_body, + **kwargs, + ) + + # Assert + assert result is not None + assert "result" in result + assert "kwargs" in result + assert result["kwargs"]["model"] == "embed-english-v3.0" + assert result["kwargs"]["custom_llm_provider"] == "cohere" + + # Verify cost calculation was called with correct parameters + mock_completion_cost.assert_called_once() + call_args = mock_completion_cost.call_args + assert call_args.kwargs["model"] == "embed-english-v3.0" + assert call_args.kwargs["custom_llm_provider"] == "cohere" + assert call_args.kwargs["call_type"] == "aembedding" + + # Verify logging object was updated + assert mock_logging_obj.model_call_details["response_cost"] == 3.6e-07 + assert mock_logging_obj.model_call_details["model"] == "embed-english-v3.0" + assert mock_logging_obj.model_call_details["custom_llm_provider"] == "cohere" + + # Verify result is an EmbeddingResponse + assert hasattr(result["result"], "data") + assert hasattr(result["result"], "model") + assert result["result"].model == "embed-english-v3.0" + + +if __name__ == "__main__": + pytest.main([__file__]) + From 84e8b9a7bf12ff415a8f66ab98e758f76cddf50e Mon Sep 17 00:00:00 2001 From: Haiyi Date: Tue, 25 Nov 2025 12:40:00 +1100 Subject: [PATCH 06/68] fix: handle None or empty contents in Gemini token counter (#17020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds null/empty check before processing contents in GoogleAIStudioTokenCounter to prevent errors when contents is None or empty. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude --- litellm/llms/gemini/count_tokens/handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index 4d6c7fd8864..fdb77452d4c 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -30,6 +30,10 @@ class GoogleAIStudioTokenCounter: from google.genai.types import FunctionResponse + # Handle None or empty contents + if not contents: + return contents + cleaned_contents = copy.deepcopy(contents) for content in cleaned_contents: From 3b6c1707393f103e2fe88ce13043dde91b21a294 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 07:10:55 +0530 Subject: [PATCH 07/68] Fix the azure auth format for videos (#17009) * fix the azure auth in correct format * Add litellm param in validate_environment method * fix lint errors --- litellm/llms/azure/videos/transformation.py | 34 ++++++------ .../llms/base_llm/videos/transformation.py | 1 + litellm/llms/custom_httpx/llm_http_handler.py | 2 + litellm/llms/gemini/videos/transformation.py | 5 ++ litellm/llms/openai/videos/transformation.py | 5 ++ .../llms/runwayml/videos/transformation.py | 5 ++ .../llms/vertex_ai/videos/transformation.py | 8 ++- .../videos/test_azure_video_transformation.py | 53 +++++++++++-------- 8 files changed, 70 insertions(+), 43 deletions(-) diff --git a/litellm/llms/azure/videos/transformation.py b/litellm/llms/azure/videos/transformation.py index 3af9e0778bc..a6fbd8cef8b 100644 --- a/litellm/llms/azure/videos/transformation.py +++ b/litellm/llms/azure/videos/transformation.py @@ -1,9 +1,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from litellm.types.videos.main import VideoCreateOptionalRequestParams -from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams from litellm.llms.azure.common_utils import BaseAzureLLM -import litellm from litellm.llms.openai.videos.transformation import OpenAIVideoConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -56,22 +55,27 @@ class AzureVideoConfig(OpenAIVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") + """ + Validate Azure environment and set up authentication headers. + Uses _base_validate_azure_environment to properly handle credentials from litellm_credential_name. + """ + # If litellm_params is provided, use it; otherwise create a new one + if litellm_params is None: + litellm_params = GenericLiteLLMParams() + + if api_key and not litellm_params.api_key: + litellm_params.api_key = api_key + + # Use the base Azure validation method which properly handles: + # 1. Credentials from litellm_credential_name via litellm_params + # 2. Sets the correct "api-key" header (not "Authorization: Bearer") + return BaseAzureLLM._base_validate_azure_environment( + headers=headers, + litellm_params=litellm_params ) - headers.update( - { - "Authorization": f"Bearer {api_key}", - } - ) - return headers - def get_complete_url( self, model: str, diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 7e990b42650..50cada42b87 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -66,6 +66,7 @@ class BaseVideoConfig(ABC): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: return {} diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 383dabb3931..fdd504e2f57 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4126,6 +4126,7 @@ class BaseLLMHTTPHandler: headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: @@ -4226,6 +4227,7 @@ class BaseLLMHTTPHandler: headers=video_generation_optional_request_params.get("extra_headers", {}) or {}, model=model, + litellm_params=litellm_params, ) if extra_headers: diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index ce2519e9177..4120d1cad22 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -160,11 +160,16 @@ class GeminiVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: """ Validate environment and add Gemini API key to headers. Gemini uses x-goog-api-key header for authentication. """ + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index d1d3fc2919e..abdcd2fbe7b 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -61,7 +61,12 @@ class OpenAIVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 651acff6fc4..5a46ebb664b 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -114,11 +114,16 @@ class RunwayMLVideoConfig(BaseVideoConfig): headers: dict, model: str, api_key: Optional[str] = None, + litellm_params: Optional[GenericLiteLLMParams] = None, ) -> dict: """ Validate environment and set up authentication headers. RunwayML uses Bearer token authentication via RUNWAYML_API_SECRET. """ + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + api_key = ( api_key or litellm.api_key diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index 2b6d43dd708..0f7b71c9262 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -160,13 +160,11 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): def validate_environment( self, - headers: Dict, + headers: dict, model: str, api_key: Optional[str] = None, - api_base: Optional[str] = None, - litellm_params: Optional[dict] = None, - **kwargs, - ) -> Dict: + litellm_params: Optional[GenericLiteLLMParams] = None, + ) -> dict: """ Validate environment and return headers for Vertex AI OCR. diff --git a/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py b/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py index 640933179a6..b3d7945db39 100644 --- a/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py +++ b/tests/test_litellm/llms/azure/videos/test_azure_video_transformation.py @@ -65,8 +65,13 @@ class TestAzureVideoConfig: assert result["size"] == "1280x720" assert result["user"] == "test_user" - def test_validate_environment_with_api_key(self): - """Test environment validation with provided API key.""" + @patch('litellm.llms.azure.common_utils.litellm') + def test_validate_environment_with_api_key(self, mock_litellm): + """Test environment validation with provided API key - should use api-key header for Azure.""" + # Since validate_environment passes litellm_params=None, it relies on litellm.api_key or litellm.azure_key + mock_litellm.api_key = self.api_key + mock_litellm.azure_key = None + headers = {"Content-Type": "application/json"} result_headers = self.config.validate_environment( @@ -75,14 +80,15 @@ class TestAzureVideoConfig: api_key=self.api_key ) - assert "Authorization" in result_headers - assert result_headers["Authorization"] == f"Bearer {self.api_key}" + # Azure uses "api-key" header, not "Authorization: Bearer" + assert "api-key" in result_headers + assert result_headers["api-key"] == self.api_key assert result_headers["Content-Type"] == "application/json" - @patch('litellm.llms.azure.videos.transformation.get_secret_str') - @patch('litellm.llms.azure.videos.transformation.litellm') + @patch('litellm.llms.azure.common_utils.get_secret_str') + @patch('litellm.llms.azure.common_utils.litellm') def test_validate_environment_without_api_key(self, mock_litellm, mock_get_secret): - """Test environment validation without provided API key.""" + """Test environment validation without provided API key - should fallback to secret manager.""" mock_litellm.api_key = None mock_litellm.azure_key = None mock_get_secret.return_value = "secret-api-key" @@ -95,8 +101,8 @@ class TestAzureVideoConfig: api_key=None ) - assert "Authorization" in result_headers - assert result_headers["Authorization"] == "Bearer secret-api-key" + assert "api-key" in result_headers + assert result_headers["api-key"] == "secret-api-key" def test_get_complete_url(self): """Test URL construction for Azure video API.""" @@ -320,23 +326,24 @@ class TestAzureVideoConfig: logging_obj=logging_obj ) - def test_azure_specific_environment_validation(self): + @patch('litellm.llms.azure.common_utils.litellm') + def test_azure_specific_environment_validation(self, mock_litellm): """Test Azure-specific environment validation with different key sources.""" + # Test with azure_key + mock_litellm.api_key = None + mock_litellm.azure_key = "azure-test-key" + mock_litellm.openai_key = None + headers = {"Content-Type": "application/json"} - # Test with azure_key - with patch('litellm.llms.azure.videos.transformation.litellm') as mock_litellm: - mock_litellm.api_key = None - mock_litellm.azure_key = "azure-test-key" - mock_litellm.openai_key = None - - result_headers = self.config.validate_environment( - headers=headers, - model=self.model, - api_key=None - ) - - assert result_headers["Authorization"] == "Bearer azure-test-key" + result_headers = self.config.validate_environment( + headers=headers, + model=self.model, + api_key=None + ) + + assert "api-key" in result_headers + assert result_headers["api-key"] == "azure-test-key" def test_usage_data_creation_in_video_create(self): """Test that usage data is created correctly in video create response.""" From c6fbdc7dc53cc483d5f06e8b1bf82e0e8cab4983 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 08:14:59 +0530 Subject: [PATCH 08/68] fix bedrock passthrough auth issue (#16879) --- .../litellm_core_utils/get_litellm_params.py | 11 ++ litellm/passthrough/main.py | 2 +- .../test_llm_pass_through_endpoints.py | 155 ++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d5675a2ac51..5279cb26b69 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -121,5 +121,16 @@ def get_litellm_params( "use_litellm_proxy": use_litellm_proxy, "litellm_request_debug": litellm_request_debug, "aws_region_name": kwargs.get("aws_region_name"), + # AWS credentials for Bedrock/Sagemaker + "aws_access_key_id": kwargs.get("aws_access_key_id"), + "aws_secret_access_key": kwargs.get("aws_secret_access_key"), + "aws_session_token": kwargs.get("aws_session_token"), + "aws_session_name": kwargs.get("aws_session_name"), + "aws_profile_name": kwargs.get("aws_profile_name"), + "aws_role_name": kwargs.get("aws_role_name"), + "aws_web_identity_token": kwargs.get("aws_web_identity_token"), + "aws_sts_endpoint": kwargs.get("aws_sts_endpoint"), + "aws_external_id": kwargs.get("aws_external_id"), + "aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"), } return litellm_params diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index cc57ceac50e..3df3037ed58 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -258,7 +258,7 @@ def llm_passthrough_route( model=model, messages=[], optional_params={}, - litellm_params={}, + litellm_params=litellm_params_dict, api_key=provider_api_key, api_base=base_target_url, ) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index ea1017e1d5a..b0e198d5e7e 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1179,6 +1179,161 @@ class TestBedrockLLMProxyRoute: in str(exc_info.value.detail) ) + @pytest.mark.asyncio + async def test_bedrock_passthrough_uses_model_specific_credentials(self): + """ + Test that Bedrock passthrough endpoints use credentials from model configuration + instead of environment variables when a router model is used. + + This test verifies the fix for the bug where passthrough endpoints were using + environment variables instead of model-specific credentials from config.yaml. + """ + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_passthrough_router_model, + ) + from litellm import Router + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + + # Model-specific credentials (different from env vars) + model_access_key = "MODEL_SPECIFIC_ACCESS_KEY" + model_secret_key = "MODEL_SPECIFIC_SECRET_KEY" + model_region = "us-west-2" + model_session_token = "MODEL_SESSION_TOKEN" + + # Environment variables (should NOT be used) + env_access_key = "ENV_ACCESS_KEY" + env_secret_key = "ENV_SECRET_KEY" + env_region = "us-east-1" + + # Set environment variables to different values + with patch.dict( + os.environ, + { + "AWS_ACCESS_KEY_ID": env_access_key, + "AWS_SECRET_ACCESS_KEY": env_secret_key, + "AWS_REGION_NAME": env_region, + }, + ): + # Test 1: Verify get_litellm_params extracts AWS credentials from kwargs + kwargs_with_creds = { + "aws_access_key_id": model_access_key, + "aws_secret_access_key": model_secret_key, + "aws_region_name": model_region, + "aws_session_token": model_session_token, + "model": "bedrock/test-model", + } + litellm_params = get_litellm_params(**kwargs_with_creds) + + # Verify credentials are extracted + assert litellm_params.get("aws_access_key_id") == model_access_key + assert litellm_params.get("aws_secret_access_key") == model_secret_key + assert litellm_params.get("aws_region_name") == model_region + assert litellm_params.get("aws_session_token") == model_session_token + + # Test 2: Verify router passes model credentials to passthrough + router = Router( + model_list=[ + { + "model_name": "claude-opus-4-1", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-opus-4-20250514-v1:0", + "aws_access_key_id": model_access_key, + "aws_secret_access_key": model_secret_key, + "aws_region_name": model_region, + "aws_session_token": model_session_token, + "custom_llm_provider": "bedrock", + }, + } + ] + ) + + # Verify router has model-specific credentials + deployments = router.get_model_list(model_name="claude-opus-4-1") + assert len(deployments) > 0 + deployment = deployments[0] + deployment_litellm_params = deployment.get("litellm_params", {}) + + # Verify model-specific credentials are in the deployment + assert deployment_litellm_params.get("aws_access_key_id") == model_access_key + assert deployment_litellm_params.get("aws_secret_access_key") == model_secret_key + assert deployment_litellm_params.get("aws_region_name") == model_region + assert deployment_litellm_params.get("aws_session_token") == model_session_token + + # Verify environment variables are NOT in the deployment + assert deployment_litellm_params.get("aws_access_key_id") != env_access_key + assert deployment_litellm_params.get("aws_secret_access_key") != env_secret_key + assert deployment_litellm_params.get("aws_region_name") != env_region + + # Test 3: Verify credentials are passed through the passthrough route + # Mock the passthrough route to capture what credentials are used + captured_kwargs = {} + + async def mock_llm_passthrough_route(**kwargs): + captured_kwargs.update(kwargs) + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aread = AsyncMock( + return_value=b'{"content": [{"text": "Hello"}]}' + ) + return mock_response + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + mock_request.url = MagicMock() + mock_request.url.path = "/bedrock/model/claude-opus-4-1/converse" + + mock_request_body = { + "messages": [{"role": "user", "content": [{"text": "Hello"}]}] + } + + mock_user_api_key_dict = Mock() + mock_user_api_key_dict.api_key = "test-key" + mock_proxy_logging_obj = Mock() + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock() + + with patch( + "litellm.passthrough.main.llm_passthrough_route", + new_callable=AsyncMock, + side_effect=mock_llm_passthrough_route, + ), patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.base_passthrough_process_llm_request", + new_callable=AsyncMock, + ) as mock_process: + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aread = AsyncMock( + return_value=b'{"content": [{"text": "Hello"}]}' + ) + mock_process.return_value = mock_response + + # Call the handler + await handle_bedrock_passthrough_router_model( + model="claude-opus-4-1", + endpoint="model/claude-opus-4-1/converse", + request=mock_request, + request_body=mock_request_body, + llm_router=router, + user_api_key_dict=mock_user_api_key_dict, + proxy_logging_obj=mock_proxy_logging_obj, + general_settings={}, + proxy_config=None, + select_data_generator=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + # Verify that the router was called (which means credentials flow through) + # The key verification is that get_litellm_params extracts the credentials + # and they're available in the router's deployment + assert mock_process.called + class TestLLMPassthroughFactoryProxyRoute: @pytest.mark.asyncio From 35bfcac3bcf9dbc7053f2f99dfe20c327e8527d2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 08:18:10 +0530 Subject: [PATCH 09/68] Add header forwarding in embedding (#16869) --- litellm/main.py | 8 +- .../bedrock/embed/test_bedrock_embedding.py | 152 +++++++++++++++++- 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index b082b491f24..4769f85d7ce 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4017,7 +4017,11 @@ def embedding( # noqa: PLR0915 azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None) aembedding: Optional[bool] = kwargs.get("aembedding", None) extra_headers = kwargs.get("extra_headers", None) - headers = kwargs.get("headers", None) + headers = kwargs.get("headers", None) or extra_headers + if headers is None: + headers = {} + if extra_headers is not None: + headers.update(extra_headers) ### CUSTOM MODEL COST ### input_cost_per_token = kwargs.get("input_cost_per_token", None) output_cost_per_token = kwargs.get("output_cost_per_token", None) @@ -4328,7 +4332,7 @@ def embedding( # noqa: PLR0915 litellm_params={}, api_base=api_base, print_verbose=print_verbose, - extra_headers=extra_headers, + extra_headers=headers, api_key=api_key, ) elif custom_llm_provider == "triton": diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index f436c66f203..a266bea3513 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -404,4 +404,154 @@ def test_twelvelabs_missing_input_type_error(): ) # Should succeed without input_type - assert isinstance(response, litellm.EmbeddingResponse) \ No newline at end of file + assert isinstance(response, litellm.EmbeddingResponse) + + +@pytest.mark.parametrize( + "model,embed_response", + [ + ("bedrock/amazon.titan-embed-text-v1", titan_embedding_response), + ("bedrock/amazon.titan-embed-text-v2:0", titan_embedding_response), + ("bedrock/cohere.embed-english-v3", cohere_embedding_response), + ], +) +def test_bedrock_embedding_header_forwarding(model, embed_response): + """ + Test that custom headers are correctly forwarded to Bedrock embedding API calls. + + This test verifies the fix for the issue where headers configured via + forward_client_headers_to_llm_api were not being passed to Bedrock embedding provider. + + Relevant Issue: https://github.com/BerriAI/litellm/pull/16042 + """ + litellm.set_verbose = True + client = HTTPHandler() + test_api_key = "test-bearer-token-12345" + + # Headers that would be set by the proxy when forwarding client headers + custom_headers = { + "X-Custom-Header": "CustomValue", + "X-BYOK-Token": "secret-token", + "Extra-Header": "foobar", + } + + with patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + try: + # Call embedding with custom headers via kwargs + # This simulates what the proxy does when forward_client_headers_to_llm_api is set + response = litellm.embedding( + model=model, + input=test_input, + client=client, + headers=custom_headers, # This is how proxy passes forwarded headers + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + api_key=test_api_key, + ) + + assert isinstance(response, litellm.EmbeddingResponse) + + # Verify that the request was made + assert mock_post.called, "HTTP client post should be called" + + # Get the actual call arguments + call_kwargs = mock_post.call_args.kwargs + headers = call_kwargs.get("headers", {}) + + # Verify our custom headers are present in the request headers + # Note: AWS SigV4 signing may modify header names to lowercase + for header_key, header_value in custom_headers.items(): + header_found = ( + header_key in headers + or header_key.lower() in headers + or any(k.lower() == header_key.lower() for k in headers.keys()) + ) + assert header_found, ( + f"Header {header_key} should be in request headers. " + f"Found headers: {list(headers.keys())}" + ) + + print(f"✓ Test passed for {model}") + print(f" Headers correctly forwarded: {list(headers.keys())}") + + except Exception as e: + pytest.fail(f"Failed to forward headers to {model}: {str(e)}") + + +def test_bedrock_embedding_extra_headers_and_headers_merge(): + """ + Test that both extra_headers and headers parameters are correctly merged for Bedrock embeddings. + + This ensures that headers from kwargs (forwarded by proxy) and extra_headers + (passed explicitly) are both included in the final headers sent to the provider. + """ + litellm.set_verbose = True + client = HTTPHandler() + test_api_key = "test-bearer-token-12345" + model = "bedrock/amazon.titan-embed-text-v1" + + # Headers from proxy (via kwargs["headers"]) + proxy_headers = {"X-Forwarded-Header": "ProxyValue"} + + # Explicit extra_headers + explicit_headers = {"X-Explicit-Header": "ExplicitValue"} + + # Mock response + embed_response = { + "embedding": [0.1, 0.2, 0.3], + "inputTextTokenCount": 10 + } + + with patch.object(client, "post") as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + try: + response = litellm.embedding( + model=model, + input=test_input, + client=client, + headers=proxy_headers, # From proxy forwarding + extra_headers=explicit_headers, # Explicitly passed + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + api_key=test_api_key, + ) + + assert isinstance(response, litellm.EmbeddingResponse) + + call_kwargs = mock_post.call_args.kwargs + headers = call_kwargs.get("headers", {}) + + # Both sets of headers should be present + # Note: AWS SigV4 signing may modify header names to lowercase + proxy_header_found = any( + k.lower() == "x-forwarded-header" for k in headers.keys() + ) + assert proxy_header_found, ( + "Proxy forwarded header should be present. " + f"Found headers: {list(headers.keys())}" + ) + + explicit_header_found = any( + k.lower() == "x-explicit-header" for k in headers.keys() + ) + assert explicit_header_found, ( + "Explicitly passed header should be present. " + f"Found headers: {list(headers.keys())}" + ) + + print("✓ Both header sources correctly merged and forwarded") + print(f" Final headers: {list(headers.keys())}") + + except Exception as e: + pytest.fail(f"Failed to merge and forward headers: {str(e)}") \ No newline at end of file From fc219c7db89a8a320c8c75137b285ba30a72aa19 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 08:19:30 +0530 Subject: [PATCH 10/68] Integrate eleven labs text-to-speech (#16573) * Add elevenlaps tts support * fix mypy error * add simple usage in docs --- docs/my-website/docs/providers/elevenlabs.md | 241 ++++++++++++- docs/my-website/docs/text_to_speech.md | 1 + .../text_to_speech/transformation.py | 332 ++++++++++++++++++ litellm/main.py | 62 +++- litellm/utils.py | 6 + tests/llm_translation/test_elevenlabs.py | 86 ++++- 6 files changed, 723 insertions(+), 5 deletions(-) create mode 100644 litellm/llms/elevenlabs/text_to_speech/transformation.py diff --git a/docs/my-website/docs/providers/elevenlabs.md b/docs/my-website/docs/providers/elevenlabs.md index e80ea534f55..5cf62f51203 100644 --- a/docs/my-website/docs/providers/elevenlabs.md +++ b/docs/my-website/docs/providers/elevenlabs.md @@ -7,10 +7,10 @@ ElevenLabs provides high-quality AI voice technology, including speech-to-text c | Property | Details | |----------|---------| -| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription capabilities that support multiple languages and speaker diarization. | +| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription and text-to-speech capabilities that support multiple languages and speaker diarization. | | Provider Route on LiteLLM | `elevenlabs/` | | Provider Doc | [ElevenLabs API ↗](https://elevenlabs.io/docs/api-reference) | -| Supported Endpoints | `/audio/transcriptions` | +| Supported Endpoints | `/audio/transcriptions`, `/audio/speech` | ## Quick Start @@ -228,4 +228,241 @@ ElevenLabs returns transcription responses in OpenAI-compatible format: 1. **Invalid API Key**: Ensure `ELEVENLABS_API_KEY` is set correctly +--- + +## Text-to-Speech (TTS) + +ElevenLabs provides high-quality text-to-speech capabilities through their TTS API, supporting multiple voices, languages, and audio formats. + +### Overview + +| Property | Details | +|----------|---------| +| Description | Convert text to natural-sounding speech using ElevenLabs' advanced TTS models | +| Provider Route on LiteLLM | `elevenlabs/` | +| Supported Operations | `/audio/speech` | +| Link to Provider Doc | [ElevenLabs TTS API ↗](https://elevenlabs.io/docs/api-reference/text-to-speech) | + +### Quick Start + +#### LiteLLM Python SDK + +```python showLineNumbers title="ElevenLabs Text-to-Speech with SDK" +import litellm +import os + +os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key" + +# Basic usage with voice mapping +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing ElevenLabs speech from LiteLLM.", + voice="alloy", # Maps to ElevenLabs voice ID automatically +) + +# Save audio to file +with open("test_output.mp3", "wb") as f: + f.write(audio.read()) +``` + +#### Advanced Usage: Overriding Parameters and ElevenLabs-Specific Features + +```python showLineNumbers title="Advanced TTS with custom parameters" +import litellm +import os + +os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key" + +# Example showing parameter overriding and ElevenLabs-specific parameters +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing ElevenLabs speech from LiteLLM.", + voice="alloy", # Can use mapped voice name or raw ElevenLabs voice_id + response_format="pcm", # Maps to ElevenLabs output_format + speed=1.1, # Maps to voice_settings.speed + # ElevenLabs-specific parameters - passed directly to API + pronunciation_dictionary_locators=[ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + model_id="eleven_multilingual_v2", # Override model if needed +) + +# Save audio to file +with open("test_output.mp3", "wb") as f: + f.write(audio.read()) +``` + +### Voice Mapping + +LiteLLM automatically maps common OpenAI voice names to ElevenLabs voice IDs: + +| OpenAI Voice | ElevenLabs Voice ID | Description | +|--------------|---------------------|-------------| +| `alloy` | `21m00Tcm4TlvDq8ikWAM` | Rachel - Neutral and balanced | +| `amber` | `5Q0t7uMcjvnagumLfvZi` | Paul - Warm and friendly | +| `ash` | `AZnzlk1XvdvUeBnXmlld` | Domi - Energetic | +| `august` | `D38z5RcWu1voky8WS1ja` | Fin - Professional | +| `blue` | `2EiwWnXFnvU5JabPnv8n` | Clyde - Deep and authoritative | +| `coral` | `9BWtsMINqrJLrRacOk9x` | Aria - Expressive | +| `lily` | `EXAVITQu4vr4xnSDxMaL` | Sarah - Friendly | +| `onyx` | `29vD33N1CtxCmqQRPOHJ` | Drew - Strong | +| `sage` | `CwhRBWXzGAHq8TQ4Fs17` | Roger - Calm | +| `verse` | `CYw3kZ02Hs0563khs1Fj` | Dave - Conversational | + +**Using Custom Voice IDs**: You can also pass any ElevenLabs voice ID directly. If the voice name is not in the mapping, LiteLLM will use it as-is: + +```python showLineNumbers title="Using custom ElevenLabs voice ID" +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="Testing with a custom voice.", + voice="21m00Tcm4TlvDq8ikWAM", # Direct ElevenLabs voice ID +) +``` + +### Response Format Mapping + +LiteLLM maps OpenAI response formats to ElevenLabs output formats: + +| OpenAI Format | ElevenLabs Format | +|---------------|-------------------| +| `mp3` | `mp3_44100_128` | +| `pcm` | `pcm_44100` | +| `opus` | `opus_48000_128` | + +You can also pass ElevenLabs-specific output formats directly using the `output_format` parameter. + +### Supported Parameters + +```python showLineNumbers title="All Supported Parameters" +audio = litellm.speech( + model="elevenlabs/eleven_multilingual_v2", # Required + input="Text to convert to speech", # Required + voice="alloy", # Required: Voice selection (mapped or raw ID) + response_format="mp3", # Optional: Audio format (mp3, pcm, opus) + speed=1.0, # Optional: Speech speed (maps to voice_settings.speed) + # ElevenLabs-specific parameters (passed directly): + model_id="eleven_multilingual_v2", # Optional: Override model + voice_settings={ # Optional: Voice customization + "stability": 0.5, + "similarity_boost": 0.75, + "speed": 1.0 + }, + pronunciation_dictionary_locators=[ # Optional: Custom pronunciation + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], +) +``` + +### LiteLLM Proxy + +#### 1. Configure your proxy + +```yaml showLineNumbers title="ElevenLabs TTS configuration in config.yaml" +model_list: + - model_name: elevenlabs-tts + litellm_params: + model: elevenlabs/eleven_multilingual_v2 + api_key: os.environ/ELEVENLABS_API_KEY + +general_settings: + master_key: your-master-key +``` + +#### 2. Make TTS requests + +##### Simple Usage (OpenAI Parameters) + +You can use standard OpenAI-compatible parameters without any provider-specific configuration: + +```bash showLineNumbers title="Simple TTS request with curl" +curl http://localhost:4000/v1/audio/speech \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "elevenlabs-tts", + "input": "Testing ElevenLabs speech via the LiteLLM proxy.", + "voice": "alloy", + "response_format": "mp3" + }' \ + --output speech.mp3 +``` + +```python showLineNumbers title="Simple TTS with OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.audio.speech.create( + model="elevenlabs-tts", + input="Testing ElevenLabs speech via the LiteLLM proxy.", + voice="alloy", + response_format="mp3" +) + +# Save audio +with open("speech.mp3", "wb") as f: + f.write(response.content) +``` + +##### Advanced Usage (ElevenLabs-Specific Parameters) + +**Note**: When using the proxy, provider-specific parameters (like `pronunciation_dictionary_locators`, `voice_settings`, etc.) must be passed in the `extra_body` field. + +```bash showLineNumbers title="Advanced TTS request with curl" +curl http://localhost:4000/v1/audio/speech \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "elevenlabs-tts", + "input": "Testing ElevenLabs speech via the LiteLLM proxy.", + "voice": "alloy", + "response_format": "pcm", + "extra_body": { + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + "voice_settings": { + "speed": 1.1, + "stability": 0.5, + "similarity_boost": 0.75 + } + } + }' \ + --output speech.mp3 +``` + +```python showLineNumbers title="Advanced TTS with OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.audio.speech.create( + model="elevenlabs-tts", + input="Testing ElevenLabs speech via the LiteLLM proxy.", + voice="alloy", + response_format="pcm", + extra_body={ + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_123", "version_id": "v1"} + ], + "voice_settings": { + "speed": 1.1, + "stability": 0.5, + "similarity_boost": 0.75 + } + } +) + +# Save audio +with open("speech.mp3", "wb") as f: + f.write(response.content) +``` + + diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md index c530e70e4be..ea2a9c2eff3 100644 --- a/docs/my-website/docs/text_to_speech.md +++ b/docs/my-website/docs/text_to_speech.md @@ -103,6 +103,7 @@ litellm --config /path/to/config.yaml | Azure AI Speech Service (AVA)| [Usage](../docs/providers/azure_ai_speech) | | Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) | | Gemini | [Usage](#gemini-text-to-speech) | +| ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) | ## `/audio/speech` to `/chat/completions` Bridge diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py new file mode 100644 index 00000000000..b78d0bafc50 --- /dev/null +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -0,0 +1,332 @@ +""" +Elevenlabs Text-to-Speech transformation + +Maps OpenAI TTS spec to Elevenlabs TTS API +""" + +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from urllib.parse import urlencode + +import httpx +from httpx import Headers + +import litellm +from litellm.types.utils import all_litellm_params +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +from ..common_utils import ElevenLabsException + + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + HttpxBinaryResponseContent = Any + + +class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): + """ + Configuration for ElevenLabs Text-to-Speech + + Reference: https://elevenlabs.io/docs/api-reference/text-to-speech/convert + """ + + TTS_BASE_URL = "https://api.elevenlabs.io" + TTS_ENDPOINT_PATH = "/v1/text-to-speech" + DEFAULT_OUTPUT_FORMAT = "pcm_44100" + VOICE_MAPPINGS = { + "alloy": "21m00Tcm4TlvDq8ikWAM", # Rachel + "amber": "5Q0t7uMcjvnagumLfvZi", # Paul + "ash": "AZnzlk1XvdvUeBnXmlld", # Domi + "august": "D38z5RcWu1voky8WS1ja", # Fin + "blue": "2EiwWnXFnvU5JabPnv8n", # Clyde + "coral": "9BWtsMINqrJLrRacOk9x", # Aria + "lily": "EXAVITQu4vr4xnSDxMaL", # Sarah + "onyx": "29vD33N1CtxCmqQRPOHJ", # Drew + "sage": "CwhRBWXzGAHq8TQ4Fs17", # Roger + "verse": "CYw3kZ02Hs0563khs1Fj", # Dave + } + + # Response format mappings from OpenAI to ElevenLabs + FORMAT_MAPPINGS = { + "mp3": "mp3_44100_128", + "pcm": "pcm_44100", + "opus": "opus_48000_128", + # ElevenLabs does not support WAV, AAC, or FLAC formats. + } + + ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__" + ELEVENLABS_VOICE_ID_KEY = "__elevenlabs_voice_id__" + + def get_supported_openai_params(self, model: str) -> list: + """ + ElevenLabs TTS supports these OpenAI parameters + """ + return ["voice", "response_format", "speed"] + + def _extract_voice_id(self, voice: str) -> str: + """ + Normalize the provided voice information into an ElevenLabs voice_id. + """ + normalized_voice = voice.strip() + mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower()) + return mapped_voice or normalized_voice + + def _resolve_voice_id( + self, + voice: Optional[Union[str, Dict[str, Any]]], + params: Dict[str, Any], + ) -> str: + """ + Determine the ElevenLabs voice_id based on provided voice input or parameters. + """ + mapped_voice: Optional[str] = None + + if isinstance(voice, str) and voice.strip(): + mapped_voice = self._extract_voice_id(voice) + elif isinstance(voice, dict): + for key in ("voice_id", "id", "name"): + candidate = voice.get(key) + if isinstance(candidate, str) and candidate.strip(): + mapped_voice = self._extract_voice_id(candidate) + break + elif voice is not None: + mapped_voice = self._extract_voice_id(str(voice)) + + if mapped_voice is None: + voice_override = params.pop("voice_id", None) + if isinstance(voice_override, str) and voice_override.strip(): + mapped_voice = self._extract_voice_id(voice_override) + + if mapped_voice is None: + raise ValueError( + "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`." + ) + + return mapped_voice + + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI parameters to ElevenLabs TTS parameters + """ + mapped_params: Dict[str, Any] = {} + query_params: Dict[str, Any] = {} + + # Work on a copy so we don't mutate the caller's dictionary + params = dict(optional_params) if optional_params else {} + passthrough_kwargs: Dict[str, Any] = kwargs if kwargs is not None else {} + + # Extract voice identifier + mapped_voice = self._resolve_voice_id(voice, params) + + # Response/output format → query parameter + response_format = params.pop("response_format", None) + if isinstance(response_format, str): + mapped_format = self.FORMAT_MAPPINGS.get(response_format, response_format) + query_params["output_format"] = mapped_format + + # ElevenLabs does not support OpenAI speed directly. + # Drop it to avoid sending unsupported keys unless caller already provided voice_settings. + speed = params.pop("speed", None) + if speed is not None: + speed_value: Optional[float] + try: + speed_value = float(speed) + except (TypeError, ValueError): + speed_value = None + if speed_value is not None: + if isinstance(params.get("voice_settings"), dict): + params["voice_settings"]["speed"] = speed_value # type: ignore[index] + else: + params["voice_settings"] = {"speed": speed_value} + + # Instructions parameter is OpenAI-specific; omit to prevent API errors. + params.pop("instructions", None) + self._add_elevenlabs_specific_params( + mapped_voice=mapped_voice, + query_params=query_params, + mapped_params=mapped_params, + kwargs=passthrough_kwargs, + remaining_params=params, + ) + + return mapped_voice, mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate Azure environment and set up authentication headers + """ + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("ELEVENLABS_API_KEY") + ) + + if api_key is None: + raise ValueError( + "ElevenLabs API key is required. Set ELEVENLABS_API_KEY environment variable." + ) + + headers.update( + { + "xi-api-key": api_key, + "Content-Type": "application/json", + } + ) + + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return ElevenLabsException( + message=error_message, status_code=status_code, headers=headers + ) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[str], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Build the ElevenLabs TTS request payload. + """ + params = dict(optional_params) if optional_params else {} + extra_body = params.pop("extra_body", None) + + request_body: Dict[str, Any] = { + "text": input, + "model_id": model, + } + + for key, value in params.items(): + if value is None: + continue + request_body[key] = value + + if isinstance(extra_body, dict): + for key, value in extra_body.items(): + if value is None: + continue + request_body[key] = value + + return TextToSpeechRequestData( + dict_body=request_body, + headers={"Content-Type": "application/json"}, + ) + + def _add_elevenlabs_specific_params( + self, + mapped_voice: str, + query_params: Dict[str, Any], + mapped_params: Dict[str, Any], + kwargs: Optional[Dict[str, Any]], + remaining_params: Dict[str, Any], + ) -> None: + if kwargs is None: + kwargs = {} + for key, value in remaining_params.items(): + if value is None: + continue + mapped_params[key] = value + + reserved_kwarg_keys = set(all_litellm_params) | { + self.ELEVENLABS_QUERY_PARAMS_KEY, + self.ELEVENLABS_VOICE_ID_KEY, + "voice", + "model", + "response_format", + "output_format", + "extra_body", + "user", + } + + extra_body_from_kwargs = kwargs.pop("extra_body", None) + if isinstance(extra_body_from_kwargs, dict): + for key, value in extra_body_from_kwargs.items(): + if value is None: + continue + mapped_params[key] = value + + for key in list(kwargs.keys()): + if key in reserved_kwarg_keys: + continue + value = kwargs[key] + if value is None: + continue + mapped_params[key] = value + kwargs.pop(key, None) + + if query_params: + kwargs[self.ELEVENLABS_QUERY_PARAMS_KEY] = query_params + else: + kwargs.pop(self.ELEVENLABS_QUERY_PARAMS_KEY, None) + + kwargs[self.ELEVENLABS_VOICE_ID_KEY] = mapped_voice + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + """ + Wrap ElevenLabs binary audio response. + """ + from litellm.types.llms.openai import HttpxBinaryResponseContent + + return HttpxBinaryResponseContent(raw_response) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Construct the ElevenLabs endpoint URL, including path voice_id and query params. + """ + base_url = ( + api_base + or get_secret_str("ELEVENLABS_API_BASE") + or self.TTS_BASE_URL + ) + base_url = base_url.rstrip("/") + + voice_id = litellm_params.get(self.ELEVENLABS_VOICE_ID_KEY) + if not isinstance(voice_id, str) or not voice_id.strip(): + raise ValueError( + "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`." + ) + + url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}" + + query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {}) + if query_params: + url = f"{url}?{urlencode(query_params)}" + + return url \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 4769f85d7ce..16516389b00 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5766,7 +5766,9 @@ def speech( # noqa: PLR0915 custom_llm_provider: Optional[str] = None, aspeech: Optional[bool] = None, **kwargs, -) -> HttpxBinaryResponseContent: +) -> Union[ + HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent] +]: user = kwargs.get("user", None) litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) proxy_server_request = kwargs.get("proxy_server_request", None) @@ -5826,7 +5828,11 @@ def speech( # noqa: PLR0915 }, custom_llm_provider=custom_llm_provider, ) - response: Optional[HttpxBinaryResponseContent] = None + response: Union[ + HttpxBinaryResponseContent, + Coroutine[Any, Any, HttpxBinaryResponseContent], + None, + ] = None if ( custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers @@ -5964,6 +5970,58 @@ def speech( # noqa: PLR0915 aspeech=aspeech, litellm_params=litellm_params_dict, ) + elif custom_llm_provider == "elevenlabs": + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + if text_to_speech_provider_config is None: + text_to_speech_provider_config = ElevenLabsTextToSpeechConfig() + + elevenlabs_config = cast( + ElevenLabsTextToSpeechConfig, text_to_speech_provider_config + ) + + voice_id = voice if isinstance(voice, str) else None + if voice_id is None or not voice_id.strip(): + raise litellm.BadRequestError( + message="'voice' must resolve to an ElevenLabs voice id for ElevenLabs TTS", + model=model, + llm_provider=custom_llm_provider, + ) + voice_id = voice_id.strip() + + query_params = kwargs.pop( + ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY, None + ) + if isinstance(query_params, dict): + litellm_params_dict[ + ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY + ] = query_params + + litellm_params_dict[ + ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY + ] = voice_id + + if api_base is not None: + litellm_params_dict["api_base"] = api_base + if api_key is not None: + litellm_params_dict["api_key"] = api_key + + response = base_llm_http_handler.text_to_speech_handler( + model=model, + input=input, + voice=voice_id, + text_to_speech_provider_config=elevenlabs_config, + text_to_speech_optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + _is_async=aspeech or False, + ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": generic_optional_params = GenericLiteLLMParams(**kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index f1f091b1719..78ed4170f49 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7865,6 +7865,12 @@ class ProviderConfigManager: ) return AzureAVATextToSpeechConfig() + elif litellm.LlmProviders.ELEVENLABS == provider: + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + return ElevenLabsTextToSpeechConfig() elif litellm.LlmProviders.RUNWAYML == provider: from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, diff --git a/tests/llm_translation/test_elevenlabs.py b/tests/llm_translation/test_elevenlabs.py index 4227c3f3c62..5128cd973e8 100644 --- a/tests/llm_translation/test_elevenlabs.py +++ b/tests/llm_translation/test_elevenlabs.py @@ -1,6 +1,8 @@ import os import sys +from typing import Any, Dict + import pytest from unittest.mock import patch, MagicMock import httpx @@ -11,6 +13,8 @@ sys.path.insert( import litellm from base_audio_transcription_unit_tests import BaseLLMAudioTranscriptionTest +os.environ.setdefault("ELEVENLABS_API_KEY", "test-elevenlabs-key") + class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest): def get_base_audio_transcription_call_args(self) -> dict: @@ -108,4 +112,84 @@ class TestElevenLabsAudioTranscription(BaseLLMAudioTranscriptionTest): except Exception as e: print(f"❌ Test failed: {e}") print(f"Captured request data: {captured_request_data}") - raise \ No newline at end of file + raise + + +class TestElevenLabsTextToSpeechTransformation: + @pytest.fixture(scope="class") + def config(self): + from litellm.llms.elevenlabs.text_to_speech.transformation import ( + ElevenLabsTextToSpeechConfig, + ) + + return ElevenLabsTextToSpeechConfig() + + def test_map_openai_params_maps_voice_and_speed(self, config): + kwargs: Dict[str, Any] = {} + mapped_voice, mapped_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={ + "response_format": "mp3", + "speed": 1.25, + "model_id": "eleven_multilingual_v2", + }, + voice="alloy", + kwargs=kwargs, + ) + + assert mapped_voice == config.VOICE_MAPPINGS["alloy"] + assert mapped_params["voice_settings"]["speed"] == pytest.approx(1.25) + assert ( + kwargs[config.ELEVENLABS_QUERY_PARAMS_KEY]["output_format"] + == "mp3_44100_128" + ) + + def test_transform_request_and_url(self, config): + kwargs: Dict[str, Any] = {} + voice_id, optional_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={ + "response_format": "pcm", + "model_id": "eleven_multilingual_v2", + "pronunciation_dictionary_locators": [ + {"pronunciation_dictionary_id": "dict_1"} + ], + }, + voice="alloy", + kwargs=kwargs, + ) + + litellm_params: Dict[str, Any] = { + config.ELEVENLABS_VOICE_ID_KEY: voice_id, + config.ELEVENLABS_QUERY_PARAMS_KEY: kwargs[ + config.ELEVENLABS_QUERY_PARAMS_KEY + ], + } + + headers = config.validate_environment( + headers={}, model="eleven_multilingual_v2", api_key="test-key" + ) + + request_data = config.transform_text_to_speech_request( + model="eleven_multilingual_v2", + input="Hello world", + voice=voice_id, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + assert request_data["dict_body"]["text"] == "Hello world" + assert request_data["dict_body"]["model_id"] == "eleven_multilingual_v2" + assert request_data["dict_body"]["pronunciation_dictionary_locators"] == [ + {"pronunciation_dictionary_id": "dict_1"} + ] + + url = config.get_complete_url( + model="eleven_multilingual_v2", + api_base=None, + litellm_params=litellm_params, + ) + + assert voice_id in url + assert "output_format=pcm_44100" in url \ No newline at end of file From 282ac87617c6005b842e43385e135c3ad335fc4c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 25 Nov 2025 08:24:22 +0530 Subject: [PATCH 11/68] Add temperature support for 5.1 models (#17011) --- .../llms/openai/chat/gpt_5_transformation.py | 25 +++- .../chat/test_azure_gpt5_transformation.py | 62 +++++++++ .../llms/openai/test_gpt5_transformation.py | 119 ++++++++++++++++++ 3 files changed, 204 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index d18f898cf1c..60a172ef817 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,15 @@ class OpenAIGPT5Config(OpenAIGPTConfig): def is_model_gpt_5_codex_model(cls, model: str) -> bool: """Check if the model is specifically a GPT-5 Codex variant.""" return "gpt-5-codex" in model + + @classmethod + def is_model_gpt_5_1_model(cls, model: str) -> bool: + """Check if the model is a gpt-5.1 variant. + + gpt-5.1 supports temperature when reasoning_effort="none", + unlike gpt-5 which only supports temperature=1. + """ + return "gpt-5.1" in model def get_supported_openai_params(self, model: str) -> list: from litellm.utils import supports_tool_choice @@ -69,14 +78,26 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "temperature" in non_default_params: temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: - if temperature_value == 1: + is_gpt_5_1 = self.is_model_gpt_5_1_model(model) + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + ) + + # gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none") + if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None): + optional_params["temperature"] = temperature_value + elif temperature_value == 1: optional_params["temperature"] = temperature_value elif litellm.drop_params or drop_params: pass else: raise litellm.utils.UnsupportedParamsError( message=( - "gpt-5 models (including gpt-5-codex) don't support temperature={}. Only temperature=1 is supported. To drop unsupported params set `litellm.drop_params = True`" + "gpt-5 models (including gpt-5-codex) don't support temperature={}. " + "Only temperature=1 is supported. " + "For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). " + "To drop unsupported params set `litellm.drop_params = True`" ).format(temperature_value), status_code=400, ) diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 2ef2020b09a..76d069733be 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -101,3 +101,65 @@ def test_azure_gpt5_codex_series_transform_request(config: AzureOpenAIGPT5Config ) assert request["model"] == "gpt-5-codex" + +# GPT-5.1 temperature handling tests for Azure +def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none'.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.5, "reasoning_effort": "none"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.5 + assert params["reasoning_effort"] == "none" + + +def test_azure_gpt5_1_temperature_without_reasoning_effort(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort is not specified.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.7 + + +def test_azure_gpt5_1_temperature_with_reasoning_effort_other_values(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 only allows temperature=1 when reasoning_effort is not 'none'.""" + # Test that temperature != 1 raises error when reasoning_effort is set to other values + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "low"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + + # Test that temperature=1 is allowed with other reasoning_effort values + params = config.map_openai_params( + non_default_params={"temperature": 1.0, "reasoning_effort": "medium"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 1.0 + assert params["reasoning_effort"] == "medium" + + +def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 with gpt5_series prefix supports temperature with reasoning_effort='none'.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.6}, + optional_params={}, + model="gpt5_series/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert params["temperature"] == 0.6 + diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 2e1eacce532..5080a7a7c59 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -209,3 +209,122 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): drop_params=False, ) assert params["reasoning_effort"] == effort + + +# GPT-5.1 temperature handling tests +def test_gpt5_1_model_detection(gpt5_config: OpenAIGPT5Config): + """Test that GPT-5.1 models are correctly detected.""" + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1") + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-codex") + assert gpt5_config.is_model_gpt_5_1_model("gpt-5.1-chat") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-mini") + assert not gpt5_config.is_model_gpt_5_1_model("gpt-5-codex") + + +def test_gpt5_1_temperature_with_reasoning_effort_none(config: OpenAIConfig): + """Test that GPT-5.1 supports any temperature when reasoning_effort='none'.""" + # Test various temperature values with reasoning_effort="none" + for temp in [0.0, 0.2, 0.5, 0.7, 0.9, 1.0, 1.5, 2.0]: + params = config.map_openai_params( + non_default_params={"temperature": temp, "reasoning_effort": "none"}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == temp + assert params["reasoning_effort"] == "none" + + +def test_gpt5_1_temperature_without_reasoning_effort(config: OpenAIConfig): + """Test that GPT-5.1 supports any temperature when reasoning_effort is not specified. + + When reasoning_effort is not provided, it defaults to "none" for gpt-5.1, + so temperature should be allowed. + """ + # Test various temperature values without reasoning_effort (defaults to "none") + for temp in [0.0, 0.2, 0.5, 0.7, 0.9, 1.0, 1.5, 2.0]: + params = config.map_openai_params( + non_default_params={"temperature": temp}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == temp + + +def test_gpt5_1_temperature_with_reasoning_effort_other_values(config: OpenAIConfig): + """Test that GPT-5.1 only allows temperature=1 when reasoning_effort is not 'none'.""" + # Test that temperature != 1 raises error when reasoning_effort is set to other values + for effort in ["low", "medium", "high"]: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": effort}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + # Test that temperature=1 is allowed with other reasoning_effort values + for effort in ["low", "medium", "high"]: + params = config.map_openai_params( + non_default_params={"temperature": 1.0, "reasoning_effort": effort}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == 1.0 + assert params["reasoning_effort"] == effort + + +def test_gpt5_1_temperature_with_reasoning_effort_in_optional_params(config: OpenAIConfig): + """Test that reasoning_effort can be in optional_params and still work correctly.""" + # Test with reasoning_effort="none" in optional_params + params = config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={"reasoning_effort": "none"}, + model="gpt-5.1", + drop_params=False, + ) + assert params["temperature"] == 0.5 + + # Test with reasoning_effort="low" in optional_params (should only allow temp=1) + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.5}, + optional_params={"reasoning_effort": "low"}, + model="gpt-5.1", + drop_params=False, + ) + +def test_gpt5_1_temperature_drop_when_not_none(config: OpenAIConfig): + """Test that GPT-5.1 drops temperature when reasoning_effort != 'none' and drop_params=True.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "low"}, + optional_params={}, + model="gpt-5.1", + drop_params=True, + ) + assert "temperature" not in params + assert params["reasoning_effort"] == "low" + + +def test_gpt5_temperature_still_restricted(config: OpenAIConfig): + """Test that regular gpt-5 (not 5.1) still only allows temperature=1.""" + # Regular gpt-5 should still only allow temperature=1 + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="gpt-5", + drop_params=False, + ) + + # temperature=1 should still work for gpt-5 + params = config.map_openai_params( + non_default_params={"temperature": 1.0}, + optional_params={}, + model="gpt-5", + drop_params=False, + ) + assert params["temperature"] == 1.0 From d53bc7b9a0b5e4861785eaa084bb0dbaf14de71b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 24 Nov 2025 20:37:33 -0800 Subject: [PATCH 12/68] Change modals to reusable component --- .../src/components/OldTeams.tsx | 4 +- .../common_components/DeleteResourceModal.tsx | 19 ++-- .../src/components/guardrails.test.tsx | 104 ++++++++++++++++++ .../src/components/guardrails.tsx | 56 ++++++---- .../src/components/organizations.tsx | 51 +++------ .../src/components/settings.tsx | 39 ++++--- .../src/components/team/team_info.tsx | 44 ++++---- 7 files changed, 212 insertions(+), 105 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/guardrails.test.tsx diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index eefb0302a89..44cfdb6aa53 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -30,8 +30,7 @@ import { Text, TextInput, } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Select as Select2, Tooltip, Typography } from "antd"; -import { AlertTriangleIcon, XIcon } from "lucide-react"; +import { Button as Button2, Form, Input, Modal, Select as Select2, Switch, Tooltip, Typography } from "antd"; import React, { useEffect, useState } from "react"; import { formatNumberWithCommas } from "../utils/dataUtils"; import { fetchTeams } from "./common_components/fetch_teams"; @@ -77,6 +76,7 @@ interface EditTeamModalProps { } import { updateExistingKeys } from "@/utils/dataUtils"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; import { Member, teamCreateCall, v2TeamListCall } from "./networking"; interface TeamInfo { diff --git a/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx b/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx index 403548e611c..93de859dd7d 100644 --- a/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DeleteResourceModal.tsx @@ -57,10 +57,6 @@ export default function DeleteResourceModal({ >
{alertMessage && } -
- {message} -
-
{resourceInformationTitle} @@ -74,18 +70,23 @@ export default function DeleteResourceModal({ ))} </Descriptions> </div> + <div> + <Text>{message}</Text> + </div> {requiredConfirmation && ( - <div className="mb-5"> + <div className="mb-6 mt-4 pt-4 border-t border-gray-200"> <Text className="block text-base font-medium text-gray-700 mb-2"> - {`Type `} - <span className="underline">{requiredConfirmation}</span> - {` to confirm deletion:`} + <Text>Type </Text> + <Text strong type="danger"> + {requiredConfirmation} + </Text> + <Text> to confirm deletion:</Text> </Text> <Input value={requiredConfirmationInput} onChange={(e) => setRequiredConfirmationInput(e.target.value)} placeholder={requiredConfirmation} - className="rounded-md" + className="rounded-md text-base border-gray-200" autoFocus /> </div> diff --git a/ui/litellm-dashboard/src/components/guardrails.test.tsx b/ui/litellm-dashboard/src/components/guardrails.test.tsx new file mode 100644 index 00000000000..8cafc18eb9a --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails.test.tsx @@ -0,0 +1,104 @@ +import { render, screen } from "@testing-library/react"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import GuardrailsPanel from "./guardrails"; +import { getGuardrailsList } from "./networking"; + +vi.mock("./networking", () => ({ + getGuardrailsList: vi.fn(), + deleteGuardrailCall: vi.fn(), +})); + +vi.mock("./guardrails/add_guardrail_form", () => ({ + __esModule: true, + default: () => <div>Mock Add Guardrail Form</div>, +})); + +vi.mock("./guardrails/guardrail_table", () => ({ + __esModule: true, + default: ({ guardrailsList, onDeleteClick }: any) => ( + <div> + <div>Mock Guardrail Table</div> + {guardrailsList.length > 0 && ( + <button + data-testid="delete-button" + onClick={() => onDeleteClick(guardrailsList[0].guardrail_id, guardrailsList[0].guardrail_name)} + > + Delete + </button> + )} + </div> + ), +})); + +vi.mock("./guardrails/guardrail_info", () => ({ + __esModule: true, + default: () => <div>Mock Guardrail Info View</div>, +})); + +vi.mock("./guardrails/GuardrailTestPlayground", () => ({ + __esModule: true, + default: () => <div>Mock Guardrail Test Playground</div>, +})); + +vi.mock("@/utils/roles", () => ({ + isAdminRole: vi.fn((role: string) => role === "admin"), +})); + +vi.mock("./guardrails/guardrail_info_helpers", () => ({ + getGuardrailLogoAndName: vi.fn(() => ({ + logo: null, + displayName: "Test Provider", + })), +})); + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: vi.fn().mockImplementation((query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), + removeListener: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), + }); +}); + +describe("GuardrailsPanel", () => { + const defaultProps = { + accessToken: "test-token", + userRole: "admin", + }; + + const mockGetGuardrailsList = vi.mocked(getGuardrailsList); + + beforeEach(() => { + vi.clearAllMocks(); + mockGetGuardrailsList.mockResolvedValue({ + guardrails: [ + { + guardrail_id: "test-guardrail-1", + guardrail_name: "Test Guardrail", + litellm_params: { + guardrail: "test-provider", + mode: "async", + default_on: true, + }, + guardrail_info: null, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + guardrail_definition_location: "database" as any, + }, + ], + }); + }); + + it("should render the component", async () => { + render(<GuardrailsPanel {...defaultProps} />); + expect(screen.getByText("Guardrails")).toBeInTheDocument(); + expect(screen.getByText("+ Add New Guardrail")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/guardrails.tsx b/ui/litellm-dashboard/src/components/guardrails.tsx index 3861545f8cd..26b9acb6a26 100644 --- a/ui/litellm-dashboard/src/components/guardrails.tsx +++ b/ui/litellm-dashboard/src/components/guardrails.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect } from "react"; import { Button, TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; -import { Modal } from "antd"; import { getGuardrailsList, deleteGuardrailCall } from "./networking"; import AddGuardrailForm from "./guardrails/add_guardrail_form"; import GuardrailTable from "./guardrails/guardrail_table"; @@ -9,6 +8,8 @@ import GuardrailInfoView from "./guardrails/guardrail_info"; import GuardrailTestPlayground from "./guardrails/GuardrailTestPlayground"; import NotificationsManager from "./molecules/notifications_manager"; import { Guardrail, GuardrailDefinitionLocation } from "./guardrails/types"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; +import { getGuardrailLogoAndName } from "./guardrails/guardrail_info_helpers"; interface GuardrailsPanelProps { accessToken: string | null; @@ -38,7 +39,8 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isLoading, setIsLoading] = useState(false); const [isDeleting, setIsDeleting] = useState(false); - const [guardrailToDelete, setGuardrailToDelete] = useState<{ id: string; name: string } | null>(null); + const [guardrailToDelete, setGuardrailToDelete] = useState<Guardrail | null>(null); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [selectedGuardrailId, setSelectedGuardrailId] = useState<string | null>(null); const [activeTab, setActiveTab] = useState<number>(0); @@ -81,7 +83,9 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole }; const handleDeleteClick = (guardrailId: string, guardrailName: string) => { - setGuardrailToDelete({ id: guardrailId, name: guardrailName }); + const guardrail = guardrailsList.find((g) => g.guardrail_id === guardrailId) || null; + setGuardrailToDelete(guardrail); + setIsDeleteModalOpen(true); }; const handleDeleteConfirm = async () => { @@ -90,22 +94,29 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole // Log removed to maintain clean production code setIsDeleting(true); try { - await deleteGuardrailCall(accessToken, guardrailToDelete.id); - NotificationsManager.success(`Guardrail "${guardrailToDelete.name}" deleted successfully`); - fetchGuardrails(); // Refresh the list + await deleteGuardrailCall(accessToken, guardrailToDelete.guardrail_id); + NotificationsManager.success(`Guardrail "${guardrailToDelete.guardrail_name}" deleted successfully`); + await fetchGuardrails(); // Refresh the list } catch (error) { console.error("Error deleting guardrail:", error); NotificationsManager.fromBackend("Failed to delete guardrail"); } finally { setIsDeleting(false); + setIsDeleteModalOpen(false); setGuardrailToDelete(null); } }; const handleDeleteCancel = () => { + setIsDeleteModalOpen(false); setGuardrailToDelete(null); }; + const providerDisplayName = + guardrailToDelete && guardrailToDelete.litellm_params + ? getGuardrailLogoAndName(guardrailToDelete.litellm_params.guardrail).displayName + : undefined; + return ( <div className="w-full mx-auto flex-auto overflow-y-auto m-8 p-2"> <TabGroup index={activeTab} onIndexChange={setActiveTab}> @@ -148,20 +159,25 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole onSuccess={handleSuccess} /> - {guardrailToDelete && ( - <Modal - title="Delete Guardrail" - open={guardrailToDelete !== null} - onOk={handleDeleteConfirm} - onCancel={handleDeleteCancel} - confirmLoading={isDeleting} - okText="Delete" - okButtonProps={{ danger: true }} - > - <p>Are you sure you want to delete guardrail: {guardrailToDelete.name} ?</p> - <p>This action cannot be undone.</p> - </Modal> - )} + <DeleteResourceModal + isOpen={isDeleteModalOpen} + title="Delete Guardrail" + message={`Are you sure you want to delete guardrail: ${guardrailToDelete?.guardrail_name}? This action cannot be undone.`} + resourceInformationTitle="Guardrail Information" + resourceInformation={[ + { label: "Name", value: guardrailToDelete?.guardrail_name }, + { label: "ID", value: guardrailToDelete?.guardrail_id, code: true }, + { label: "Provider", value: providerDisplayName }, + { label: "Mode", value: guardrailToDelete?.litellm_params.mode }, + { + label: "Default On", + value: guardrailToDelete?.litellm_params.default_on ? "Yes" : "No", + }, + ]} + onCancel={handleDeleteCancel} + onOk={handleDeleteConfirm} + confirmLoading={isDeleting} + /> </TabPanel> <TabPanel> diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 71e9030a24e..5f7275091e1 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -32,6 +32,7 @@ import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; import { formatNumberWithCommas } from "../utils/dataUtils"; import NotificationsManager from "./molecules/notifications_manager"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; interface OrganizationsTableProps { organizations: Organization[]; @@ -70,6 +71,7 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({ const [editOrg, setEditOrg] = useState(false); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [orgToDelete, setOrgToDelete] = useState<string | null>(null); + const [isDeleting, setIsDeleting] = useState(false); const [isOrgModalVisible, setIsOrgModalVisible] = useState(false); const [form] = Form.useForm(); const [expandedAccordions, setExpandedAccordions] = useState<Record<string, boolean>>({}); @@ -91,15 +93,18 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({ if (!orgToDelete || !accessToken) return; try { + setIsDeleting(true); await organizationDeleteCall(accessToken, orgToDelete); NotificationsManager.success("Organization deleted successfully"); setIsDeleteModalOpen(false); setOrgToDelete(null); // Refresh organizations list - fetchOrganizations(accessToken, setOrganizations); + await fetchOrganizations(accessToken, setOrganizations); } catch (error) { console.error("Error deleting organization:", error); + } finally { + setIsDeleting(false); } }; @@ -506,40 +511,16 @@ const OrganizationsTable: React.FC<OrganizationsTableProps> = ({ </Form> </Modal> - {isDeleteModalOpen ? ( - <div className="fixed z-10 inset-0 overflow-y-auto"> - <div className="flex items-end justify-center min-h-screen pt-4 px-4 pb-20 text-center sm:block sm:p-0"> - <div className="fixed inset-0 transition-opacity" aria-hidden="true"> - <div className="absolute inset-0 bg-gray-500 opacity-75"></div> - </div> - - <span className="hidden sm:inline-block sm:align-middle sm:h-screen" aria-hidden="true"> - ​ - </span> - - <div className="inline-block align-bottom bg-white rounded-lg text-left overflow-hidden shadow-xl transform transition-all sm:my-8 sm:align-middle sm:max-w-lg sm:w-full"> - <div className="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4"> - <div className="sm:flex sm:items-start"> - <div className="mt-3 text-center sm:mt-0 sm:ml-4 sm:text-left"> - <h3 className="text-lg leading-6 font-medium text-gray-900">Delete Organization</h3> - <div className="mt-2"> - <p className="text-sm text-gray-500">Are you sure you want to delete this organization?</p> - </div> - </div> - </div> - </div> - <div className="bg-gray-50 px-4 py-3 sm:px-6 sm:flex sm:flex-row-reverse"> - <Button onClick={confirmDelete} color="red" className="ml-2"> - Delete - </Button> - <Button onClick={cancelDelete}>Cancel</Button> - </div> - </div> - </div> - </div> - ) : ( - <></> - )} + <DeleteResourceModal + isOpen={isDeleteModalOpen} + title="Delete Organization?" + message="Are you sure you want to delete this organization? This action cannot be undone." + resourceInformationTitle="Organization Information" + resourceInformation={[{ label: "Organization ID", value: orgToDelete, code: true }]} + onCancel={cancelDelete} + onOk={confirmDelete} + confirmLoading={isDeleting} + /> </div> ); }; diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index f8a7bafce51..80b5653e75a 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -38,6 +38,7 @@ import { import { LoggingCallbacksTable } from "./Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable"; import { AlertingObject } from "./Settings/LoggingAndAlerts/LoggingCallbacks/types"; import { parseErrorMessage } from "./shared/errorUtils"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; interface SettingsPageProps { accessToken: string | null; userRole: string | null; @@ -240,9 +241,10 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, const [showEditCallback, setShowEditCallback] = useState(false); const [selectedEditCallback, setSelectedEditCallback] = useState<any | null>(null); const [showDeleteConfirmModal, setShowDeleteConfirmModal] = useState(false); - const [callbackToDelete, setCallbackToDelete] = useState<string | null>(null); + const [callbackToDelete, setCallbackToDelete] = useState<any | null>(null); const [isUpdatingCallback, setIsUpdatingCallback] = useState(false); const [isAddingCallback, setIsAddingCallback] = useState(false); + const [isDeletingCallback, setIsDeletingCallback] = useState(false); useEffect(() => { if (!accessToken) { @@ -525,8 +527,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, }); }; - const handleDeleteCallback = (callbackName: string) => { - setCallbackToDelete(callbackName); + const handleDeleteCallback = (callback: any) => { + setCallbackToDelete(callback); setShowDeleteConfirmModal(true); }; @@ -536,8 +538,9 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, } try { - await deleteCallback(accessToken, callbackToDelete); - NotificationsManager.success(`Callback ${callbackToDelete} deleted successfully`); + setIsDeletingCallback(true); + await deleteCallback(accessToken, callbackToDelete.name); + NotificationsManager.success(`Callback ${callbackToDelete.name} deleted successfully`); // Refresh the callbacks list if (userID && userRole) { @@ -550,6 +553,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, } catch (error) { console.error("Failed to delete callback:", error); NotificationsManager.fromBackend(error); + } finally { + setIsDeletingCallback(false); } }; @@ -577,7 +582,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, setSelectedEditCallback(cb); setShowEditCallback(true); }} - onDelete={(cb) => handleDeleteCallback(cb.name)} + onDelete={(cb) => handleDeleteCallback(cb)} onTest={async (cb) => { try { await serviceHealthCheck(accessToken, cb.name); @@ -804,20 +809,22 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID, </Form> </Modal> - <Modal - title="Confirm Delete" - open={showDeleteConfirmModal} - onOk={confirmDeleteCallback} + <DeleteResourceModal + isOpen={showDeleteConfirmModal} + title="Delete Callback" + message="Are you sure you want to delete this callback? This action cannot be undone." + resourceInformationTitle="Callback Information" + resourceInformation={[ + { label: "Callback Name", value: callbackToDelete?.name }, + { label: "Mode", value: callbackToDelete?.mode || "success" }, + ]} onCancel={() => { setShowDeleteConfirmModal(false); setCallbackToDelete(null); }} - okText="Delete" - cancelText="Cancel" - okButtonProps={{ danger: true }} - > - <p>Are you sure you want to delete the callback - {callbackToDelete}? This action cannot be undone.</p> - </Modal> + onOk={confirmDeleteCallback} + confirmLoading={isDeletingCallback} + /> </div> ); }; diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 0dada8bb79f..6558889d284 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -25,7 +25,7 @@ import { teamUpdateCall, getGuardrailsList, } from "@/components/networking"; -import { Button, Form, Input, Select, Switch, message, Modal, Tooltip } from "antd"; +import { Button, Form, Input, Select, Switch, message, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { ArrowLeftIcon } from "@heroicons/react/outline"; import MemberModal from "./edit_membership"; @@ -44,6 +44,7 @@ import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import NotificationsManager from "../molecules/notifications_manager"; import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; export interface TeamMembership { user_id: string; @@ -139,6 +140,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({ const [copiedStates, setCopiedStates] = useState<Record<string, boolean>>({}); const [guardrailsList, setGuardrailsList] = useState<string[]>([]); const [memberToDelete, setMemberToDelete] = useState<Member | null>(null); + const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [isDeleting, setIsDeleting] = useState(false); console.log("userModels in team info", userModels); @@ -272,6 +274,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({ const handleMemberDelete = (member: Member) => { setMemberToDelete(member); + setIsDeleteModalOpen(true); }; const handleDeleteConfirm = async () => { @@ -294,11 +297,13 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({ console.error("Error removing team member:", error); } finally { setIsDeleting(false); + setIsDeleteModalOpen(false); setMemberToDelete(null); } }; const handleDeleteCancel = () => { + setIsDeleteModalOpen(false); setMemberToDelete(null); }; @@ -924,28 +929,21 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({ /> {/* Delete Member Confirmation Modal */} - {memberToDelete && ( - <Modal - title="Delete Team Member" - open={memberToDelete !== null} - onOk={handleDeleteConfirm} - onCancel={handleDeleteCancel} - confirmLoading={isDeleting} - okText={isDeleting ? "Deleting..." : "Delete"} - okButtonProps={{ danger: true }} - > - <p>Are you sure you want to remove this member from the team?</p> - <p className="mt-2"> - <strong>User ID:</strong> {memberToDelete.user_id} - </p> - {memberToDelete.user_email && ( - <p> - <strong>Email:</strong> {memberToDelete.user_email} - </p> - )} - <p className="mt-2 text-red-600">This action cannot be undone.</p> - </Modal> - )} + <DeleteResourceModal + isOpen={isDeleteModalOpen} + title="Delete Team Member" + alertMessage="Removing team members will also delete any keys created by or created for this member." + message="Are you sure you want to remove this member from the team? This action cannot be undone." + resourceInformationTitle="Team Member Information" + resourceInformation={[ + { label: "User ID", value: memberToDelete?.user_id, code: true }, + { label: "Email", value: memberToDelete?.user_email }, + { label: "Role", value: memberToDelete?.role }, + ]} + onCancel={handleDeleteCancel} + onOk={handleDeleteConfirm} + confirmLoading={isDeleting} + /> </div> ); }; From bd8196f982e8f51ff887a76d5da6bfeed2be28ef Mon Sep 17 00:00:00 2001 From: Raghav Jhavar <156360524+raghav-stripe@users.noreply.github.com> Date: Mon, 24 Nov 2025 23:40:43 -0500 Subject: [PATCH 13/68] (fix) propagate x-litellm-model-id in responses (#16986) * propagate model id on errors too * make it work for messages and streaming * fix * cleanup * cleanup * final * cleanup * clean up method name and fix responses api streaming * remove comment --- .../proxy/anthropic_endpoints/endpoints.py | 27 +- litellm/proxy/common_request_processing.py | 64 +++++ litellm/responses/streaming_iterator.py | 19 ++ .../proxy/test_model_id_header_propagation.py | 250 ++++++++++++++++++ 4 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/test_model_id_header_propagation.py diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index c450b655a2c..abea9e6fee1 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -154,8 +154,13 @@ async def anthropic_response( # noqa: PLR0915 response = responses[1] + # Extract model_id from request metadata (set by router during routing) + litellm_metadata = data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + + # Get other metadata from hidden_params hidden_params = getattr(response, "_hidden_params", {}) or {} - model_id = hidden_params.get("model_id", None) or "" cache_key = hidden_params.get("cache_key", None) or "" api_base = hidden_params.get("api_base", None) or "" response_cost = hidden_params.get("response_cost", None) or "" @@ -216,12 +221,32 @@ async def anthropic_response( # noqa: PLR0915 str(e) ) ) + + # Extract model_id from request metadata (same as success path) + litellm_metadata = data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + + # Get headers + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=data.get("litellm_call_id", ""), + model_id=model_id, + version=version, + response_cost=0, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + request_data=data, + timeout=getattr(e, "timeout", None), + litellm_logging_obj=None, + ) + error_msg = f"{str(e)}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), + headers=headers, ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1cdeb3b99ea..0143a6e6cec 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -344,6 +344,7 @@ class ProxyBaseLLMRequestProcessing: user_max_tokens: Optional[int] = None, user_api_base: Optional[str] = None, model: Optional[str] = None, + llm_router: Optional[Router] = None, ) -> Tuple[dict, LiteLLMLoggingObj]: start_time = datetime.now() # start before calling guardrail hooks @@ -498,6 +499,7 @@ class ProxyBaseLLMRequestProcessing: user_api_base=user_api_base, model=model, route_type=route_type, + llm_router=llm_router, ) tasks = [] @@ -536,6 +538,13 @@ class ProxyBaseLLMRequestProcessing: hidden_params = getattr(response, "_hidden_params", {}) or {} model_id = hidden_params.get("model_id", None) or "" + + # Fallback: extract model_id from litellm_metadata if not in hidden_params + if not model_id: + litellm_metadata = self.data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", "") or "" + cache_key = hidden_params.get("cache_key", None) or "" api_base = hidden_params.get("api_base", None) or "" response_cost = hidden_params.get("response_cost", None) or "" @@ -756,11 +765,19 @@ class ProxyBaseLLMRequestProcessing: _litellm_logging_obj: Optional[LiteLLMLoggingObj] = self.data.get( "litellm_logging_obj", None ) + + # Attempt to get model_id from logging object + # + # Note: We check the direct model_info path first (not nested in metadata) because that's where the router sets it. + # The nested metadata path is only a fallback for cases where model_info wasn't set at the top level. + model_id = self.maybe_get_model_id(_litellm_logging_obj) + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, call_id=( _litellm_logging_obj.litellm_call_id if _litellm_logging_obj else None ), + model_id=model_id, version=version, response_cost=0, model_region=getattr(user_api_key_dict, "allowed_model_region", ""), @@ -1073,3 +1090,50 @@ class ProxyBaseLLMRequestProcessing: obj.setdefault("usage", {})["cost"] = cost_val return obj return None + + def maybe_get_model_id(self, _logging_obj: Optional[LiteLLMLoggingObj]) -> Optional[str]: + """ + Get model_id from logging object or request metadata. + + The router sets model_info.id when selecting a deployment. This tries multiple locations + where the ID might be stored depending on the request lifecycle stage. + """ + model_id = None + if _logging_obj: + # 1. Try getting from litellm_params (updated during call) + if ( + hasattr(_logging_obj, "litellm_params") + and _logging_obj.litellm_params + ): + # First check direct model_info path (set by router.py with selected deployment) + model_info = _logging_obj.litellm_params.get("model_info") or {} + model_id = model_info.get("id", None) + + # Fallback to nested metadata path + if not model_id: + metadata = _logging_obj.litellm_params.get("metadata") or {} + model_info = metadata.get("model_info") or {} + model_id = model_info.get("id", None) + + # 2. Fallback to kwargs (initial) + if not model_id: + _kwargs = getattr(_logging_obj, "kwargs", None) + if _kwargs: + litellm_params = _kwargs.get("litellm_params", {}) + # First check direct model_info path + model_info = litellm_params.get("model_info") or {} + model_id = model_info.get("id", None) + + # Fallback to nested metadata path + if not model_id: + metadata = litellm_params.get("metadata") or {} + model_info = metadata.get("model_info") or {} + model_id = model_info.get("id", None) + + # 3. Final fallback to self.data["litellm_metadata"] (for routes like /v1/responses that populate data before error) + if not model_id: + litellm_metadata = self.data.get("litellm_metadata", {}) or {} + model_info = litellm_metadata.get("model_info", {}) or {} + model_id = model_info.get("id", None) + + return model_id diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8eecc3e8211..0407776029d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,7 +8,9 @@ import httpx import litellm from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils @@ -51,6 +53,23 @@ class BaseResponsesAPIStreamingIterator: self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider + # set hidden params for response headers (e.g., x-litellm-model-id) + # This matches ths stream wrapper in litellm/litellm_core_utils/streaming_handler.py + _api_base = get_api_base( + model=model or "", + optional_params=self.logging_obj.model_call_details.get( + "litellm_params", {} + ), + ) + _model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {} + self._hidden_params = { + "model_id": _model_info.get("id", None), + "api_base": _api_base, + } + self._hidden_params["additional_headers"] = process_response_headers( + self.response.headers or {} + ) # GUARANTEE OPENAI HEADERS IN RESPONSE + def _process_chunk(self, chunk) -> Optional[ResponsesAPIStreamingResponse]: """Process a single chunk of data from the stream""" if not chunk: diff --git a/tests/test_litellm/proxy/test_model_id_header_propagation.py b/tests/test_litellm/proxy/test_model_id_header_propagation.py new file mode 100644 index 00000000000..cc4e7c084d6 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_id_header_propagation.py @@ -0,0 +1,250 @@ +""" +Test that x-litellm-model-id header is propagated correctly on error responses. + +This test suite verifies the `maybe_get_model_id` method +which is responsible for extracting model_id from different locations +depending on the request lifecycle stage. +""" + +import pytest +from unittest.mock import MagicMock + +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy._types import UserAPIKeyAuth + + +def test_maybe_get_model_id_from_litellm_params(): + """ + Test extraction of model_id from logging_obj.litellm_params (used by /v1/chat/completions). + """ + # Create a ProxyBaseLLMRequestProcessing instance + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info in litellm_params + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "model_info": { + "id": "test-model-id-from-litellm-params" + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-litellm-params" + + +def test_maybe_get_model_id_from_litellm_params_nested(): + """ + Test extraction of model_id from nested metadata in logging_obj.litellm_params. + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info nested in metadata + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "metadata": { + "model_info": { + "id": "test-model-id-nested" + } + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-nested" + + +def test_maybe_get_model_id_from_kwargs(): + """ + Test extraction of model_id from logging_obj.kwargs (fallback path). + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object with model_info in kwargs + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = None + mock_logging_obj.kwargs = { + "litellm_params": { + "model_info": { + "id": "test-model-id-from-kwargs" + } + } + } + + # Test extraction + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-kwargs" + + +def test_maybe_get_model_id_from_data(): + """ + Test extraction of model_id from self.data (used by /v1/messages and /v1/responses). + """ + # Create a processor with model_info in data + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "test-model-id-from-data" + } + } + }) + + # Create a mock logging object without model_info + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = {} + mock_logging_obj.kwargs = {} + + # Test extraction - should fall back to self.data + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "test-model-id-from-data" + + +def test_maybe_get_model_id_no_logging_obj(): + """ + Test extraction of model_id when logging_obj is None (should use self.data). + """ + # Create a processor with model_info in data + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "test-model-id-no-logging-obj" + } + } + }) + + # Test extraction with None logging_obj + model_id = processor.maybe_get_model_id(None) + + assert model_id == "test-model-id-no-logging-obj" + + +def test_maybe_get_model_id_not_found(): + """ + Test extraction of model_id when it's not available anywhere (should return None). + """ + processor = ProxyBaseLLMRequestProcessing(data={}) + + # Create a mock logging object without model_info anywhere + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = {} + mock_logging_obj.kwargs = {} + + # Test extraction - should return None + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id is None + + +def test_maybe_get_model_id_priority_litellm_params_over_data(): + """ + Test that model_id from logging_obj.litellm_params takes priority over self.data. + """ + # Create a processor with model_info in both places + processor = ProxyBaseLLMRequestProcessing(data={ + "litellm_metadata": { + "model_info": { + "id": "model-id-from-data" + } + } + }) + + # Create a mock logging object with model_info + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_params = { + "model_info": { + "id": "model-id-from-litellm-params" + } + } + + # Test extraction - should prefer litellm_params + model_id = processor.maybe_get_model_id(mock_logging_obj) + + assert model_id == "model-id-from-litellm-params" + + +def test_get_custom_headers_includes_model_id(): + """ + Test that get_custom_headers includes x-litellm-model-id when model_id is provided. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers with a model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id="test-model-123", + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # Verify model_id is in headers + assert "x-litellm-model-id" in headers + assert headers["x-litellm-model-id"] == "test-model-123" + + +def test_get_custom_headers_without_model_id(): + """ + Test that get_custom_headers works correctly when model_id is None or empty. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers without a model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id=None, + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # x-litellm-model-id should not be in headers (or should be empty/None) + if "x-litellm-model-id" in headers: + assert headers["x-litellm-model-id"] in [None, ""] + + +def test_get_custom_headers_with_empty_string_model_id(): + """ + Test that get_custom_headers handles empty string model_id correctly. + """ + # Create mock user_api_key_dict with all required attributes + mock_user_api_key_dict = MagicMock() + mock_user_api_key_dict.user_id = "test-user" + mock_user_api_key_dict.team_id = "test-team" + mock_user_api_key_dict.tpm_limit = 1000 + mock_user_api_key_dict.rpm_limit = 100 + + # Call get_custom_headers with empty string model_id + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + model_id="", + cache_key="test-cache-key", + api_base="https://api.example.com", + version="1.0.0", + response_cost=0.001, + request_data={}, + hidden_params={} + ) + + # x-litellm-model-id should not be in headers (or should be empty) + if "x-litellm-model-id" in headers: + assert headers["x-litellm-model-id"] == "" From 262fb742d2f727cc689812b4200e8b116f86b31f Mon Sep 17 00:00:00 2001 From: yuya_matsuba <61488647+yuya2017@users.noreply.github.com> Date: Tue, 25 Nov 2025 13:41:44 +0900 Subject: [PATCH 14/68] Fix: Distinguish permission errors from idempotent errors in Prisma migrations (#17064) * fix: distinguish permission errors from idempotent errors in Prisma migrations * style: apply Black formatting and fix line length issues --- .../litellm_proxy_extras/utils.py | 142 +++++++++++++++--- .../test_litellm_proxy_extras_utils.py | 107 ++++++++++++- 2 files changed, 224 insertions(+), 25 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 73065b050b7..96e1a5106ac 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -130,6 +130,60 @@ class ProxyExtrasDBManager: capture_output=True, ) + @staticmethod + def _is_permission_error(error_message: str) -> bool: + """ + Check if the error message indicates a database permission error. + + Permission errors should NOT be marked as applied, as the migration + did not actually execute successfully. + + Args: + error_message: The error message from Prisma migrate + + Returns: + bool: True if this is a permission error, False otherwise + """ + permission_patterns = [ + r"Database error code: 42501", # PostgreSQL insufficient privilege + r"must be owner of table", + r"permission denied for schema", + r"permission denied for table", + r"must be owner of schema", + ] + + for pattern in permission_patterns: + if re.search(pattern, error_message, re.IGNORECASE): + return True + return False + + @staticmethod + def _is_idempotent_error(error_message: str) -> bool: + """ + Check if the error message indicates an idempotent operation error. + + Idempotent errors (like "column already exists") mean the migration + has effectively already been applied, so it's safe to mark as applied. + + Args: + error_message: The error message from Prisma migrate + + Returns: + bool: True if this is an idempotent error, False otherwise + """ + idempotent_patterns = [ + r"already exists", + r"column .* already exists", + r"duplicate key value violates", + r"relation .* already exists", + r"constraint .* already exists", + ] + + for pattern in idempotent_patterns: + if re.search(pattern, error_message, re.IGNORECASE): + return True + return False + @staticmethod def _resolve_all_migrations( migrations_dir: str, schema_path: str, mark_all_applied: bool = True @@ -320,29 +374,79 @@ class ProxyExtrasDBManager: ) logger.info("✅ All migrations resolved.") return True - elif ( - "P3018" in e.stderr - ): # PostgreSQL error code for duplicate column - logger.info( - "Migration already exists, resolving specific migration" - ) - # Extract the migration name from the error message - migration_match = re.search( - r"Migration name: (\d+_.*)", e.stderr - ) - if migration_match: - migration_name = migration_match.group(1) - logger.info(f"Rolling back migration {migration_name}") - ProxyExtrasDBManager._roll_back_migration( - migration_name + elif "P3018" in e.stderr: + # Check if this is a permission error or idempotent error + if ProxyExtrasDBManager._is_permission_error(e.stderr): + # Permission errors should NOT be marked as applied + # Extract migration name for logging + migration_match = re.search( + r"Migration name: (\d+_.*)", e.stderr ) + migration_name = ( + migration_match.group(1) + if migration_match + else "unknown" + ) + + logger.error( + f"❌ Migration {migration_name} failed due to insufficient permissions. " + f"Please check database user privileges. Error: {e.stderr}" + ) + + # Mark as rolled back and exit with error + if migration_match: + try: + ProxyExtrasDBManager._roll_back_migration( + migration_name + ) + logger.info( + f"Migration {migration_name} marked as rolled back" + ) + except Exception as rollback_error: + logger.warning( + f"Failed to mark migration as rolled back: {rollback_error}" + ) + + # Re-raise the error to prevent silent failures + raise RuntimeError( + f"Migration failed due to permission error. Migration {migration_name} " + f"was NOT applied. Please grant necessary database permissions and retry." + ) from e + + elif ProxyExtrasDBManager._is_idempotent_error(e.stderr): + # Idempotent errors mean the migration has effectively been applied logger.info( - f"Resolving migration {migration_name} that failed due to existing columns" + "Migration failed due to idempotent error (e.g., column already exists), " + "resolving as applied" ) - ProxyExtrasDBManager._resolve_specific_migration( - migration_name + # Extract the migration name from the error message + migration_match = re.search( + r"Migration name: (\d+_.*)", e.stderr ) - logger.info("✅ Migration resolved.") + if migration_match: + migration_name = migration_match.group(1) + logger.info( + f"Rolling back migration {migration_name}" + ) + ProxyExtrasDBManager._roll_back_migration( + migration_name + ) + logger.info( + f"Resolving migration {migration_name} that failed " + f"due to existing schema objects" + ) + ProxyExtrasDBManager._resolve_specific_migration( + migration_name + ) + logger.info("✅ Migration resolved.") + else: + # Unknown P3018 error - log and re-raise for safety + logger.warning( + f"P3018 error encountered but could not classify " + f"as permission or idempotent error. " + f"Error: {e.stderr}" + ) + raise else: # Use prisma db push with increased timeout subprocess.run( diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 3bcb5b25da8..5714cd5c487 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -1,11 +1,5 @@ -import json import os import sys -import httpx -import pytest -import respx - -from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../..") @@ -13,8 +7,10 @@ sys.path.insert( from litellm_proxy_extras.utils import ProxyExtrasDBManager + def test_custom_prisma_dir(monkeypatch): import tempfile + # create a temp directory temp_dir = tempfile.mkdtemp() monkeypatch.setenv("LITELLM_MIGRATION_DIR", temp_dir) @@ -30,3 +26,102 @@ def test_custom_prisma_dir(monkeypatch): migrations_dir = os.path.join(temp_dir, "migrations") assert os.path.exists(migrations_dir) + +class TestPermissionErrorDetection: + """Test cases for permission error detection in Prisma migrations""" + + def test_is_permission_error_postgres_42501(self): + """Test detection of PostgreSQL 42501 error code (insufficient privilege)""" + error_message = "Database error code: 42501 - permission denied for table users" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_must_be_owner(self): + """Test detection of 'must be owner of table' error""" + error_message = "ERROR: must be owner of table my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_permission_denied_schema(self): + """Test detection of 'permission denied for schema' error""" + error_message = "permission denied for schema public" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_permission_denied_table(self): + """Test detection of 'permission denied for table' error""" + error_message = "permission denied for table my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_must_be_owner_schema(self): + """Test detection of 'must be owner of schema' error""" + error_message = "must be owner of schema public" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_case_insensitive(self): + """Test that permission error detection is case insensitive""" + error_message = "PERMISSION DENIED FOR TABLE my_table" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + + def test_is_permission_error_negative(self): + """Test that non-permission errors are not detected as permission errors""" + error_message = "column 'id' already exists" + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + + +class TestIdempotentErrorDetection: + """Test cases for idempotent error detection in Prisma migrations""" + + def test_is_idempotent_error_already_exists(self): + """Test detection of generic 'already exists' error""" + error_message = "object already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_column_already_exists(self): + """Test detection of 'column already exists' error""" + error_message = "column 'email' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_duplicate_key(self): + """Test detection of duplicate key violation error""" + error_message = "duplicate key value violates unique constraint" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_relation_already_exists(self): + """Test detection of 'relation already exists' error""" + error_message = "relation 'users_pkey' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_constraint_already_exists(self): + """Test detection of 'constraint already exists' error""" + error_message = "constraint 'fk_user_id' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_case_insensitive(self): + """Test that idempotent error detection is case insensitive""" + error_message = "COLUMN 'ID' ALREADY EXISTS" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + + def test_is_idempotent_error_negative(self): + """Test that non-idempotent errors are not detected as idempotent errors""" + error_message = "Database error code: 42501 - permission denied" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + + +class TestErrorClassificationPriority: + """Test cases to ensure errors are correctly classified""" + + def test_permission_error_not_classified_as_idempotent(self): + """Ensure permission errors are not mistakenly classified as idempotent""" + error_message = "Database error code: 42501 - must be owner of table users" + assert ProxyExtrasDBManager._is_permission_error(error_message) is True + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False + + def test_idempotent_error_not_classified_as_permission(self): + """Ensure idempotent errors are not mistakenly classified as permission errors""" + error_message = "column 'created_at' already exists" + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + + def test_unknown_error_classified_as_neither(self): + """Ensure unknown errors are classified as neither permission nor idempotent""" + error_message = "connection timeout" + assert ProxyExtrasDBManager._is_permission_error(error_message) is False + assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False From e371ff454a4479c63d0fa02acb99f47401893769 Mon Sep 17 00:00:00 2001 From: yuneng-jiang <yuneng.jiang@gmail.com> Date: Mon, 24 Nov 2025 20:45:56 -0800 Subject: [PATCH 15/68] Non root docker build fix (#17060) --- docker/Dockerfile.non_root | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 3fa0ab69e3b..2dcb7cb4787 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -20,27 +20,33 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI -RUN mkdir -p /tmp/litellm_ui && \ - npm install -g npm@latest && \ - npm cache clean --force && \ - cd ui/litellm-dashboard && \ - if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \ - cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ - fi && \ - rm -f package-lock.json && \ - npm install --legacy-peer-deps && \ - npm run build && \ - cp -r ./out/* /tmp/litellm_ui/ && \ - cd /tmp/litellm_ui && \ +RUN mkdir -p /tmp/litellm_ui + +RUN npm install -g npm@latest && npm cache clean --force + +RUN cd /app/ui/litellm-dashboard && \ + if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ + cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ + fi + +RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json + +RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps + +RUN cd /app/ui/litellm-dashboard && npm run build + +RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ + +RUN cd /tmp/litellm_ui && \ for html_file in *.html; do \ if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ folder_name="${html_file%.html}" && \ mkdir -p "$folder_name" && \ mv "$html_file" "$folder_name/index.html"; \ fi; \ - done && \ - cd /app/ui/litellm-dashboard && \ - rm -rf ./out + done + +RUN cd /app/ui/litellm-dashboard && rm -rf ./out # Build package and wheel dependencies RUN rm -rf dist/* && python -m build && \ From 3f5a34d72c12746c00502822b00d0b923a891014 Mon Sep 17 00:00:00 2001 From: yuneng-jiang <yuneng.jiang@gmail.com> Date: Mon, 24 Nov 2025 20:47:43 -0800 Subject: [PATCH 16/68] Deleting a user from team deletes key user created for team (#17057) --- .../management_endpoints/team_endpoints.py | 9 ++++ .../test_team_endpoints.py | 52 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 3a66f95e812..6d4faae5fd8 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1878,6 +1878,15 @@ async def team_member_delete( where={"team_id": data.team_id, "user_id": _uid} ) + ## DELETE KEYS CREATED BY USER FOR THIS TEAM + if user_ids_to_delete: + await prisma_client.db.litellm_verificationtoken.delete_many( + where={ + "user_id": {"in": list(user_ids_to_delete)}, + "team_id": data.team_id, + } + ) + return existing_team_row diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index c9b7e057904..86b23c98ba5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -1783,6 +1783,10 @@ async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_a mock_db_client.db.litellm_teammembership = MagicMock() mock_db_client.db.litellm_teammembership.delete_many = AsyncMock(return_value=MagicMock()) + # Verification token deletion should be called + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock(return_value=MagicMock()) + # Execute await team_member_delete( data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), @@ -1795,6 +1799,54 @@ async def test_team_member_delete_cleans_membership(mock_db_client, mock_admin_a ) +@pytest.mark.asyncio +async def test_team_member_delete_cleans_verification_tokens(mock_db_client, mock_admin_auth): + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-tokens-123" + test_user_id = "user-tokens@example.com" + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [ + {"user_id": test_user_id, "user_email": None, "role": "user"} + ], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=mock_team_row) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.teams = [test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock(return_value=[mock_user_row]) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock(return_value=MagicMock()) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + + mock_db_client.db.litellm_verificationtoken.delete_many.assert_awaited_once_with( + where={ + "user_id": {"in": [test_user_id]}, + "team_id": test_team_id, + } + ) + + @pytest.mark.asyncio async def test_new_team_max_budget_exceeds_user_max_budget(): """ From 1ae80955e8d70e81c62c420d3f37936df966eb11 Mon Sep 17 00:00:00 2001 From: Krish Dholakia <krrishdholakia@gmail.com> Date: Mon, 24 Nov 2025 20:48:10 -0800 Subject: [PATCH 17/68] Docs: Add link to logging payload spec (#17049) Co-authored-by: Cursor Agent <cursoragent@cursor.com> --- docs/my-website/docs/observability/custom_callback.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md index cfe97ca42c0..ae892621270 100644 --- a/docs/my-website/docs/observability/custom_callback.md +++ b/docs/my-website/docs/observability/custom_callback.md @@ -203,7 +203,11 @@ asyncio.run(test_chat_openai()) ## What's Available in kwargs? -The kwargs dictionary contains all the details about your API call: +The kwargs dictionary contains all the details about your API call. + +:::info +For the complete logging payload specification, see the [Standard Logging Payload Spec](https://docs.litellm.ai/docs/proxy/logging_spec). +::: ```python def custom_callback(kwargs, completion_response, start_time, end_time): From d2b3ef0667db4a2d13ec729d9bf1c7219c63bfa8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang <yuneng.jiang@gmail.com> Date: Mon, 24 Nov 2025 20:48:51 -0800 Subject: [PATCH 18/68] Add aws_bedrock_runtime_endpoint into Credential Types (#17053) --- litellm/types/router.py | 1 + tests/test_litellm/test_router.py | 32 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 2bf126211c3..002792d0490 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -159,6 +159,7 @@ class CredentialLiteLLMParams(BaseModel): aws_access_key_id: Optional[str] = None aws_secret_access_key: Optional[str] = None aws_region_name: Optional[str] = None + aws_bedrock_runtime_endpoint: Optional[str] = None ## IBM WATSONX ## watsonx_region_name: Optional[str] = None diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8851264db07..032616849bd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1692,3 +1692,35 @@ async def test_router_acompletion_with_unknown_model_and_no_fallback(): # Check that the error message is correct. # The router returns 'no healthy deployments' because get_model_list returns [] not None. assert "no healthy deployments for this model" in str(excinfo.value) + + +def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): + """ + Test that get_deployment_credentials_with_provider correctly copies + aws_bedrock_runtime_endpoint from deployment litellm_params to credentials. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-claude-model", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-east-1.amazonaws.com", + }, + } + ], + ) + + credentials = router.get_deployment_credentials_with_provider( + model_id="bedrock-claude-model" + ) + + assert credentials is not None + assert credentials["aws_bedrock_runtime_endpoint"] == "https://bedrock-runtime.us-east-1.amazonaws.com" + assert credentials["aws_access_key_id"] == "test-access-key" + assert credentials["aws_secret_access_key"] == "test-secret-key" + assert credentials["aws_region_name"] == "us-east-1" + assert credentials["custom_llm_provider"] == "bedrock" From 597fa4d35cf792eb03e1492445a88808ce34d150 Mon Sep 17 00:00:00 2001 From: Emerson Gomes <emerson.gomes@thalesgroup.com> Date: Mon, 24 Nov 2025 22:52:35 -0600 Subject: [PATCH 19/68] Fix image edit endpoint (#17046) * Fix image edit endpoint * Update litellm/proxy/image_endpoints/endpoints.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- litellm/proxy/image_endpoints/endpoints.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 16aa8f16571..a1453e10dbf 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -215,9 +215,11 @@ async def image_generation( async def image_edit_api( request: Request, fastapi_response: Response, - image: List[UploadFile] = File(...), - mask: Optional[List[UploadFile]] = File(None), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + image: Optional[List[UploadFile]] = File(None), + image_array: Optional[List[UploadFile]] = File(None, alias="image[]"), + mask: Optional[List[UploadFile]] = File(None), + mask_array: Optional[List[UploadFile]] = File(None, alias="mask[]"), model: Optional[str] = None, ): """ @@ -233,6 +235,18 @@ async def image_edit_api( -F 'prompt=Create a studio ghibli image of this' ``` """ + if image is not None and image_array is not None: + raise HTTPException(status_code=422, detail="Cannot specify both 'image' and 'image[]'") + if mask is not None and mask_array is not None: + raise HTTPException(status_code=422, detail="Cannot specify both 'mask' and 'mask[]'") + if image is None and image_array is not None: + image = image_array + if mask is None and mask_array is not None: + mask = mask_array + + if image is None: + raise HTTPException(status_code=422, detail="Field required: image") + from litellm.proxy.proxy_server import ( _read_request_body, general_settings, From 777ef628d2fc1247e16c9f0c41e7217bf2ac1181 Mon Sep 17 00:00:00 2001 From: Saar wintrov <saar1122@gmail.com> Date: Tue, 25 Nov 2025 06:53:02 +0200 Subject: [PATCH 20/68] Enhancement(helm): ServiceMonitor template rendering (#17038) * Metadata: fix 401 when audio/transcriptions * check if str, CR fixes * Added new helmchart functionality * . * . * adding new tests --- .../litellm-helm/templates/deployment.yaml | 9 ++ .../templates/servicemonitor.yaml | 39 +++++ .../templates/tests/test-servicemonitor.yaml | 152 ++++++++++++++++++ deploy/charts/litellm-helm/values.yaml | 26 +++ 4 files changed, 226 insertions(+) create mode 100644 deploy/charts/litellm-helm/templates/servicemonitor.yaml create mode 100644 deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 6a5a6e87577..316323be99a 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -129,6 +129,10 @@ spec: args: - --config - /etc/litellm/config.yaml + {{ if .Values.numWorkers }} + - --num_workers + - {{ .Values.numWorkers | quote }} + {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} @@ -208,3 +212,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }} + {{- if .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/deploy/charts/litellm-helm/templates/servicemonitor.yaml b/deploy/charts/litellm-helm/templates/servicemonitor.yaml new file mode 100644 index 00000000000..743098deb3f --- /dev/null +++ b/deploy/charts/litellm-helm/templates/servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- with .Values.serviceMonitor }} +{{- if and (eq .enabled true) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.fullname" $ }} + labels: + {{- include "litellm.labels" $ | nindent 4 }} + {{- if .labels }} + {{- toYaml .labels | nindent 4 }} + {{- end }} + {{- if .annotations }} + annotations: + {{- toYaml .annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.selectorLabels" $ | nindent 6 }} + namespaceSelector: + matchNames: + # if not set, use the release namespace + {{- if not .namespaceSelector.matchNames }} + - {{ $.Release.Namespace | quote }} + {{- else }} + {{- toYaml .namespaceSelector.matchNames | nindent 4 }} + {{- end }} + endpoints: + - port: http + path: /metrics/ + interval: {{ .interval }} + scrapeTimeout: {{ .scrapeTimeout }} + scheme: http + {{- if .relabelings }} + relabelings: +{{- toYaml .relabelings | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml b/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml new file mode 100644 index 00000000000..c2a4f84ec21 --- /dev/null +++ b/deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml @@ -0,0 +1,152 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "litellm.fullname" . }}-test-servicemonitor" + labels: + {{- include "litellm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: test + image: bitnami/kubectl:latest + command: ['sh', '-c'] + args: + - | + set -e + echo "🔍 Testing ServiceMonitor configuration..." + + # Check if ServiceMonitor exists + if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then + echo "❌ ServiceMonitor not found" + exit 1 + fi + echo "✅ ServiceMonitor exists" + + # Get ServiceMonitor YAML + SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml) + + # Test endpoint configuration + ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}') + if [ "$ENDPOINT_PORT" != "http" ]; then + echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT" + exit 1 + fi + echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT" + + # Test endpoint path + ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}') + if [ "$ENDPOINT_PATH" != "/metrics/" ]; then + echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH" + exit 1 + fi + echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH" + + # Test interval + INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}') + if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then + echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL" + exit 1 + fi + echo "✅ Interval is correctly set to: $INTERVAL" + + # Test scrapeTimeout + TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}') + if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then + echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT" + exit 1 + fi + echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT" + + # Test scheme + SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}') + if [ "$SCHEME" != "http" ]; then + echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME" + exit 1 + fi + echo "✅ Scheme is correctly set to: $SCHEME" + + {{- if .Values.serviceMonitor.labels }} + # Test custom labels + echo "🔍 Checking custom labels..." + {{- range $key, $value := .Values.serviceMonitor.labels }} + LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}') + if [ "$LABEL_VALUE" != "{{ $value }}" ]; then + echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE" + exit 1 + fi + echo "✅ Label {{ $key }} is correctly set to: {{ $value }}" + {{- end }} + {{- end }} + + {{- if .Values.serviceMonitor.annotations }} + # Test annotations + echo "🔍 Checking annotations..." + {{- range $key, $value := .Values.serviceMonitor.annotations }} + ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}') + if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then + echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE" + exit 1 + fi + echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}" + {{- end }} + {{- end }} + + {{- if .Values.serviceMonitor.namespaceSelector.matchNames }} + # Test namespace selector + echo "🔍 Checking namespace selector..." + {{- range .Values.serviceMonitor.namespaceSelector.matchNames }} + if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then + echo "❌ Namespace {{ . }} not found in namespaceSelector" + exit 1 + fi + echo "✅ Namespace {{ . }} found in namespaceSelector" + {{- end }} + {{- else }} + # Test default namespace selector (should be release namespace) + if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then + echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector" + exit 1 + fi + echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}" + {{- end }} + + {{- if .Values.serviceMonitor.relabelings }} + # Test relabelings + echo "🔍 Checking relabelings configuration..." + if ! echo "$SM" | grep -q "relabelings:"; then + echo "❌ Relabelings section not found" + exit 1 + fi + echo "✅ Relabelings section exists" + {{- range .Values.serviceMonitor.relabelings }} + {{- if .targetLabel }} + if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then + echo "❌ Relabeling targetLabel {{ .targetLabel }} not found" + exit 1 + fi + echo "✅ Relabeling targetLabel {{ .targetLabel }} found" + {{- end }} + {{- if .action }} + if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then + echo "❌ Relabeling action {{ .action }} not found" + exit 1 + fi + echo "✅ Relabeling action {{ .action }} found" + {{- end }} + {{- end }} + {{- end }} + + # Test selector labels match the service + echo "🔍 Checking selector labels match service..." + SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}') + echo "Service labels: $SVC_LABELS" + echo "✅ Selector labels validation passed" + + echo "" + echo "🎉 All ServiceMonitor tests passed successfully!" + serviceAccountName: {{ include "litellm.serviceAccountName" . }} + restartPolicy: Never +{{- end }} + diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index c1792497d29..acb8c9ca32f 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -3,6 +3,7 @@ # Declare variables to be passed into your templates. replicaCount: 1 +# numWorkers: 2 image: # Use "ghcr.io/berriai/litellm-database" for optimized image with database @@ -33,6 +34,15 @@ deploymentAnnotations: {} podAnnotations: {} podLabels: {} +terminationGracePeriodSeconds: 90 +topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: litellm + # At the time of writing, the litellm docker image requires write access to the # filesystem on startup so that prisma can install some dependencies. podSecurityContext: {} @@ -248,3 +258,19 @@ pdb: maxUnavailable: null # e.g. 1 or "20%" annotations: {} labels: {} + +serviceMonitor: + enabled: false + labels: {} + # test: test + annotations: {} + # kubernetes.io/test: test + interval: 15s + scrapeTimeout: 10s + relabelings: [] + # - targetLabel: __meta_kubernetes_pod_node_name + # replacement: $1 + # action: replace + namespaceSelector: + matchNames: [] + # - test-namespace \ No newline at end of file From 3aba6d96fd88122b0d3af394587ed53907bb72f3 Mon Sep 17 00:00:00 2001 From: yuneng-jiang <yuneng.jiang@gmail.com> Date: Mon, 24 Nov 2025 20:53:17 -0800 Subject: [PATCH 21/68] [Fix] UI - Add No Default Models for Team and User Settings (#17037) * Add No Default Models to Team and User settings * Removing unused imports * Adding to Create User and Team flow --- .../src/components/OldTeams.tsx | 7 ++- .../src/components/SSOSettings.tsx | 4 +- .../src/components/TeamSSOSettings.test.tsx | 63 +++++++++++++++++++ .../src/components/TeamSSOSettings.tsx | 3 + .../src/components/create_user_button.tsx | 3 + .../src/components/team/team_info.tsx | 3 + 6 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index eefb0302a89..cc66a23eb48 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -30,8 +30,7 @@ import { Text, TextInput, } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Select as Select2, Tooltip, Typography } from "antd"; -import { AlertTriangleIcon, XIcon } from "lucide-react"; +import { Button as Button2, Form, Input, Modal, Select as Select2, Switch, Tooltip, Typography } from "antd"; import React, { useEffect, useState } from "react"; import { formatNumberWithCommas } from "../utils/dataUtils"; import { fetchTeams } from "./common_components/fetch_teams"; @@ -77,6 +76,7 @@ interface EditTeamModalProps { } import { updateExistingKeys } from "@/utils/dataUtils"; +import DeleteResourceModal from "./common_components/DeleteResourceModal"; import { Member, teamCreateCall, v2TeamListCall } from "./networking"; interface TeamInfo { @@ -1145,6 +1145,9 @@ const Teams: React.FC<TeamProps> = ({ <Select2.Option key="all-proxy-models" value="all-proxy-models"> All Proxy Models </Select2.Option> + <Select2.Option key="no-default-models" value="no-default-models"> + No Default Models + </Select2.Option> {modelsToPick.map((model) => ( <Select2.Option key={model} value={model}> {getModelDisplayName(model)} diff --git a/ui/litellm-dashboard/src/components/SSOSettings.tsx b/ui/litellm-dashboard/src/components/SSOSettings.tsx index 917aa1864e7..6402220f374 100644 --- a/ui/litellm-dashboard/src/components/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/SSOSettings.tsx @@ -274,7 +274,9 @@ const SSOSettings: React.FC<SSOSettingsProps> = ({ accessToken, possibleUIRoles, onChange={(value) => handleTextInputChange(key, value)} className="mt-2" > - <Option value="no-default-models">No Default Models</Option> + <Option key="no-default-models" value="no-default-models"> + No Default Models + </Option> {availableModels.map((model: string) => ( <Option key={model} value={model}> {getModelDisplayName(model)} diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx new file mode 100644 index 00000000000..f5e43fc3d5f --- /dev/null +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.test.tsx @@ -0,0 +1,63 @@ +import { screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../tests/test-utils"; +import TeamSSOSettings from "./TeamSSOSettings"; +import * as networking from "./networking"; + +// Mock the networking functions +vi.mock("./networking"); + +// Mock the budget duration dropdown +vi.mock("./common_components/budget_duration_dropdown", () => ({ + default: ({ value, onChange }: { value: string | null; onChange: (value: string) => void }) => ( + <select data-testid="budget-duration-dropdown" value={value || ""} onChange={(e) => onChange(e.target.value)}> + <option value="">Select duration</option> + <option value="daily">Daily</option> + <option value="monthly">Monthly</option> + </select> + ), + getBudgetDurationLabel: vi.fn((value: string) => value), +})); + +// Mock the model display name helper +vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ + getModelDisplayName: vi.fn((model: string) => model), +})); + +describe("TeamSSOSettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("renders the component", async () => { + // Mock successful API responses + vi.mocked(networking.getDefaultTeamSettings).mockResolvedValue({ + values: { + budget_duration: "monthly", + max_budget: 1000, + }, + field_schema: { + description: "Default team settings", + properties: { + budget_duration: { + type: "string", + description: "Budget duration", + }, + max_budget: { + type: "number", + description: "Maximum budget", + }, + }, + }, + }); + + vi.mocked(networking.modelAvailableCall).mockResolvedValue({ + data: [{ id: "gpt-4" }, { id: "claude-3" }], + }); + + renderWithProviders(<TeamSSOSettings accessToken="test-token" userID="test-user" userRole="admin" />); + + const container = await screen.findByText("Default Team Settings"); + expect(container).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx index 1c4d7f62400..8537b108cdc 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx @@ -123,6 +123,9 @@ const TeamSSOSettings: React.FC<TeamSSOSettingsProps> = ({ accessToken, userID, onChange={(value) => handleTextInputChange(key, value)} className="mt-2" > + <Option key="no-default-models" value="no-default-models"> + No Default Models + </Option> {availableModels.map((model: string) => ( <Option key={model} value={model}> {getModelDisplayName(model)} diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index 6fb6f80c4b4..935c34321ef 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -299,6 +299,9 @@ const Createuser: React.FC<CreateuserProps> = ({ <Select2.Option key="all-proxy-models" value="all-proxy-models"> All Proxy Models </Select2.Option> + <Select2.Option key="no-default-models" value="no-default-models"> + No Default Models + </Select2.Option> {userModels.map((model) => ( <Select2.Option key={model} value={model}> {getModelDisplayName(model)} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 0dada8bb79f..1906a6fce01 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -586,6 +586,9 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({ <Select.Option key="all-proxy-models" value="all-proxy-models"> All Proxy Models </Select.Option> + <Select.Option key="no-default-models" value="no-default-models"> + No Default Models + </Select.Option> {Array.from(new Set(userModels)).map((model, idx) => ( <Select.Option key={idx} value={model}> {getModelDisplayName(model)} From 650b18974fb35e41675548130b16e6b7824289f7 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 25 Nov 2025 01:54:12 -0300 Subject: [PATCH 22/68] fix(gemini): skip thinking config for image models (#17027) * fix(gemini): exclude image models from automatic thinking_level parameter (#17013) - gemini-3-pro-image-preview does not support thinking_level parameter - Added check to skip adding thinkingConfig for models containing "image" - Fixes BadRequestError: "Thinking level is not supported for this model" - Only affects automatic default behavior, user can still pass reasoning_effort explicitly Fixes #17013 * test: add tests for gemini-3 image models thinking_level exclusion * update docs --- docs/my-website/docs/providers/gemini.md | 4 + .../vertex_and_google_ai_studio_gemini.py | 16 ++-- ...test_vertex_and_google_ai_studio_gemini.py | 95 +++++++++++++++++++ 3 files changed, 108 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index e04225e1f85..1b21ed8d03c 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -74,6 +74,10 @@ Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. For **Gemini 3+ models** (e.g., `gemini-3-pro-preview`), LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter instead of `thinking_budget`. The `thinking_level` parameter uses `"low"` or `"high"` values for better control over reasoning depth. ::: +:::warning Image Models +**Gemini image models** (e.g., `gemini-3-pro-image-preview`, `gemini-2.0-flash-exp-image-generation`) do **not** support the `thinking_level` parameter. LiteLLM automatically excludes image models from receiving thinking configuration to prevent API errors. +::: + **Mapping for Gemini 2.5 and earlier models** | reasoning_effort | thinking | Notes | diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ab594c79ef4..5fef8c1ec49 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -904,13 +904,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if VertexGeminiConfig._is_gemini_3_or_newer(model): if "temperature" not in optional_params: optional_params["temperature"] = 1.0 - thinking_config = optional_params.get("thinkingConfig", {}) - if ( - "thinkingLevel" not in thinking_config - and "thinkingBudget" not in thinking_config - ): - thinking_config["thinkingLevel"] = "low" - optional_params["thinkingConfig"] = thinking_config + # Only add thinkingLevel if model supports it (exclude image models) + if "image" not in model.lower(): + thinking_config = optional_params.get("thinkingConfig", {}) + if ( + "thinkingLevel" not in thinking_config + and "thinkingBudget" not in thinking_config + ): + thinking_config["thinkingLevel"] = "low" + optional_params["thinkingConfig"] = thinking_config return optional_params diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 8942239bb21..2b305dbade1 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1967,3 +1967,98 @@ def test_media_resolution_per_part(): assert "inline_data" in image2_part assert image2_part["inline_data"]["mediaResolution"] == "high" + +def test_gemini_3_image_models_no_thinking_config(): + """ + Test that Gemini 3 image models do NOT receive automatic thinkingConfig. + + Related issue: https://github.com/BerriAI/litellm/issues/17013 + gemini-3-pro-image-preview does not support thinking_level parameter + and returns BadRequestError: "Thinking level is not supported for this model" + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test gemini-3-pro-image-preview (the specific model from the bug report) + model = "gemini-3-pro-image-preview" + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # Should NOT have thinkingConfig automatically added + assert "thinkingConfig" not in result + # But should still get temperature=1.0 for Gemini 3 + assert result["temperature"] == 1.0 + + +def test_gemini_3_text_models_get_thinking_config(): + """ + Test that Gemini 3 text models DO receive automatic thinkingConfig. + This ensures we didn't break the existing behavior for non-image models. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test gemini-3-pro-preview (text model, should get thinking) + model = "gemini-3-pro-preview" + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # Should have thinkingConfig automatically added + assert "thinkingConfig" in result + assert result["thinkingConfig"]["thinkingLevel"] == "low" + assert result["temperature"] == 1.0 + + +def test_gemini_image_models_excluded_from_thinking(): + """ + Test that any Gemini model with 'image' in the name is excluded from thinking config. + This covers current and future image models. + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + v = VertexGeminiConfig() + + # Test various image model patterns + image_models = [ + "gemini-3-pro-image-preview", + "gemini-3-pro-image-generation", + "gemini-3-flash-image-preview", + "gemini/gemini-3-image-edit", + ] + + for model in image_models: + optional_params = {} + non_default_params = {} + + result = v.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=False, + ) + + # None of these should have thinkingConfig + assert "thinkingConfig" not in result, f"Model {model} should not have thinkingConfig" + From cfd35d3b146b0c183b4353201f90ad3241d1d54e Mon Sep 17 00:00:00 2001 From: Saar wintrov <saar1122@gmail.com> Date: Tue, 25 Nov 2025 06:56:27 +0200 Subject: [PATCH 23/68] Metadata: fix 401 when audio/transcriptions (#17023) * Metadata: fix 401 when audio/transcriptions * check if str, CR fixes --- .../proxy/common_utils/http_parsing_utils.py | 2 + .../common_utils/test_http_parsing_utils.py | 202 ++++++++++++++++++ 2 files changed, 204 insertions(+) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 8b602c525d6..8d8d176e232 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -39,6 +39,8 @@ async def _read_request_body(request: Optional[Request]) -> Dict: if "form" in content_type: parsed_body = dict(await request.form()) + if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str): + parsed_body["metadata"] = json.loads(parsed_body["metadata"]) else: # Read the request body body = await request.body() diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index a8df4273765..85858866dda 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -93,6 +93,208 @@ async def test_form_data_parsing(): assert not hasattr(mock_request, "body") or not mock_request.body.called +@pytest.mark.asyncio +async def test_form_data_with_json_metadata(): + """ + Test that form data with a JSON-encoded metadata field is correctly parsed. + + When form data includes a 'metadata' field, it comes as a JSON string that needs + to be parsed into a Python dictionary (lines 42-43 of http_parsing_utils.py). + """ + # Create a mock request with form data containing JSON metadata + mock_request = MagicMock() + + # Metadata is sent as a JSON string in form data + metadata_json_string = json.dumps({ + "user_id": "12345", + "request_type": "audio_transcription", + "tags": ["urgent", "production"], + "custom_field": {"nested": "value"} + }) + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": metadata_json_string # This is a JSON string, not a dict + } + + # Mock the form method to return the test data as an awaitable + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata was parsed from JSON string to dict + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"]["user_id"] == "12345" + assert result["metadata"]["request_type"] == "audio_transcription" + assert result["metadata"]["tags"] == ["urgent", "production"] + assert result["metadata"]["custom_field"] == {"nested": "value"} + + # Verify other fields remain unchanged + assert result["model"] == "whisper-1" + assert result["file"] == "audio.mp3" + + # Verify form() was called + mock_request.form.assert_called_once() + + +@pytest.mark.asyncio +async def test_form_data_with_invalid_json_metadata(): + """ + Test that form data with invalid JSON in metadata field raises an exception. + + This tests error handling when the metadata field contains malformed JSON. + """ + # Create a mock request with form data containing invalid JSON metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": '{"invalid": json}' # Invalid JSON - unquoted value + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Should raise JSONDecodeError when trying to parse invalid JSON metadata + with pytest.raises(json.JSONDecodeError): + await _read_request_body(mock_request) + + +@pytest.mark.asyncio +async def test_form_data_without_metadata(): + """ + Test that form data without metadata field works correctly. + + Ensures the metadata parsing logic doesn't break when metadata is absent. + """ + # Create a mock request with form data without metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "language": "en" + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "application/x-www-form-urlencoded"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify all fields are preserved as-is + assert result == test_data + assert "metadata" not in result + assert result["model"] == "whisper-1" + assert result["file"] == "audio.mp3" + assert result["language"] == "en" + + +@pytest.mark.asyncio +async def test_form_data_with_empty_metadata(): + """ + Test that form data with empty JSON object in metadata field is parsed correctly. + """ + # Create a mock request with form data containing empty metadata + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": "{}" # Empty JSON object as string + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata was parsed to an empty dict + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"] == {} + assert result["model"] == "whisper-1" + + +@pytest.mark.asyncio +async def test_form_data_with_dict_metadata(): + """ + Test that form data with metadata already as a dict is not parsed again. + + This handles edge cases where metadata might already be a dictionary + (shouldn't happen in normal form data, but defensive coding). + """ + # Create a mock request with form data where metadata is already a dict + mock_request = MagicMock() + + metadata_dict = { + "user_id": "12345", + "tags": ["test"] + } + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": metadata_dict # Already a dict, not a string + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata remains as a dict and is not parsed + assert "metadata" in result + assert isinstance(result["metadata"], dict) + assert result["metadata"] == metadata_dict + assert result["metadata"]["user_id"] == "12345" + assert result["model"] == "whisper-1" + + +@pytest.mark.asyncio +async def test_form_data_with_none_metadata(): + """ + Test that form data with None metadata value is handled gracefully. + """ + # Create a mock request with form data where metadata is None + mock_request = MagicMock() + + test_data = { + "model": "whisper-1", + "file": "audio.mp3", + "metadata": None # None value + } + + # Mock the form method to return the test data + mock_request.form = AsyncMock(return_value=test_data) + mock_request.headers = {"content-type": "multipart/form-data"} + mock_request.scope = {} + + # Parse the form data + result = await _read_request_body(mock_request) + + # Verify the metadata remains None (not parsed) + assert "metadata" in result + assert result["metadata"] is None + assert result["model"] == "whisper-1" + + @pytest.mark.asyncio async def test_empty_request_body(): """ From 046b7efbbeb1928c3f94d64eb85c0b7cd7cc5b13 Mon Sep 17 00:00:00 2001 From: Dmitrii Komarov <dmitrii.k@miro.com> Date: Tue, 25 Nov 2025 05:58:01 +0100 Subject: [PATCH 24/68] Make Bedrock image generation more consistent (#17021) --- .../amazon_nova_canvas_transformation.py | 20 +++++ .../image/amazon_stability1_transformation.py | 59 +++++++++++++ .../image/amazon_stability3_transformation.py | 31 ++++++- .../image/amazon_titan_transformation.py | 6 +- litellm/llms/bedrock/image/cost_calculator.py | 41 ++------- litellm/llms/bedrock/image/image_handler.py | 85 +++++-------------- litellm/utils.py | 11 +-- .../test_bedrock_image_gen_unit_tests.py | 35 +++++--- 8 files changed, 164 insertions(+), 124 deletions(-) diff --git a/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py index cd33e62af16..f2b94b617c0 100644 --- a/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional from openai.types.image import Image +from litellm import get_model_info from litellm.types.llms.bedrock import ( AmazonNovaCanvasColorGuidedGenerationParams, AmazonNovaCanvasColorGuidedRequest, @@ -197,3 +198,22 @@ class AmazonNovaCanvasConfig: model_response.data = openai_images return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images \ No newline at end of file diff --git a/litellm/llms/bedrock/image/amazon_stability1_transformation.py b/litellm/llms/bedrock/image/amazon_stability1_transformation.py index 698ecca94ba..63af32f3f56 100644 --- a/litellm/llms/bedrock/image/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image/amazon_stability1_transformation.py @@ -1,8 +1,11 @@ +import copy +import os import types from typing import List, Optional from openai.types.image import Image +from litellm import get_model_info from litellm.types.utils import ImageResponse @@ -90,6 +93,31 @@ class AmazonStabilityConfig: return optional_params + @classmethod + def transform_request_body( + cls, + text: str, + optional_params: dict, + ) -> dict: + inference_params = copy.deepcopy(optional_params) + inference_params.pop( + "user", None + ) # make sure user is not passed in for bedrock call + + prompt = text.replace(os.linesep, " ") + ## LOAD CONFIG + config = cls.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + + return { + "text_prompts": [{"text": prompt, "weight": 1}], + **inference_params, + } + @classmethod def transform_response_dict_to_openai_response( cls, model_response: ImageResponse, response_dict: dict @@ -102,3 +130,34 @@ class AmazonStabilityConfig: model_response.data = image_list return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + optional_params = optional_params or {} + + # see model_prices_and_context_window.json for details on how steps is used + # Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/ + _steps = optional_params.get("steps", 50) + steps = "max-steps" if _steps > 50 else "50-steps" + + # size is stored in model_prices_and_context_window.json as 1024-x-1024 + # current size has 1024x1024 + size = size or "1024-x-1024" + model = f"{size}/{steps}/{model}" + + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images \ No newline at end of file diff --git a/litellm/llms/bedrock/image/amazon_stability3_transformation.py b/litellm/llms/bedrock/image/amazon_stability3_transformation.py index 06e06209791..445a2fe1100 100644 --- a/litellm/llms/bedrock/image/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image/amazon_stability3_transformation.py @@ -3,6 +3,8 @@ from typing import List, Optional from openai.types.image import Image +from litellm import get_model_info +from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.llms.bedrock import ( AmazonStability3TextToImageRequest, AmazonStability3TextToImageResponse, @@ -66,12 +68,12 @@ class AmazonStability3Config: @classmethod def transform_request_body( - cls, prompt: str, optional_params: dict + cls, text: str, optional_params: dict ) -> AmazonStability3TextToImageRequest: """ Transform the request body for the Stability 3 models """ - data = AmazonStability3TextToImageRequest(prompt=prompt, **optional_params) + data = AmazonStability3TextToImageRequest(prompt=text, **optional_params) return data @classmethod @@ -92,9 +94,34 @@ class AmazonStability3Config: """ stability_3_response = AmazonStability3TextToImageResponse(**response_dict) + + finish_reasons = stability_3_response.get("finish_reasons", []) + finish_reasons = [reason for reason in finish_reasons if reason] + if len(finish_reasons) > 0: + raise BedrockError(status_code=400, message="; ".join(finish_reasons)) + openai_images: List[Image] = [] for _img in stability_3_response.get("images", []): openai_images.append(Image(b64_json=_img)) model_response.data = openai_images return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + model_info = get_model_info( + model=model, + custom_llm_provider="bedrock", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images diff --git a/litellm/llms/bedrock/image/amazon_titan_transformation.py b/litellm/llms/bedrock/image/amazon_titan_transformation.py index 2709f406dfd..bed9ad0c300 100644 --- a/litellm/llms/bedrock/image/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image/amazon_titan_transformation.py @@ -103,16 +103,16 @@ class AmazonTitanImageGenerationConfig: return optional_params @classmethod - def _transform_request( + def transform_request_body( cls, - input: str, + text: str, optional_params: dict, ) -> AmazonTitanImageGenerationRequestBody: from typing import Any, Dict image_generation_config = optional_params.pop("imageGenerationConfig", {}) negative_text = optional_params.pop("negativeText", None) - text_to_image_params: Dict[str, Any] = {"text": input} + text_to_image_params: Dict[str, Any] = {"text": text} if negative_text: text_to_image_params["negativeText"] = negative_text task_type = optional_params.pop("taskType", "TEXT_IMAGE") diff --git a/litellm/llms/bedrock/image/cost_calculator.py b/litellm/llms/bedrock/image/cost_calculator.py index 9b2ae8782cb..bc1a57b8aec 100644 --- a/litellm/llms/bedrock/image/cost_calculator.py +++ b/litellm/llms/bedrock/image/cost_calculator.py @@ -1,9 +1,6 @@ from typing import Optional -import litellm -from litellm.llms.bedrock.image.amazon_titan_transformation import ( - AmazonTitanImageGenerationConfig, -) +from litellm.llms.bedrock.image.image_handler import BedrockImageGeneration from litellm.types.utils import ImageResponse @@ -18,36 +15,10 @@ def cost_calculator( Handles both Stability 1 and Stability 3 models """ - if litellm.AmazonStability3Config()._is_stability_3_model(model=model): - pass - elif AmazonTitanImageGenerationConfig._is_titan_model(model=model): - return AmazonTitanImageGenerationConfig.cost_calculator( - model=model, - image_response=image_response, - size=size, - optional_params=optional_params, - ) - else: - # Stability 1 models - optional_params = optional_params or {} - - # see model_prices_and_context_window.json for details on how steps is used - # Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/ - _steps = optional_params.get("steps", 50) - steps = "max-steps" if _steps > 50 else "50-steps" - - # size is stored in model_prices_and_context_window.json as 1024-x-1024 - # current size has 1024x1024 - size = size or "1024-x-1024" - model = f"{size}/{steps}/{model}" - - _model_info = litellm.get_model_info( + config_class = BedrockImageGeneration.get_config_class(model=model) + return config_class.cost_calculator( model=model, - custom_llm_provider="bedrock", + image_response=image_response, + size=size, + optional_params=optional_params, ) - - output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = 0 - if image_response.data: - num_images = len(image_response.data) - return output_cost_per_image * num_images diff --git a/litellm/llms/bedrock/image/image_handler.py b/litellm/llms/bedrock/image/image_handler.py index 313a1dc17bd..0825aecc856 100644 --- a/litellm/llms/bedrock/image/image_handler.py +++ b/litellm/llms/bedrock/image/image_handler.py @@ -1,13 +1,10 @@ -import copy import json -import os from typing import TYPE_CHECKING, Any, Optional, Union import httpx from pydantic import BaseModel import litellm -from litellm import BEDROCK_INVOKE_PROVIDERS_LITERAL from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.llms.bedrock.image.amazon_nova_canvas_transformation import ( @@ -47,11 +44,30 @@ class BedrockImagePreparedRequest(BaseModel): data: dict +BedrockImageConfigClass = Union[ + type[AmazonTitanImageGenerationConfig], + type[AmazonNovaCanvasConfig], + type[AmazonStability3Config], + type[litellm.AmazonStabilityConfig], +] + + class BedrockImageGeneration(BaseAWSLLM): """ Bedrock Image Generation handler """ + @classmethod + def get_config_class(cls, model: str | None) -> BedrockImageConfigClass: + if AmazonTitanImageGenerationConfig._is_titan_model(model): + return AmazonTitanImageGenerationConfig + elif AmazonNovaCanvasConfig._is_nova_model(model): + return AmazonNovaCanvasConfig + elif AmazonStability3Config._is_stability_3_model(model): + return AmazonStability3Config + else: + return litellm.AmazonStabilityConfig + def image_generation( self, model: str, @@ -202,7 +218,6 @@ class BedrockImageGeneration(BaseAWSLLM): model=model, prompt=prompt, optional_params=optional_params, - bedrock_provider=bedrock_provider, ) # Make POST Request @@ -241,7 +256,6 @@ class BedrockImageGeneration(BaseAWSLLM): def _get_request_body( self, model: str, - bedrock_provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL], prompt: str, optional_params: dict, ) -> dict: @@ -253,49 +267,9 @@ class BedrockImageGeneration(BaseAWSLLM): Returns: dict: The request body to use for the Bedrock Image Generation API """ - if bedrock_provider == "amazon" or bedrock_provider == "nova": - # Handle Amazon Nova Canvas models - provider = "amazon" - elif bedrock_provider == "stability": - provider = "stability" - else: - # Fallback to original logic for backward compatibility - provider = model.split(".")[0] - inference_params = copy.deepcopy(optional_params) - inference_params.pop( - "user", None - ) # make sure user is not passed in for bedrock call - data = {} - if provider == "stability": - if litellm.AmazonStability3Config._is_stability_3_model(model): - request_body = litellm.AmazonStability3Config.transform_request_body( - prompt=prompt, optional_params=optional_params - ) - return dict(request_body) - else: - prompt = prompt.replace(os.linesep, " ") - ## LOAD CONFIG - config = litellm.AmazonStabilityConfig.get_config() - for k, v in config.items(): - if ( - k not in inference_params - ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in - inference_params[k] = v - data = { - "text_prompts": [{"text": prompt, "weight": 1}], - **inference_params, - } - elif provider == "amazon": - return dict( - litellm.AmazonNovaCanvasConfig.transform_request_body( - text=prompt, optional_params=optional_params - ) - ) - else: - raise BedrockError( - status_code=422, message=f"Unsupported model={model}, passed in" - ) - return data + config_class = self.get_config_class(model=model) + request_body = config_class.transform_request_body(text=prompt, optional_params=optional_params) + return dict(request_body) def _transform_response_dict_to_openai_response( self, @@ -323,20 +297,7 @@ class BedrockImageGeneration(BaseAWSLLM): if response_dict is None: raise ValueError("Error in response object format, got None") - config_class: Union[ - type[AmazonTitanImageGenerationConfig], - type[AmazonNovaCanvasConfig], - type[AmazonStability3Config], - type[litellm.AmazonStabilityConfig], - ] - if AmazonTitanImageGenerationConfig._is_titan_model(model=model): - config_class = AmazonTitanImageGenerationConfig - elif AmazonNovaCanvasConfig._is_nova_model(model=model): - config_class = AmazonNovaCanvasConfig - elif AmazonStability3Config._is_stability_3_model(model=model): - config_class = AmazonStability3Config - else: - config_class = litellm.AmazonStabilityConfig + config_class = self.get_config_class(model=model) config_class.transform_response_dict_to_openai_response( model_response=model_response, diff --git a/litellm/utils.py b/litellm/utils.py index 78ed4170f49..302e2ec6308 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2631,16 +2631,7 @@ def get_optional_params_image_gen( ): optional_params = non_default_params elif custom_llm_provider == "bedrock": - # use stability3 config class if model is a stability3 model - config_class = ( - litellm.AmazonStability3Config - if litellm.AmazonStability3Config._is_stability_3_model(model=model) - else ( - litellm.AmazonNovaCanvasConfig - if litellm.AmazonNovaCanvasConfig._is_nova_model(model=model) - else litellm.AmazonStabilityConfig - ) - ) + config_class = litellm.BedrockImageGeneration.get_config_class(model=model) supported_params = config_class.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) optional_params = config_class.map_openai_params( diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index d3a5ade1cef..5526f22cd5e 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -119,6 +119,20 @@ def test_transform_response_dict_to_openai_response(): assert [img.b64_json for img in result.data] == response_dict["images"] +def test_transform_response_dict_to_openai_response_from_stability_3_models_with_no_null_finish_reason(): + # Create a mock response + response_dict = {"finish_reasons": ["Filter reason: prompt"]} + model_response = ImageResponse() + + with pytest.raises(BedrockError) as exc_info: + AmazonStability3Config.transform_response_dict_to_openai_response( + model_response, response_dict + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.message == "Filter reason: prompt" + + def test_amazon_stability_get_supported_openai_params(): result = AmazonStabilityConfig.get_supported_openai_params() assert result == ["size"] @@ -168,7 +182,7 @@ def test_get_request_body_stability3(): model = "stability.sd3-large" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["prompt"] == prompt @@ -181,7 +195,7 @@ def test_get_request_body_stability(): model = "stability.stable-diffusion-xl-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["text_prompts"][0]["text"] == prompt @@ -239,7 +253,7 @@ def test_get_request_body_nova_canvas_default(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -254,7 +268,7 @@ def test_get_request_body_nova_canvas_text_image(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -273,7 +287,7 @@ def test_get_request_body_nova_canvas_color_guided_generation(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "COLOR_GUIDED_GENERATION" @@ -437,7 +451,7 @@ def test_get_request_body_nova_canvas_inference_profile_arn(): bedrock_provider = handler.get_bedrock_invoke_provider(model=nova_model) result = handler._get_request_body( - model=nova_model, bedrock_provider=bedrock_provider, prompt=prompt, optional_params=optional_params + model=nova_model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -453,7 +467,7 @@ def test_get_request_body_nova_canvas_with_model_id_param(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) # After fix, model_id should not appear in the result @@ -488,12 +502,9 @@ def test_get_request_body_cross_region_inference_profile(): # Cross-region inference profile format model = "us.amazon.nova-canvas-v1:0" - # Get the provider using the method from the handler - bedrock_provider = handler.get_bedrock_invoke_provider(model=model) - # This should work after the fix - cross-region format should be detected as 'nova' result = handler._get_request_body( - model=model, bedrock_provider=bedrock_provider, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" @@ -508,7 +519,7 @@ def test_backward_compatibility_regular_nova_model(): model = "amazon.nova-canvas-v1" result = handler._get_request_body( - model=model, bedrock_provider=None, prompt=prompt, optional_params=optional_params + model=model, prompt=prompt, optional_params=optional_params ) assert result["taskType"] == "TEXT_IMAGE" From 6dcb5425a535e806de86c36a207afed916d1c1f8 Mon Sep 17 00:00:00 2001 From: wcyat <wcyat@wcyat.me> Date: Tue, 25 Nov 2025 13:24:29 +0800 Subject: [PATCH 25/68] fix(vertex): fix CreateCachedContentRequest enum error (#16965) * feat: add _fix_enum_types function to remove enums from non-string fields in schema * test: add test for _fix_enum_types function to validate enum removal from non-string fields --- litellm/llms/vertex_ai/common_utils.py | 54 ++++++++ .../vertex_ai/test_vertex_ai_common_utils.py | 118 ++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 2c534577366..dc6a3170afe 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -274,6 +274,57 @@ def _fix_enum_empty_strings(schema, depth=0): _fix_enum_empty_strings(items, depth=depth + 1) +def _fix_enum_types(schema, depth=0): + """Remove `enum` fields when the schema type is not string. + + Gemini / Vertex APIs only allow enums for string-typed fields. When an enum + is present on a non-string typed property (or when `anyOf` types do not + include a string type), remove the enum to avoid provider validation errors. + """ + if depth > DEFAULT_MAX_RECURSE_DEPTH: + raise ValueError( + f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema." + ) + + if not isinstance(schema, dict): + return + + # If enum exists but type is not string (and anyOf doesn't include string), drop enum + if "enum" in schema and isinstance(schema["enum"], list): + schema_type = schema.get("type") + keep_enum = False + if isinstance(schema_type, str) and schema_type.lower() == "string": + keep_enum = True + else: + anyof = schema.get("anyOf") + if isinstance(anyof, list): + for item in anyof: + if isinstance(item, dict): + item_type = item.get("type") + if isinstance(item_type, str) and item_type.lower() == "string": + keep_enum = True + break + + if not keep_enum: + schema.pop("enum", None) + + # Recurse into nested structures + properties = schema.get("properties", None) + if properties is not None: + for _, value in properties.items(): + _fix_enum_types(value, depth=depth + 1) + + items = schema.get("items", None) + if items is not None: + _fix_enum_types(items, depth=depth + 1) + + anyof = schema.get("anyOf", None) + if anyof is not None and isinstance(anyof, list): + for item in anyof: + if isinstance(item, dict): + _fix_enum_types(item, depth=depth + 1) + + def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): """ This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419 @@ -307,6 +358,9 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False): # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums _fix_enum_empty_strings(parameters) + # Remove enums for non-string typed fields (Gemini requires enum only on strings) + _fix_enum_types(parameters) + # Handle empty items objects process_items(parameters) add_object_type(parameters) diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index 4ea1d81c266..a5eee9e37b1 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -803,6 +803,124 @@ def test_fix_enum_empty_strings(): assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent" +def test_fix_enum_types(): + """ + Test _fix_enum_types function removes enum fields when type is not string. + + This test verifies the fix for the issue where Gemini rejects cached content + with function parameter enums on non-string types, causing API failures. + + Relevant issue: Gemini only allows enums for string-typed fields + """ + from litellm.llms.vertex_ai.common_utils import _fix_enum_types + + # Input: Schema with enum on non-string type (the problematic case) + input_schema = { + "type": "object", + "properties": { + "truncateMode": { + "enum": ["auto", "none", "start", "end"], + "type": "string", # This should keep the enum + "description": "How to truncate content" + }, + "maxLength": { + "enum": [100, 200, 500], # This should be removed + "type": "integer", + "description": "Maximum length" + }, + "enabled": { + "enum": [True, False], # This should be removed + "type": "boolean", + "description": "Whether feature is enabled" + }, + "nested": { + "type": "object", + "properties": { + "innerEnum": { + "enum": ["a", "b", "c"], # This should be kept + "type": "string" + }, + "innerNonStringEnum": { + "enum": [1, 2, 3], # This should be removed + "type": "integer" + } + } + }, + "anyOfField": { + "anyOf": [ + {"type": "string", "enum": ["option1", "option2"]}, # This should be kept + {"type": "integer", "enum": [1, 2, 3]} # This should be removed + ] + } + } + } + + # Expected output: Non-string enums removed, string enums kept + expected_output = { + "type": "object", + "properties": { + "truncateMode": { + "enum": ["auto", "none", "start", "end"], # Kept - string type + "type": "string", + "description": "How to truncate content" + }, + "maxLength": { # enum removed + "type": "integer", + "description": "Maximum length" + }, + "enabled": { # enum removed + "type": "boolean", + "description": "Whether feature is enabled" + }, + "nested": { + "type": "object", + "properties": { + "innerEnum": { + "enum": ["a", "b", "c"], # Kept - string type + "type": "string" + }, + "innerNonStringEnum": { # enum removed + "type": "integer" + } + } + }, + "anyOfField": { + "anyOf": [ + {"type": "string", "enum": ["option1", "option2"]}, # Kept - has string type + {"type": "integer"} # enum removed + ] + } + } + } + + # Apply the transformation + _fix_enum_types(input_schema) + + # Verify the transformation + assert input_schema == expected_output + + # Verify specific transformations: + # 1. String enums are preserved + assert "enum" in input_schema["properties"]["truncateMode"] + assert input_schema["properties"]["truncateMode"]["enum"] == ["auto", "none", "start", "end"] + + assert "enum" in input_schema["properties"]["nested"]["properties"]["innerEnum"] + assert input_schema["properties"]["nested"]["properties"]["innerEnum"]["enum"] == ["a", "b", "c"] + + # 2. Non-string enums are removed + assert "enum" not in input_schema["properties"]["maxLength"] + assert "enum" not in input_schema["properties"]["enabled"] + assert "enum" not in input_schema["properties"]["nested"]["properties"]["innerNonStringEnum"] + + # 3. anyOf with string type keeps enum, non-string removes it + assert "enum" in input_schema["properties"]["anyOfField"]["anyOf"][0] + assert "enum" not in input_schema["properties"]["anyOfField"]["anyOf"][1] + + # 4. Other properties preserved + assert input_schema["properties"]["maxLength"]["type"] == "integer" + assert input_schema["properties"]["enabled"]["type"] == "boolean" + + def test_get_token_url(): from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexLLM, From 29ab291cf5c6e4895e587b982b26792d730da34b Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:31:16 +0530 Subject: [PATCH 26/68] Add vertex ai image support --- litellm/images/main.py | 45 +-- .../vertex_ai/image_generation/__init__.py | 43 +++ .../vertex_gemini_transformation.py | 266 ++++++++++++++++++ .../vertex_imagen_transformation.py | 231 +++++++++++++++ litellm/utils.py | 8 +- 5 files changed, 550 insertions(+), 43 deletions(-) create mode 100644 litellm/llms/vertex_ai/image_generation/__init__.py create mode 100644 litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py create mode 100644 litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 333a751b045..786136e6699 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -19,6 +19,8 @@ from litellm.llms.custom_llm import CustomLLM #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() +from openai.types.audio.transcription_create_params import FileTypes # type: ignore + from litellm.main import ( azure_chat_completions, base_llm_aiohttp_handler, @@ -26,7 +28,6 @@ from litellm.main import ( bedrock_image_generation, openai_chat_completions, openai_image_variations, - vertex_image_generation, ) ########################################### @@ -36,7 +37,6 @@ from litellm.types.llms.openai import ImageGenerationRequestQuality from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, - FileTypes, LlmProviders, all_litellm_params, ) @@ -344,6 +344,7 @@ def image_generation( # noqa: PLR0915 litellm.LlmProviders.GEMINI, litellm.LlmProviders.FAL_AI, litellm.LlmProviders.RUNWAYML, + litellm.LlmProviders.VERTEX_AI, ): if image_generation_config is None: raise ValueError( @@ -430,46 +431,6 @@ def image_generation( # noqa: PLR0915 api_base=api_base, api_key=api_key, ) - elif custom_llm_provider == "vertex_ai": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret_str("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret_str("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret_str("VERTEXAI_CREDENTIALS") - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERTEXAI_API_BASE") - or get_secret_str("VERTEX_API_BASE") - ) - - model_response = vertex_image_generation.image_generation( - model=model, - prompt=prompt, - timeout=timeout, - logging_obj=litellm_logging_obj, - optional_params=optional_params, - model_response=model_response, - vertex_project=vertex_ai_project, - vertex_location=vertex_ai_location, - vertex_credentials=vertex_credentials, - aimg_generation=aimg_generation, - api_base=api_base, - client=client, - ) elif ( custom_llm_provider in litellm._custom_providers ): # Assume custom LLM provider diff --git a/litellm/llms/vertex_ai/image_generation/__init__.py b/litellm/llms/vertex_ai/image_generation/__init__.py new file mode 100644 index 00000000000..a6f6156167a --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/__init__.py @@ -0,0 +1,43 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.common_utils import ( + VertexAIModelRoute, + get_vertex_ai_model_route, +) + +from .vertex_gemini_transformation import VertexAIGeminiImageGenerationConfig +from .vertex_imagen_transformation import VertexAIImagenImageGenerationConfig + +__all__ = [ + "VertexAIGeminiImageGenerationConfig", + "VertexAIImagenImageGenerationConfig", + "get_vertex_ai_image_generation_config", +] + + +def get_vertex_ai_image_generation_config(model: str) -> BaseImageGenerationConfig: + """ + Get the appropriate image generation config for a Vertex AI model. + + Routes to the correct transformation class based on the model type: + - Gemini image generation models use generateContent API (VertexAIGeminiImageGenerationConfig) + - Imagen models use predict API (VertexAIImagenImageGenerationConfig) + + Args: + model: The model name (e.g., "gemini-2.5-flash-image", "imagegeneration@006") + + Returns: + BaseImageGenerationConfig: The appropriate configuration class + """ + # Determine the model route + model_route = get_vertex_ai_model_route(model) + + if model_route == VertexAIModelRoute.GEMINI: + # Gemini models use generateContent API + return VertexAIGeminiImageGenerationConfig() + else: + # Default to Imagen for other models (imagegeneration, etc.) + # This includes NON_GEMINI models like imagegeneration@006 + return VertexAIImagenImageGenerationConfig() + diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py new file mode 100644 index 00000000000..0a87dea997a --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -0,0 +1,266 @@ +import json +import os +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import httpx + +import litellm +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse +from litellm.utils import ModelResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): + """ + Vertex AI Gemini Image Generation Configuration + + Uses generateContent API for Gemini image generation models on Vertex AI + Supports models like gemini-2.5-flash-image, gemini-3-pro-image-preview, etc. + """ + + def __init__(self) -> None: + BaseImageGenerationConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Gemini image generation supported parameters + """ + return [ + "n", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + mapped_params = {} + + for k, v in non_default_params.items(): + if k not in optional_params.keys(): + if k in supported_params: + # Map OpenAI parameters to Gemini format + if k == "n": + mapped_params["candidate_count"] = v + elif k == "size": + # Map OpenAI size format to Gemini aspectRatio + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) + else: + mapped_params[k] = v + + return mapped_params + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Gemini aspect ratio format + """ + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4" + } + return aspect_ratio_map.get(size, "1:1") + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Vertex AI Gemini generateContent API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Gemini format + + Uses generateContent API with responseModalities: ["IMAGE"] + """ + # Prepare messages with the prompt + contents = [ + { + "role": "user", + "parts": [{"text": prompt}] + } + ] + + # Prepare generation config + generation_config: Dict[str, Any] = { + "responseModalities": ["IMAGE"] + } + + # Handle image-specific config parameters + image_config: Dict[str, Any] = {} + + # Map aspectRatio + if "aspectRatio" in optional_params: + image_config["aspectRatio"] = optional_params["aspectRatio"] + elif "aspect_ratio" in optional_params: + image_config["aspectRatio"] = optional_params["aspect_ratio"] + + # Map imageSize (for Gemini 3 Pro) + if "imageSize" in optional_params: + image_config["imageSize"] = optional_params["imageSize"] + elif "image_size" in optional_params: + image_config["imageSize"] = optional_params["image_size"] + + if image_config: + generation_config["imageConfig"] = image_config + + # Handle candidate_count (n parameter) + if "candidate_count" in optional_params: + generation_config["candidateCount"] = optional_params["candidate_count"] + elif "n" in optional_params: + generation_config["candidateCount"] = optional_params["n"] + + request_body: Dict[str, Any] = { + "contents": contents, + "generationConfig": generation_config + } + + return request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform Gemini image generation response to litellm ImageResponse format + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Gemini image generation models return in candidates format + candidates = response_data.get("candidates", []) + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + # Look for inlineData with image + if "inlineData" in part: + inline_data = part["inlineData"] + if "data" in inline_data: + model_response.data.append(ImageObject( + b64_json=inline_data["data"], + url=None, + )) + + return model_response + diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py new file mode 100644 index 00000000000..275c547c8dd --- /dev/null +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -0,0 +1,231 @@ +import json +import os +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import httpx + +import litellm +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): + """ + Vertex AI Imagen Image Generation Configuration + + Uses predict API for Imagen models on Vertex AI + Supports models like imagegeneration@006 + """ + + def __init__(self) -> None: + BaseImageGenerationConfig.__init__(self) + VertexLLM.__init__(self) + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Imagen API supported parameters + """ + return [ + "n", + "size" + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + mapped_params = {} + + for k, v in non_default_params.items(): + if k not in optional_params.keys(): + if k in supported_params: + # Map OpenAI parameters to Imagen format + if k == "n": + mapped_params["sampleCount"] = v + elif k == "size": + # Map OpenAI size format to Imagen aspectRatio + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) + else: + mapped_params[k] = v + + return mapped_params + + def _map_size_to_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Imagen aspect ratio format + """ + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4" + } + return aspect_ratio_map.get(size, "1:1") + + def _resolve_vertex_project(self) -> Optional[str]: + return ( + getattr(self, "_vertex_project", None) + or os.environ.get("VERTEXAI_PROJECT") + or getattr(litellm, "vertex_project", None) + or get_secret_str("VERTEXAI_PROJECT") + ) + + def _resolve_vertex_location(self) -> Optional[str]: + return ( + getattr(self, "_vertex_location", None) + or os.environ.get("VERTEXAI_LOCATION") + or os.environ.get("VERTEX_LOCATION") + or getattr(litellm, "vertex_location", None) + or get_secret_str("VERTEXAI_LOCATION") + or get_secret_str("VERTEX_LOCATION") + ) + + def _resolve_vertex_credentials(self) -> Optional[str]: + return ( + getattr(self, "_vertex_credentials", None) + or os.environ.get("VERTEXAI_CREDENTIALS") + or getattr(litellm, "vertex_credentials", None) + or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Vertex AI Imagen predict API + """ + vertex_project = self._resolve_vertex_project() + vertex_location = self._resolve_vertex_location() + + if not vertex_project or not vertex_location: + raise ValueError("vertex_project and vertex_location are required for Vertex AI") + + # Use the model name as provided, handling vertex_ai prefix + model_name = model + if model.startswith("vertex_ai/"): + model_name = model.replace("vertex_ai/", "") + + if api_base: + base_url = api_base.rstrip("/") + else: + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers = headers or {} + vertex_project = self._resolve_vertex_project() + vertex_credentials = self._resolve_vertex_credentials() + access_token, _ = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + custom_llm_provider="vertex_ai", + ) + return self.set_headers(access_token, headers) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Imagen format + + Uses predict API with instances and parameters + """ + # Default parameters + default_params = { + "sampleCount": 1, + } + + # Merge with optional params + parameters = {**default_params, **optional_params} + + request_body = { + "instances": [{"prompt": prompt}], + "parameters": parameters, + } + + return request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform Imagen image generation response to litellm ImageResponse format + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Imagen format - predictions with generated images + predictions = response_data.get("predictions", []) + for prediction in predictions: + # Imagen returns images as bytesBase64Encoded + if "bytesBase64Encoded" in prediction: + model_response.data.append(ImageObject( + b64_json=prediction["bytesBase64Encoded"], + url=None, + )) + + return model_response + diff --git a/litellm/utils.py b/litellm/utils.py index 1ec2576d356..3bc570bd700 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6866,7 +6866,7 @@ def convert_to_dict(message: Union[BaseModel, dict]) -> dict: dict: The converted message. """ if isinstance(message, BaseModel): - return message.model_dump(exclude_none=True) + return message.model_dump(exclude_none=True) # type: ignore elif isinstance(message, dict): return message else: @@ -7671,6 +7671,12 @@ class ProviderConfigManager: ) return get_runwayml_image_generation_config(model) + elif LlmProviders.VERTEX_AI == provider: + from litellm.llms.vertex_ai.image_generation import ( + get_vertex_ai_image_generation_config, + ) + + return get_vertex_ai_image_generation_config(model) return None @staticmethod From f52f05748dc688895c7a26a4b65d8ea8d8c0b59c Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:31:50 +0530 Subject: [PATCH 27/68] Update docs related to vertex ai image gen --- .../my-website/docs/providers/vertex_image.md | 73 +++++++++++++++++-- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/providers/vertex_image.md b/docs/my-website/docs/providers/vertex_image.md index 27e584cb222..c4d5d554088 100644 --- a/docs/my-website/docs/providers/vertex_image.md +++ b/docs/my-website/docs/providers/vertex_image.md @@ -1,18 +1,65 @@ # Vertex AI Image Generation -Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. +Vertex AI supports two types of image generation: + +1. **Gemini Image Generation Models** (Nano Banana 🍌) - Conversational image generation using `generateContent` API +2. **Imagen Models** - Traditional image generation using `predict` API | Property | Details | |----------|---------| -| Description | Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. | +| Description | Vertex AI Image Generation supports both Gemini image generation models | | Provider Route on LiteLLM | `vertex_ai/` | | Provider Doc | [Google Cloud Vertex AI Image Generation ↗](https://cloud.google.com/vertex-ai/docs/generative-ai/image/generate-images) | +| Gemini Image Generation Docs | [Gemini Image Generation ↗](https://ai.google.dev/gemini-api/docs/image-generation) | ## Quick Start -### LiteLLM Python SDK +### Gemini Image Generation Models -```python showLineNumbers title="Basic Image Generation" +Gemini image generation models support conversational image creation with features like: +- Text-to-Image generation +- Image editing (text + image → image) +- Multi-turn image refinement +- High-fidelity text rendering +- Up to 4K resolution (Gemini 3 Pro) + +```python showLineNumbers title="Gemini 2.5 Flash Image" +import litellm + +# Generate a single image +response = await litellm.aimage_generation( + prompt="A nano banana dish in a fancy restaurant with a Gemini theme", + model="vertex_ai/gemini-2.5-flash-image", + vertex_ai_project="your-project-id", + vertex_ai_location="us-central1", + n=1, + size="1024x1024", +) + +print(response.data[0].b64_json) # Gemini returns base64 images +``` + +```python showLineNumbers title="Gemini 3 Pro Image Preview (4K output)" +import litellm + +# Generate high-resolution image +response = await litellm.aimage_generation( + prompt="Da Vinci style anatomical sketch of a dissected Monarch butterfly", + model="vertex_ai/gemini-3-pro-image-preview", + vertex_ai_project="your-project-id", + vertex_ai_location="us-central1", + n=1, + size="1024x1024", + # Optional: specify image size for Gemini 3 Pro + # imageSize="4K", # Options: "1K", "2K", "4K" +) + +print(response.data[0].b64_json) +``` + +### Imagen Models + +```python showLineNumbers title="Imagen Image Generation" import litellm # Generate a single image @@ -21,9 +68,11 @@ response = await litellm.aimage_generation( model="vertex_ai/imagen-4.0-generate-001", vertex_ai_project="your-project-id", vertex_ai_location="us-central1", + n=1, + size="1024x1024", ) -print(response.data[0].url) +print(response.data[0].b64_json) # Imagen also returns base64 images ``` ### LiteLLM Proxy @@ -70,6 +119,18 @@ print(response.data[0].url) ## Supported Models +### Gemini Image Generation Models + +- `vertex_ai/gemini-2.5-flash-image` - Fast, efficient image generation (1024px resolution) +- `vertex_ai/gemini-3-pro-image-preview` - Advanced model with 4K output, Google Search grounding, and thinking mode +- `vertex_ai/gemini-2.0-flash-preview-image` - Preview model +- `vertex_ai/gemini-2.5-flash-image-preview` - Preview model + +### Imagen Models + +- `vertex_ai/imagegeneration@006` - Legacy Imagen model +- `vertex_ai/imagen-4.0-generate-001` - Latest Imagen model +- `vertex_ai/imagen-3.0-generate-001` - Imagen 3.0 model :::tip @@ -77,7 +138,5 @@ print(response.data[0].url) ::: -LiteLLM supports all Vertex AI Imagen models available through Google Cloud. - For the complete and up-to-date list of supported models, visit: [https://models.litellm.ai/](https://models.litellm.ai/) From 883cfaeeafa2ecb16f91a540be63dfbfa644001c Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:32:13 +0530 Subject: [PATCH 28/68] Add tests --- .../image_gen_tests/test_image_generation.py | 32 ++ ...rtex_ai_image_generation_transformation.py | 457 ++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 24d5843293b..add60c755be 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -119,6 +119,38 @@ class TestVertexImageGeneration(BaseImageGenTest): } +class TestVertexAIGeminiImageGeneration(BaseImageGenTest): + """Test Gemini image generation models (Nano Banana)""" + def get_base_image_generation_call_args(self) -> dict: + # comment this when running locally + load_vertex_ai_credentials() + + litellm.in_memory_llm_clients_cache = InMemoryCache() + return { + "model": "vertex_ai/gemini-2.5-flash-image", + "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_location": "us-central1", + "n": 1, + "size": "1024x1024", + } + + +class TestVertexAIGemini3ProImageGeneration(BaseImageGenTest): + """Test Gemini 3 Pro image generation model""" + def get_base_image_generation_call_args(self) -> dict: + # comment this when running locally + load_vertex_ai_credentials() + + litellm.in_memory_llm_clients_cache = InMemoryCache() + return { + "model": "vertex_ai/gemini-3-pro-image-preview", + "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_location": "us-central1", + "n": 1, + "size": "1024x1024", + } + + class TestBedrockNovaCanvasTextToImage(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.in_memory_llm_clients_cache = InMemoryCache() diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py new file mode 100644 index 00000000000..9f33400594b --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -0,0 +1,457 @@ +import os +import sys +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.llms.vertex_ai.image_generation import ( + get_vertex_ai_image_generation_config, +) +from litellm.llms.vertex_ai.image_generation.vertex_gemini_transformation import ( + VertexAIGeminiImageGenerationConfig, +) +from litellm.llms.vertex_ai.image_generation.vertex_imagen_transformation import ( + VertexAIImagenImageGenerationConfig, +) + + +class TestVertexAIGeminiImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIGeminiImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("gemini-2.5-flash-image") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to candidate_count""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("candidate_count") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("aspectRatio") == "1:1" + + def test_map_openai_params_size_16_9(self): + """Test mapping 16:9 size""" + non_default_params = {"size": "1792x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "gemini-2.5-flash-image", False + ) + assert result.get("aspectRatio") == "16:9" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("1024x1792") == "9:16" + assert self.config._map_size_to_aspect_ratio("1280x896") == "4:3" + assert self.config._map_size_to_aspect_ratio("896x1280") == "3:4" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "contents" in request + assert "generationConfig" in request + assert request["generationConfig"]["responseModalities"] == ["IMAGE"] + assert request["contents"][0]["parts"][0]["text"] == "A nano banana" + + def test_transform_image_generation_request_with_aspect_ratio(self): + """Test request transformation with aspectRatio""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_request_with_image_size(self): + """Test request transformation with imageSize (Gemini 3 Pro)""" + request = self.config.transform_image_generation_request( + model="gemini-3-pro-image-preview", + prompt="A nano banana", + optional_params={"imageSize": "4K"}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["imageConfig"]["imageSize"] == "4K" + + def test_transform_image_generation_request_with_candidate_count(self): + """Test request transformation with candidate_count""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"candidate_count": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_request_with_n(self): + """Test request transformation with n parameter""" + request = self.config.transform_image_generation_request( + model="gemini-2.5-flash-image", + prompt="A nano banana", + optional_params={"n": 2}, + litellm_params={}, + headers={}, + ) + assert request["generationConfig"]["candidateCount"] == 2 + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "base64_encoded_image_data", + } + } + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": "image1", + } + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "image2", + } + }, + ] + } + } + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="gemini-2.5-flash-image", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + +class TestVertexAIImagenImageGenerationConfig: + def setup_method(self): + """Set up test fixtures""" + self.config = VertexAIImagenImageGenerationConfig() + + def test_get_supported_openai_params(self): + """Test get_supported_openai_params returns correct params""" + supported = self.config.get_supported_openai_params("imagegeneration@006") + assert "n" in supported + assert "size" in supported + + def test_map_openai_params_n(self): + """Test mapping n parameter to sampleCount""" + non_default_params = {"n": 3} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "imagegeneration@006", False + ) + assert result.get("sampleCount") == 3 + + def test_map_openai_params_size(self): + """Test mapping size parameter to aspectRatio""" + non_default_params = {"size": "1024x1024"} + optional_params = {} + result = self.config.map_openai_params( + non_default_params, optional_params, "imagegeneration@006", False + ) + assert result.get("aspectRatio") == "1:1" + + def test_map_size_to_aspect_ratio(self): + """Test size to aspect ratio mapping""" + assert self.config._map_size_to_aspect_ratio("1024x1024") == "1:1" + assert self.config._map_size_to_aspect_ratio("1792x1024") == "16:9" + assert self.config._map_size_to_aspect_ratio("unknown") == "1:1" # default + + def test_transform_image_generation_request_basic(self): + """Test basic request transformation""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "instances" in request + assert "parameters" in request + assert request["instances"][0]["prompt"] == "A cat" + assert request["parameters"]["sampleCount"] == 1 + + def test_transform_image_generation_request_with_params(self): + """Test request transformation with parameters""" + request = self.config.transform_image_generation_request( + model="imagegeneration@006", + prompt="A cat", + optional_params={"sampleCount": 2, "aspectRatio": "16:9"}, + litellm_params={}, + headers={}, + ) + assert request["parameters"]["sampleCount"] == 2 + assert request["parameters"]["aspectRatio"] == "16:9" + + def test_transform_image_generation_response(self): + """Test response transformation""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + {"bytesBase64Encoded": "base64_encoded_image_data"} + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "base64_encoded_image_data" + assert result.data[0].url is None + + def test_transform_image_generation_response_multiple_images(self): + """Test response transformation with multiple images""" + mock_response = MagicMock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + {"bytesBase64Encoded": "image1"}, + {"bytesBase64Encoded": "image2"}, + ] + } + mock_response.headers = {} + + from litellm.types.utils import ImageResponse + + model_response = ImageResponse() + result = self.config.transform_image_generation_response( + model="imagegeneration@006", + raw_response=mock_response, + model_response=model_response, + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].b64_json == "image1" + assert result.data[1].b64_json == "image2" + + +class TestGetVertexAIImageGenerationConfig: + """Test the router function that selects the correct config""" + + def test_get_gemini_model_config(self): + """Test that Gemini models return Gemini config""" + config = get_vertex_ai_image_generation_config("gemini-2.5-flash-image") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("gemini-3-pro-image-preview") + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + config = get_vertex_ai_image_generation_config( + "vertex_ai/gemini-2.5-flash-image" + ) + assert isinstance(config, VertexAIGeminiImageGenerationConfig) + + def test_get_imagen_model_config(self): + """Test that Imagen models return Imagen config""" + config = get_vertex_ai_image_generation_config("imagegeneration@006") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config("imagen-4.0-generate-001") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + config = get_vertex_ai_image_generation_config( + "vertex_ai/imagegeneration@006" + ) + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + def test_get_non_gemini_model_config(self): + """Test that non-Gemini models default to Imagen config""" + config = get_vertex_ai_image_generation_config("some-other-model") + assert isinstance(config, VertexAIImagenImageGenerationConfig) + + +class TestVertexAIImageGenerationIntegration: + """Integration tests for Vertex AI image generation""" + + @pytest.mark.skipif( + not os.getenv("VERTEXAI_PROJECT"), + reason="Vertex AI credentials not set", + ) + def test_gemini_image_generation_config_validation(self): + """Test that Gemini config can validate environment""" + config = VertexAIGeminiImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ), patch.object( + config, "_ensure_access_token", return_value=("token", None) + ): + headers = config.validate_environment( + headers={}, + model="gemini-2.5-flash-image", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert "Authorization" in headers + + @pytest.mark.skipif( + not os.getenv("VERTEXAI_PROJECT"), + reason="Vertex AI credentials not set", + ) + def test_imagen_image_generation_config_validation(self): + """Test that Imagen config can validate environment""" + config = VertexAIImagenImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ), patch.object( + config, "_ensure_access_token", return_value=("token", None) + ): + headers = config.validate_environment( + headers={}, + model="imagegeneration@006", + messages=[], + optional_params={}, + litellm_params={}, + ) + assert "Authorization" in headers + + def test_gemini_get_complete_url(self): + """Test Gemini config URL generation""" + config = VertexAIGeminiImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="gemini-2.5-flash-image", + optional_params={}, + litellm_params={}, + ) + assert "test-project" in url + assert "us-central1" in url + assert "gemini-2.5-flash-image" in url + assert "generateContent" in url + + def test_imagen_get_complete_url(self): + """Test Imagen config URL generation""" + config = VertexAIImagenImageGenerationConfig() + with patch.object( + config, "_resolve_vertex_project", return_value="test-project" + ), patch.object( + config, "_resolve_vertex_location", return_value="us-central1" + ): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="imagegeneration@006", + optional_params={}, + litellm_params={}, + ) + assert "test-project" in url + assert "us-central1" in url + assert "imagegeneration@006" in url + assert "predict" in url + From b0d511143c952d523ab5302ac6031618af1d69d9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:36:20 +0530 Subject: [PATCH 29/68] remove unsused imports --- .../vertex_ai/image_generation/vertex_gemini_transformation.py | 2 -- .../vertex_ai/image_generation/vertex_imagen_transformation.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 0a87dea997a..149e0850bf0 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -1,4 +1,3 @@ -import json import os from typing import TYPE_CHECKING, Any, Dict, List, Optional @@ -15,7 +14,6 @@ from litellm.types.llms.openai import ( OpenAIImageGenerationOptionalParams, ) from litellm.types.utils import ImageObject, ImageResponse -from litellm.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 275c547c8dd..8c4ad5dd423 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -1,6 +1,5 @@ -import json import os -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional import httpx From a50083a87b024887327e1e67e516f2fe641d5a34 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:56:30 +0530 Subject: [PATCH 30/68] Remove none support from reasoning param --- .../llms/azure/chat/gpt_5_transformation.py | 36 ++++++++++++++++++- .../chat/test_azure_gpt5_transformation.py | 10 ++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index d563a2889ca..209475730f8 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -2,6 +2,8 @@ from typing import List +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.types.llms.openai import AllMessageValues @@ -33,7 +35,34 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", ) -> dict: - return OpenAIGPT5Config.map_openai_params( + reasoning_effort_value = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + ) + + if reasoning_effort_value == "none": + if litellm.drop_params is True or ( + drop_params is not None and drop_params is True + ): + non_default_params = non_default_params.copy() + optional_params = optional_params.copy() + if non_default_params.get("reasoning_effort") == "none": + non_default_params.pop("reasoning_effort") + if optional_params.get("reasoning_effort") == "none": + optional_params.pop("reasoning_effort") + else: + raise UnsupportedParamsError( + status_code=400, + message=( + "Azure OpenAI does not support reasoning_effort='none'. " + "Supported values are: 'low', 'medium', and 'high'. " + "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + "Issue: https://github.com/BerriAI/litellm/issues/16704" + ), + ) + + result = OpenAIGPT5Config.map_openai_params( self, non_default_params=non_default_params, optional_params=optional_params, @@ -41,6 +70,11 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params=drop_params, ) + if result.get("reasoning_effort") == "none": + result.pop("reasoning_effort") + + return result + def transform_request( self, model: str, diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 76d069733be..f48d1a1e93e 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -104,7 +104,12 @@ def test_azure_gpt5_codex_series_transform_request(config: AzureOpenAIGPT5Config # GPT-5.1 temperature handling tests for Azure def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAIGPT5Config): - """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none'.""" + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none'. + + Note: Azure OpenAI doesn't support reasoning_effort='none', so it's dropped from the params. + However, the temperature logic still works correctly because the parent treats missing + reasoning_effort the same as 'none' for gpt-5.1. + """ params = config.map_openai_params( non_default_params={"temperature": 0.5, "reasoning_effort": "none"}, optional_params={}, @@ -113,7 +118,8 @@ def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAI api_version="2024-05-01-preview", ) assert params["temperature"] == 0.5 - assert params["reasoning_effort"] == "none" + # Azure doesn't support reasoning_effort="none", so it should be dropped + assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" def test_azure_gpt5_1_temperature_without_reasoning_effort(config: AzureOpenAIGPT5Config): From c149ade6a8de9c4ab69b905c4d75b136cfa686c3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 13:57:15 +0530 Subject: [PATCH 31/68] Add tests related to reasoning param none --- .../chat/test_azure_gpt5_transformation.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index f48d1a1e93e..3095ff87f5a 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -104,17 +104,17 @@ def test_azure_gpt5_codex_series_transform_request(config: AzureOpenAIGPT5Config # GPT-5.1 temperature handling tests for Azure def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAIGPT5Config): - """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none'. + """Test that Azure GPT-5.1 supports any temperature when reasoning_effort='none' and drop_params=True. - Note: Azure OpenAI doesn't support reasoning_effort='none', so it's dropped from the params. - However, the temperature logic still works correctly because the parent treats missing - reasoning_effort the same as 'none' for gpt-5.1. + Note: Azure OpenAI doesn't support reasoning_effort='none', so it's dropped from the params + when drop_params=True. The temperature logic still works correctly because the parent treats + missing reasoning_effort the same as 'none' for gpt-5.1. """ params = config.map_openai_params( non_default_params={"temperature": 0.5, "reasoning_effort": "none"}, optional_params={}, model="azure/gpt-5.1", - drop_params=False, + drop_params=True, api_version="2024-05-01-preview", ) assert params["temperature"] == 0.5 @@ -122,6 +122,18 @@ def test_azure_gpt5_1_temperature_with_reasoning_effort_none(config: AzureOpenAI assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" +def test_azure_gpt5_1_reasoning_effort_none_error_when_drop_params_false(config: AzureOpenAIGPT5Config): + """Test that Azure GPT-5.1 raises error for reasoning_effort='none' when drop_params=False.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="azure/gpt-5.1", + drop_params=False, + api_version="2024-05-01-preview", + ) + + def test_azure_gpt5_1_temperature_without_reasoning_effort(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5.1 supports any temperature when reasoning_effort is not specified.""" params = config.map_openai_params( From 67d69d12b059777701d96e4ad60592f333e0ffc2 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 17:14:59 +0530 Subject: [PATCH 32/68] Add cost tracking and logging support --- litellm/cost_calculator.py | 51 +++++ litellm/litellm_core_utils/litellm_logging.py | 5 + litellm/proxy/_types.py | 2 + litellm/proxy/proxy_server.py | 36 +++- litellm/proxy/search_endpoints/endpoints.py | 21 ++ .../spend_tracking/spend_tracking_utils.py | 3 +- .../test_search_api_logging.py | 202 ++++++++++++++++++ 7 files changed, 310 insertions(+), 10 deletions(-) create mode 100644 tests/proxy_unit_tests/test_search_api_logging.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 0f5195e31af..9ef26d23ce2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1031,6 +1031,57 @@ def completion_cost( # noqa: PLR0915 billed_units.get("search_units") or 1 ) # cohere charges per request by default. completion_tokens = search_units + elif ( + call_type == CallTypes.search.value + or call_type == CallTypes.asearch.value + ): + from litellm.search import search_provider_cost_per_query + + # Extract number_of_queries from optional_params or default to 1 + number_of_queries = 1 + if optional_params is not None: + # Check if query is a list (multiple queries) + query = optional_params.get("query") + if isinstance(query, list): + number_of_queries = len(query) + elif query is not None: + number_of_queries = 1 + + search_model = model or "" + if custom_llm_provider and "/" not in search_model: + # If model is like "tavily-search", construct "tavily/search" for cost lookup + search_model = f"{custom_llm_provider}/search" + + prompt_cost, completion_cost_result = search_provider_cost_per_query( + model=search_model, + custom_llm_provider=custom_llm_provider, + number_of_queries=number_of_queries, + optional_params=optional_params, + ) + + # Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost) + _final_cost = prompt_cost + completion_cost_result + + # Apply discount + original_cost = _final_cost + _final_cost, discount_percent, discount_amount = _apply_cost_discount( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + + # Store cost breakdown in logging object if available + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=prompt_cost, + completion_tokens_cost_usd_dollar=completion_cost_result, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=_final_cost, + original_cost=original_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + ) + + return _final_cost elif call_type == CallTypes.arealtime.value and isinstance( completion_response, LiteLLMRealtimeStreamLoggingObject ): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9c4a7e38768..5b4fa6b7e24 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -70,6 +70,7 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, ) from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.containers.main import ContainerObject from litellm.types.llms.openai import ( @@ -1298,6 +1299,7 @@ class Logging(LiteLLMLoggingBaseClass): OpenAIFileObject, LiteLLMRealtimeStreamLoggingObject, OpenAIModerationResponse, + "SearchResponse", ], cache_hit: Optional[bool] = None, litellm_model_name: Optional[str] = None, @@ -1710,8 +1712,11 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) or isinstance(logging_result, OCRResponse) # OCR + or isinstance(logging_result, SearchResponse) # Search API or isinstance(logging_result, dict) and logging_result.get("object") == "vector_store.search_results.page" + or isinstance(logging_result, dict) + and logging_result.get("object") == "search" # Search API (dict format) or isinstance(logging_result, VideoObject) or isinstance(logging_result, ContainerObject) or (self.call_type == CallTypes.call_mcp_tool.value) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8a8bbdfe2ef..9bfd118aba2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -348,6 +348,8 @@ class LiteLLMRoutes(enum.Enum): # search "/search", "/v1/search", + "/search/{search_tool_name}", + "/v1/search/{search_tool_name}", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0c0ced82ba1..13307742860 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2010,6 +2010,16 @@ class ProxyConfig: ) print(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # noqa + # Handle os.environ/ variables in litellm_params + litellm_params = search_tool.get("litellm_params", {}) + if litellm_params: + for k, v in litellm_params.items(): + if isinstance(v, str) and v.startswith("os.environ/"): + _v = v.replace("os.environ/", "") + v = get_secret(_v) + litellm_params[k] = v + search_tool["litellm_params"] = litellm_params + # Cast to SearchToolTypedDict for type safety try: search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore @@ -3761,6 +3771,7 @@ class ProxyConfig: async def _init_search_tools_in_db(self, prisma_client: PrismaClient): """ Initialize search tools from database into the router on startup. + Only updates router if there are tools in the database, otherwise preserves config-loaded tools. """ global llm_router @@ -3778,17 +3789,24 @@ class ProxyConfig: f"Loading {len(search_tools)} search tool(s) from database into router" ) - if llm_router is not None: - # Add search tools to the router - await SearchAPIRouter.update_router_search_tools( - router_instance=llm_router, search_tools=search_tools - ) - verbose_proxy_logger.info( - f"Successfully loaded {len(search_tools)} search tool(s) into router" - ) + # Only update router if there are tools in the database + # This prevents overwriting config-loaded tools with an empty list + if len(search_tools) > 0: + if llm_router is not None: + # Add search tools to the router + await SearchAPIRouter.update_router_search_tools( + router_instance=llm_router, search_tools=search_tools + ) + verbose_proxy_logger.info( + f"Successfully loaded {len(search_tools)} search tool(s) into router" + ) + else: + verbose_proxy_logger.debug( + "Router not initialized yet, search tools will be added when router is created" + ) else: verbose_proxy_logger.debug( - "Router not initialized yet, search tools will be added when router is created" + "No search tools found in database, keeping config-loaded search tools (if any)" ) except Exception as e: diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index da5388f389d..8cfc8bb5106 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -131,6 +131,27 @@ async def search( if search_tool_name is not None: data["search_tool_name"] = search_tool_name + if "search_tool_name" in data and data["search_tool_name"]: + data["model"] = data["search_tool_name"] + + if llm_router is not None and hasattr(llm_router, "search_tools"): + search_tool_name_value = data["search_tool_name"] + matching_tools = [ + tool for tool in llm_router.search_tools + if tool.get("search_tool_name") == search_tool_name_value + ] + + if matching_tools: + search_tool = matching_tools[0] + search_provider = search_tool.get("litellm_params", {}).get("search_provider") + + if search_provider: + data["custom_llm_provider"] = search_provider + + if "metadata" not in data: + data["metadata"] = {} + data["metadata"]["model_group"] = search_tool_name_value + # Process request using ProxyBaseLLMRequestProcessing processor = ProxyBaseLLMRequestProcessing(data=data) try: diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 08e56248853..59f712e5b6f 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -228,7 +228,8 @@ def get_logging_payload( # noqa: PLR0915 if call_type in ["ocr", "aocr"]: usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) else: - usage = cast(dict, response_obj).get("usage", None) or {} + # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models + usage = response_obj_dict.get("usage", None) or {} if isinstance(usage, litellm.Usage): usage = dict(usage) diff --git a/tests/proxy_unit_tests/test_search_api_logging.py b/tests/proxy_unit_tests/test_search_api_logging.py new file mode 100644 index 00000000000..7ac22e51ef2 --- /dev/null +++ b/tests/proxy_unit_tests/test_search_api_logging.py @@ -0,0 +1,202 @@ +""" +Test search API logging and cost tracking in proxy. + +Tests that search API requests are properly logged to LiteLLM_SpendLogs +with correct fields populated (call_type, model, custom_llm_provider, +model_group, spend, etc.) +""" +import asyncio +import os +import sys +import time +from datetime import datetime +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) +import litellm +from litellm import Router +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger +from litellm.proxy.spend_tracking.spend_management_endpoints import view_spend_logs +from litellm.proxy.utils import ProxyLogging, hash_token, update_spend +from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult + + +@pytest.fixture +def prisma_client(): + from litellm.proxy import proxy_server + from litellm.proxy.proxy_cli import append_query_params + from litellm.proxy.utils import PrismaClient + + params = {"connection_limit": 100, "pool_timeout": 60} + database_url = os.getenv("DATABASE_URL") + if database_url is None: + pytest.skip("DATABASE_URL not set") + + modified_url = append_query_params(database_url, params) + os.environ["DATABASE_URL"] = modified_url + + user_api_key_cache = DualCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) + + prisma_client = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj + ) + + proxy_server.litellm_proxy_budget_name = ( + f"litellm-proxy-budget-{time.time()}" + ) + proxy_server.user_custom_key_generate = None + + return prisma_client + + +@pytest.mark.asyncio +async def test_search_api_logging_and_cost_tracking(prisma_client): + """ + Test that search API requests are logged with correct fields and cost tracking. + + Verifies: + 1. Search request creates a spend log entry + 2. call_type is set to "asearch" + 3. model is set to search_tool_name + 4. custom_llm_provider is set correctly + 5. model_group is set to search_tool_name + 6. spend is calculated and logged + """ + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + await litellm.proxy.proxy_server.prisma_client.connect() + + # Setup router with search tool + search_tool_name = "tavily-search" + search_provider = "tavily" + + router = Router(model_list=[]) + router.search_tools = [ + { + "search_tool_name": search_tool_name, + "litellm_params": { + "search_provider": search_provider, + }, + } + ] + + setattr(litellm.proxy.proxy_server, "llm_router", router) + + # Generate a test API key + from litellm.proxy.management_endpoints.key_management_endpoints import generate_key_fn + from litellm.proxy._types import GenerateKeyRequest + + from litellm.proxy._types import LitellmUserRoles + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="test_user", + ) + + key_request = GenerateKeyRequest(models=[], duration=None) + key_response = await generate_key_fn( + data=key_request, user_api_key_dict=user_api_key_dict + ) + generated_key = key_response.key + user_id = key_response.user_id + + # Create mock search response + mock_search_result = SearchResult( + title="Test Result", + url="https://example.com", + snippet="Test snippet", + ) + + mock_search_response = SearchResponse( + object="search", + results=[mock_search_result], + ) + + # Mock the search function to return our mock response + with patch("litellm.search.main.asearch", new_callable=AsyncMock) as mock_asearch: + mock_asearch.return_value = mock_search_response + + # Setup proxy logging + user_api_key_cache = DualCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) + setattr(litellm.proxy.proxy_server, "proxy_logging_obj", proxy_logging_obj) + + # Call the track_cost_callback directly to simulate what happens after a search + proxy_db_logger = _ProxyDBLogger() + + # Simulate the kwargs that would be passed from the search endpoint + request_id = "search_test_123" + kwargs = { + "call_type": "asearch", + "model": search_tool_name, + "custom_llm_provider": search_provider, + "litellm_call_id": request_id, # Set request_id in kwargs + "litellm_params": { + "metadata": { + "user_api_key": hash_token(generated_key), + "user_api_key_user_id": user_id, + "model_group": search_tool_name, + } + }, + "metadata": { + "user_api_key": hash_token(generated_key), + "user_api_key_user_id": user_id, + "model_group": search_tool_name, + }, + "response_cost": 0.008, # Mock cost for tavily search + } + + # Set id on the response object + mock_search_response.id = request_id + + await proxy_db_logger._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=mock_search_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + # Wait for async operations + await asyncio.sleep(2) + await update_spend( + prisma_client=prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging_obj, + ) + + # Query spend logs + spend_logs = await view_spend_logs( + request_id=request_id, + user_api_key_dict=UserAPIKeyAuth(api_key=generated_key), + ) + + # Verify spend log was created + assert len(spend_logs) == 1, f"Expected 1 spend log, got {len(spend_logs)}" + + spend_log = spend_logs[0] + + # Verify all fields are populated correctly + assert spend_log.request_id == request_id + assert spend_log.call_type == "asearch" + assert spend_log.model == search_tool_name + assert spend_log.custom_llm_provider == search_provider + assert spend_log.model_group == search_tool_name + assert spend_log.spend == 0.008 + # API key should be hashed (either the generated key or the one from metadata) + assert spend_log.api_key != "" # Should be populated + # Note: user field may be empty if not set in the request, but user_id should be in metadata + assert spend_log.metadata.get("user_api_key_user_id") == user_id or spend_log.user == user_id + + print(f"✅ Search API logging test passed!") + print(f" - call_type: {spend_log.call_type}") + print(f" - model: {spend_log.model}") + print(f" - custom_llm_provider: {spend_log.custom_llm_provider}") + print(f" - model_group: {spend_log.model_group}") + print(f" - spend: {spend_log.spend}") + From afe540e88d7681e91036c716ad389b259247d4d6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 18:26:25 +0530 Subject: [PATCH 33/68] Fix auth issue --- .../llms/azure/anthropic/transformation.py | 14 ++--- litellm/main.py | 16 ++++-- ...odel_prices_and_context_window_backup.json | 54 +++++++++++++++++++ model_prices_and_context_window.json | 54 +++++++++++++++++++ 4 files changed, 127 insertions(+), 11 deletions(-) diff --git a/litellm/llms/azure/anthropic/transformation.py b/litellm/llms/azure/anthropic/transformation.py index 81beeb74ae1..9bc4f130563 100644 --- a/litellm/llms/azure/anthropic/transformation.py +++ b/litellm/llms/azure/anthropic/transformation.py @@ -1,11 +1,10 @@ """ Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication """ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Union -import litellm from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams @@ -56,6 +55,11 @@ class AzureAnthropicConfig(AnthropicConfig): headers = BaseAzureLLM._base_validate_azure_environment( headers=headers, litellm_params=litellm_params_obj ) + + # Azure Anthropic uses x-api-key header (not api-key) + # Convert api-key to x-api-key if present + if "api-key" in headers and "x-api-key" not in headers: + headers["x-api-key"] = headers.pop("api-key") # Get tools and other anthropic-specific setup tools = optional_params.get("tools") @@ -81,10 +85,6 @@ class AzureAnthropicConfig(AnthropicConfig): user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, ) - - # Remove x-api-key from anthropic headers since Azure uses different auth - anthropic_headers.pop("x-api-key", None) - # Merge headers - Azure auth (api-key or Authorization) takes precedence headers = {**anthropic_headers, **headers} diff --git a/litellm/main.py b/litellm/main.py index 4857f1b9754..c155dc31db3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2379,10 +2379,18 @@ def completion( # type: ignore # noqa: PLR0915 ) # Ensure the URL ends with /v1/messages - if not api_base.endswith("/v1/messages"): - if not api_base.endswith("/anthropic"): - api_base = api_base.rstrip("/") + "/anthropic" - api_base = api_base.rstrip("/") + "/v1/messages" + api_base = api_base.rstrip("/") + if api_base.endswith("/v1/messages"): + pass + elif api_base.endswith("/anthropic/v1/messages"): + pass + else: + if "/anthropic" in api_base: + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + api_base = api_base + "/anthropic" + api_base = api_base + "/v1/messages" response = azure_anthropic_chat_completions.completion( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 351e01bc45e..bc1a1950664 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1087,6 +1087,60 @@ "output_cost_per_token": 1.5e-05, "supports_function_calling": true }, + "azure/claude-haiku-4-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-opus-4-1": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-sonnet-4-5": { + "input_cost_per_token": 3e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 351e01bc45e..bc1a1950664 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1087,6 +1087,60 @@ "output_cost_per_token": 1.5e-05, "supports_function_calling": true }, + "azure/claude-haiku-4-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-opus-4-1": { + "input_cost_per_token": 1.5e-05, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "azure/claude-sonnet-4-5": { + "input_cost_per_token": 3e-06, + "litellm_provider": "azure_anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure/computer-use-preview": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", From dd4c8ecbef4ea0e5dda8925553261d5978de9172 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 18:36:39 +0530 Subject: [PATCH 34/68] Add v1/messages support for azure anthropic models --- .../docs/providers/azure/azure_anthropic.md | 2 +- litellm/llms/azure/anthropic/__init__.py | 6 +- .../anthropic/messages_transformation.py | 125 ++++++++++++++++++ litellm/utils.py | 6 + 4 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/azure/anthropic/messages_transformation.py diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 9a45e1db599..6e0f2b60ef2 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -19,7 +19,7 @@ Azure Foundry supports the following Claude models: | Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) | | Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | | API Endpoint | `https://<resource-name>.services.ai.azure.com/anthropic/v1/messages` | -| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages` (passthrough) | +| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`| ## Key Features diff --git a/litellm/llms/azure/anthropic/__init__.py b/litellm/llms/azure/anthropic/__init__.py index b40c4dfa9b3..233f22999f0 100644 --- a/litellm/llms/azure/anthropic/__init__.py +++ b/litellm/llms/azure/anthropic/__init__.py @@ -4,5 +4,9 @@ Azure Anthropic provider - supports Claude models via Azure Foundry from .handler import AzureAnthropicChatCompletion from .transformation import AzureAnthropicConfig -__all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"] +try: + from .messages_transformation import AzureAnthropicMessagesConfig + __all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig", "AzureAnthropicMessagesConfig"] +except ImportError: + __all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"] diff --git a/litellm/llms/azure/anthropic/messages_transformation.py b/litellm/llms/azure/anthropic/messages_transformation.py new file mode 100644 index 00000000000..dd6aae4a769 --- /dev/null +++ b/litellm/llms/azure/anthropic/messages_transformation.py @@ -0,0 +1,125 @@ +""" +Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication +""" +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj, verbose_logger +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + pass + + +class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): + """ + Azure Anthropic messages configuration that extends AnthropicMessagesConfig. + The only difference is authentication - Azure uses x-api-key header (not api-key) + and Azure endpoint format. + """ + + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + """ + Validate environment and set up Azure authentication headers for /v1/messages endpoint. + Azure Anthropic uses x-api-key header (not api-key). + """ + from litellm.secret_managers.main import get_secret_str + + # Convert dict to GenericLiteLLMParams if needed + if isinstance(litellm_params, dict): + if api_key and "api_key" not in litellm_params: + litellm_params = {**litellm_params, "api_key": api_key} + litellm_params_obj = GenericLiteLLMParams(**litellm_params) + else: + litellm_params_obj = litellm_params or GenericLiteLLMParams() + if api_key and not litellm_params_obj.api_key: + litellm_params_obj.api_key = api_key + + # Use Azure authentication logic + headers = BaseAzureLLM._base_validate_azure_environment( + headers=headers, litellm_params=litellm_params_obj + ) + + # Azure Anthropic uses x-api-key header (not api-key) + # Convert api-key to x-api-key if present + if "api-key" in headers and "x-api-key" not in headers: + headers["x-api-key"] = headers.pop("api-key") + + # Set anthropic-version header + if "anthropic-version" not in headers: + headers["anthropic-version"] = "2023-06-01" + + # Set content-type header + if "content-type" not in headers: + headers["content-type"] = "application/json" + + # Update headers with optional anthropic beta features + headers = self._update_headers_with_optional_anthropic_beta( + headers=headers, + context_management=optional_params.get("context_management"), + ) + + return headers, api_base + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Azure Anthropic /v1/messages endpoint. + Azure Foundry endpoint format: https://<resource-name>.services.ai.azure.com/anthropic/v1/messages + """ + from litellm.secret_managers.main import get_secret_str + + api_base = api_base or get_secret_str("AZURE_API_BASE") + if api_base is None: + raise ValueError( + "Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. " + "Expected format: https://<resource-name>.services.ai.azure.com/anthropic" + ) + + # Ensure the URL ends with /v1/messages + api_base = api_base.rstrip("/") + if api_base.endswith("/v1/messages"): + # Already correct + pass + elif api_base.endswith("/anthropic/v1/messages"): + # Already correct + pass + else: + # Check if /anthropic is already in the path + if "/anthropic" in api_base: + # /anthropic exists, ensure we end with /anthropic/v1/messages + # Extract the base URL up to and including /anthropic + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + # /anthropic not in path, add it + api_base = api_base + "/anthropic" + # Add /v1/messages + api_base = api_base + "/v1/messages" + + return api_base + diff --git a/litellm/utils.py b/litellm/utils.py index e114e4cc051..2245487c6d5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7341,6 +7341,12 @@ class ProviderConfigManager: ) return VertexAIPartnerModelsAnthropicMessagesConfig() + elif litellm.LlmProviders.AZURE_ANTHROPIC == provider: + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + return AzureAnthropicMessagesConfig() return None @staticmethod From e2f2ccd913954cdca833b4be2d890b54c8bb1f28 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 18:45:51 +0530 Subject: [PATCH 35/68] Add tests related messages api --- .../anthropic/messages_transformation.py | 3 +- .../anthropic/test_azure_anthropic_handler.py | 6 +- ...azure_anthropic_messages_transformation.py | 241 ++++++++++++++++++ .../test_azure_anthropic_provider_config.py | 59 +++++ .../test_azure_anthropic_transformation.py | 16 +- 5 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py diff --git a/litellm/llms/azure/anthropic/messages_transformation.py b/litellm/llms/azure/anthropic/messages_transformation.py index dd6aae4a769..4640f68a3e0 100644 --- a/litellm/llms/azure/anthropic/messages_transformation.py +++ b/litellm/llms/azure/anthropic/messages_transformation.py @@ -5,7 +5,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple import httpx -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj, verbose_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import verbose_logger from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py index 34aed42478d..bb5d1f9933d 100644 --- a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py @@ -32,7 +32,7 @@ class TestAzureAnthropicChatCompletion: mock_config.transform_request.return_value = {"model": "claude-sonnet-4-5", "messages": []} mock_config.transform_response.return_value = ModelResponse() mock_config_instance = MagicMock() - mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} mock_azure_config.return_value = mock_config_instance mock_provider_manager.get_provider_chat_config.return_value = mock_config @@ -90,7 +90,7 @@ class TestAzureAnthropicChatCompletion: "stream": True, } mock_config_instance = MagicMock() - mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} mock_azure_config.return_value = mock_config_instance mock_provider_manager.get_provider_chat_config.return_value = mock_config @@ -151,7 +151,7 @@ class TestAzureAnthropicChatCompletion: mock_response = ModelResponse() mock_config.transform_response.return_value = mock_response mock_config_instance = MagicMock() - mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_config_instance.validate_environment.return_value = {"x-api-key": "test-api-key", "anthropic-version": "2023-06-01"} mock_azure_config.return_value = mock_config_instance mock_provider_manager.get_provider_chat_config.return_value = mock_config diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py new file mode 100644 index 00000000000..abed1a7852e --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_messages_transformation.py @@ -0,0 +1,241 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, +) +from litellm.types.router import GenericLiteLLMParams + + +class TestAzureAnthropicMessagesConfig: + def test_inherits_from_anthropic_messages_config(self): + """Test that AzureAnthropicMessagesConfig inherits from AnthropicMessagesConfig""" + config = AzureAnthropicMessagesConfig() + assert isinstance(config, AzureAnthropicMessagesConfig) + # Check that it has methods from parent class + assert hasattr(config, "get_supported_anthropic_messages_params") + assert hasattr(config, "get_complete_url") + assert hasattr(config, "validate_anthropic_messages_environment") + assert hasattr(config, "transform_anthropic_messages_request") + assert hasattr(config, "transform_anthropic_messages_response") + + def test_validate_anthropic_messages_environment_with_dict_litellm_params(self): + """Test validate_anthropic_messages_environment with dict litellm_params""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that dict was converted to GenericLiteLLMParams + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert call_args[1]["litellm_params"].api_key == "test-api-key" + assert "anthropic-version" in result + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result + + def test_validate_anthropic_messages_environment_converts_api_key_to_x_api_key(self): + """Test that api-key header is converted to x-api-key""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Verify api-key was converted to x-api-key + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result + + def test_validate_anthropic_messages_environment_sets_headers(self): + """Test that required headers are set""" + config = AzureAnthropicMessagesConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result, api_base = config.validate_anthropic_messages_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert "anthropic-version" in result + assert result["anthropic-version"] == "2023-06-01" + assert "content-type" in result + assert result["content-type"] == "application/json" + assert "x-api-key" in result + + def test_get_complete_url_with_base_url(self): + """Test get_complete_url with base URL""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_ending_with_slash(self): + """Test get_complete_url with base URL ending with slash""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic/" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_already_containing_v1_messages(self): + """Test get_complete_url with base URL already containing /v1/messages""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_containing_anthropic(self): + """Test get_complete_url with base URL already containing /anthropic""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com/anthropic" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_with_base_url_without_anthropic(self): + """Test get_complete_url with base URL without /anthropic""" + config = AzureAnthropicMessagesConfig() + api_base = "https://test.services.ai.azure.com" + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + url = config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + + def test_get_complete_url_raises_error_when_api_base_missing(self): + """Test get_complete_url raises error when api_base is None""" + config = AzureAnthropicMessagesConfig() + api_base = None + api_key = "test-api-key" + model = "claude-sonnet-4-5" + optional_params = {} + litellm_params = {} + + with patch("litellm.secret_managers.main.get_secret_str", return_value=None): + with pytest.raises(ValueError, match="Missing Azure API Base"): + config.get_complete_url( + api_base=api_base, + api_key=api_key, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + def test_get_supported_anthropic_messages_params(self): + """Test get_supported_anthropic_messages_params returns correct params""" + config = AzureAnthropicMessagesConfig() + model = "claude-sonnet-4-5" + params = config.get_supported_anthropic_messages_params(model) + + assert "messages" in params + assert "model" in params + assert "max_tokens" in params + assert "temperature" in params + assert "tools" in params + assert "tool_choice" in params + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py new file mode 100644 index 00000000000..db118154eee --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_config.py @@ -0,0 +1,59 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from unittest.mock import patch + +import pytest + +import litellm +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +class TestAzureAnthropicProviderConfig: + def test_get_provider_anthropic_messages_config_returns_azure_config(self): + """Test that get_provider_anthropic_messages_config returns AzureAnthropicMessagesConfig for azure_anthropic provider""" + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + config = ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude-sonnet-4-5", + provider=LlmProviders.AZURE_ANTHROPIC, + ) + + assert config is not None + assert isinstance(config, AzureAnthropicMessagesConfig) + + def test_get_provider_anthropic_messages_config_returns_anthropic_config_for_anthropic_provider(self): + """Test that get_provider_anthropic_messages_config returns AnthropicMessagesConfig for anthropic provider""" + from litellm.llms.azure.anthropic.messages_transformation import ( + AzureAnthropicMessagesConfig, + ) + + config = ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude-sonnet-4-5", + provider=LlmProviders.ANTHROPIC, + ) + + # Should return AnthropicMessagesConfig, not AzureAnthropicMessagesConfig + assert config is not None + assert not isinstance(config, AzureAnthropicMessagesConfig) + assert isinstance(config, litellm.AnthropicMessagesConfig) + + def test_get_provider_chat_config_returns_azure_anthropic_config(self): + """Test that get_provider_chat_config returns AzureAnthropicConfig for azure_anthropic provider""" + from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig + + config = ProviderConfigManager.get_provider_chat_config( + model="claude-sonnet-4-5", + provider=LlmProviders.AZURE_ANTHROPIC, + ) + + assert config is not None + assert isinstance(config, AzureAnthropicConfig) + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py index 43f0b5c439d..f26831e9195 100644 --- a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py @@ -5,9 +5,10 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) -import pytest from unittest.mock import MagicMock, patch +import pytest + from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig from litellm.types.router import GenericLiteLLMParams @@ -102,8 +103,8 @@ class TestAzureAnthropicConfig: call_args = mock_validate.call_args assert call_args[1]["litellm_params"].api_key == "provided-api-key" - def test_validate_environment_removes_x_api_key(self): - """Test that x-api-key header is removed (Azure uses api-key instead)""" + def test_validate_environment_converts_api_key_to_x_api_key(self): + """Test that api-key header is converted to x-api-key (Azure Anthropic uses x-api-key)""" config = AzureAnthropicConfig() headers = {} model = "claude-sonnet-4-5" @@ -116,7 +117,7 @@ class TestAzureAnthropicConfig: ) as mock_validate: mock_validate.return_value = {"api-key": "test-api-key"} with patch.object( - config, "get_anthropic_headers", return_value={"x-api-key": "should-be-removed"} + config, "get_anthropic_headers", return_value={} ): result = config.validate_environment( headers=headers, @@ -126,9 +127,10 @@ class TestAzureAnthropicConfig: litellm_params=litellm_params, ) - # Verify x-api-key was removed - assert "x-api-key" not in result - assert "api-key" in result + # Verify api-key was converted to x-api-key + assert "x-api-key" in result + assert result["x-api-key"] == "test-api-key" + assert "api-key" not in result def test_validate_environment_sets_anthropic_version(self): """Test that anthropic-version header is set""" From 255d1bc2398646a10be00a9979992070f48c4d07 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 18:53:50 +0530 Subject: [PATCH 36/68] fix lint errors --- litellm/llms/azure/anthropic/handler.py | 6 ++---- .../llms/azure/anthropic/messages_transformation.py | 11 +---------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure/anthropic/handler.py index cfa5eeddaa9..e0aec942502 100644 --- a/litellm/llms/azure/anthropic/handler.py +++ b/litellm/llms/azure/anthropic/handler.py @@ -3,7 +3,7 @@ Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authen """ import copy import json -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Union import httpx @@ -12,7 +12,6 @@ from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, - get_async_httpx_client, ) from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper @@ -20,8 +19,7 @@ from litellm.utils import CustomStreamWrapper from .transformation import AzureAnthropicConfig if TYPE_CHECKING: - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as CustomStreamWrapperType - from litellm.llms.base_llm.chat.transformation import BaseConfig + pass class AzureAnthropicChatCompletion(AnthropicChatCompletion): diff --git a/litellm/llms/azure/anthropic/messages_transformation.py b/litellm/llms/azure/anthropic/messages_transformation.py index 4640f68a3e0..55818cc07d6 100644 --- a/litellm/llms/azure/anthropic/messages_transformation.py +++ b/litellm/llms/azure/anthropic/messages_transformation.py @@ -1,19 +1,12 @@ """ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication """ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, List, Optional, Tuple -import httpx - -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.litellm_logging import verbose_logger from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM -from litellm.types.llms.anthropic_messages.anthropic_response import ( - AnthropicMessagesResponse, -) from litellm.types.router import GenericLiteLLMParams if TYPE_CHECKING: @@ -41,8 +34,6 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): Validate environment and set up Azure authentication headers for /v1/messages endpoint. Azure Anthropic uses x-api-key header (not api-key). """ - from litellm.secret_managers.main import get_secret_str - # Convert dict to GenericLiteLLMParams if needed if isinstance(litellm_params, dict): if api_key and "api_key" not in litellm_params: From 1c612288bc4d69880e3c93fea63f579368374754 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Tue, 25 Nov 2025 20:19:55 +0530 Subject: [PATCH 37/68] fix lint errors --- litellm/llms/azure/anthropic/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure/anthropic/handler.py index e0aec942502..cf4765190c2 100644 --- a/litellm/llms/azure/anthropic/handler.py +++ b/litellm/llms/azure/anthropic/handler.py @@ -176,7 +176,9 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion): timeout=timeout, json_mode=json_mode, ) - from litellm.llms.anthropic.common_utils import process_anthropic_headers + from litellm.llms.anthropic.common_utils import ( + process_anthropic_headers, + ) return CustomStreamWrapper( completion_stream=completion_stream, From 00e17c81a1f3d47b2b44ce1ffa350ef3bcbdc7ee Mon Sep 17 00:00:00 2001 From: Krish Dholakia <krrishdholakia@gmail.com> Date: Tue, 25 Nov 2025 09:36:24 -0800 Subject: [PATCH 38/68] Add enforce user param functionality (#17088) * feat: Add reject_metadata_tags to proxy config Co-authored-by: krrishdholakia <krrishdholakia@gmail.com> * Refactor: Rename reject_metadata_tags to reject_clientside_metadata_tags Co-authored-by: krrishdholakia <krrishdholakia@gmail.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> --- docs/my-website/docs/proxy/config_settings.md | 2 + .../proxy/reject_clientside_metadata_tags.md | 120 ++++++++++++ litellm/proxy/_types.py | 4 + litellm/proxy/auth/auth_checks.py | 18 ++ ...eject_clientside_metadata_tags_config.yaml | 14 ++ .../proxy/auth/test_auth_checks.py | 179 ++++++++++++++++++ 6 files changed, 337 insertions(+) create mode 100644 docs/my-website/docs/proxy/reject_clientside_metadata_tags.md create mode 100644 litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4d1bc549e05..5a586035bcf 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -104,6 +104,7 @@ general_settings: disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param + reject_clientside_metadata_tags: boolean # if true, rejects requests with client-side 'metadata.tags' to prevent users from influencing budgets allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) key_management_system: google_kms # either google_kms or azure_kms master_key: string @@ -201,6 +202,7 @@ router_settings: | disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints | | enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) | | enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)| +| reject_clientside_metadata_tags | boolean | If true, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. | | allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)| | key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) | | master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) | diff --git a/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md b/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md new file mode 100644 index 00000000000..534c65939eb --- /dev/null +++ b/docs/my-website/docs/proxy/reject_clientside_metadata_tags.md @@ -0,0 +1,120 @@ +# Reject Client-Side Metadata Tags + +## Overview + +The `reject_clientside_metadata_tags` setting allows you to prevent users from passing client-side `metadata.tags` in their API requests. This ensures that tags are only inherited from the API key metadata and cannot be overridden by users to potentially influence budget tracking or routing decisions. + +## Use Case + +This feature is particularly useful in multi-tenant scenarios where: +- You want to enforce strict budget tracking based on API key tags +- You want to prevent users from manipulating routing decisions by sending custom client-side tags +- You need to ensure consistent tag-based filtering and reporting + +## Configuration + +Add the following to your `config.yaml`: + +```yaml +general_settings: + reject_clientside_metadata_tags: true # Default is false/null +``` + +## Behavior + +### When `reject_clientside_metadata_tags: true` + +**Rejected Request Example:** +```bash +curl -X POST http://localhost:4000/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": { + "tags": ["custom-tag"] # This will be rejected + } + }' +``` + +**Error Response:** +```json +{ + "error": { + "message": "Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'=True. Tags can only be set via API key metadata.", + "type": "bad_request_error", + "param": "metadata.tags", + "code": 400 + } +} +``` + +**Allowed Request Example:** +```bash +curl -X POST http://localhost:4000/chat/completions \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": { + "custom_field": "value" # Other metadata fields are allowed + } + }' +``` + +### When `reject_clientside_metadata_tags: false` or not set + +All requests are allowed, including those with client-side `metadata.tags`. + +## Setting Tags via API Key + +When `reject_clientside_metadata_tags` is enabled, tags should be set on the API key metadata: + +```bash +curl -X POST http://localhost:4000/key/generate \ + -H "Authorization: Bearer sk-master-key" \ + -H "Content-Type: application/json" \ + -d '{ + "metadata": { + "tags": ["team-a", "production"] + } + }' +``` + +These tags will be automatically inherited by all requests made with that API key. + +## Complete Example Configuration + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 + database_url: "postgresql://user:password@localhost:5432/litellm" + + # Reject client-side tags + reject_clientside_metadata_tags: true + + # Optional: Also enforce user parameter + enforce_user_param: true +``` + +## Similar Features + +- `enforce_user_param` - Requires all requests to include a 'user' parameter +- Tag-based routing - Use tags for intelligent request routing +- Budget tracking - Track spending per tag + +## Notes + +- This check only applies to LLM API routes (e.g., `/chat/completions`, `/embeddings`) +- Management endpoints (e.g., `/key/generate`) are not affected +- The check validates that client-side `metadata.tags` is not present in the request body +- Other metadata fields can still be passed in requests +- Tags set on API keys will still be applied to all requests diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8a8bbdfe2ef..d209ac52ae9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1889,6 +1889,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): allowed_routes: Optional[List] = Field( None, description="Proxy API Endpoints you want users to be able to access" ) + reject_clientside_metadata_tags: Optional[bool] = Field( + None, + description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", + ) enable_public_model_hub: bool = Field( default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 32795a1874d..c9774b18b88 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -186,6 +186,24 @@ async def common_checks( raise Exception( f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}" ) + + # 6.1 [OPTIONAL] If 'reject_clientside_metadata_tags' enabled - reject request if it has client-side 'metadata.tags' + if ( + general_settings.get("reject_clientside_metadata_tags", None) is not None + and general_settings["reject_clientside_metadata_tags"] is True + ): + if ( + RouteChecks.is_llm_api_route(route=route) + and "metadata" in request_body + and isinstance(request_body["metadata"], dict) + and "tags" in request_body["metadata"] + ): + raise ProxyException( + message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.", + type=ProxyErrorTypes.bad_request_error, + param="metadata.tags", + code=status.HTTP_400_BAD_REQUEST, + ) # 7. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget if ( litellm.max_budget > 0 diff --git a/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml b/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml new file mode 100644 index 00000000000..3c43c3c5374 --- /dev/null +++ b/litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml @@ -0,0 +1,14 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 + database_url: "postgresql://user:password@localhost:5432/litellm" + + # Reject requests that contain client-side metadata.tags + # This prevents users from influencing budgets by sending different tags + # Tags can only be inherited from the API key metadata + reject_clientside_metadata_tags: true diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 057b56ce317..6cd4bec3f18 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -732,3 +732,182 @@ async def test_get_team_object_raises_404_when_not_found(): assert exc_info.value.status_code == 404 assert "Team doesn't exist in db" in str(exc_info.value.detail) + + +# Reject Client-Side Metadata Tags Tests + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_enabled_with_tags(): + """Test that common_checks rejects request when reject_clientside_metadata_tags is True and metadata.tags is present""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + with pytest.raises(ProxyException) as exc_info: + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert exc_info.value.type == ProxyErrorTypes.bad_request_error + assert "metadata.tags" in exc_info.value.message + assert exc_info.value.param == "metadata.tags" + assert exc_info.value.code == 400 + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_enabled_without_tags(): + """Test that common_checks allows request when reject_clientside_metadata_tags is True but no metadata.tags is present""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"custom_field": "value"}, # No tags field + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_disabled_with_tags(): + """Test that common_checks allows request with metadata.tags when reject_clientside_metadata_tags is False""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": False} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_not_set_with_tags(): + """Test that common_checks allows request with metadata.tags when reject_clientside_metadata_tags is not set""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {} # No reject_clientside_metadata_tags setting + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True + + +@pytest.mark.asyncio +async def test_reject_clientside_metadata_tags_non_llm_route(): + """Test that reject_clientside_metadata_tags check only applies to LLM API routes""" + from litellm.proxy.auth.auth_checks import common_checks + from fastapi import Request + + request_body = { + "metadata": {"tags": ["custom-tag"]}, + } + + general_settings = {"reject_clientside_metadata_tags": True} + + # Create a mock request object + mock_request = MagicMock(spec=Request) + + # Should not raise an exception for non-LLM route + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/key/generate", # Management route, not LLM route + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=None, + request=mock_request, + ) + + assert result is True From 59b4b9a07cc3e9f9f9619534d619b44597177d62 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Wed, 26 Nov 2025 00:02:48 +0530 Subject: [PATCH 39/68] fix documentation of anthropic azure --- docs/my-website/docs/providers/azure/azure.md | 2 +- docs/my-website/docs/providers/azure/azure_anthropic.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 971cd9fd492..0b9fd29e680 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem'; |-------|-------| | Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. | | Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) (Claude passthrough) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) | | Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) ## API Keys, Params diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 6e0f2b60ef2..771912646b5 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -231,9 +231,9 @@ print(response) </TabItem> </Tabs> -## Messages API Passthrough +## Messages API -Azure Anthropic also supports the native Anthropic Messages API via passthrough. The endpoint structure is the same as Anthropic's `/v1/messages` API. +Azure Anthropic also supports the native Anthropic Messages API. The endpoint structure is the same as Anthropic's `/v1/messages` API. ### Using Anthropic SDK @@ -256,7 +256,7 @@ response = client.messages.create( print(response) ``` -### Using LiteLLM Proxy Passthrough +### Using LiteLLM Proxy ```bash curl --request POST \ From 67622fb0404877bf07865bc1be60932adc315376 Mon Sep 17 00:00:00 2001 From: Sameer Kankute <sameer@berri.ai> Date: Wed, 26 Nov 2025 00:58:47 +0530 Subject: [PATCH 40/68] Add day 0 support for anthropic new feat (#17091) * Added tool search support for anthropic * Add programtic tool calling support * Add tool use input examples support * Add anthropic effort param support * Add anthropic effort param support * Add blog for new features * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * Add better handling * Add better handling --- .../blog/anthropic_advanced_features/index.md | 655 ++++++++++++++++ .../docs/providers/anthropic_effort.md | 276 +++++++ .../anthropic_programmatic_tool_calling.md | 430 ++++++++++ .../anthropic_tool_input_examples.md | 438 +++++++++++ .../docs/providers/anthropic_tool_search.md | 397 ++++++++++ litellm/llms/anthropic/chat/handler.py | 81 +- litellm/llms/anthropic/chat/transformation.py | 266 ++++++- litellm/llms/anthropic/common_utils.py | 101 +++ .../adapters/transformation.py | 2 +- litellm/types/llms/anthropic.py | 74 ++ litellm/types/llms/openai.py | 13 +- litellm/types/utils.py | 3 +- .../test_anthropic_chat_transformation.py | 741 ++++++++++++++++++ 13 files changed, 3420 insertions(+), 57 deletions(-) create mode 100644 docs/my-website/blog/anthropic_advanced_features/index.md create mode 100644 docs/my-website/docs/providers/anthropic_effort.md create mode 100644 docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md create mode 100644 docs/my-website/docs/providers/anthropic_tool_input_examples.md create mode 100644 docs/my-website/docs/providers/anthropic_tool_search.md diff --git a/docs/my-website/blog/anthropic_advanced_features/index.md b/docs/my-website/blog/anthropic_advanced_features/index.md new file mode 100644 index 00000000000..71e5d9b1925 --- /dev/null +++ b/docs/my-website/blog/anthropic_advanced_features/index.md @@ -0,0 +1,655 @@ +--- +slug: anthropic_advanced_features +title: "Advanced Anthropic Features in LiteLLM: Tool Search, Programmatic Tool Calling, Input Examples, and Effort Control" +date: 2025-01-25T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::info + +This guide covers Anthropic's latest advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. + +::: + +We're excited to announce support for Anthropic's latest advanced features in LiteLLM! These powerful capabilities enable you to build more efficient, scalable, and cost-effective AI applications with Claude. + +## Table of Contents + +1. [Tool Search](#tool-search) +2. [Programmatic Tool Calling](#programmatic-tool-calling) +3. [Tool Input Examples](#tool-input-examples) +4. [Effort Parameter: Control Token Usage](#effort-parameter) +5. [Cost Tracking: Monitor Tool Search Usage](#cost-tracking) +6. [Combining Features](#combining-features) + +--- + +## Tool Search {#tool-search} + +### Usage Example + +```python +import litellm +import os + +# Configure your API key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +# Define your tools with defer_loading +tools = [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } +] + +# Make a request - Claude will search for and use relevant tools +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message.content) +print("Tool calls:", response.choices[0].message.tool_calls) + +# Check tool search usage +if hasattr(response.usage, 'server_tool_use'): + print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}") +``` + +### BM25 Variant (Natural Language Search) + +For natural language queries instead of regex patterns: + +```python +tools = [ + { + "type": "tool_search_tool_bm25_20251119", # Natural language variant + "name": "tool_search_tool_bm25" + }, + # ... your deferred tools +] +``` + +--- + +## Programmatic Tool Calling {#programmatic-tool-calling} + +### Usage Example + +```python +import litellm +import json + +# Define tools that can be called programmatically +tools = [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } +] + +# First request +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message) + +# Handle tool calls +messages = [ + {"role": "user", "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue"}, + {"role": "assistant", "content": response.choices[0].message.content, "tool_calls": response.choices[0].message.tool_calls} +] + +# Process each tool call +for tool_call in response.choices[0].message.tool_calls: + # Check if it's a programmatic call + if hasattr(tool_call, 'caller') and tool_call.caller: + print(f"Programmatic call to {tool_call.function.name}") + print(f"Called from: {tool_call.caller}") + + # Simulate tool execution + if tool_call.function.name == "query_database": + args = json.loads(tool_call.function.arguments) + # Simulate database query + result = json.dumps([ + {"region": "West", "revenue": 150000}, + {"region": "East", "revenue": 180000}, + {"region": "Central", "revenue": 120000} + ]) + + messages.append({ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": tool_call.id, + "content": result + }] + }) + +# Get final response +final_response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=tools +) + +print("\nFinal answer:", final_response.choices[0].message.content) +``` + +--- + +## Tool Input Examples {#tool-input-examples} + +### Usage Example + +```python +import litellm + +tools = [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + tools=tools +) + +print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments) +``` + +--- + +## Effort Parameter: Control Token Usage {#effort-parameter} + +### Usage Example + +```python +import litellm + +message = "Analyze the trade-offs between microservices and monolithic architectures" + +# High effort (default) - Maximum capability +response_high = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "high"} +) + +print("High effort response:") +print(response_high.choices[0].message.content) +print(f"Tokens used: {response_high.usage.completion_tokens}\n") + +# Medium effort - Balanced approach +response_medium = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "medium"} +) + +print("Medium effort response:") +print(response_medium.choices[0].message.content) +print(f"Tokens used: {response_medium.usage.completion_tokens}\n") + +# Low effort - Maximum efficiency +response_low = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "low"} +) + +print("Low effort response:") +print(response_low.choices[0].message.content) +print(f"Tokens used: {response_low.usage.completion_tokens}\n") + +# Compare token usage +print("Token Comparison:") +print(f"High: {response_high.usage.completion_tokens} tokens") +print(f"Medium: {response_medium.usage.completion_tokens} tokens") +print(f"Low: {response_low.usage.completion_tokens} tokens") +``` + +### Effort with Tool Use + +Lower effort affects both explanations and tool calls: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +# Low effort = fewer tool calls, more direct +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Check weather in San Francisco, New York, and London" + }], + tools=tools, + output_config={"effort": "low"} # May combine into fewer calls +) +``` + +--- + +## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} + +### Understanding Tool Search Costs + +Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. + +### Tracking Example + +```python +import litellm + +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + tools=tools +) + +# Standard token usage +print("Token Usage:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +# Tool search specific usage +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f"\nTool Search Usage:") + print(f" Search requests: {response.usage.server_tool_use.tool_search_requests}") + + # Calculate cost (example pricing) + input_cost = response.usage.prompt_tokens * 0.000003 # $3 per 1M tokens + output_cost = response.usage.completion_tokens * 0.000015 # $15 per 1M tokens + search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001 # Example + + total_cost = input_cost + output_cost + search_cost + + print(f"\nCost Breakdown:") + print(f" Input tokens: ${input_cost:.6f}") + print(f" Output tokens: ${output_cost:.6f}") + print(f" Tool searches: ${search_cost:.6f}") + print(f" Total: ${total_cost:.6f}") +``` + +### Cost Optimization Tips + +1. **Keep frequently used tools non-deferred** (3-5 tools) +2. **Use tool search for large catalogs** (10+ tools) +3. **Monitor search requests** to identify optimization opportunities +4. **Combine with effort parameter** for maximum efficiency + +```python +# Optimized for cost +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Simple query"}], + tools=tools_with_search, + output_config={"effort": "low"} # Reduce output tokens +) +``` + +--- + +## Combining Features {#combining-features} + +### The Power of Integration + +These features work together seamlessly. Here's a real-world example combining all of them: + +```python +import litellm +import json + +# Large tool catalog with search, programmatic calling, and examples +tools = [ + # Enable tool search + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Enable programmatic calling + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Database tool with all features + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the analytics database. Returns JSON array of results.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL SELECT statement" + }, + "limit": { + "type": "integer", + "description": "Maximum rows to return" + } + }, + "required": ["sql"] + } + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + { + "sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region", + "limit": 100 + } + ] + }, + # ... 50 more tools with defer_loading +] + +# Make request with effort control +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + tools=tools, + output_config={"effort": "medium"} # Balanced efficiency +) + +# Track comprehensive usage +print("Complete Usage Metrics:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f" Tool searches: {response.usage.server_tool_use.tool_search_requests}") + +print(f"\nResponse: {response.choices[0].message.content}") +``` + +### Real-World Benefits + +This combination enables: + +1. **Massive scale** - Handle 1000+ tools efficiently +2. **Low latency** - Programmatic calling reduces round trips +3. **High accuracy** - Input examples ensure correct tool usage +4. **Cost control** - Effort parameter optimizes token spend +5. **Full visibility** - Track all usage metrics + +--- + +## Getting Started + +### Installation + +```bash +pip install litellm --upgrade +``` + +### Configuration + +```python +import os +import litellm + +# Set your API key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +# LiteLLM automatically handles beta headers for all features +``` + +### Supported Models + +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + +### Supported Endpoints + +**Note**: All features are supported on the `/chat/completions` endpoint only. + +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + +### Provider Support + +All features work across: +- ✅ Standard Anthropic API +- ✅ Azure Anthropic +- ✅ Vertex AI Anthropic +- ✅ LiteLLM Proxy + +--- + +## Conclusion + +These advanced Anthropic features in LiteLLM enable you to build more sophisticated, efficient, and cost-effective AI applications: + +- **Tool Search** scales to thousands of tools +- **Programmatic Tool Calling** reduces latency and tokens +- **Input Examples** improve accuracy +- **Effort Parameter** controls costs + +All features work seamlessly together and are supported across all Anthropic providers through LiteLLM's unified interface. + +### Resources + +- [LiteLLM Documentation](https://docs.litellm.ai/) +- [Anthropic Tool Search Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_search) +- [Anthropic Programmatic Tool Calling Docs](https://docs.litellm.ai/docs/providers/anthropic_programmatic_tool_calling) +- [Anthropic Input Examples Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_input_examples) +- [Anthropic Effort Parameter Docs](https://docs.litellm.ai/docs/providers/anthropic_effort) + +### Get Started Today + +```bash +pip install litellm --upgrade +``` + +Happy building! 🚀 + diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md new file mode 100644 index 00000000000..d1116ad5be4 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -0,0 +1,276 @@ +# Anthropic Effort Parameter + +Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency. + +## Overview + +The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model. + +**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected). + +## How Effort Works + +By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability. + +**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely. + +The effort parameter affects **all tokens** in the response, including: +- Text responses and explanations +- Tool calls and function arguments +- Extended thinking (when enabled) + +This approach has two major advantages: +1. It doesn't require thinking to be enabled in order to use it. +2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls. + +This gives a much greater degree of control over efficiency. + +## Effort Levels + +| Level | Description | Typical use case | +|-------|-------------|------------------| +| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks | +| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance | +| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents | + +## Quick Start + +### Using LiteLLM SDK + +<Tabs> +<TabItem value="python" label="Python"> + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config={ + "effort": "medium" + } +) + +print(response.choices[0].message.content) +``` + +</TabItem> +<TabItem value="typescript" label="TypeScript"> + +```typescript +import Anthropic from "@anthropic-ai/sdk"; + +const client = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, +}); + +const response = await client.messages.create({ + model: "claude-opus-4-5-20251101", + max_tokens: 4096, + messages: [{ + role: "user", + content: "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config: { + effort: "medium" + } +}); + +console.log(response.content[0].text); +``` + +</TabItem> +</Tabs> + +### Using LiteLLM Proxy + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-5-20251101", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +### Direct Anthropic API Call + +```bash +curl https://api.anthropic.com/v1/messages \ + --header "x-api-key: $ANTHROPIC_API_KEY" \ + --header "anthropic-version: 2023-06-01" \ + --header "anthropic-beta: effort-2025-11-24" \ + --header "content-type: application/json" \ + --data '{ + "model": "claude-opus-4-5-20251101", + "max_tokens": 4096, + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +## Model Compatibility + +The effort parameter is currently only supported by: +- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) + +## When Should I Adjust the Effort Parameter? + +- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority. + +- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort. + +- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend. + +## Effort with Tool Use + +When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to: +- Combine multiple operations into fewer tool calls +- Make fewer tool calls +- Proceed directly to action + +Example with tools: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Check the weather in multiple cities" + }], + tools=[{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + }], + output_config={ + "effort": "low" # Will make fewer tool calls + } +) +``` + +## Effort with Extended Thinking + +The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Solve this complex problem" + }], + thinking={ + "type": "enabled", + "budget_tokens": 5000 + }, + output_config={ + "effort": "medium" # Affects both thinking and response tokens + } +) +``` + +## Best Practices + +1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs. + +2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency. + +3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses. + +4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases. + +5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity. + +## Provider Support + +The effort parameter is supported across all Anthropic-compatible providers: + +- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5) + +LiteLLM automatically handles the beta header injection for all providers. + +## Usage and Pricing + +Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs: + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": "Analyze this"}], + output_config={"effort": "low"} +) + +print(f"Output tokens: {response.usage.completion_tokens}") +print(f"Total tokens: {response.usage.total_tokens}") +``` + +## Troubleshooting + +### Beta header not being added + +LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header: + +1. Ensure you're using `output_config` with an `effort` field +2. Verify the model is Claude Opus 4.5 +3. Check that LiteLLM version supports this feature + +### Invalid effort value error + +Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error: + +```python +# ❌ This will raise an error +output_config={"effort": "very_low"} + +# ✅ Use one of the valid values +output_config={"effort": "low"} +``` + +### Model not supported + +Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error. + +## Related Features + +- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process +- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions +- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools +- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs + +## Additional Resources + +- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort) +- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic) +- [Cost Optimization Best Practices](/docs/guides/cost_optimization) + diff --git a/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md new file mode 100644 index 00000000000..6d3e15785e5 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md @@ -0,0 +1,430 @@ +# Anthropic Programmatic Tool Calling + +Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window. + +:::info +Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field. + +This feature requires the code execution tool to be enabled. +::: + +## Model Compatibility + +Programmatic tool calling is available on the following models: + +| Model | Tool Version | +|-------|--------------| +| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` | +| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` | + +## Quick Start + +Here's a simple example where Claude programmatically queries a database multiple times and aggregates results: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + { + "role": "user", + "content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue" + } + ], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) + +print(response) +``` + +## How It Works + +When you configure a tool to be callable from code execution and Claude decides to use that tool: + +1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic +2. Claude runs this code in a sandboxed container via code execution +3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field +4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window) +5. Once all code execution completes, Claude receives the final output and continues working on the task + +This approach is particularly useful for: + +- **Large data processing**: Filter or aggregate tool results before they reach Claude's context +- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls +- **Conditional logic**: Make decisions based on intermediate tool results + +## The `allowed_callers` Field + +The `allowed_callers` field specifies which contexts can invoke a tool: + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the database", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"] +} +``` + +**Possible values:** + +- `["direct"]` - Only Claude can call this tool directly (default if omitted) +- `["code_execution_20250825"]` - Only callable from within code execution +- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution + +:::tip +We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool. +::: + +## The `caller` Field in Responses + +Every tool use block includes a `caller` field indicating how it was invoked: + +**Direct invocation (traditional tool use):** + +```python +{ + "type": "tool_use", + "id": "toolu_abc123", + "name": "query_database", + "input": {"sql": "<sql>"}, + "caller": {"type": "direct"} +} +``` + +**Programmatic invocation:** + +```python +{ + "type": "tool_use", + "id": "toolu_xyz789", + "name": "query_database", + "input": {"sql": "<sql>"}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } +} +``` + +The `tool_id` references the code execution tool that made the programmatic call. + +## Container Lifecycle + +Programmatic tool calling uses code execution containers: + +- **Container creation**: A new container is created for each session unless you reuse an existing one +- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change) +- **Container ID**: Pass the `container` parameter to reuse an existing container +- **Reuse**: Pass the container ID to maintain state across requests + +```python +# First request - creates a new container +response1 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Query the database"}], + tools=[...] +) + +# Get container ID from response (if available in response metadata) +container_id = response1.get("container", {}).get("id") + +# Second request - reuse the same container +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[...], + tools=[...], + container=container_id # Reuse container +) +``` + +:::warning +When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it. +::: + +## Example Workflow + +### Step 1: Initial Request + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue" + }], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string", "description": "SQL query to execute"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) +``` + +### Step 2: API Response with Tool Call + +Claude writes code that calls your tool. The response includes: + +```python +{ + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the purchase history and analyze the results." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_abc123", + "name": "code_execution", + "input": { + "code": "results = await query_database('<sql>')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]" + } + }, + { + "type": "tool_use", + "id": "toolu_def456", + "name": "query_database", + "input": {"sql": "<sql>"}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } + } + ], + "stop_reason": "tool_use" +} +``` + +### Step 3: Provide Tool Result + +```python +# Add assistant's response and tool result to conversation +messages = [ + {"role": "user", "content": "Query customer purchase history..."}, + { + "role": "assistant", + "content": response.choices[0].message.content, + "tool_calls": response.choices[0].message.tool_calls + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_def456", + "content": '[{"customer_id": "C1", "revenue": 45000}, ...]' + } + ] + } +] + +# Continue the conversation +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=[...] +) +``` + +### Step 4: Final Response + +Once code execution completes, Claude provides the final response: + +```python +{ + "content": [ + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_abc123", + "content": { + "type": "code_execution_result", + "stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...", + "stderr": "", + "return_code": 0 + } + }, + { + "type": "text", + "text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..." + } + ], + "stop_reason": "end_turn" +} +``` + +## Advanced Patterns + +### Batch Processing with Loops + +Claude can write code that processes multiple items efficiently: + +```python +# Claude writes code like this: +regions = ["West", "East", "Central", "North", "South"] +results = {} +for region in regions: + data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'") + results[region] = data[0]["total"] + +top_region = max(results.items(), key=lambda x: x[1]) +print(f"Top region: {top_region[0]} with ${top_region[1]:,}") +``` + +This pattern: +- Reduces model round-trips from N (one per region) to 1 +- Processes large result sets programmatically before returning to Claude +- Saves tokens by only returning aggregated conclusions + +### Early Termination + +Claude can stop processing as soon as success criteria are met: + +```python +endpoints = ["us-east", "eu-west", "apac"] +for endpoint in endpoints: + status = await check_health(endpoint) + if status == "healthy": + print(f"Found healthy endpoint: {endpoint}") + break # Stop early +``` + +### Data Filtering + +```python +logs = await fetch_logs(server_id) +errors = [log for log in logs if "ERROR" in log] +print(f"Found {len(errors)} errors") +for error in errors[-10:]: # Only return last 10 errors + print(error) +``` + +## Best Practices + +### Tool Design + +- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.) +- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing +- **Keep responses concise**: Return only necessary data to minimize processing overhead + +### When to Use Programmatic Calling + +**Good use cases:** + +- Processing large datasets where you only need aggregates or summaries +- Multi-step workflows with 3+ dependent tool calls +- Operations requiring filtering, sorting, or transformation of tool results +- Tasks where intermediate data shouldn't influence Claude's reasoning +- Parallel operations across many items (e.g., checking 50 endpoints) + +**Less ideal use cases:** + +- Single tool calls with simple responses +- Tools that need immediate user feedback +- Very fast operations where code execution overhead would outweigh the benefit + +## Token Efficiency + +Programmatic tool calling can significantly reduce token consumption: + +- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is +- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens +- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns + +For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary. + +## Provider Support + +LiteLLM supports programmatic tool calling across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field. + +## Limitations + +### Feature Incompatibilities + +- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling +- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice` +- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling + +### Tool Restrictions + +The following tools cannot currently be called programmatically: + +- Web search +- Web fetch +- Tools provided by an MCP connector + +## Troubleshooting + +### Common Issues + +**"Tool not allowed" error** + +- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]` +- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5) + +**Container expiration** + +- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes) +- Consider implementing faster tool execution + +**Beta header not added** + +- LiteLLM automatically adds the beta header when it detects `allowed_callers` +- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20` + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_input_examples.md b/docs/my-website/docs/providers/anthropic_tool_input_examples.md new file mode 100644 index 00000000000..d0b7cc1762c --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_input_examples.md @@ -0,0 +1,438 @@ +# Anthropic Tool Input Examples + +Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs. + +:::info +Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field. +::: + +## When to Use Input Examples + +Input examples are most helpful for: + +- **Complex nested objects**: Tools with deeply nested parameter structures +- **Optional parameters**: Showing when optional parameters should be included +- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.) +- **Enum values**: Illustrating valid enum choices in context +- **Edge cases**: Showing how to handle special cases + +:::tip +**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient. +::: + +## Quick Start + +Add an `input_examples` field to your tool definition with an array of example input objects: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What's the weather like in San Francisco?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature" + } + }, + "required": ["location"] + } + }, + "input_examples": [ + { + "location": "San Francisco, CA", + "unit": "fahrenheit" + }, + { + "location": "Tokyo, Japan", + "unit": "celsius" + }, + { + "location": "New York, NY" # 'unit' is optional + } + ] + } + ] +) + +print(response) +``` + +## How It Works + +When you provide `input_examples`: + +1. **LiteLLM detects** the `input_examples` field in your tool definition +2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected +3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema +4. **Claude learns patterns**: The model uses examples to understand proper tool usage +5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats + +## Example Formats + +### Simple Tool with Examples + +```python +{ + "type": "function", + "function": { + "name": "send_email", + "description": "Send an email to a recipient", + "parameters": { + "type": "object", + "properties": { + "to": {"type": "string", "description": "Email address"}, + "subject": {"type": "string"}, + "body": {"type": "string"} + }, + "required": ["to", "subject", "body"] + } + }, + "input_examples": [ + { + "to": "user@example.com", + "subject": "Meeting Reminder", + "body": "Don't forget our meeting tomorrow at 2 PM." + }, + { + "to": "team@company.com", + "subject": "Weekly Update", + "body": "Here's this week's progress report..." + } + ] +} +``` + +### Complex Nested Objects + +```python +{ + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start": { + "type": "object", + "properties": { + "date": {"type": "string"}, + "time": {"type": "string"} + } + }, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + } + }, + "required": ["title", "start"] + } + }, + "input_examples": [ + { + "title": "Team Standup", + "start": { + "date": "2025-01-15", + "time": "09:00" + }, + "attendees": [ + {"email": "alice@example.com", "optional": False}, + {"email": "bob@example.com", "optional": True} + ] + }, + { + "title": "Lunch Break", + "start": { + "date": "2025-01-15", + "time": "12:00" + } + # No attendees - showing optional field + } + ] +} +``` + +### Format-Sensitive Parameters + +```python +{ + "type": "function", + "function": { + "name": "search_flights", + "description": "Search for available flights", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string", "description": "Airport code"}, + "destination": {"type": "string", "description": "Airport code"}, + "date": {"type": "string", "description": "Date in YYYY-MM-DD format"}, + "passengers": {"type": "integer"} + }, + "required": ["origin", "destination", "date"] + } + }, + "input_examples": [ + { + "origin": "SFO", + "destination": "JFK", + "date": "2025-03-15", + "passengers": 2 + }, + { + "origin": "LAX", + "destination": "ORD", + "date": "2025-04-20", + "passengers": 1 + } + ] +} +``` + +## Requirements and Limitations + +### Schema Validation + +- Each example **must be valid** according to the tool's `input_schema` +- Invalid examples will return a **400 error** from Anthropic +- Validation happens server-side (LiteLLM passes examples through) + +### Server-Side Tools Not Supported + +Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`: + +- `web_search` (web search tool) +- `code_execution` (code execution tool) +- `computer_use` (computer use tool) +- `bash_tool` (bash execution tool) +- `text_editor` (text editor tool) + +### Token Costs + +Examples add to your prompt tokens: + +- **Simple examples**: ~20-50 tokens per example +- **Complex nested objects**: ~100-200 tokens per example +- **Trade-off**: Higher token cost for better tool call accuracy + +### Model Compatibility + +Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header: + +- Claude Opus 4.5 (`claude-opus-4-5-20251101`) +- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) +- Claude Opus 4.1 (`claude-opus-4-1-20250805`) + +:::note +On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples. +::: + +## Best Practices + +### 1. Show Diverse Examples + +Include examples that demonstrate different use cases: + +```python +"input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, # US city + {"location": "Tokyo, Japan", "unit": "celsius"}, # International + {"location": "New York, NY"} # Optional param omitted +] +``` + +### 2. Demonstrate Optional Parameters + +Show when optional parameters should and shouldn't be included: + +```python +"input_examples": [ + { + "query": "machine learning", + "filters": {"year": 2024, "category": "research"} # With optional filters + }, + { + "query": "artificial intelligence" # Without optional filters + } +] +``` + +### 3. Illustrate Format Requirements + +Make format expectations clear through examples: + +```python +"input_examples": [ + { + "phone": "+1-555-123-4567", # Shows expected phone format + "date": "2025-01-15", # Shows date format (YYYY-MM-DD) + "time": "14:30" # Shows time format (HH:MM) + } +] +``` + +### 4. Keep Examples Realistic + +Use realistic, production-like examples rather than placeholder data: + +```python +# ✅ Good - realistic examples +"input_examples": [ + {"email": "alice@company.com", "role": "admin"}, + {"email": "bob@company.com", "role": "user"} +] + +# ❌ Bad - placeholder examples +"input_examples": [ + {"email": "test@test.com", "role": "role1"}, + {"email": "example@example.com", "role": "role2"} +] +``` + +### 5. Limit Example Count + +Provide 2-5 examples per tool: + +- **Too few** (1): May not show enough variation +- **Just right** (2-5): Demonstrates patterns without bloating tokens +- **Too many** (10+): Wastes tokens, diminishing returns + +## Integration with Other Features + +Input examples work seamlessly with other Anthropic tool features: + +### With Tool Search + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "input_examples": [ # Input examples + {"sql": "SELECT * FROM users WHERE id = 1"} + ] +} +``` + +### With Programmatic Tool Calling + +```python +{ + "type": "function", + "function": { + "name": "fetch_data", + "description": "Fetch data from API", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"endpoint": "/api/users", "method": "GET"} + ] +} +``` + +### All Features Combined + +```python +{ + "type": "function", + "function": { + "name": "advanced_tool", + "description": "A complex tool", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"param1": "value1", "param2": "value2"} + ] +} +``` + +## Provider Support + +LiteLLM supports input examples across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `input_examples` field. + +## Troubleshooting + +### "Invalid request" error with examples + +**Problem**: Receiving 400 error when using input examples + +**Solution**: Ensure each example is valid according to your `input_schema`: + +```python +# Check that: +# 1. All required fields are present in examples +# 2. Field types match the schema +# 3. Enum values are valid +# 4. Nested objects follow the schema structure +``` + +### Examples not improving tool calls + +**Problem**: Adding examples doesn't seem to help + +**Solution**: +1. **Check descriptions first**: Ensure tool descriptions are detailed and clear +2. **Review example quality**: Make sure examples are realistic and diverse +3. **Verify schema**: Confirm examples actually match your schema +4. **Add more variation**: Include examples showing different use cases + +### Token usage too high + +**Problem**: Input examples consuming too many tokens + +**Solution**: +1. **Reduce example count**: Use 2-3 examples instead of 5+ +2. **Simplify examples**: Remove unnecessary fields from examples +3. **Consider descriptions**: If descriptions are clear, examples may not be needed + +## When NOT to Use Input Examples + +Skip input examples if: + +- **Tool is simple**: Single parameter tools with clear descriptions +- **Schema is self-explanatory**: Well-structured schema with good descriptions +- **Token budget is tight**: Examples add 20-200 tokens each +- **Server-side tools**: web_search, code_execution, etc. don't support examples + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md new file mode 100644 index 00000000000..3d61022b26f --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -0,0 +1,397 @@ +# Anthropic Tool Search + +Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs. + +## Benefits + +- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions +- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools +- **On-demand loading**: Tools are only loaded when Claude needs them + +## Supported Models + +Tool search is available on: +- Claude Opus 4.5 +- Claude Sonnet 4.5 + +## Supported Platforms + +- Anthropic API (direct) +- Azure Anthropic (Microsoft Foundry) +- Google Cloud Vertex AI +- Amazon Bedrock (invoke API only, not converse API) + +## Tool Search Variants + +LiteLLM supports both tool search variants: + +### 1. Regex Tool Search (`tool_search_tool_regex_20251119`) + +Claude constructs regex patterns to search for tools. + +### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`) + +Claude uses natural language queries to search for tools using the BM25 algorithm. + +## Quick Start + +### Basic Example with Regex Tool Search + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What is the weather in San Francisco?"} + ], + tools=[ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tool - will be loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather at a specific location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Mark for deferred loading + }, + # Another deferred tool + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) + +print(response.choices[0].message.content) +``` + +### BM25 Tool Search Example + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Search for Python files containing 'authentication'"} + ], + tools=[ + # Tool search tool (BM25 variant) + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Deferred tools... + { + "type": "function", + "function": { + "name": "search_codebase", + "description": "Search through codebase files by content and filename", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_pattern": {"type": "string"} + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Azure Anthropic + +```python +import litellm + +response = litellm.completion( + model="azure_anthropic/claude-sonnet-4-5", + api_base="https://<your-resource>.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "What's the weather like?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Vertex AI + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/claude-sonnet-4-5", + vertex_project="your-project-id", + vertex_location="us-central1", + messages=[ + {"role": "user", "content": "Search my documents"} + ], + tools=[ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Your deferred tools... + ] +) +``` + +## Streaming Support + +Tool search works with streaming: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Get the weather"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +## LiteLLM Proxy + +Tool search works automatically through the LiteLLM proxy: + +### Proxy Config + +```yaml +model_list: + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +### Client Request + +```python +import openai + +client = openai.OpenAI( + api_key="your-litellm-proxy-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet", + messages=[ + {"role": "user", "content": "What's the weather?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Important Notes + +### Beta Header + +LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it. + +### Deferred Loading + +- Tools with `defer_loading: true` are only loaded when Claude discovers them via search +- At least one tool must be non-deferred (the tool search tool itself) +- Keep your 3-5 most frequently used tools as non-deferred for optimal performance + +### Tool Descriptions + +Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses: +- Tool names +- Tool descriptions +- Argument names +- Argument descriptions + +### Usage Tracking + +Tool search requests are tracked in the usage object: + +```python +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Search for tools"}], + tools=[...] +) + +# Check tool search usage +if response.usage.server_tool_use: + print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}") +``` + +## Error Handling + +### All Tools Deferred + +```python +# ❌ This will fail - at least one tool must be non-deferred +tools = [ + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] + +# ✅ Correct - tool search tool is non-deferred +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] +``` + +### Missing Tool Definition + +If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`. + +## Best Practices + +1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true` + +2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries + +3. **Choose the right variant**: + - Use **regex** for exact pattern matching (faster) + - Use **BM25** for natural language semantic search + +4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns + +5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality + +## When to Use Tool Search + +**Good use cases:** +- 10+ tools available in your system +- Tool definitions consuming >10K tokens +- Experiencing tool selection accuracy issues +- Building systems with multiple tool categories +- Tool library growing over time + +**When traditional tool calling is better:** +- Less than 10 tools total +- All tools are frequently used +- Very small tool definitions (<100 tokens total) + +## Limitations + +- Not compatible with tool use examples +- Requires Claude Opus 4.5 or Sonnet 4.5 +- On Bedrock, only available via invoke API (not converse API) +- Maximum 10,000 tools in catalog +- Returns 3-5 most relevant tools per search + +## Additional Resources + +- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search) +- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call) + diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index b7b39f10395..b363b747de5 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -42,6 +42,7 @@ from litellm.types.llms.openai import ( ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ) from litellm.types.utils import ( Delta, @@ -550,15 +551,18 @@ class ModelResponseIterator: if "text" in content_block["delta"]: text = content_block["delta"]["text"] elif "partial_json" in content_block["delta"]: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": content_block["delta"]["partial_json"], + tool_use = cast( + ChatCompletionToolCallChunk, + { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": content_block["delta"]["partial_json"], + }, + "index": self.tool_index, }, - "index": self.tool_index, - } + ) elif "citation" in content_block["delta"]: provider_specific_fields["citation"] = content_block["delta"]["citation"] elif ( @@ -569,7 +573,7 @@ class ModelResponseIterator: ChatCompletionThinkingBlock( type="thinking", thinking=content_block["delta"].get("thinking") or "", - signature=content_block["delta"].get("signature"), + signature=str(content_block["delta"].get("signature") or ""), ) ] provider_specific_fields["thinking_blocks"] = thinking_blocks @@ -625,7 +629,7 @@ class ModelResponseIterator: return content_block_start - def chunk_parser(self, chunk: dict) -> ModelResponseStream: + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915 try: type_chunk = chunk.get("type", "") or "" @@ -672,15 +676,32 @@ class ModelResponseIterator: text = content_block_start["content_block"]["text"] elif content_block_start["content_block"]["type"] == "tool_use": self.tool_index += 1 - tool_use = { - "id": content_block_start["content_block"]["id"], - "type": "function", - "function": { - "name": content_block_start["content_block"]["name"], - "arguments": "", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content_block_start["content_block"]: + caller_data = content_block_start["content_block"]["caller"] + if caller_data: + tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item] + elif content_block_start["content_block"]["type"] == "server_tool_use": + # Handle server tool use (for tool search) + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) elif ( content_block_start["content_block"]["type"] == "redacted_thinking" ): @@ -696,17 +717,21 @@ class ModelResponseIterator: # check if tool call content block is_empty = self.check_empty_tool_call_args() if is_empty: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": "{}", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=None, # type: ignore[typeddict-item] + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, # type: ignore[typeddict-item] + arguments="{}", + ), + index=self.tool_index, + ) # Reset response_format tool tracking when block stops self.is_response_format_tool = False + elif type_chunk == "tool_result": + # Handle tool_result blocks (for tool search results with tool_reference) + # These are automatically handled by Anthropic API, we just pass them through + pass elif type_chunk == "message_delta": finish_reason, usage = self._handle_message_delta(chunk) elif type_chunk == "message_start": diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 623a98c132f..ac1c9b1e000 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -54,7 +54,10 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, ) from litellm.types.utils import Message as LitellmMessage -from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse +from litellm.types.utils import ( + PromptTokensDetailsWrapper, + ServerToolUse, +) from litellm.utils import ( ModelResponse, Usage, @@ -187,7 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _tool_choice - def _map_tool_helper( + def _map_tool_helper( # noqa: PLR0915 self, tool: ChatCompletionToolParam ) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]: returned_tool: Optional[AllAnthropicToolsValues] = None @@ -250,9 +253,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): returned_tool = _computer_tool elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS): - function_name = tool.get("name", tool.get("function", {}).get("name")) - if function_name is None or not isinstance(function_name, str): + function_name_obj = tool.get("name", tool.get("function", {}).get("name")) + if function_name_obj is None or not isinstance(function_name_obj, str): raise ValueError("Missing required parameter: name") + function_name = function_name_obj additional_tool_params = {} for k, v in tool.items(): @@ -268,6 +272,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_server = self._map_openai_mcp_server_tool( cast(OpenAIMcpServerTool, tool) ) + elif tool["type"] == "tool_search_tool_regex_20251119": + # Tool search tool using regex + from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex + + tool_name_obj = tool.get("name", "tool_search_tool_regex") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolRegex( + type="tool_search_tool_regex_20251119", + name=tool_name, + ) + elif tool["type"] == "tool_search_tool_bm25_20251119": + # Tool search tool using BM25 + from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25 + + tool_name_obj = tool.get("name", "tool_search_tool_bm25") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolBM25( + type="tool_search_tool_bm25_20251119", + name=tool_name, + ) if returned_tool is None and mcp_server is None: raise ValueError(f"Unsupported tool type: {tool['type']}") @@ -275,14 +303,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _cache_control = tool.get("cache_control", None) _cache_control_function = tool.get("function", {}).get("cache_control", None) if returned_tool is not None: - if _cache_control is not None: - returned_tool["cache_control"] = _cache_control - elif _cache_control_function is not None and isinstance( - _cache_control_function, dict - ): - returned_tool["cache_control"] = ChatCompletionCachedContent( - **_cache_control_function # type: ignore - ) + # Only set cache_control on tools that support it (not tool search tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"): + if _cache_control is not None: + returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item] + elif _cache_control_function is not None and isinstance( + _cache_control_function, dict + ): + returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item] + **_cache_control_function # type: ignore + ) + + ## check if defer_loading is set in the tool + _defer_loading = tool.get("defer_loading", None) + _defer_loading_function = tool.get("function", {}).get("defer_loading", None) + if returned_tool is not None: + # Only set defer_loading on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _defer_loading is not None: + if not isinstance(_defer_loading, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading # type: ignore[typeddict-item] + elif _defer_loading_function is not None: + if not isinstance(_defer_loading_function, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item] + + ## check if allowed_callers is set in the tool + _allowed_callers = tool.get("allowed_callers", None) + _allowed_callers_function = tool.get("function", {}).get("allowed_callers", None) + if returned_tool is not None: + # Only set allowed_callers on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _allowed_callers is not None: + if not isinstance(_allowed_callers, list) or not all( + isinstance(item, str) for item in _allowed_callers + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers # type: ignore[typeddict-item] + elif _allowed_callers_function is not None: + if not isinstance(_allowed_callers_function, list) or not all( + isinstance(item, str) for item in _allowed_callers_function + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item] + + ## check if input_examples is set in the tool + _input_examples = tool.get("input_examples", None) + _input_examples_function = tool.get("function", {}).get("input_examples", None) + if returned_tool is not None: + # Only set input_examples on user-defined tools (type "custom" or no type) + tool_type = returned_tool.get("type", "") + if tool_type == "custom" or (tool_type == "" and "name" in returned_tool): + if _input_examples is not None and isinstance(_input_examples, list): + returned_tool["input_examples"] = _input_examples # type: ignore[typeddict-item] + elif _input_examples_function is not None and isinstance( + _input_examples_function, list + ): + returned_tool["input_examples"] = _input_examples_function # type: ignore[typeddict-item] return returned_tool, mcp_server @@ -334,6 +415,82 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_servers.append(mcp_server_tool) return anthropic_tools, mcp_servers + def _detect_tool_search_tools(self, tools: Optional[List]) -> bool: + """Check if tool search tools are present in the tools list.""" + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def _separate_deferred_tools( + self, tools: List + ) -> Tuple[List, List]: + """ + Separate tools into deferred and non-deferred lists. + + Returns: + Tuple of (non_deferred_tools, deferred_tools) + """ + non_deferred = [] + deferred = [] + + for tool in tools: + if tool.get("defer_loading", False): + deferred.append(tool) + else: + non_deferred.append(tool) + + return non_deferred, deferred + + def _expand_tool_references( + self, + content: List, + deferred_tools: List, + ) -> List: + """ + Expand tool_reference blocks to full tool definitions. + + When Anthropic's tool search returns results, it includes tool_reference blocks + that reference tools by name. This method expands those references to full + tool definitions from the deferred_tools catalog. + + Args: + content: Response content that may contain tool_reference blocks + deferred_tools: List of deferred tools that can be referenced + + Returns: + Content with tool_reference blocks expanded to full tool definitions + """ + if not deferred_tools: + return content + + # Create a mapping of tool names to tool definitions + tool_map = {} + for tool in deferred_tools: + tool_name = tool.get("name") or tool.get("function", {}).get("name") + if tool_name: + tool_map[tool_name] = tool + + # Expand tool references in content + expanded_content = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "tool_reference": + tool_name = item.get("tool_name") + if tool_name and tool_name in tool_map: + # Replace reference with full tool definition + expanded_content.append(tool_map[tool_name]) + else: + # Keep the reference if we can't find the tool + expanded_content.append(item) + else: + expanded_content.append(item) + + return expanded_content + def _map_stop_sequences( self, stop: Optional[Union[str, List[str]]] ) -> Optional[List[str]]: @@ -822,6 +979,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "messages": anthropic_messages, **optional_params, } + + ## Handle output_config (Anthropic-specific parameter) + if "output_config" in optional_params: + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and effort not in ["high", "medium", "low"]: + raise ValueError( + f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'" + ) + data["output_config"] = output_config return data @@ -870,18 +1038,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text_content += content["text"] ## TOOL CALLING elif content["type"] == "tool_use": - tool_calls.append( - ChatCompletionToolCallChunk( - id=content["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content["name"], - arguments=json.dumps(content["input"]), - ), - index=idx, - ) + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content["input"]), + ), + index=idx, ) - + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## SERVER TOOL USE (for tool search) + elif content["type"] == "server_tool_use": + # Server tool use blocks are for tool search - treat as tool calls + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content.get("input", {})), + ), + index=idx, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery) + elif content["type"] == "tool_search_tool_result": + # This block contains tool_references that were discovered + # We don't need to include this in the response as it's internal metadata + pass elif content.get("thinking", None) is not None: if thinking_blocks is None: thinking_blocks = [] @@ -916,7 +1106,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return text_content, citations, thinking_blocks, reasoning_content, tool_calls def calculate_usage( - self, usage_object: dict, reasoning_content: Optional[str] + self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -926,6 +1116,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens: int = 0 cache_creation_token_details: Optional[CacheCreationTokenDetails] = None web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None if ( "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None @@ -946,6 +1137,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): web_search_requests = cast( int, _usage["server_tool_use"]["web_search_requests"] ) + if ( + "tool_search_requests" in _usage["server_tool_use"] + and _usage["server_tool_use"]["tool_search_requests"] is not None + ): + tool_search_requests = cast( + int, _usage["server_tool_use"]["tool_search_requests"] + ) + + # Count tool_search_requests from content blocks if not in usage + # Anthropic doesn't always include tool_search_requests in the usage object + if tool_search_requests is None and completion_response is not None: + tool_search_count = 0 + for content in completion_response.get("content", []): + if content.get("type") == "server_tool_use": + tool_name = content.get("name", "") + if "tool_search" in tool_name: + tool_search_count += 1 + if tool_search_count > 0: + tool_search_requests = tool_search_count if "cache_creation" in _usage and _usage["cache_creation"] is not None: cache_creation_token_details = CacheCreationTokenDetails( @@ -982,8 +1192,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens=cache_read_input_tokens, completion_tokens_details=completion_token_details, server_tool_use=( - ServerToolUse(web_search_requests=web_search_requests) - if web_search_requests is not None + ServerToolUse( + web_search_requests=web_search_requests, + tool_search_requests=tool_search_requests, + ) + if (web_search_requests is not None or tool_search_requests is not None) else None ), ) @@ -1077,6 +1290,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage = self.calculate_usage( usage_object=completion_response["usage"], reasoning_content=reasoning_content, + completion_response=completion_response, ) setattr(model_response, "usage", usage) # type: ignore diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0d00a3b4632..9f5688f9e01 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -88,6 +88,86 @@ class AnthropicModelInfo(BaseLLMModelInfo): return True return False + def is_tool_search_used(self, tools: Optional[List]) -> bool: + """ + Check if tool search tools are present in the tools list. + """ + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool: + """ + Check if programmatic tool calling is being used (tools with allowed_callers field). + + Returns True if any tool has allowed_callers containing 'code_execution_20250825'. + """ + if not tools: + return False + + for tool in tools: + # Check top-level allowed_callers + allowed_callers = tool.get("allowed_callers", None) + if allowed_callers and isinstance(allowed_callers, list): + if "code_execution_20250825" in allowed_callers: + return True + + # Check function.allowed_callers for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_allowed_callers = function.get("allowed_callers", None) + if function_allowed_callers and isinstance(function_allowed_callers, list): + if "code_execution_20250825" in function_allowed_callers: + return True + + return False + + def is_input_examples_used(self, tools: Optional[List]) -> bool: + """ + Check if input_examples is being used in any tools. + + Returns True if any tool has input_examples field. + """ + if not tools: + return False + + for tool in tools: + # Check top-level input_examples + input_examples = tool.get("input_examples", None) + if input_examples and isinstance(input_examples, list) and len(input_examples) > 0: + return True + + # Check function.input_examples for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_input_examples = function.get("input_examples", None) + if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0: + return True + + return False + + def is_effort_used(self, optional_params: Optional[dict]) -> bool: + """ + Check if effort parameter is being used via output_config. + + Returns True if output_config with effort field is present. + """ + if not optional_params: + return False + + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and isinstance(effort, str): + return True + + return False + def _get_user_anthropic_beta_headers( self, anthropic_beta_header: Optional[str] ) -> Optional[List[str]]: @@ -122,6 +202,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): pdf_used: bool = False, file_id_used: bool = False, mcp_server_used: bool = False, + tool_search_used: bool = False, + programmatic_tool_calling_used: bool = False, + input_examples_used: bool = False, + effort_used: bool = False, is_vertex_request: bool = False, user_anthropic_beta_headers: Optional[List[str]] = None, ) -> dict: @@ -138,6 +222,15 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.add("code-execution-2025-05-22") if mcp_server_used: betas.add("mcp-client-2025-04-04") + # Tool search, programmatic tool calling, and input_examples all use the same beta header + if tool_search_used or programmatic_tool_calling_used or input_examples_used: + from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER + betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) + + # Effort parameter uses a separate beta header + if effort_used: + from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER + betas.add(ANTHROPIC_EFFORT_BETA_HEADER) headers = { "anthropic-version": anthropic_version or "2023-06-01", @@ -182,6 +275,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) pdf_used = self.is_pdf_used(messages=messages) file_id_used = self.is_file_id_used(messages=messages) + tool_search_used = self.is_tool_search_used(tools=tools) + programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) + input_examples_used = self.is_input_examples_used(tools=tools) + effort_used = self.is_effort_used(optional_params=optional_params) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -194,6 +291,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + tool_search_used=tool_search_used, + programmatic_tool_calling_used=programmatic_tool_calling_used, + input_examples_used=input_examples_used, + effort_used=effort_used, ) headers = {**headers, **anthropic_headers} diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 0e905014fe2..98e57f279cf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -645,7 +645,7 @@ class LiteLLMAnthropicMessagesAdapter: type="tool_use", id=choice.delta.tool_calls[0].id or str(uuid.uuid4()), name=choice.delta.tool_calls[0].function.name or "", - input={}, + input={}, # type: ignore[typeddict-item] ) elif isinstance(choice, StreamingChoices) and hasattr( choice.delta, "thinking_blocks" diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index fc210a7d084..507b382f785 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -36,12 +36,20 @@ class AnthropicOutputSchema(TypedDict, total=False): schema: Required[dict] +class AnthropicOutputConfig(TypedDict, total=False): + """Configuration for controlling Claude's output behavior.""" + effort: Literal["high", "medium", "low"] + + class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str input_schema: Optional[AnthropicInputSchema] type: Literal["custom"] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: bool + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicComputerTool(TypedDict, total=False): @@ -67,24 +75,78 @@ class AnthropicWebSearchTool(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] max_uses: Optional[int] user_location: Optional[AnthropicWebSearchUserLocation] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_editor type: Required[str] name: Required[str] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicCodeExecutionTool(TypedDict, total=False): type: Required[str] name: Required[Literal["code_execution"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicMemoryTool(TypedDict, total=False): type: Required[str] name: Required[Literal["memory"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class AnthropicToolSearchToolRegex(TypedDict, total=False): + """Tool search tool using regex patterns for tool discovery.""" + type: Required[Literal["tool_search_tool_regex_20251119"]] + name: Required[str] + + +class AnthropicToolSearchToolBM25(TypedDict, total=False): + """Tool search tool using BM25 algorithm for tool discovery.""" + type: Required[Literal["tool_search_tool_bm25_20251119"]] + name: Required[str] + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class ToolReference(TypedDict, total=False): + """Reference to a tool that should be expanded from deferred tools.""" + type: Required[Literal["tool_reference"]] + tool_name: Required[str] + + +class DirectToolCaller(TypedDict, total=False): + """Indicates a tool was called directly by Claude.""" + type: Required[Literal["direct"]] + + +class CodeExecutionToolCaller(TypedDict, total=False): + """Indicates a tool was called programmatically from code execution.""" + type: Required[Literal["code_execution_20250825"]] + tool_id: Required[str] # ID of the code execution tool that made the call + + +ToolCaller = Union[DirectToolCaller, CodeExecutionToolCaller] + + +class AnthropicContainer(TypedDict, total=False): + """Container metadata for code execution.""" + id: Required[str] + expires_at: Optional[str] # ISO 8601 timestamp AllAnthropicToolsValues = Union[ @@ -94,6 +156,8 @@ AllAnthropicToolsValues = Union[ AnthropicWebSearchTool, AnthropicCodeExecutionTool, AnthropicMemoryTool, + AnthropicToolSearchToolRegex, + AnthropicToolSearchToolBM25, ] @@ -121,6 +185,7 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): name: str input: dict cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + caller: Optional[ToolCaller] AnthropicMessagesAssistantMessageValues = Union[ @@ -372,6 +437,7 @@ class ToolUseBlock(TypedDict): name: str type: Literal["tool_use"] + caller: Optional[ToolCaller] class TextBlock(TypedDict): @@ -565,3 +631,11 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" + ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" + + +# Tool search beta header constant +ANTHROPIC_TOOL_SEARCH_BETA_HEADER = "advanced-tool-use-2025-11-20" + +# Effort beta header constant +ANTHROPIC_EFFORT_BETA_HEADER = "effort-2025-11-24" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c29b2f32ea5..61d58e4c86d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,6 +1,17 @@ from enum import Enum from os import PathLike -from typing import IO, Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Union +from typing import ( + IO, + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import httpx from openai._legacy_response import ( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0f73407610b..b0d081d8f87 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -999,7 +999,8 @@ class PromptTokensDetailsWrapper( class ServerToolUse(BaseModel): - web_search_requests: Optional[int] + web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None class Usage(CompletionUsage): diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 8ff2f0a4474..09c16add77a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -556,3 +556,744 @@ def test_anthropic_structured_output_beta_header(): "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] ) + + +# ============ Tool Search Tests ============ + + +def test_tool_search_regex_detection(): + """Test that tool search regex tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search regex tool + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + ] + assert config.is_tool_search_used(tools) is True + + # Test without tool search + tools = [ + { + "type": "function", + "function": {"name": "get_weather"} + } + ] + assert config.is_tool_search_used(tools) is False + + +def test_tool_search_bm25_detection(): + """Test that tool search BM25 tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search BM25 tool + tools = [ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + ] + assert config.is_tool_search_used(tools) is True + + +def test_tool_search_beta_header(): + """Test that tool search beta header is automatically added""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + headers = config.get_anthropic_headers( + api_key="test-key", + tool_search_used=True, + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_tool_search_regex_mapping(): + """Test that tool search regex tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_regex_20251119" + assert mapped_tool["name"] == "tool_search_tool_regex" + assert mcp_server is None + + +def test_tool_search_bm25_mapping(): + """Test that tool search BM25 tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_bm25_20251119" + assert mapped_tool["name"] == "tool_search_tool_bm25" + assert mcp_server is None + + +def test_deferred_tools_separation(): + """Test that deferred and non-deferred tools are properly separated""" + config = AnthropicConfig() + + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {"name": "get_weather"}, + "defer_loading": True + }, + { + "type": "function", + "function": {"name": "search_files"}, + "defer_loading": False + } + ] + + non_deferred, deferred = config._separate_deferred_tools(tools) + + assert len(non_deferred) == 2 # tool_search and search_files + assert len(deferred) == 1 # get_weather + + +def test_server_tool_use_in_response(): + """Test that server_tool_use blocks are parsed correctly""" + config = AnthropicConfig() + + completion_response = { + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC123", + "name": "tool_search_tool_regex", + "input": {"query": "weather"} + } + ] + } + + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "srvtoolu_01ABC123" + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + + +def test_tool_search_usage_tracking(): + """Test that tool_search_requests are tracked in usage""" + config = AnthropicConfig() + + usage_object = { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": { + "tool_search_requests": 2 + } + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.tool_search_requests == 2 + + +def test_tool_reference_expansion(): + """Test that tool_reference blocks are expanded correctly""" + config = AnthropicConfig() + + deferred_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather" + } + } + ] + + content = [ + {"type": "text", "text": "I'll search for tools"}, + {"type": "tool_reference", "tool_name": "get_weather"} + ] + + expanded = config._expand_tool_references(content, deferred_tools) + + assert len(expanded) == 2 + assert expanded[0]["type"] == "text" + assert expanded[1]["type"] == "function" + assert expanded[1]["function"]["name"] == "get_weather" + + +def test_defer_loading_preserved_in_transformation(): + """Test that defer_loading parameter is preserved when transforming tools""" + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool.get("defer_loading") is True + assert mapped_tool["name"] == "get_weather" + assert mcp_server is None + + +def test_tool_search_complete_response_parsing(): + """Test parsing a complete tool search response with server_tool_use and tool_search_tool_result blocks""" + config = AnthropicConfig() + + # Simulating actual Anthropic API response with tool search + completion_response = { + "content": [ + { + "type": "text", + "text": "I'll search for weather-related tools that can help you." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather", "limit": 5}, + "caller": {"type": "direct"} + }, + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}] + } + }, + { + "type": "text", + "text": "Great! I found a weather tool." + }, + { + "type": "tool_use", + "id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ", + "name": "get_weather", + "input": {"location": "San Francisco"} + } + ], + "usage": { + "input_tokens": 1639, + "output_tokens": 170, + "server_tool_use": {"web_search_requests": 0} + } + } + + # Extract content + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + # Verify text extraction (should concatenate both text blocks) + assert "I'll search for weather-related tools" in text + assert "Great! I found a weather tool" in text + + # Verify tool calls (should have both server_tool_use and tool_use) + assert len(tool_calls) == 2 + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + assert tool_calls[1]["function"]["name"] == "get_weather" + + # Verify usage calculation counts tool_search_requests from content + usage = config.calculate_usage( + usage_object=completion_response["usage"], + reasoning_content=None, + completion_response=completion_response + ) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.web_search_requests == 0 + assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + + +def test_allowed_callers_field_preservation(): + """Test that allowed_callers field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level allowed_callers + tool_with_allowed_callers = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_programmatic_tool_calling_beta_header(): + """Test that beta header is automatically added when programmatic tool calling is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with allowed_callers + tools = [ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {"type": "object", "properties": {}} + }, + "allowed_callers": ["code_execution_20250825"] + } + ] + + is_programmatic = model_info.is_programmatic_tool_calling_used(tools) + assert is_programmatic is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + programmatic_tool_calling_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_caller_field_in_response(): + """Test that caller field is correctly parsed from tool_use blocks.""" + config = AnthropicConfig() + + # Mock response with programmatic tool call + completion_response = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the database." + }, + { + "type": "tool_use", + "id": "toolu_123", + "name": "query_database", + "input": {"sql": "SELECT * FROM users"}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc" + } + } + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 100, "output_tokens": 50} + } + + text, citations, thinking, reasoning, tool_calls = config.extract_response_content(completion_response) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "toolu_123" + assert tool_calls[0]["function"]["name"] == "query_database" + assert "caller" in tool_calls[0] + assert tool_calls[0]["caller"]["type"] == "code_execution_20250825" + assert tool_calls[0]["caller"]["tool_id"] == "srvtoolu_abc" + + +def test_code_execution_20250825_tool_type(): + """Test that code_execution_20250825 tool type is handled correctly.""" + config = AnthropicConfig() + + tool = { + "type": "code_execution_20250825", + "name": "code_execution" + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert transformed_tool["type"] == "code_execution_20250825" + assert transformed_tool["name"] == "code_execution" + + +def test_allowed_callers_in_function_field(): + """Test that allowed_callers in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + }, + "allowed_callers": ["code_execution_20250825"] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_field_preservation(): + """Test that input_examples field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level input_examples + tool_with_examples = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + }, + "required": ["location"] + } + }, + "input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, + {"location": "Tokyo, Japan", "unit": "celsius"} + ] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_examples) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + assert transformed_tool["input_examples"][0]["location"] == "San Francisco, CA" + + +def test_input_examples_beta_header(): + """Test that beta header is automatically added when input_examples is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with input_examples + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": {"type": "object", "properties": {}} + }, + "input_examples": [ + {"location": "San Francisco, CA"} + ] + } + ] + + is_examples_used = model_info.is_input_examples_used(tools) + assert is_examples_used is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + input_examples_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_input_examples_in_function_field(): + """Test that input_examples in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "input_examples": [ + {"location": "Paris, France"}, + {"location": "London, UK"} + ] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + + +def test_input_examples_with_other_features(): + """Test that input_examples works alongside other tool features.""" + config = AnthropicConfig() + + # Tool with input_examples, defer_loading, and allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "input_examples": [ + {"sql": "SELECT * FROM users WHERE id = 1"} + ], + "defer_loading": True, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert "defer_loading" in transformed_tool + assert "allowed_callers" in transformed_tool + assert transformed_tool["defer_loading"] is True + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_empty_list_not_added(): + """Test that empty input_examples list is not added to transformed tool.""" + config = AnthropicConfig() + + # Tool with empty input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "input_examples": [] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + # Empty list should not be added + assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + + +# ============ Effort Parameter Tests ============ + + +def test_effort_output_config_preservation(): + """Test that output_config with effort is preserved in transformation.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Analyze this code"}] + optional_params = { + "output_config": { + "effort": "medium" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "medium" + + +def test_effort_beta_header_injection(): + """Test that effort beta header is automatically added when output_config is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test with effort parameter + optional_params = { + "output_config": { + "effort": "low" + } + } + + effort_used = model_info.is_effort_used(optional_params=optional_params) + assert effort_used is True + + headers = model_info.get_anthropic_headers( + api_key="test-key", + effort_used=effort_used + ) + + assert "anthropic-beta" in headers + assert "effort-2025-11-24" in headers["anthropic-beta"] + + +def test_effort_validation(): + """Test that only valid effort values are accepted.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Test"}] + + # Valid values should work + for effort in ["high", "medium", "low"]: + optional_params = {"output_config": {"effort": effort}} + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + assert result["output_config"]["effort"] == effort + + # Invalid value should raise error + with pytest.raises(ValueError, match="Invalid effort value"): + optional_params = {"output_config": {"effort": "invalid"}} + config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + +def test_effort_with_claude_opus_45(): + """Test effort parameter works with Claude Opus 4.5 model.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Complex analysis task"}] + optional_params = { + "output_config": { + "effort": "high" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "high" + assert result["model"] == "claude-opus-4-5-20251101" + + +def test_effort_with_other_features(): + """Test effort works alongside other features (thinking, tools).""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Use tools efficiently"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_data", + "description": "Get data", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + } + } + ] + optional_params = { + "output_config": { + "effort": "low" + }, + "tools": tools, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Verify all features are present + assert "output_config" in result + assert result["output_config"]["effort"] == "low" + assert "tools" in result + assert len(result["tools"]) > 0 + assert "thinking" in result From db2c8e363175b6ca155fd159f166e2a7f17a0565 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia <krrishdholakia@gmail.com> Date: Tue, 25 Nov 2025 11:57:51 -0800 Subject: [PATCH 41/68] docs: initial doc cleanup --- .../index.md | 205 +++++++++++++++++- .../docs/providers/anthropic_tool_search.md | 2 +- 2 files changed, 194 insertions(+), 13 deletions(-) rename docs/my-website/blog/{anthropic_advanced_features => anthropic_opus_4_5_and_advanced_features}/index.md (79%) diff --git a/docs/my-website/blog/anthropic_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md similarity index 79% rename from docs/my-website/blog/anthropic_advanced_features/index.md rename to docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 71e5d9b1925..0b0f4a54167 100644 --- a/docs/my-website/blog/anthropic_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -1,7 +1,7 @@ --- slug: anthropic_advanced_features -title: "Advanced Anthropic Features in LiteLLM: Tool Search, Programmatic Tool Calling, Input Examples, and Effort Control" -date: 2025-01-25T10:00:00 +title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" +date: 2025-11-25T10:00:00 authors: - name: Sameer Kankute title: SWE @ LiteLLM (LLM Translation) @@ -22,24 +22,205 @@ hide_table_of_contents: false import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. + +--- + +## Usage + +<Tabs> +<TabItem value="sdk" label="LiteLLM Python SDK"> + + +```python +import os +from litellm import completion + +# set env - [OPTIONAL] replace with your anthropic key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "Hey! how's it going?"}] + +## OPENAI /chat/completions API format +response = completion(model="claude-opus-4-5-20251101", messages=messages) +print(response) + +``` + +</TabItem> +<TabItem value="proxy" label="LiteLLM Proxy"> + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: claude-opus-4-5-20251101 ### MODEL NAME sent to `litellm.completion()` ### + api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY") +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + +<Tabs> +<TabItem value="curl" label="OpenAI Chat Completions"> +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` +</TabItem> +<TabItem value="anthropic" label="Anthropic /v1/messages API"> +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` +</TabItem> +</Tabs> +</TabItem> +</Tabs> + +## Usage - Bedrock + :::info -This guide covers Anthropic's latest advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. +LiteLLM uses the boto3 library to authenticate with Bedrock. + +For more ways to authenticate with Bedrock, see the [Bedrock documentation](../../docs/providers/bedrock#authentication). ::: -We're excited to announce support for Anthropic's latest advanced features in LiteLLM! These powerful capabilities enable you to build more efficient, scalable, and cost-effective AI applications with Claude. +<Tabs> +<TabItem value="sdk" label="LiteLLM Python SDK"> -## Table of Contents -1. [Tool Search](#tool-search) -2. [Programmatic Tool Calling](#programmatic-tool-calling) -3. [Tool Input Examples](#tool-input-examples) -4. [Effort Parameter: Control Token Usage](#effort-parameter) -5. [Cost Tracking: Monitor Tool Search Usage](#cost-tracking) -6. [Combining Features](#combining-features) +```python +import os +from litellm import completion + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +## OPENAI /chat/completions API format +response = completion( + model="bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + +</TabItem> +<TabItem value="proxy" label="LiteLLM Proxy"> + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0 ### MODEL NAME sent to `litellm.completion()` ### + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + +<Tabs> +<TabItem value="curl" label="OpenAI Chat Completions"> +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` +</TabItem> +<TabItem value="anthropic" label="Anthropic /v1/messages API"> +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` +</TabItem> +<TabItem value="invoke" label="Bedrock /invoke API"> +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/invoke' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` +</TabItem> +<TabItem value="converse" label="Bedrock /converse API"> +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/converse' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` +</TabItem> +</Tabs> +</TabItem> +</Tabs> ---- ## Tool Search {#tool-search} diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md index 3d61022b26f..7b9e7cfaa72 100644 --- a/docs/my-website/docs/providers/anthropic_tool_search.md +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -380,7 +380,7 @@ If Claude references a tool that isn't in your deferred tools list, you'll get a **When traditional tool calling is better:** - Less than 10 tools total - All tools are frequently used -- Very small tool definitions (<100 tokens total) +- Very small tool definitions (\<100 tokens total) ## Limitations From 44cde2e48fe6d2365f8cf3c972d1be8de7bbceec Mon Sep 17 00:00:00 2001 From: yuneng-jiang <yuneng.jiang@gmail.com> Date: Tue, 25 Nov 2025 12:03:01 -0800 Subject: [PATCH 42/68] Disable edit, delete, info, for dynamically generated spend tags --- .../tag_management/TagTable.test.tsx | 101 ++++++++++++++++++ .../components/tag_management/TagTable.tsx | 86 +++++++++++---- .../components/CreateTagModal.test.tsx | 64 +++++++++++ .../components/CreateTagModal.tsx | 44 ++------ .../components/tag_management/tag_info.tsx | 61 ++++++----- 5 files changed, 279 insertions(+), 77 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx create mode 100644 ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx new file mode 100644 index 00000000000..a56721787d5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx @@ -0,0 +1,101 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import TagTable from "./TagTable"; +import { Tag } from "./types"; + +describe("TagTable", () => { + const mockOnEdit = vi.fn(); + const mockOnDelete = vi.fn(); + const mockOnSelectTag = vi.fn(); + + const mockTag: Tag = { + name: "test-tag", + description: "Test description", + models: ["model-1", "model-2"], + model_info: { + "model-1": "GPT-4", + "model-2": "Claude-3", + }, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }; + + const mockDynamicSpendTag: Tag = { + name: "dynamic-spend-tag", + description: + "This is just a spend tag that was passed dynamically in a request. It does not control any LLM models.", + models: [], + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }; + + const defaultProps = { + data: [], + onEdit: mockOnEdit, + onDelete: mockOnDelete, + onSelectTag: mockOnSelectTag, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + render(<TagTable {...defaultProps} />); + expect(screen.getByText("Tag Name")).toBeInTheDocument(); + expect(screen.getByText("Description")).toBeInTheDocument(); + expect(screen.getByText("Allowed Models")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + it("should display no tags found message when data is empty", () => { + render(<TagTable {...defaultProps} />); + expect(screen.getByText("No tags found")).toBeInTheDocument(); + }); + + it("should display tag name", () => { + render(<TagTable {...defaultProps} data={[mockTag]} />); + expect(screen.getByText("test-tag")).toBeInTheDocument(); + }); + + it("should display tag description", () => { + render(<TagTable {...defaultProps} data={[mockTag]} />); + expect(screen.getByText("Test description")).toBeInTheDocument(); + }); + + it("should display All Models badge when models array is empty", () => { + const tagWithNoModels: Tag = { + ...mockTag, + models: [], + }; + render(<TagTable {...defaultProps} data={[tagWithNoModels]} />); + expect(screen.getByText("All Models")).toBeInTheDocument(); + }); + + it("should display formatted created date", () => { + render(<TagTable {...defaultProps} data={[mockTag]} />); + const formattedDate = new Date(mockTag.created_at).toLocaleDateString(); + expect(screen.getByText(formattedDate)).toBeInTheDocument(); + }); + + it("should disable tag name button for dynamic spend tags", () => { + render(<TagTable {...defaultProps} data={[mockDynamicSpendTag]} />); + const tagButton = screen.getByRole("button", { name: "dynamic-spend-tag" }); + expect(tagButton).toBeDisabled(); + }); + + it("should disable edit icon for dynamic spend tags", () => { + render(<TagTable {...defaultProps} data={[mockDynamicSpendTag]} />); + const editIcon = screen.getByLabelText("Edit tag (disabled)"); + expect(editIcon).toBeInTheDocument(); + expect(editIcon).toHaveClass("cursor-not-allowed"); + }); + + it("should disable delete icon for dynamic spend tags", () => { + render(<TagTable {...defaultProps} data={[mockDynamicSpendTag]} />); + const deleteIcon = screen.getByLabelText("Delete tag (disabled)"); + expect(deleteIcon).toBeInTheDocument(); + expect(deleteIcon).toHaveClass("cursor-not-allowed"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx index aa43388893e..ce28ac6e6f2 100644 --- a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx @@ -1,18 +1,4 @@ -import React from "react"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Icon, - Button, - Badge, - Text, -} from "@tremor/react"; -import { PencilAltIcon, TrashIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Tooltip } from "antd"; +import { ChevronDownIcon, ChevronUpIcon, PencilAltIcon, SwitchVerticalIcon, TrashIcon } from "@heroicons/react/outline"; import { ColumnDef, flexRender, @@ -21,6 +7,20 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; +import { + Badge, + Button, + Icon, + Table, + TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, + Text, +} from "@tremor/react"; +import { Tooltip } from "antd"; +import React from "react"; import { Tag } from "./types"; interface TagTableProps { @@ -30,6 +30,9 @@ interface TagTableProps { onSelectTag: (tagName: string) => void; } +const DYNAMIC_SPEND_TAG_DESCRIPTION = + "This is just a spend tag that was passed dynamically in a request. It does not control any LLM models."; + const TagTable: React.FC<TagTableProps> = ({ data, onEdit, onDelete, onSelectTag }) => { const [sorting, setSorting] = React.useState<SortingState>([{ id: "created_at", desc: true }]); @@ -39,14 +42,20 @@ const TagTable: React.FC<TagTableProps> = ({ data, onEdit, onDelete, onSelectTag accessorKey: "name", cell: ({ row }) => { const tag = row.original; + const isDynamicSpendTag = tag.description === DYNAMIC_SPEND_TAG_DESCRIPTION; return ( <div className="overflow-hidden"> - <Tooltip title={tag.name}> + <Tooltip + title={ + isDynamicSpendTag ? "You cannot view the information of a dynamically generated spend tag" : tag.name + } + > <Button size="xs" variant="light" className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5" onClick={() => onSelectTag(tag.name)} + disabled={isDynamicSpendTag} > {tag.name} </Button> @@ -68,7 +77,7 @@ const TagTable: React.FC<TagTableProps> = ({ data, onEdit, onDelete, onSelectTag }, }, { - header: "Allowed LLMs", + header: "Allowed Models", accessorKey: "models", cell: ({ row }) => { const tag = row.original; @@ -102,13 +111,50 @@ const TagTable: React.FC<TagTableProps> = ({ data, onEdit, onDelete, onSelectTag }, { id: "actions", - header: "", + header: "Actions", cell: ({ row }) => { const tag = row.original; + const isDynamicSpendTag = tag.description === DYNAMIC_SPEND_TAG_DESCRIPTION; return ( <div className="flex space-x-2"> - <Icon icon={PencilAltIcon} size="sm" onClick={() => onEdit(tag)} className="cursor-pointer" /> - <Icon icon={TrashIcon} size="sm" onClick={() => onDelete(tag.name)} className="cursor-pointer" /> + {isDynamicSpendTag ? ( + <Tooltip title="Dynamically generated spend tags cannot be edited"> + <Icon + icon={PencilAltIcon} + size="sm" + className="opacity-50 cursor-not-allowed" + aria-label="Edit tag (disabled)" + /> + </Tooltip> + ) : ( + <Tooltip title="Edit tag"> + <Icon + icon={PencilAltIcon} + size="sm" + onClick={() => onEdit(tag)} + className="cursor-pointer hover:text-blue-500" + /> + </Tooltip> + )} + {isDynamicSpendTag ? ( + <Tooltip title="Dynamically generated spend tags cannot be deleted"> + <Icon + icon={TrashIcon} + size="sm" + className="opacity-50 cursor-not-allowed" + aria-label="Delete tag (disabled)" + /> + </Tooltip> + ) : ( + <Tooltip title="Delete tag"> + <Icon + icon={TrashIcon} + size="sm" + onClick={() => onDelete(tag.name)} + className="cursor-pointer hover:text-red-500" + /> + </Tooltip> + )} </div> ); }, diff --git a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx new file mode 100644 index 00000000000..997faf4a004 --- /dev/null +++ b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx @@ -0,0 +1,64 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import CreateTagModal from "./CreateTagModal"; + +describe("CreateTagModal", () => { + const mockOnCancel = vi.fn(); + const mockOnSubmit = vi.fn(); + const mockAvailableModels = [ + { + model_name: "GPT-4", + litellm_params: { model: "gpt-4" }, + model_info: { id: "model-1" }, + }, + { + model_name: "Claude-3", + litellm_params: { model: "claude-3" }, + model_info: { id: "model-2" }, + }, + ]; + + const defaultProps = { + visible: true, + onCancel: mockOnCancel, + onSubmit: mockOnSubmit, + availableModels: mockAvailableModels, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render the modal", () => { + render(<CreateTagModal {...defaultProps} />); + expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(screen.getByText("Create New Tag")).toBeInTheDocument(); + }); + + it("should submit form with required tag name", async () => { + const user = userEvent.setup(); + render(<CreateTagModal {...defaultProps} />); + + const tagNameInput = screen.getByLabelText("Tag Name"); + await user.type(tagNameInput, "test-tag"); + + const submitButton = screen.getByRole("button", { name: /Create Tag/i }); + await user.click(submitButton); + + expect(mockOnSubmit).toHaveBeenCalledWith({ + tag_name: "test-tag", + }); + }); + + it("should not submit form when tag name is missing", async () => { + const user = userEvent.setup(); + render(<CreateTagModal {...defaultProps} />); + + const submitButton = screen.getByRole("button", { name: /Create Tag/i }); + await user.click(submitButton); + + // Form validation should prevent submission + expect(mockOnSubmit).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx index 4d1909abd93..3412d684524 100644 --- a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx @@ -1,9 +1,9 @@ -import React from "react"; -import { Button, TextInput, Accordion, AccordionHeader, AccordionBody, Title } from "@tremor/react"; -import { Modal, Form, Select as Select2, Tooltip, Input } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import NumericalInput from "../../shared/numerical_input"; +import { Accordion, AccordionBody, AccordionHeader, Button, TextInput, Title } from "@tremor/react"; +import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; +import React from "react"; import BudgetDurationDropdown from "../../common_components/budget_duration_dropdown"; +import NumericalInput from "../../shared/numerical_input"; interface ModelInfo { model_name: string; @@ -22,12 +22,7 @@ interface CreateTagModalProps { availableModels: ModelInfo[]; } -const CreateTagModal: React.FC<CreateTagModalProps> = ({ - visible, - onCancel, - onSubmit, - availableModels, -}) => { +const CreateTagModal: React.FC<CreateTagModalProps> = ({ visible, onCancel, onSubmit, availableModels }) => { const [form] = Form.useForm(); const handleFinish = (values: any) => { @@ -41,25 +36,9 @@ const CreateTagModal: React.FC<CreateTagModalProps> = ({ }; return ( - <Modal - title="Create New Tag" - visible={visible} - width={800} - footer={null} - onCancel={handleCancel} - > - <Form - form={form} - onFinish={handleFinish} - labelCol={{ span: 8 }} - wrapperCol={{ span: 16 }} - labelAlign="left" - > - <Form.Item - label="Tag Name" - name="tag_name" - rules={[{ required: true, message: "Please input a tag name" }]} - > + <Modal title="Create New Tag" visible={visible} width={800} footer={null} onCancel={handleCancel}> + <Form form={form} onFinish={handleFinish} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} labelAlign="left"> + <Form.Item label="Tag Name" name="tag_name" rules={[{ required: true, message: "Please input a tag name" }]}> <TextInput /> </Form.Item> @@ -70,15 +49,15 @@ const CreateTagModal: React.FC<CreateTagModalProps> = ({ <Form.Item label={ <span> - Allowed Models{" "} - <Tooltip title="Select which LLMs are allowed to process requests from this tag"> + Allowed Models + <Tooltip title="Select which models are allowed to process requests from this tag"> <InfoCircleOutlined style={{ marginLeft: "4px" }} /> </Tooltip> </span> } name="allowed_llms" > - <Select2 mode="multiple" placeholder="Select LLMs"> + <Select2 mode="multiple" placeholder="Select Models"> {availableModels.map((model) => ( <Select2.Option key={model.model_info.id} value={model.model_info.id}> <div> @@ -150,4 +129,3 @@ const CreateTagModal: React.FC<CreateTagModalProps> = ({ }; export default CreateTagModal; - diff --git a/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx b/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx index 60cde134e60..1c66a107db7 100644 --- a/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx @@ -1,5 +1,15 @@ import React, { useState, useEffect } from "react"; -import { Card, Text, Title, Button, Badge, Accordion, AccordionHeader, AccordionBody, Title as TremorTitle } from "@tremor/react"; +import { + Card, + Text, + Title, + Button, + Badge, + Accordion, + AccordionHeader, + AccordionBody, + Title as TremorTitle, +} from "@tremor/react"; import { Form, Input, Select as Select2, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { fetchUserModels } from "../organisms/create_key_button"; @@ -131,7 +141,7 @@ const TagInfoView: React.FC<TagInfoViewProps> = ({ tagId, onClose, accessToken, <Card> <Form form={form} onFinish={handleSave} layout="vertical" initialValues={tagDetails}> <Form.Item label="Tag Name" name="name" rules={[{ required: true, message: "Please input a tag name" }]}> - <Input /> + <Input className="rounded-md border-gray-300" /> </Form.Item> <Form.Item label="Description" name="description"> @@ -141,15 +151,15 @@ const TagInfoView: React.FC<TagInfoViewProps> = ({ tagId, onClose, accessToken, <Form.Item label={ <span> - Allowed LLMs{" "} - <Tooltip title="Select which LLMs are allowed to process this type of data"> + Allowed Models + <Tooltip title="Select which models are allowed to process this type of data"> <InfoCircleOutlined style={{ marginLeft: "4px" }} /> </Tooltip> </span> } name="models" > - <Select2 mode="multiple" placeholder="Select LLMs"> + <Select2 mode="multiple" placeholder="Select Models"> {userModels.map((modelId) => ( <Select2.Option key={modelId} value={modelId}> {getModelDisplayName(modelId)} @@ -228,7 +238,7 @@ const TagInfoView: React.FC<TagInfoViewProps> = ({ tagId, onClose, accessToken, <Text>{tagDetails.description || "-"}</Text> </div> <div> - <Text className="font-medium">Allowed LLMs</Text> + <Text className="font-medium">Allowed Models</Text> <div className="flex flex-wrap gap-2 mt-2"> {!tagDetails.models || tagDetails.models.length === 0 ? ( <Badge color="red">All Models</Badge> @@ -256,30 +266,33 @@ const TagInfoView: React.FC<TagInfoViewProps> = ({ tagId, onClose, accessToken, <Card> <Title>Budget & Rate Limits
- {tagDetails.litellm_budget_table.max_budget !== undefined && tagDetails.litellm_budget_table.max_budget !== null && ( -
- Max Budget - ${tagDetails.litellm_budget_table.max_budget} -
- )} + {tagDetails.litellm_budget_table.max_budget !== undefined && + tagDetails.litellm_budget_table.max_budget !== null && ( +
+ Max Budget + ${tagDetails.litellm_budget_table.max_budget} +
+ )} {tagDetails.litellm_budget_table.budget_duration && (
Budget Duration {tagDetails.litellm_budget_table.budget_duration}
)} - {tagDetails.litellm_budget_table.tpm_limit !== undefined && tagDetails.litellm_budget_table.tpm_limit !== null && ( -
- TPM Limit - {tagDetails.litellm_budget_table.tpm_limit.toLocaleString()} -
- )} - {tagDetails.litellm_budget_table.rpm_limit !== undefined && tagDetails.litellm_budget_table.rpm_limit !== null && ( -
- RPM Limit - {tagDetails.litellm_budget_table.rpm_limit.toLocaleString()} -
- )} + {tagDetails.litellm_budget_table.tpm_limit !== undefined && + tagDetails.litellm_budget_table.tpm_limit !== null && ( +
+ TPM Limit + {tagDetails.litellm_budget_table.tpm_limit.toLocaleString()} +
+ )} + {tagDetails.litellm_budget_table.rpm_limit !== undefined && + tagDetails.litellm_budget_table.rpm_limit !== null && ( +
+ RPM Limit + {tagDetails.litellm_budget_table.rpm_limit.toLocaleString()} +
+ )}
)} From be712908a3ea1f625826b261e5d55ecd51890ee8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Nov 2025 12:20:39 -0800 Subject: [PATCH 43/68] [Feat] Add OpenAI compatible bedrock imported models. - qwen etc (#17097) * test_bedrock_openai_imported_model * AmazonBedrockOpenAIConfig * add openai route for bedrock * docs fix * fix code qa check --- docs/my-website/docs/providers/bedrock.md | 202 +--------- .../docs/providers/bedrock_imported.md | 369 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 3 + .../amazon_openai_transformation.py | 186 +++++++++ litellm/llms/bedrock/common_utils.py | 18 +- .../prompt_security/prompt_security.py | 19 +- litellm/proxy/proxy_config.yaml | 19 +- litellm/utils.py | 12 +- .../test_bedrock_completion.py | 97 +++++ 10 files changed, 699 insertions(+), 227 deletions(-) create mode 100644 docs/my-website/docs/providers/bedrock_imported.md create mode 100644 litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index f0b89615a0d..9e22f67527e 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -1598,206 +1598,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -## Bedrock Imported Models (Deepseek, Deepseek R1) - -### Deepseek R1 - -This is a separate route, as the chat template is different. - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/deepseek_r1/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - -### Deepseek (not R1) - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/llama/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - -Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec - - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - -### Qwen3 Imported Models - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/qwen3/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], - max_tokens=100, - temperature=0.7 -) -``` - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: Qwen3-32B - litellm_params: - model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "Qwen3-32B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - ### OpenAI GPT OSS | Property | Details | diff --git a/docs/my-website/docs/providers/bedrock_imported.md b/docs/my-website/docs/providers/bedrock_imported.md new file mode 100644 index 00000000000..8b0dd721c3c --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_imported.md @@ -0,0 +1,369 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bedrock Imported Models + +Bedrock Imported Models (Deepseek, Deepseek R1, Qwen, OpenAI-compatible models) + +### Deepseek R1 + +This is a separate route, as the chat template is different. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/deepseek_r1/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + +### Deepseek (not R1) + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/llama/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + +Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec + + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### Qwen3 Imported Models + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/qwen3/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=100, + temperature=0.7 +) +``` + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: Qwen3-32B + litellm_params: + model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "Qwen3-32B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.) + +Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/openai/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) | +| Supported Features | Vision (images), tool calling, streaming, system messages | + +#### LiteLLMSDK Usage + +**Basic Usage** + +```python +from litellm import completion + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", # bedrock/openai/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=300, + temperature=0.5 +) +``` + +**With Vision (Images)** + +```python +import base64 +from litellm import completion + +# Load and encode image +with open("image.jpg", "rb") as f: + image_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +**Comparing Multiple Images** + +```python +import base64 +from litellm import completion + +# Load images +with open("image1.jpg", "rb") as f: + image1_base64 = base64.b64encode(f.read()).decode("utf-8") +with open("image2.jpg", "rb") as f: + image2_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Spot the difference between these two images?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image1_base64}"} + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image2_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +#### LiteLLM Proxy Usage (AI Gateway) + +**1. Add to config** + +```yaml +model_list: + - model_name: qwen-25vl-72b + litellm_params: + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +Basic text request: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "max_tokens": 300 + }' +``` + +With vision (image): + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZ..."} + } + ] + } + ], + "max_tokens": 300, + "temperature": 0.5 + }' +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6fa5fdeced0..104c7541d67 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -530,6 +530,7 @@ const sidebars = { items: [ "providers/bedrock", "providers/bedrock_embedding", + "providers/bedrock_imported", "providers/bedrock_image_gen", "providers/bedrock_rerank", "providers/bedrock_agentcore", diff --git a/litellm/__init__.py b/litellm/__init__.py index 768ba39a47a..0048f4c29cf 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1225,6 +1225,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation impor from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, +) from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py new file mode 100644 index 00000000000..ee07b71ef15 --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py @@ -0,0 +1,186 @@ +""" +Transformation for Bedrock imported models that use OpenAI Chat Completions format. + +Use this for models imported into Bedrock that accept the OpenAI API format. +Model format: bedrock/openai/ + +Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123 +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM): + """ + Configuration for Bedrock imported models that use OpenAI Chat Completions format. + + This class handles the transformation of requests and responses for Bedrock + imported models that accept the OpenAI API format directly. + + Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling + and response transformation, while adding Bedrock-specific URL generation + and AWS request signing. + + Usage: + model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123" + """ + + def __init__(self, **kwargs): + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_openai_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Input format: bedrock/openai/ + Returns: + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove openai/ prefix + if model.startswith("openai/"): + model = model[7:] + + return model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the Bedrock invoke endpoint. + + Uses the standard Bedrock invoke endpoint format. + """ + model_id = self._get_openai_model_id(model) + + # Get AWS region + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, model=model + ) + + # Get runtime endpoint + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + + # Build the invoke URL + if stream: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream" + else: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign the request using AWS Signature Version 4. + """ + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to OpenAI Chat Completions format for Bedrock imported models. + + Removes AWS-specific params and stream param (handled separately in URL), + then delegates to parent class for standard OpenAI request transformation. + """ + # Remove stream from optional_params as it's handled separately in URL + optional_params.pop("stream", None) + + # Remove AWS-specific params that shouldn't be in the request body + inference_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } + + # Use parent class transform_request for OpenAI format + return super().transform_request( + model=self._get_openai_model_id(model), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate the environment and return headers. + + For Bedrock, we don't need Bearer token auth since we use AWS SigV4. + """ + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index baaec996535..35d3d736a1c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -403,6 +403,9 @@ class BedrockModelInfo(BaseLLMModelInfo): if model.startswith("invoke/"): model = model.split("/", 1)[1] + if model.startswith("openai/"): + model = model.split("/", 1)[1] + return model @staticmethod @@ -446,12 +449,12 @@ class BedrockModelInfo(BaseLLMModelInfo): @staticmethod def get_bedrock_route( model: str, - ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke"]: + ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke", "openai"]: """ Get the bedrock route for the given model. """ route_mappings: Dict[ - str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke"] + str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke", "openai"] ] = { "invoke/": "invoke", "converse_like/": "converse_like", @@ -459,6 +462,7 @@ class BedrockModelInfo(BaseLLMModelInfo): "agent/": "agent", "agentcore/": "agentcore", "async_invoke/": "async_invoke", + "openai/": "openai", } # Check explicit routes first @@ -517,6 +521,14 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "async_invoke/" in model + @staticmethod + def _explicit_openai_route(model: str) -> bool: + """ + Check if the model is an explicit openai route. + Used for Bedrock imported models that use OpenAI Chat Completions format. + """ + return "openai/" in model + @staticmethod def get_bedrock_provider_config_for_messages_api( model: str, @@ -566,6 +578,8 @@ def get_bedrock_chat_config(model: str): # Handle explicit routes first if bedrock_route == "converse" or bedrock_route == "converse_like": return litellm.AmazonConverseConfig() + elif bedrock_route == "openai": + return litellm.AmazonBedrockOpenAIConfig() elif bedrock_route == "agent": from litellm.llms.bedrock.chat.invoke_agent.transformation import ( AmazonInvokeAgentConfig, diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index daee50f30cc..23b9da4714c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -1,13 +1,18 @@ -import os -import re import asyncio import base64 +import os +import re from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Type, Union + from fastapi import HTTPException + from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, httpxSpecialProvider +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( Choices, @@ -15,7 +20,7 @@ from litellm.types.utils import ( EmbeddingResponse, ImageResponse, ModelResponse, - ModelResponseStream + ModelResponseStream, ) if TYPE_CHECKING: @@ -267,8 +272,10 @@ class PromptSecurityGuardrail(CustomGuardrail): content = msg.get('content', '') # Handle both string and list content types if isinstance(content, str): - if content.startswith('### '): return False - if '"follow_ups": [' in content: return False + if content.startswith('### '): + return False + if '"follow_ups": [' in content: + return False return True messages = list(filter(lambda msg: good_msg(msg), messages)) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 014bcdc1670..26e867dc33e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,22 +1,7 @@ model_list: - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 + - model_name: qwen-25vl-72b litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-east-1 - custom_llm_provider: bedrock - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-west-2 - custom_llm_provider: bedrock - - model_name: bedrock/* - litellm_params: - model: bedrock/* - custom_llm_provider: bedrock - aws_region_name: us-west-2 - - model_name: runwayml/* - litellm_params: - model: runwayml/* + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z diff --git a/litellm/utils.py b/litellm/utils.py index f683a59f501..1b4df689959 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3719,7 +3719,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) - + elif bedrock_route == "openai": + optional_params = litellm.AmazonBedrockOpenAIConfig().map_openai_params( + model=model, + non_default_params=non_default_params, + optional_params=optional_params, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif "anthropic" in bedrock_base_model and bedrock_route == "invoke": if bedrock_base_model.startswith("anthropic.claude-3"): optional_params = ( diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 9242950daac..f43e939c681 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3434,3 +3434,100 @@ async def test_bedrock_streaming_passthrough_test1(monkeypatch): print(mock_callback.call_args.kwargs.keys()) assert "standard_logging_object" in mock_callback.call_args.kwargs["kwargs"] assert "response_cost" in mock_callback.call_args.kwargs["kwargs"] + + +def test_bedrock_openai_imported_model(): + """ + Test that Bedrock imported models using OpenAI format work correctly. + + This test validates: + 1. The request body follows OpenAI Chat Completions format + 2. The URL is correctly constructed for Bedrock invoke endpoint + 3. Messages with system, user roles and image_url content are preserved + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + # Sample base64 image data (truncated for test) + sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Spot the difference between the two images?", + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + ], + }, + ] + + with patch.object(client, "post") as mock_post: + try: + response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy", + messages=messages, + max_tokens=300, + temperature=0.5, + client=client, + ) + except Exception as e: + print(f"Exception (expected during mock): {e}") + + mock_post.assert_called_once() + + # Validate URL + url = mock_post.call_args.kwargs["url"] + print(f"URL: {url}") + assert "bedrock-runtime.us-east-1.amazonaws.com" in url + assert "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + assert "/invoke" in url + + # Validate request body follows OpenAI format + request_body = json.loads(mock_post.call_args.kwargs["data"]) + print(f"Request body: {json.dumps(request_body, indent=2)}") + + # Check messages structure + assert "messages" in request_body + assert len(request_body["messages"]) == 2 + + # Check system message + system_msg = request_body["messages"][0] + assert system_msg["role"] == "system" + assert "helpful assistant" in system_msg["content"] + + # Check user message with image content + user_msg = request_body["messages"][1] + assert user_msg["role"] == "user" + assert isinstance(user_msg["content"], list) + assert len(user_msg["content"]) == 3 + + # Check text content + assert user_msg["content"][0]["type"] == "text" + assert "Spot the difference" in user_msg["content"][0]["text"] + + # Check image_url content + assert user_msg["content"][1]["type"] == "image_url" + assert "image_url" in user_msg["content"][1] + assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + + assert user_msg["content"][2]["type"] == "image_url" + assert "image_url" in user_msg["content"][2] + + # Check max_tokens and temperature + assert request_body["max_tokens"] == 300 + assert request_body["temperature"] == 0.5 From 52f1bf1a800bf76d3896fdf9d714f160f45d408b Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Wed, 26 Nov 2025 07:33:38 +0900 Subject: [PATCH 44/68] fix: missing await (#17103) --- litellm/proxy/_experimental/mcp_server/mcp_server_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 54c79fc696c..25b5211464c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1976,7 +1976,7 @@ class MCPServerManager: verbose_logger.debug( f"Adding server to registry: {server.server_id} ({server.server_name})" ) - self.add_update_server(server) + await self.add_update_server(server) verbose_logger.debug( f"Registry now contains {len(self.get_registry())} servers" @@ -2270,7 +2270,7 @@ class MCPServerManager: server.status = "unhealthy" ## try adding server to registry to get error try: - self.add_update_server(server) + await self.add_update_server(server) except Exception as e: server.health_check_error = str(e) server.health_check_error = "Server is not in in memory registry yet. This could be a temporary sync issue." From f3d577592023d374ce532058c1634e63a69f5009 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 14:40:17 -0800 Subject: [PATCH 45/68] fix: fix doc load issue --- docs/my-website/docs/providers/anthropic_effort.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md index d1116ad5be4..0015162a95b 100644 --- a/docs/my-website/docs/providers/anthropic_effort.md +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Anthropic Effort Parameter Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency. From db587926a473f51a21bc96935d10495ae7fdab7e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 14:46:46 -0800 Subject: [PATCH 46/68] Sorting changes, pending tests and loading state --- .../src/components/view_users/columns.tsx | 14 +++++++-- .../src/components/view_users/table.tsx | 31 +++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_users/columns.tsx b/ui/litellm-dashboard/src/components/view_users/columns.tsx index 32bfa0ed6d2..20df4fc246e 100644 --- a/ui/litellm-dashboard/src/components/view_users/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_users/columns.tsx @@ -22,10 +22,12 @@ export const columns = ( handleUserClick: (userId: string, openInEditMode?: boolean) => void, selectionOptions?: SelectionOptions, ): ColumnDef[] => { + // Backend sortable columns: user_id, user_email, created_at, spend, user_alias, user_role const baseColumns: ColumnDef[] = [ { header: "User ID", accessorKey: "user_id", + enableSorting: true, cell: ({ row }) => ( {row.original.user_id ? `${row.original.user_id.slice(0, 7)}...` : "-"} @@ -35,16 +37,19 @@ export const columns = ( { header: "Email", accessorKey: "user_email", + enableSorting: true, cell: ({ row }) => {row.original.user_email || "-"}, }, { header: "Global Proxy Role", accessorKey: "user_role", + enableSorting: true, cell: ({ row }) => {possibleUIRoles?.[row.original.user_role]?.ui_label || "-"}, }, { header: "Spend (USD)", accessorKey: "spend", + enableSorting: true, cell: ({ row }) => ( {row.original.spend ? formatNumberWithCommas(row.original.spend, 4) : "-"} ), @@ -52,6 +57,7 @@ export const columns = ( { header: "Budget (USD)", accessorKey: "max_budget", + enableSorting: false, cell: ({ row }) => ( {row.original.max_budget !== null ? row.original.max_budget : "Unlimited"} ), @@ -66,6 +72,7 @@ export const columns = (
), accessorKey: "sso_user_id", + enableSorting: false, cell: ({ row }) => ( {row.original.sso_user_id !== null ? row.original.sso_user_id : "-"} ), @@ -73,6 +80,7 @@ export const columns = ( { header: "API Keys", accessorKey: "key_count", + enableSorting: false, cell: ({ row }) => ( {row.original.key_count > 0 ? ( @@ -90,7 +98,7 @@ export const columns = ( { header: "Created At", accessorKey: "created_at", - sortingFn: "datetime", + enableSorting: true, cell: ({ row }) => ( {row.original.created_at ? new Date(row.original.created_at).toLocaleDateString() : "-"} @@ -100,7 +108,7 @@ export const columns = ( { header: "Updated At", accessorKey: "updated_at", - sortingFn: "datetime", + enableSorting: false, cell: ({ row }) => ( {row.original.updated_at ? new Date(row.original.updated_at).toLocaleDateString() : "-"} @@ -110,6 +118,7 @@ export const columns = ( { id: "actions", header: "Actions", + enableSorting: false, cell: ({ row }) => (
@@ -148,6 +157,7 @@ export const columns = ( return [ { id: "select", + enableSorting: false, header: () => ( { + onSortingChange: (updaterOrValue: any) => { + const newSorting = typeof updaterOrValue === "function" ? updaterOrValue(sorting) : updaterOrValue; setSorting(newSorting); - if (newSorting.length > 0) { + if (newSorting && Array.isArray(newSorting) && newSorting.length > 0 && newSorting[0]) { const sortState = newSorting[0]; - const sortBy = sortState.id; - const sortOrder = sortState.desc ? "desc" : "asc"; - onSortChange?.(sortBy, sortOrder); + if (sortState.id) { + const sortBy = sortState.id; + const sortOrder = sortState.desc ? "desc" : "asc"; + onSortChange?.(sortBy, sortOrder); + } + } else { + // Reset to default sort when no sorting is selected + onSortChange?.("created_at", "desc"); } }, getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), + manualSorting: true, enableSorting: true, }); @@ -403,7 +402,7 @@ export function UserDataTable({ header.id === "actions" ? "sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]" : "" - }`} + } ${header.column.getCanSort() ? "cursor-pointer hover:bg-gray-50" : ""}`} onClick={header.column.getToggleSortingHandler()} >
@@ -412,7 +411,7 @@ export function UserDataTable({ ? null : flexRender(header.column.columnDef.header, header.getContext())}
- {header.id !== "actions" && ( + {header.id !== "actions" && header.column.getCanSort() && (
{header.column.getIsSorted() ? ( { From c0288d81aa4ef31b9e1f529ce032c827c3087c9f Mon Sep 17 00:00:00 2001 From: Sam Chou Date: Tue, 25 Nov 2025 14:49:12 -0800 Subject: [PATCH 47/68] Fix bedrock claude opus 4.5 inference profile - only global currently (#17101) --- .../model_prices_and_context_window_backup.json | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3dc11e206c..f9cc8bfa067 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2687,7 +2687,7 @@ "/v1/audio/transcriptions" ] }, - "azure/gpt-5.1-2025-11-13": { + "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2723,7 +2723,7 @@ "supports_service_tier": true, "supports_vision": true }, - "azure/gpt-5.1-chat-2025-11-13": { + "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2758,7 +2758,7 @@ "supports_tool_choice": false, "supports_vision": true }, - "azure/gpt-5.1-codex-2025-11-13": { + "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -19521,7 +19521,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/deepseek-v3.1:671b-cloud" : { + "ollama/deepseek-v3.1:671b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 163840, @@ -19531,7 +19531,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:120b-cloud" : { + "ollama/gpt-oss:120b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -19541,7 +19541,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:20b-cloud" : { + "ollama/gpt-oss:20b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -22037,7 +22037,6 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, @@ -23232,7 +23231,7 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "us.anthropic.claude-opus-4-5-20251101-v1:0": { + "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, From 8637d74e170b52c23af6d780f9d7d20eea167069 Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Wed, 26 Nov 2025 01:50:17 +0300 Subject: [PATCH 48/68] include `server_tool_use` in streaming usage (#16826) * include server_tool_use in streaming usage * add test --- .../streaming_chunk_builder_utils.py | 12 ++- .../streaming_chunk_builder_utils.py | 3 +- .../test_streaming_chunk_builder_utils.py | 81 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ddcf81b5ba5..c332e5f88f7 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -18,6 +18,7 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, Usage, + ServerToolUse ) from litellm.utils import print_verbose, token_counter @@ -418,7 +419,8 @@ class ChunkProcessor: ## anthropic prompt caching information ## cache_creation_input_tokens: Optional[int] = None cache_read_input_tokens: Optional[int] = None - + + server_tool_use: Optional[ServerToolUse] = None web_search_requests: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None @@ -462,6 +464,8 @@ class ChunkProcessor: completion_tokens_details = usage_chunk_dict[ "completion_tokens_details" ] + if hasattr(usage_chunk, 'server_tool_use') and usage_chunk.server_tool_use is not None: + server_tool_use = usage_chunk.server_tool_use if ( usage_chunk_dict["prompt_tokens_details"] is not None and getattr( @@ -483,6 +487,7 @@ class ChunkProcessor: completion_tokens=completion_tokens, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, + server_tool_use=server_tool_use, web_search_requests=web_search_requests, completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, @@ -513,6 +518,9 @@ class ChunkProcessor: "cache_read_input_tokens" ] + server_tool_use: Optional[ServerToolUse] = calculated_usage_per_chunk[ + "server_tool_use" + ] web_search_requests: Optional[int] = calculated_usage_per_chunk[ "web_search_requests" ] @@ -576,6 +584,8 @@ class ChunkProcessor: if prompt_tokens_details is not None: returned_usage.prompt_tokens_details = prompt_tokens_details + if server_tool_use is not None: + returned_usage.server_tool_use = server_tool_use if web_search_requests is not None: if returned_usage.prompt_tokens_details is None: returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper( diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index aa879e14c34..a1f89dac5cf 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional from typing_extensions import TypedDict -from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper +from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper, ServerToolUse class UsagePerChunk(TypedDict): @@ -10,6 +10,7 @@ class UsagePerChunk(TypedDict): completion_tokens: int cache_creation_input_tokens: Optional[int] cache_read_input_tokens: Optional[int] + server_tool_use: Optional[ServerToolUse] web_search_requests: Optional[int] completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index f6636874336..2164a3b82e3 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -16,6 +16,7 @@ from litellm.types.utils import ( Function, ModelResponseStream, PromptTokensDetails, + ServerToolUse, StreamingChoices, Usage, ) @@ -325,3 +326,83 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 assert usage.total_tokens == 77 + + +def test_stream_chunk_builder_anthropic_web_search(): + # Prepare two mocked streaming chunks with usage split across them + chunk1 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513206, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=0, + prompt_tokens=50, + total_tokens=50, + completion_tokens_details=None, + server_tool_use=ServerToolUse(web_search_requests=2), + prompt_tokens_details=None, + ), + ) + + chunk2 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513207, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content=None, + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=27, + prompt_tokens=0, + total_tokens=27, + completion_tokens_details=None, + prompt_tokens_details=None, + ), + ) + + chunks = [chunk1, chunk2] + processor = ChunkProcessor(chunks=chunks) + + usage = processor.calculate_usage( + chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" + ) + + assert usage.prompt_tokens == 50 + assert usage.completion_tokens == 27 + assert usage.total_tokens == 77 + assert usage.server_tool_use['web_search_requests'] == 2 \ No newline at end of file From 70a13258477f2868cbae00919205af433fa68625 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 15:01:15 -0800 Subject: [PATCH 49/68] docs: more doc cleanup --- .../index.md | 188 ++++++++++-------- ...odel_prices_and_context_window_backup.json | 52 +++++ model_prices_and_context_window.json | 52 +++++ 3 files changed, 209 insertions(+), 83 deletions(-) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 0b0f4a54167..051235bc74d 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -26,6 +26,13 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe --- +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + ## Usage @@ -222,6 +229,104 @@ curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/converse' \ +## Usage - Vertex AI + + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +## RUN ## +# !gcloud auth application-default login - run this to add vertex credentials to your env +## OR ## +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + +## COMPLETION CALL +response = completion( + model="vertex_ai/claude-opus-4-5@20251101", + messages=[{ "content": "Hello, how are you?","role": "user"}], + vertex_credentials=vertex_credentials_json, + vertex_project="your-project-id", + vertex_location="us-east5" +) +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: + model: vertex_ai/claude-opus-4-5@20251101 + vertex_credentials: "/path/to/service_account.json" + vertex_project: "your-project-id" + vertex_location: "us-east5" +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + + + + + ## Tool Search {#tool-search} ### Usage Example @@ -336,8 +441,6 @@ tools = [ ## Programmatic Tool Calling {#programmatic-tool-calling} -### Usage Example - ```python import litellm import json @@ -428,8 +531,6 @@ print("\nFinal answer:", final_response.choices[0].message.content) ## Tool Input Examples {#tool-input-examples} -### Usage Example - ```python import litellm @@ -755,82 +856,3 @@ This combination enables: 4. **Cost control** - Effort parameter optimizes token spend 5. **Full visibility** - Track all usage metrics ---- - -## Getting Started - -### Installation - -```bash -pip install litellm --upgrade -``` - -### Configuration - -```python -import os -import litellm - -# Set your API key -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -# LiteLLM automatically handles beta headers for all features -``` - -### Supported Models - -| Feature | Supported Models | -|---------|-----------------| -| Tool Search | Claude Opus 4.5, Sonnet 4.5 | -| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | -| Input Examples | Claude Opus 4.5, Sonnet 4.5 | -| Effort Parameter | Claude Opus 4.5 only | - -### Supported Endpoints - -**Note**: All features are supported on the `/chat/completions` endpoint only. - -| Feature | Supported Models | -|---------|-----------------| -| Tool Search | Claude Opus 4.5, Sonnet 4.5 | -| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | -| Input Examples | Claude Opus 4.5, Sonnet 4.5 | -| Effort Parameter | Claude Opus 4.5 only | - -### Provider Support - -All features work across: -- ✅ Standard Anthropic API -- ✅ Azure Anthropic -- ✅ Vertex AI Anthropic -- ✅ LiteLLM Proxy - ---- - -## Conclusion - -These advanced Anthropic features in LiteLLM enable you to build more sophisticated, efficient, and cost-effective AI applications: - -- **Tool Search** scales to thousands of tools -- **Programmatic Tool Calling** reduces latency and tokens -- **Input Examples** improve accuracy -- **Effort Parameter** controls costs - -All features work seamlessly together and are supported across all Anthropic providers through LiteLLM's unified interface. - -### Resources - -- [LiteLLM Documentation](https://docs.litellm.ai/) -- [Anthropic Tool Search Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_search) -- [Anthropic Programmatic Tool Calling Docs](https://docs.litellm.ai/docs/providers/anthropic_programmatic_tool_calling) -- [Anthropic Input Examples Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_input_examples) -- [Anthropic Effort Parameter Docs](https://docs.litellm.ai/docs/providers/anthropic_effort) - -### Get Started Today - -```bash -pip install litellm --upgrade -``` - -Happy building! 🚀 - diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f9cc8bfa067..e51e5bb4b2b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24604,6 +24604,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3dc11e206c..ff8766003c4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24605,6 +24605,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, From 3da9974a8770a5d05f839d78a7e15739b0664217 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 15:54:55 -0800 Subject: [PATCH 50/68] Tests --- .../src/components/view_users/table.test.tsx | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_users/table.test.tsx b/ui/litellm-dashboard/src/components/view_users/table.test.tsx index 5b612b27329..278a42e8963 100644 --- a/ui/litellm-dashboard/src/components/view_users/table.test.tsx +++ b/ui/litellm-dashboard/src/components/view_users/table.test.tsx @@ -1,6 +1,5 @@ -import { render } from "@testing-library/react"; +import { act, fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; -import React from "react"; import { UserDataTable } from "./table"; @@ -21,7 +20,7 @@ describe("UserDataTable", () => { const updateFilters = vi.fn(); - const { getByText } = render( + render( { />, ); - expect(getByText("Filters")).toBeInTheDocument(); + expect(screen.getByText("Filters")).toBeInTheDocument(); + }); + + it("should call onSortChange when clicking a sortable header", () => { + const filters = { + email: "", + user_id: "", + user_role: "", + sso_user_id: "", + team: "", + model: "", + min_spend: null, + max_spend: null, + sort_by: "created_at", + sort_order: "desc" as const, + }; + + const updateFilters = vi.fn(); + const onSortChange = vi.fn(); + + const possibleUIRoles = { + admin: { ui_label: "Admin" }, + user: { ui_label: "User" }, + }; + + render( + , + ); + + const emailHeader = screen.getByRole("columnheader", { name: /email/i }); + act(() => { + fireEvent.click(emailHeader); + }); + + expect(onSortChange).toHaveBeenCalledWith("user_email", "desc"); }); }); From 8ee6812edff5d1e79d5efbabd5c801a45439b1cf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 15:58:51 -0800 Subject: [PATCH 51/68] docs: cleanup launch post --- .../index.md | 503 ++++++++++++++++-- ...odel_prices_and_context_window_backup.json | 15 +- 2 files changed, 470 insertions(+), 48 deletions(-) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 051235bc74d..9753529f572 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -329,8 +329,13 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ ## Tool Search {#tool-search} +This lets Claude work with thousands of tools, by dynamically loading tools on-demand, instead of loading all tools into the context window upfront. + ### Usage Example + + + ```python import litellm import os @@ -407,7 +412,7 @@ tools = [ # Make a request - Claude will search for and use relevant tools response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", + model="anthropic/claude-opus-4-5-20251101", messages=[{ "role": "user", "content": "What's the weather like in San Francisco?" @@ -422,6 +427,108 @@ print("Tool calls:", response.choices[0].message.tool_calls) if hasattr(response.usage, 'server_tool_use'): print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + "tools": [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } + ] +} +' +``` + + ### BM25 Variant (Natural Language Search) @@ -441,6 +548,11 @@ tools = [ ## Programmatic Tool Calling {#programmatic-tool-calling} +Programmatic tool calling allows Claude to write code that calls your tools programmatically. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/programmatic-tool-calling) + + + + ```python import litellm import json @@ -527,10 +639,80 @@ final_response = litellm.completion( print("\nFinal answer:", final_response.choices[0].message.content) ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + "tools": [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } + ] +} +' +``` + + + --- ## Tool Input Examples {#tool-input-examples} +You can now provide Claude with examples of how to use your tools. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-input-examples) + + + + + ```python import litellm @@ -609,12 +791,124 @@ response = litellm.completion( print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments) ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + "tools": [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] +} +' +``` + + + --- ## Effort Parameter: Control Token Usage {#effort-parameter} +Controls aspects like how much effort the model puts into its response, via `output_config={"effort": ..}`. + +:::info + +Soon, we will map OpenAI's `reasoning_effort` parameter to this. +::: + +Potential Values for `effort` parameter: `"high"`, `"medium"`, `"low"`. + ### Usage Example + + + ```python import litellm @@ -660,41 +954,46 @@ print(f"Medium: {response_medium.usage.completion_tokens} tokens") print(f"Low: {response_low.usage.completion_tokens} tokens") ``` -### Effort with Tool Use + + -Lower effort affects both explanations and tool calls: +1. Setup config.yaml -```python -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } - } - } -] - -# Low effort = fewer tool calls, more direct -response = litellm.completion( - model="anthropic/claude-opus-4-5-20251101", - messages=[{ - "role": "user", - "content": "Check weather in San Francisco, New York, and London" - }], - tools=tools, - output_config={"effort": "low"} # May combine into fewer calls -) +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY ``` ---- +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "high" + } + } +' +``` + + + ## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} @@ -702,8 +1001,15 @@ response = litellm.completion( Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. +It is available in the `usage` object, under `server_tool_use.tool_search_requests`. + +Anthropic charges $0.0001 per tool search request. + ### Tracking Example + + + ```python import litellm @@ -749,6 +1055,65 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use print(f" Total: ${total_cost:.6f}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ] + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + ### Cost Optimization Tips 1. **Keep frequently used tools non-deferred** (3-5 tools) @@ -756,15 +1121,6 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use 3. **Monitor search requests** to identify optimization opportunities 4. **Combine with effort parameter** for maximum efficiency -```python -# Optimized for cost -response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Simple query"}], - tools=tools_with_search, - output_config={"effort": "low"} # Reduce output tokens -) -``` --- @@ -774,6 +1130,9 @@ response = litellm.completion( These features work together seamlessly. Here's a real-world example combining all of them: + + + ```python import litellm import json @@ -846,6 +1205,68 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use print(f"\nResponse: {response.choices[0].message.content}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ], + "output_config": { + "effort": "medium" + } + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + ### Real-World Benefits This combination enables: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e51e5bb4b2b..ff8766003c4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2687,7 +2687,7 @@ "/v1/audio/transcriptions" ] }, - "azure/gpt-5.1-2025-11-13": { + "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2723,7 +2723,7 @@ "supports_service_tier": true, "supports_vision": true }, - "azure/gpt-5.1-chat-2025-11-13": { + "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2758,7 +2758,7 @@ "supports_tool_choice": false, "supports_vision": true }, - "azure/gpt-5.1-codex-2025-11-13": { + "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -19521,7 +19521,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/deepseek-v3.1:671b-cloud": { + "ollama/deepseek-v3.1:671b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 163840, @@ -19531,7 +19531,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:120b-cloud": { + "ollama/gpt-oss:120b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -19541,7 +19541,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:20b-cloud": { + "ollama/gpt-oss:20b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -22037,6 +22037,7 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, @@ -23231,7 +23232,7 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, From 5cb5c2a7b7fcd80de1caa803701af2d596965fbb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 16:04:27 -0800 Subject: [PATCH 52/68] docs: more doc cleanup --- .../blog/anthropic_opus_4_5_and_advanced_features/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 9753529f572..b545e936186 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -33,6 +33,8 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe | Input Examples | Claude Opus 4.5, Sonnet 4.5 | | Effort Parameter | Claude Opus 4.5 only | +Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude). + ## Usage From 6e5c7c0008f6c157a27ffc4530fe509b57687976 Mon Sep 17 00:00:00 2001 From: Otavio Brito <69211663+otaviofbrito@users.noreply.github.com> Date: Tue, 25 Nov 2025 21:41:35 -0300 Subject: [PATCH 53/68] fix transcription exception handling - /audio/transcriptions (#16791) * fix transcription exception handling * reraise the exception --- litellm/proxy/proxy_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 13307742860..7c415b8106e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5483,6 +5483,7 @@ async def audio_transcriptions( file_object = io.BytesIO(file_content) file_object.name = file.filename data["file"] = file_object + try: ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook( @@ -5500,7 +5501,7 @@ async def audio_transcriptions( ) response = await llm_call except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + raise e finally: file_object.close() # close the file read in by io library From 5ec3f19a53dbf6028df7279468552b9db9442320 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 16:57:38 -0800 Subject: [PATCH 54/68] Make model select required for team, add checks for all-proxy-models --- .../src/components/OldTeams.test.tsx | 103 ++++++++++++++---- .../src/components/OldTeams.tsx | 14 ++- .../src/components/team/team_info.test.tsx | 88 ++++++++++++++- .../src/components/team/team_info.tsx | 90 ++++++++------- 4 files changed, 225 insertions(+), 70 deletions(-) diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index 261178191f8..7f4ec3b09ca 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1,5 +1,6 @@ -import { act, fireEvent, render, screen } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { fetchAvailableModelsForTeamOrKey } from "./key_team_helpers/fetch_available_models_team_key"; import { teamCreateCall } from "./networking"; import OldTeams from "./OldTeams"; @@ -23,6 +24,28 @@ vi.mock("./molecules/notifications_manager", () => ({ }, })); +vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ + fetchAvailableModelsForTeamOrKey: vi.fn(), + getModelDisplayName: vi.fn((model: string) => model), + unfurlWildcardModelsInList: vi.fn((teamModels: string[], allModels: string[]) => { + const wildcardDisplayNames: string[] = []; + const expandedModels: string[] = []; + + teamModels.forEach((teamModel) => { + if (teamModel.endsWith("/*")) { + const provider = teamModel.replace("/*", ""); + const matchingModels = allModels.filter((model) => model.startsWith(provider + "/")); + expandedModels.push(...matchingModels); + wildcardDisplayNames.push(teamModel); + } else { + expandedModels.push(teamModel); + } + }); + + return [...wildcardDisplayNames, ...expandedModels].filter((item, index, array) => array.indexOf(item) === index); + }), +})); + describe("OldTeams - handleCreate organization handling", () => { beforeEach(() => { vi.clearAllMocks(); @@ -236,7 +259,7 @@ describe("OldTeams - handleCreate organization handling", () => { }); it("should clear the delete modal when the cancel button is clicked", async () => { - const { getByRole, getByTestId } = render( + render( { organizations={[]} />, ); - const deleteTeamButton = getByTestId("delete-team-button"); + const deleteTeamButton = screen.getByTestId("delete-team-button"); act(() => { fireEvent.click(deleteTeamButton); }); @@ -275,7 +298,7 @@ describe("OldTeams - empty state", () => { }); it("should display empty state message when teams array is empty", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should display empty state message when teams is null", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should not display empty state when teams array has items", () => { - const { queryByText, getByText } = render( + render( { />, ); - expect(queryByText("No teams found")).not.toBeInTheDocument(); - expect(queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); - expect(getByText("Test Team")).toBeInTheDocument(); + expect(screen.queryByText("No teams found")).not.toBeInTheDocument(); + expect(screen.queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); + expect(screen.getByText("Test Team")).toBeInTheDocument(); }); }); @@ -473,7 +496,7 @@ describe("OldTeams - Default Team Settings tab visibility", () => { }); it("should show Default Team Settings tab for Admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should show Default Team Settings tab for proxy_admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should not show Default Team Settings tab for proxy_admin_viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); }); it("should not show Default Team Settings tab for Admin Viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + }); +}); + +describe("OldTeams - all-proxy-models dropdown visibility", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + }); + + it("should not show all-proxy-models option when user has no access to it", async () => { + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + + render( + , + ); + + await waitFor(() => { + expect(fetchAvailableModelsForTeamOrKey).toHaveBeenCalled(); + }); + + const createButton = screen.getByRole("button", { name: /create new team/i }); + act(() => { + fireEvent.click(createButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText(/models/i)).toBeInTheDocument(); + }); + const allProxyModelsOption = screen.queryByText("All Proxy Models"); + expect(allProxyModelsOption).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index cc66a23eb48..83ec28a5177 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -1139,12 +1139,20 @@ const Teams: React.FC = ({ } + rules={[ + { + required: true, + message: "Please select at least one model", + }, + ]} name="models" > - - All Proxy Models - + {(isProxyAdminRole(userRole || "") || userModels.includes("all-proxy-models")) && ( + + All Proxy Models + + )} No Default Models diff --git a/ui/litellm-dashboard/src/components/team/team_info.test.tsx b/ui/litellm-dashboard/src/components/team/team_info.test.tsx index 526f0972d98..17041659cec 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.test.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.test.tsx @@ -1,7 +1,7 @@ +import * as networking from "@/components/networking"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; import TeamInfoView from "./team_info"; -import { render, waitFor } from "@testing-library/react"; -import * as networking from "@/components/networking"; // Mock the networking module vi.mock("@/components/networking", () => ({ @@ -61,7 +61,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue([]); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - const { getByText } = render( + render( {}} @@ -75,7 +75,87 @@ describe("TeamInfoView", () => { />, ); await waitFor(() => { - expect(getByText("User ID")).toBeInTheDocument(); + expect(screen.queryByText("User ID")).not.toBeNull(); }); }); + + it("should not show all-proxy-models option when user has no access to it", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue({ + team_id: "123", + team_info: { + team_alias: "Test Team", + team_id: "123", + organization_id: null, + admins: ["admin@test.com"], + members: ["user1@test.com", "user2@test.com"], + members_with_roles: [ + { + user_id: "user1@test.com", + user_email: "user1@test.com", + role: "member", + spend: 0, + budget_id: "budget1", + }, + ], + metadata: {}, + tpm_limit: null, + rpm_limit: null, + max_budget: null, + budget_duration: null, + models: ["gpt-4"], + blocked: false, + spend: 0, + max_parallel_requests: null, + budget_reset_at: null, + model_id: null, + litellm_model_table: null, + created_at: "2024-01-01T00:00:00Z", + team_member_budget_table: null, + }, + keys: [], + team_memberships: [], + }); + + vi.mocked(networking.getGuardrailsList).mockResolvedValue([]); + vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); + + render( + {}} + onClose={() => {}} + accessToken="123" + is_team_admin={true} + is_proxy_admin={true} + userModels={["gpt-4", "gpt-3.5-turbo"]} + editTeam={false} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getAllByText("Test Team")).not.toBeNull(); + }); + + const settingsTab = screen.getByRole("tab", { name: "Settings" }); + act(() => { + fireEvent.click(settingsTab); + }); + + await waitFor(() => { + expect(screen.getByText("Team Settings")).toBeInTheDocument(); + }); + + const editButton = screen.getByRole("button", { name: "Edit Settings" }); + act(() => { + fireEvent.click(editButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText("Models")).toBeInTheDocument(); + }); + + const allProxyModelsOption = screen.queryByText("All Proxy Models"); + expect(allProxyModelsOption).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index bd52b5aef4d..1c6ba629ef7 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -1,50 +1,50 @@ -import React, { useState, useEffect } from "react"; -import NumericalInput from "../shared/numerical_input"; +import UserSearchModal from "@/components/common_components/user_search_modal"; import { - Card, - Title, - Text, - Tab, - TabList, - TabGroup, - TabPanel, - TabPanels, - Grid, - Badge, - Button as TremorButton, - TextInput, -} from "@tremor/react"; -import TeamMembersComponent from "./team_member_view"; -import MemberPermissions from "./member_permissions"; -import { - teamInfoCall, - teamMemberDeleteCall, - teamMemberAddCall, - teamMemberUpdateCall, - Member, - teamUpdateCall, getGuardrailsList, + Member, + teamInfoCall, + teamMemberAddCall, + teamMemberDeleteCall, + teamMemberUpdateCall, + teamUpdateCall, } from "@/components/networking"; -import { Button, Form, Input, Select, Switch, message, Tooltip } from "antd"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { ArrowLeftIcon } from "@heroicons/react/outline"; -import MemberModal from "./edit_membership"; -import UserSearchModal from "@/components/common_components/user_search_modal"; +import { + Badge, + Card, + Grid, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, + Text, + TextInput, + Title, + Button as TremorButton, +} from "@tremor/react"; +import { Button, Form, Input, message, Select, Switch, Tooltip } from "antd"; +import { CheckIcon, CopyIcon } from "lucide-react"; +import React, { useEffect, useState } from "react"; +import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; -import ObjectPermissionsView from "../object_permissions_view"; -import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import LoggingSettingsView from "../logging_settings_view"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import EditLoggingSettings from "./EditLoggingSettings"; -import LoggingSettingsView from "../logging_settings_view"; -import { fetchMCPAccessGroups } from "../networking"; -import { CheckIcon, CopyIcon } from "lucide-react"; -import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import NotificationsManager from "../molecules/notifications_manager"; -import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; -import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import { fetchMCPAccessGroups } from "../networking"; +import ObjectPermissionsView from "../object_permissions_view"; +import NumericalInput from "../shared/numerical_input"; +import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import MemberModal from "./edit_membership"; +import EditLoggingSettings from "./EditLoggingSettings"; +import MemberPermissions from "./member_permissions"; +import TeamMembersComponent from "./team_member_view"; export interface TeamMembership { user_id: string; @@ -586,11 +586,17 @@ const TeamInfoView: React.FC = ({ - +