From 29057ba6add3dd17288e06d90424f9f2d71f4a2c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 19 Nov 2025 11:55:38 +0530 Subject: [PATCH 001/223] Added support for azure anthopic models via chat completion --- docs/my-website/docs/providers/anthropic.md | 26 +- docs/my-website/docs/providers/azure/azure.md | 14 +- .../docs/providers/azure/azure_anthropic.md | 378 ++++++++++++++++++ litellm/__init__.py | 1 + .../get_llm_provider_logic.py | 18 + litellm/llms/azure/anthropic/__init__.py | 8 + litellm/llms/azure/anthropic/handler.py | 236 +++++++++++ .../llms/azure/anthropic/transformation.py | 96 +++++ litellm/main.py | 58 +++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + .../anthropic/test_azure_anthropic_handler.py | 216 ++++++++++ .../test_azure_anthropic_provider_routing.py | 82 ++++ .../test_azure_anthropic_transformation.py | 191 +++++++++ 14 files changed, 1318 insertions(+), 9 deletions(-) create mode 100644 docs/my-website/docs/providers/azure/azure_anthropic.md create mode 100644 litellm/llms/azure/anthropic/__init__.py create mode 100644 litellm/llms/azure/anthropic/handler.py create mode 100644 litellm/llms/azure/anthropic/transformation.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py create mode 100644 tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index afcb6a34d9c..08b52066e16 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -17,11 +17,11 @@ LiteLLM supports all anthropic models. | Property | Details | |-------|-------| -| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. | -| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) | -| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) | -| API Endpoint for Provider | https://api.anthropic.com | -| Supported Endpoints | `/chat/completions` | +| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. Also available via Azure Foundry. | +| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`). For Azure Foundry deployments, use `azure/claude-*` (see [Azure Anthropic documentation](../providers/azure/azure_anthropic)) | +| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint for Provider | https://api.anthropic.com (or Azure Foundry endpoint: `https://.services.ai.azure.com/anthropic`) | +| Supported Endpoints | `/chat/completions`, `/v1/messages` (passthrough) | ## Supported OpenAI Parameters @@ -59,6 +59,22 @@ os.environ["ANTHROPIC_API_KEY"] = "your-api-key" # os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending ``` +:::tip Azure Foundry Support + +Claude models are also available via Microsoft Azure Foundry. Use the `azure/` prefix instead of `anthropic/` and configure Azure authentication. See the [Azure Anthropic documentation](../providers/azure/azure_anthropic) for details. + +Example: +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +::: + ### Custom API Base When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL. diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 2f845357328..d338a0287e0 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -9,10 +9,10 @@ import TabItem from '@theme/TabItem'; | Property | Details | |-------|-------| -| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series | -| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | -| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) +| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. | +| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) (Claude passthrough) | +| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) ## API Keys, Params api_key, api_base, api_version etc can be passed directly to `litellm.completion` - see here or set as `litellm.api_key` params see here @@ -27,6 +27,12 @@ os.environ["AZURE_AD_TOKEN"] = "" os.environ["AZURE_API_TYPE"] = "" ``` +:::info Azure Foundry Claude Models + +Azure also supports Claude models via Azure Foundry. Use `azure/claude-*` model names (e.g., `azure/claude-sonnet-4-5`) with Azure authentication. See the [Azure Anthropic documentation](./azure_anthropic) for details. + +::: + ## **Usage - LiteLLM Python SDK** Open In Colab diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md new file mode 100644 index 00000000000..9a45e1db599 --- /dev/null +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -0,0 +1,378 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure Anthropic (Claude via Azure Foundry) + +LiteLLM supports Claude models deployed via Microsoft Azure Foundry, including Claude Sonnet 4.5, Claude Haiku 4.5, and Claude Opus 4.1. + +## Available Models + +Azure Foundry supports the following Claude models: + +- `claude-sonnet-4-5` - Anthropic's most capable model for building real-world agents and handling complex, long-horizon tasks +- `claude-haiku-4-5` - Near-frontier performance with the right speed and cost for high-volume use cases +- `claude-opus-4-1` - Industry leader for coding, delivering sustained performance on long-running tasks + +| Property | Details | +|-------|-------| +| Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. | +| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) | +| Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) | +| API Endpoint | `https://.services.ai.azure.com/anthropic/v1/messages` | +| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages` (passthrough) | + +## Key Features + +- **Extended thinking**: Enhanced reasoning capabilities for complex tasks +- **Image and text input**: Strong vision capabilities for analyzing charts, graphs, technical diagrams, and reports +- **Code generation**: Advanced thinking with code generation, analysis, and debugging (Claude Sonnet 4.5 and Claude Opus 4.1) +- **Same API as Anthropic**: All request/response transformations are identical to the main Anthropic provider + +## Authentication + +Azure Anthropic supports two authentication methods: + +1. **API Key**: Use the `api-key` header +2. **Azure AD Token**: Use `Authorization: Bearer ` header (Microsoft Entra ID) + +## API Keys and Configuration + +```python +import os + +# Option 1: API Key authentication +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Option 2: Azure AD Token authentication +os.environ["AZURE_AD_TOKEN"] = "your-azure-ad-token" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Optional: Azure AD Token Provider (for automatic token refresh) +os.environ["AZURE_TENANT_ID"] = "your-tenant-id" +os.environ["AZURE_CLIENT_ID"] = "your-client-id" +os.environ["AZURE_CLIENT_SECRET"] = "your-client-secret" +os.environ["AZURE_SCOPE"] = "https://cognitiveservices.azure.com/.default" +``` + +## Usage - LiteLLM Python SDK + +### Basic Completion + +```python +from litellm import completion + +# Set environment variables +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" + +# Make a completion request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What are 3 things to visit in Seattle?"} + ], + max_tokens=1000, + temperature=0.7, +) + +print(response) +``` + +### Completion with API Key Parameter + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Completion with Azure AD Token + +```python +import litellm + +response = litellm.completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + azure_ad_token="your-azure-ad-token", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000, +) +``` + +### Streaming + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Write a short story"} + ], + stream=True, + max_tokens=1000, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + +### Tool Calling + +```python +from litellm import completion + +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What's the weather in Seattle?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } + } + ], + tool_choice="auto", + max_tokens=1000, +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export AZURE_API_KEY="your-azure-api-key" +export AZURE_API_BASE="https://.services.ai.azure.com/anthropic" +``` + +### 2. Configure the proxy + +```yaml +model_list: + - model_name: claude-sonnet-4-5 + litellm_params: + model: azure/claude-sonnet-4-5 + api_base: https://.services.ai.azure.com/anthropic + api_key: os.environ/AZURE_API_KEY +``` + +### 3. Test it + + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "claude-sonnet-4-5", + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ], + "max_tokens": 1000 +}' +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Hello!"} + ], + max_tokens=1000 +) + +print(response) +``` + + + + +## Messages API Passthrough + +Azure Anthropic also supports the native Anthropic Messages API via passthrough. The endpoint structure is the same as Anthropic's `/v1/messages` API. + +### Using Anthropic SDK + +```python +from anthropic import Anthropic + +client = Anthropic( + api_key="your-azure-api-key", + base_url="https://.services.ai.azure.com/anthropic" +) + +response = client.messages.create( + model="claude-sonnet-4-5", + max_tokens=1000, + messages=[ + {"role": "user", "content": "Hello, world"} + ] +) + +print(response) +``` + +### Using LiteLLM Proxy Passthrough + +```bash +curl --request POST \ + --url http://0.0.0.0:4000/anthropic/v1/messages \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --header "Authorization: bearer sk-anything" \ + --data '{ + "model": "claude-sonnet-4-5", + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": "Hello, world"} + ] +}' +``` + +## Supported OpenAI Parameters + +Azure Anthropic supports the same parameters as the main Anthropic provider: + +``` +"stream", +"stop", +"temperature", +"top_p", +"max_tokens", +"max_completion_tokens", +"tools", +"tool_choice", +"extra_headers", +"parallel_tool_calls", +"response_format", +"user", +"thinking", +"reasoning_effort" +``` + +:::info + +Azure Anthropic API requires `max_tokens` to be passed. LiteLLM automatically passes `max_tokens=4096` when no `max_tokens` are provided. + +::: + +## Differences from Standard Anthropic Provider + +The only difference between Azure Anthropic and the standard Anthropic provider is authentication: + +- **Standard Anthropic**: Uses `x-api-key` header +- **Azure Anthropic**: Uses `api-key` header or `Authorization: Bearer ` for Azure AD authentication + +All other request/response transformations, tool calling, streaming, and feature support are identical. + +## API Base URL Format + +The API base URL should follow this format: + +``` +https://.services.ai.azure.com/anthropic +``` + +LiteLLM will automatically append `/v1/messages` if not already present in the URL. + +## Example: Full Configuration + +```python +import os +from litellm import completion + +# Configure Azure Anthropic +os.environ["AZURE_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic" + +# Make a request +response = completion( + model="azure/claude-sonnet-4-5", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Explain quantum computing in simple terms."} + ], + max_tokens=1000, + temperature=0.7, + stream=False, +) + +print(response.choices[0].message.content) +``` + +## Troubleshooting + +### Missing API Base Error + +If you see an error about missing API base, ensure you've set: + +```python +os.environ["AZURE_API_BASE"] = "https://.services.ai.azure.com/anthropic" +``` + +Or pass it directly: + +```python +response = completion( + model="azure/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + # ... +) +``` + +### Authentication Errors + +- **API Key**: Ensure `AZURE_API_KEY` is set or passed as `api_key` parameter +- **Azure AD Token**: Ensure `AZURE_AD_TOKEN` is set or passed as `azure_ad_token` parameter +- **Token Provider**: For automatic token refresh, configure `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` + +## Related Documentation + +- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models +- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup + diff --git a/litellm/__init__.py b/litellm/__init__.py index c86768490f3..929835c547e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1108,6 +1108,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo +from .llms.azure.anthropic.transformation import AzureAnthropicConfig from .llms.groq.stt.transformation import GroqSTTConfig from .llms.anthropic.completion.transformation import AnthropicTextConfig from .llms.triton.completion.transformation import TritonConfig diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ef0ebe074d7..de616a332d7 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -22,6 +22,19 @@ def _is_non_openai_azure_model(model: str) -> bool: return False +def _is_azure_anthropic_model(model: str) -> Optional[str]: + try: + model_parts = model.split("/", 1) + if len(model_parts) > 1: + model_name = model_parts[1].lower() + # Check if model name contains claude + if "claude" in model_name or model_name.startswith("claude"): + return model_parts[1] # Return model name without "azure/" prefix + except Exception: + pass + return None + + def handle_cohere_chat_model_custom_llm_provider( model: str, custom_llm_provider: Optional[str] = None ) -> Tuple[str, Optional[str]]: @@ -123,6 +136,11 @@ def get_llm_provider( # noqa: PLR0915 # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus if model.split("/", 1)[0] == "azure": + # Check if it's an Azure Anthropic model (claude models) + azure_anthropic_model = _is_azure_anthropic_model(model) + if azure_anthropic_model: + custom_llm_provider = "azure_anthropic" + return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base if _is_non_openai_azure_model(model): custom_llm_provider = "openai" return model, custom_llm_provider, dynamic_api_key, api_base diff --git a/litellm/llms/azure/anthropic/__init__.py b/litellm/llms/azure/anthropic/__init__.py new file mode 100644 index 00000000000..b40c4dfa9b3 --- /dev/null +++ b/litellm/llms/azure/anthropic/__init__.py @@ -0,0 +1,8 @@ +""" +Azure Anthropic provider - supports Claude models via Azure Foundry +""" +from .handler import AzureAnthropicChatCompletion +from .transformation import AzureAnthropicConfig + +__all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"] + diff --git a/litellm/llms/azure/anthropic/handler.py b/litellm/llms/azure/anthropic/handler.py new file mode 100644 index 00000000000..cfa5eeddaa9 --- /dev/null +++ b/litellm/llms/azure/anthropic/handler.py @@ -0,0 +1,236 @@ +""" +Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authentication +""" +import copy +import json +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +import httpx + +import litellm +from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) +from litellm.types.utils import ModelResponse +from litellm.utils import CustomStreamWrapper + +from .transformation import AzureAnthropicConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as CustomStreamWrapperType + from litellm.llms.base_llm.chat.transformation import BaseConfig + + +class AzureAnthropicChatCompletion(AnthropicChatCompletion): + """ + Azure Anthropic chat completion handler. + Reuses all Anthropic logic but with Azure authentication. + """ + + def __init__(self) -> None: + super().__init__() + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_llm_provider: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + optional_params: dict, + timeout: Union[float, httpx.Timeout], + litellm_params: dict, + acompletion=None, + logger_fn=None, + headers={}, + client=None, + ): + """ + Completion method that uses Azure authentication instead of Anthropic's x-api-key. + All other logic is the same as AnthropicChatCompletion. + """ + from litellm.utils import ProviderConfigManager + + optional_params = copy.deepcopy(optional_params) + stream = optional_params.pop("stream", None) + json_mode: bool = optional_params.pop("json_mode", False) + is_vertex_request: bool = optional_params.pop("is_vertex_request", False) + _is_function_call = False + messages = copy.deepcopy(messages) + + # Use AzureAnthropicConfig instead of AnthropicConfig + headers = AzureAnthropicConfig().validate_environment( + api_key=api_key, + headers=headers, + model=model, + messages=messages, + optional_params={**optional_params, "is_vertex_request": is_vertex_request}, + litellm_params=litellm_params, + ) + + config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=litellm.types.utils.LlmProviders(custom_llm_provider), + ) + if config is None: + raise ValueError( + f"Provider config not found for model: {model} and provider: {custom_llm_provider}" + ) + + data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + print_verbose(f"_is_function_call: {_is_function_call}") + if acompletion is True: + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + print_verbose("makes async azure anthropic streaming POST request") + data["stream"] = stream + return self.acompletion_stream_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + json_mode=json_mode, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=( + client + if client is not None and isinstance(client, AsyncHTTPHandler) + else None + ), + ) + else: + return self.acompletion_function( + model=model, + messages=messages, + data=data, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + provider_config=config, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + _is_function_call=_is_function_call, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + client=client, + json_mode=json_mode, + timeout=timeout, + ) + else: + ## COMPLETION CALL + if ( + stream is True + ): # if function call - fake the streaming (need complete blocks for output parsing in openai format) + data["stream"] = stream + # Import the make_sync_call from parent + from litellm.llms.anthropic.chat.handler import make_sync_call + + completion_stream, response_headers = make_sync_call( + client=client, + api_base=api_base, + headers=headers, # type: ignore + data=json.dumps(data), + model=model, + messages=messages, + logging_obj=logging_obj, + timeout=timeout, + json_mode=json_mode, + ) + from litellm.llms.anthropic.common_utils import process_anthropic_headers + + return CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="azure_anthropic", + logging_obj=logging_obj, + _response_headers=process_anthropic_headers(response_headers), + ) + + else: + if client is None or not isinstance(client, HTTPHandler): + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client(params={"timeout": timeout}) + else: + client = client + + try: + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout, + ) + except Exception as e: + from litellm.llms.anthropic.common_utils import AnthropicError + + status_code = getattr(e, "status_code", 500) + error_headers = getattr(e, "headers", None) + error_text = getattr(e, "text", str(e)) + error_response = getattr(e, "response", None) + if error_headers is None and error_response: + error_headers = getattr(error_response, "headers", None) + if error_response and hasattr(error_response, "text"): + error_text = getattr(error_response, "text", error_text) + raise AnthropicError( + message=error_text, + status_code=status_code, + headers=error_headers, + ) + + return config.transform_response( + model=model, + raw_response=response, + model_response=model_response, + logging_obj=logging_obj, + api_key=api_key, + request_data=data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + json_mode=json_mode, + ) + diff --git a/litellm/llms/azure/anthropic/transformation.py b/litellm/llms/azure/anthropic/transformation.py new file mode 100644 index 00000000000..81beeb74ae1 --- /dev/null +++ b/litellm/llms/azure/anthropic/transformation.py @@ -0,0 +1,96 @@ +""" +Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication +""" +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import litellm +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + pass + + +class AzureAnthropicConfig(AnthropicConfig): + """ + Azure Anthropic configuration that extends AnthropicConfig. + The only difference is authentication - Azure uses api-key header or Azure AD token + instead of x-api-key header. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "azure_anthropic" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: Union[dict, GenericLiteLLMParams], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Dict: + """ + Validate environment and set up Azure authentication headers. + Azure supports: + 1. API key via 'api-key' header + 2. Azure AD token via 'Authorization: Bearer ' header + """ + # Convert dict to GenericLiteLLMParams if needed + if isinstance(litellm_params, dict): + # Ensure api_key is included if provided + if api_key and "api_key" not in litellm_params: + litellm_params = {**litellm_params, "api_key": api_key} + litellm_params_obj = GenericLiteLLMParams(**litellm_params) + else: + litellm_params_obj = litellm_params or GenericLiteLLMParams() + # Set api_key if provided and not already set + if api_key and not litellm_params_obj.api_key: + litellm_params_obj.api_key = api_key + + # Use Azure authentication logic + headers = BaseAzureLLM._base_validate_azure_environment( + headers=headers, litellm_params=litellm_params_obj + ) + + # Get tools and other anthropic-specific setup + tools = optional_params.get("tools") + prompt_caching_set = self.is_cache_control_set(messages=messages) + computer_tool_used = self.is_computer_tool_used(tools=tools) + mcp_server_used = self.is_mcp_server_used( + mcp_servers=optional_params.get("mcp_servers") + ) + pdf_used = self.is_pdf_used(messages=messages) + file_id_used = self.is_file_id_used(messages=messages) + user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( + anthropic_beta_header=headers.get("anthropic-beta") + ) + + # Get anthropic headers (but we'll replace x-api-key with Azure auth) + anthropic_headers = self.get_anthropic_headers( + computer_tool_used=computer_tool_used, + prompt_caching_set=prompt_caching_set, + pdf_used=pdf_used, + api_key=api_key or "", # Azure auth is already in headers + file_id_used=file_id_used, + is_vertex_request=optional_params.get("is_vertex_request", False), + user_anthropic_beta_headers=user_anthropic_beta_headers, + mcp_server_used=mcp_server_used, + ) + + # Remove x-api-key from anthropic headers since Azure uses different auth + anthropic_headers.pop("x-api-key", None) + + # Merge headers - Azure auth (api-key or Authorization) takes precedence + headers = {**anthropic_headers, **headers} + + # Ensure anthropic-version header is set + if "anthropic-version" not in headers: + headers["anthropic-version"] = "2023-06-01" + + return headers + diff --git a/litellm/main.py b/litellm/main.py index 88c3f7bc55b..4857f1b9754 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -152,6 +152,7 @@ from .litellm_core_utils.prompt_templates.factory import ( ) from .litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from .llms.anthropic.chat import AnthropicChatCompletion +from .llms.azure.anthropic.handler import AzureAnthropicChatCompletion from .llms.azure.audio_transcriptions import AzureAudioTranscription from .llms.azure.azure import AzureChatCompletion, _check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion @@ -253,6 +254,7 @@ openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions = GroqChatCompletion() azure_ai_embedding = AzureAIEmbedding() anthropic_chat_completions = AnthropicChatCompletion() +azure_anthropic_chat_completions = AzureAnthropicChatCompletion() azure_chat_completions = AzureChatCompletion() azure_o1_chat_completions = AzureOpenAIO1ChatCompletion() azure_text_completions = AzureTextCompletion() @@ -2353,6 +2355,62 @@ def completion( # type: ignore # noqa: PLR0915 original_response=response, ) response = response + elif custom_llm_provider == "azure_anthropic": + # Azure Anthropic uses same API as Anthropic but with Azure authentication + api_key = ( + api_key + or litellm.azure_key + or litellm.api_key + or get_secret("AZURE_API_KEY") + or get_secret("AZURE_OPENAI_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # Azure Foundry endpoint format: https://.services.ai.azure.com/anthropic/v1/messages + api_base = ( + api_base + or litellm.api_base + or get_secret("AZURE_API_BASE") + ) + + if api_base is None: + raise ValueError( + "Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. " + "Expected format: https://.services.ai.azure.com/anthropic" + ) + + # Ensure the URL ends with /v1/messages + if not api_base.endswith("/v1/messages"): + if not api_base.endswith("/anthropic"): + api_base = api_base.rstrip("/") + "/anthropic" + api_base = api_base.rstrip("/") + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response elif custom_llm_provider == "nlp_cloud": nlp_cloud_key = ( api_key diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9bef6ebef7d..3c8fa885f67 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2545,6 +2545,7 @@ class LlmProviders(str, Enum): AZURE = "azure" AZURE_TEXT = "azure_text" AZURE_AI = "azure_ai" + AZURE_ANTHROPIC = "azure_anthropic" SAGEMAKER = "sagemaker" SAGEMAKER_CHAT = "sagemaker_chat" BEDROCK = "bedrock" diff --git a/litellm/utils.py b/litellm/utils.py index 47fd06a9f27..e114e4cc051 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7148,6 +7148,8 @@ class ProviderConfigManager: return litellm.AzureAIStudioConfig() elif litellm.LlmProviders.AZURE_TEXT == provider: return litellm.AzureOpenAITextConfig() + elif litellm.LlmProviders.AZURE_ANTHROPIC == provider: + return litellm.AzureAnthropicConfig() elif litellm.LlmProviders.HOSTED_VLLM == provider: return litellm.HostedVLLMChatConfig() elif litellm.LlmProviders.NLP_CLOUD == provider: diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py new file mode 100644 index 00000000000..34aed42478d --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_handler.py @@ -0,0 +1,216 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.azure.anthropic.handler import AzureAnthropicChatCompletion +from litellm.types.utils import ModelResponse + + +class TestAzureAnthropicChatCompletion: + def test_inherits_from_anthropic_chat_completion(self): + """Test that AzureAnthropicChatCompletion inherits from AnthropicChatCompletion""" + handler = AzureAnthropicChatCompletion() + assert isinstance(handler, AzureAnthropicChatCompletion) + # Check that it has methods from parent class + assert hasattr(handler, "acompletion_function") + assert hasattr(handler, "acompletion_stream_function") + + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_uses_azure_anthropic_config(self, mock_azure_config, mock_provider_manager): + """Test that completion method uses AzureAnthropicConfig""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = {"model": "claude-sonnet-4-5", "messages": []} + mock_config.transform_response.return_value = ModelResponse() + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + with patch.object( + handler, "acompletion_function", return_value=ModelResponse() + ) as mock_acompletion: + handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=True, + ) + + # Verify AzureAnthropicConfig was used + mock_azure_config.assert_called_once() + mock_config_instance.validate_environment.assert_called_once() + + @patch("litellm.llms.anthropic.chat.handler.make_sync_call") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_streaming(self, mock_azure_config, mock_provider_manager, mock_make_sync_call): + # Note: decorators are applied in reverse order + """Test completion with streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + "stream": True, + } + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + # Mock streaming response + mock_stream = MagicMock() + mock_headers = MagicMock() + mock_make_sync_call.return_value = (mock_stream, mock_headers) + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {"stream": True} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + acompletion=False, + ) + + # Verify streaming was handled + mock_make_sync_call.assert_called_once() + assert result is not None + + @patch("litellm.llms.custom_httpx.http_handler._get_httpx_client") + @patch("litellm.utils.ProviderConfigManager") + @patch("litellm.llms.azure.anthropic.handler.AzureAnthropicConfig") + def test_completion_non_streaming(self, mock_azure_config, mock_provider_manager, mock_get_client): + # Note: decorators are applied in reverse order + """Test completion without streaming""" + handler = AzureAnthropicChatCompletion() + mock_config = MagicMock() + mock_config.transform_request.return_value = { + "model": "claude-sonnet-4-5", + "messages": [], + } + mock_response = ModelResponse() + mock_config.transform_response.return_value = mock_response + mock_config_instance = MagicMock() + mock_config_instance.validate_environment.return_value = {"api-key": "test-api-key", "anthropic-version": "2023-06-01"} + mock_azure_config.return_value = mock_config_instance + mock_provider_manager.get_provider_chat_config.return_value = mock_config + + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + api_base = "https://test.services.ai.azure.com/anthropic/v1/messages" + custom_llm_provider = "azure_anthropic" + custom_prompt_dict = {} + model_response = ModelResponse() + print_verbose = MagicMock() + encoding = MagicMock() + api_key = "test-api-key" + logging_obj = MagicMock() + optional_params = {} + timeout = 60.0 + litellm_params = {"api_key": "test-api-key"} + headers = {} + + # Mock HTTP client + mock_client = MagicMock() + mock_response_obj = MagicMock() + mock_response_obj.status_code = 200 + mock_response_obj.text = json.dumps({ + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }) + mock_response_obj.json.return_value = { + "id": "test-id", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Hello!"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + mock_client.post.return_value = mock_response_obj + mock_get_client.return_value = mock_client + + result = handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + headers=headers, + client=None, # Let it create the client + acompletion=False, + ) + + # Verify non-streaming was handled + mock_client.post.assert_called_once() + assert result is not None + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py new file mode 100644 index 00000000000..a7aa2983175 --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_provider_routing.py @@ -0,0 +1,82 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import pytest + +from litellm.litellm_core_utils.get_llm_provider_logic import _is_azure_anthropic_model, get_llm_provider + + +class TestAzureAnthropicProviderRouting: + def test_is_azure_anthropic_model_with_claude(self): + """Test _is_azure_anthropic_model detects Claude models""" + # Test various Claude model names + assert _is_azure_anthropic_model("azure/claude-sonnet-4-5") == "claude-sonnet-4-5" + assert _is_azure_anthropic_model("azure/claude-opus-4-1") == "claude-opus-4-1" + assert _is_azure_anthropic_model("azure/claude-haiku-4-5") == "claude-haiku-4-5" + assert _is_azure_anthropic_model("azure/claude-3-5-sonnet") == "claude-3-5-sonnet" + assert _is_azure_anthropic_model("azure/claude-3-opus") == "claude-3-opus" + + def test_is_azure_anthropic_model_case_insensitive(self): + """Test _is_azure_anthropic_model is case insensitive""" + assert _is_azure_anthropic_model("azure/CLAUDE-sonnet-4-5") == "CLAUDE-sonnet-4-5" + assert _is_azure_anthropic_model("azure/Claude-Sonnet-4-5") == "Claude-Sonnet-4-5" + + def test_is_azure_anthropic_model_with_non_claude(self): + """Test _is_azure_anthropic_model returns None for non-Claude models""" + assert _is_azure_anthropic_model("azure/gpt-4") is None + assert _is_azure_anthropic_model("azure/gpt-35-turbo") is None + assert _is_azure_anthropic_model("azure/command-r-plus") is None + + def test_is_azure_anthropic_model_with_invalid_format(self): + """Test _is_azure_anthropic_model handles invalid formats""" + assert _is_azure_anthropic_model("azure") is None + assert _is_azure_anthropic_model("claude-sonnet-4-5") is None + assert _is_azure_anthropic_model("") is None + + def test_get_llm_provider_routes_azure_claude_to_azure_anthropic(self): + """Test that get_llm_provider routes azure/claude-* models to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-sonnet-4-5" # Should strip "azure/" prefix + + def test_get_llm_provider_routes_azure_claude_opus(self): + """Test routing for Claude Opus models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-opus-4-1" + ) + assert provider == "azure_anthropic" + assert model == "claude-opus-4-1" + + def test_get_llm_provider_routes_azure_claude_haiku(self): + """Test routing for Claude Haiku models""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-haiku-4-5" + ) + assert provider == "azure_anthropic" + assert model == "claude-haiku-4-5" + + def test_get_llm_provider_does_not_route_non_claude_azure_models(self): + """Test that non-Claude Azure models are not routed to azure_anthropic""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/gpt-4" + ) + assert provider != "azure_anthropic" + # Should be routed to regular azure provider + assert provider == "azure" or provider == "openai" + + def test_get_llm_provider_with_custom_llm_provider_override(self): + """Test that custom_llm_provider parameter can override routing""" + model, provider, dynamic_api_key, api_base = get_llm_provider( + model="azure/claude-sonnet-4-5", custom_llm_provider="azure" + ) + # When custom_llm_provider is explicitly set, it should be respected + # But the routing logic should still detect it as azure_anthropic + # This depends on the order of checks in get_llm_provider + assert provider in ["azure_anthropic", "azure"] + diff --git a/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py new file mode 100644 index 00000000000..43f0b5c439d --- /dev/null +++ b/tests/test_litellm/llms/azure/anthropic/test_azure_anthropic_transformation.py @@ -0,0 +1,191 @@ +import os +import sys + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +import pytest +from unittest.mock import MagicMock, patch + +from litellm.llms.azure.anthropic.transformation import AzureAnthropicConfig +from litellm.types.router import GenericLiteLLMParams + + +class TestAzureAnthropicConfig: + def test_custom_llm_provider(self): + """Test that custom_llm_provider returns 'azure_anthropic'""" + config = AzureAnthropicConfig() + assert config.custom_llm_provider == "azure_anthropic" + + def test_validate_environment_with_dict_litellm_params(self): + """Test validate_environment with dict litellm_params""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that dict was converted to GenericLiteLLMParams + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert call_args[1]["litellm_params"].api_key == "test-api-key" + assert "anthropic-version" in result + + def test_validate_environment_with_generic_litellm_params(self): + """Test validate_environment with GenericLiteLLMParams object""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = GenericLiteLLMParams(api_key="test-api-key") + api_key = "test-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that GenericLiteLLMParams was passed through + call_args = mock_validate.call_args + assert isinstance(call_args[1]["litellm_params"], GenericLiteLLMParams) + assert "anthropic-version" in result + + def test_validate_environment_sets_api_key_in_litellm_params(self): + """Test that api_key parameter is set in litellm_params if provided""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {} # Empty dict, no api_key + api_key = "provided-api-key" + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "provided-api-key"} + config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + ) + + # Verify that api_key was set in litellm_params + call_args = mock_validate.call_args + assert call_args[1]["litellm_params"].api_key == "provided-api-key" + + def test_validate_environment_removes_x_api_key(self): + """Test that x-api-key header is removed (Azure uses api-key instead)""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object( + config, "get_anthropic_headers", return_value={"x-api-key": "should-be-removed"} + ): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Verify x-api-key was removed + assert "x-api-key" not in result + assert "api-key" in result + + def test_validate_environment_sets_anthropic_version(self): + """Test that anthropic-version header is set""" + config = AzureAnthropicConfig() + headers = {} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key"} + with patch.object(config, "get_anthropic_headers", return_value={}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2023-06-01" + + def test_validate_environment_preserves_existing_anthropic_version(self): + """Test that existing anthropic-version header is preserved""" + config = AzureAnthropicConfig() + headers = {"anthropic-version": "2024-01-01"} + model = "claude-sonnet-4-5" + messages = [{"role": "user", "content": "Hello"}] + optional_params = {} + litellm_params = {"api_key": "test-api-key"} + + with patch( + "litellm.llms.azure.common_utils.BaseAzureLLM._base_validate_azure_environment" + ) as mock_validate: + mock_validate.return_value = {"api-key": "test-api-key", "anthropic-version": "2024-01-01"} + with patch.object(config, "get_anthropic_headers", return_value={"anthropic-version": "2024-01-01"}): + result = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + assert result["anthropic-version"] == "2024-01-01" + + def test_inherits_anthropic_config_methods(self): + """Test that AzureAnthropicConfig inherits methods from AnthropicConfig""" + config = AzureAnthropicConfig() + + # Test that it has AnthropicConfig methods + assert hasattr(config, "get_anthropic_headers") + assert hasattr(config, "is_cache_control_set") + assert hasattr(config, "is_computer_tool_used") + assert hasattr(config, "transform_request") + assert hasattr(config, "transform_response") + From 3e58fe42b7a4e9ddf334525e18677714d68d3af2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 19 Nov 2025 17:07:46 -0300 Subject: [PATCH 002/223] fix: Support response_format parameter in completion -> responses bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #16810 ## Problem When using completion() with models that have mode: "responses" (like o3-pro, gpt-5-codex), the response_format parameter with JSON schemas was being ignored or incorrectly handled, causing: - Large schemas (>512 chars) to fail with "metadata.schema_dict_json: string too long" error - Structured outputs to be silently dropped - Users' code to break unexpectedly ## Root Cause The completion -> responses bridge in litellm/completion_extras/litellm_responses_transformation/transformation.py was missing the conversion of response_format (Chat Completion format) to text.format (Responses API format). The inverse bridge (responses -> completion) already had this conversion implemented in commit 29f0ed223a, but the completion -> responses direction was incomplete. ## Solution Added _transform_response_format_to_text_format() method that converts: - response_format with json_schema → text.format with json_schema - response_format with json_object → text.format with json_object - response_format with text → text.format with text Updated transform_request() to detect and convert response_format parameter before sending to litellm.responses(). ## Changes - Added _transform_response_format_to_text_format() method (lines 592-647) - Modified transform_request() to handle response_format (lines 199-203) - Added comprehensive tests to validate the conversion ## Testing - 5 new unit tests covering all conversion scenarios - Real API test with OpenAI confirming large schemas (>512 chars) work - No more metadata.schema_dict_json errors ## Impact Users can now use completion() with models that have mode: "responses" and: - Use large JSON schemas without hitting metadata 512 char limit - Get proper structured outputs - Have their existing code continue working --- .../transformation.py | 62 ++++++++ ...responses_transformation_transformation.py | 136 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 54d32fe3466..a6dbd5d0dcd 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -196,6 +196,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(List[Dict[str, Any]], value) ) ) + elif key == "response_format": + # Convert response_format to text.format + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key in ("metadata"): @@ -589,6 +594,63 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return Reasoning(effort="minimal") return None + def _transform_response_format_to_text_format( + self, response_format: Union[Dict[str, Any], Any] + ) -> Optional[Dict[str, Any]]: + """ + Transform Chat Completion response_format parameter to Responses API text.format parameter. + + Chat Completion response_format structure: + { + "type": "json_schema", + "json_schema": { + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + + Responses API text parameter structure: + { + "format": { + "type": "json_schema", + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + """ + if not response_format: + return None + + if isinstance(response_format, dict): + format_type = response_format.get("type") + + if format_type == "json_schema": + json_schema = response_format.get("json_schema", {}) + return { + "format": { + "type": "json_schema", + "name": json_schema.get("name", "response_schema"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", False), + } + } + elif format_type == "json_object": + return { + "format": { + "type": "json_object" + } + } + elif format_type == "text": + return { + "format": { + "type": "text" + } + } + + return None + def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str: """Map responses API status to chat completion finish_reason""" if not status: diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py new file mode 100644 index 00000000000..adbaf219079 --- /dev/null +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -0,0 +1,136 @@ +""" +Test for response_format to text.format conversion in completion -> responses bridge +""" +import pytest +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) + + +def test_transform_response_format_to_text_format_json_schema(): + """Test conversion of response_format with json_schema to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + # Chat Completion format + response_format = { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + + # Convert to Responses API format + result = handler._transform_response_format_to_text_format(response_format) + + # Verify conversion + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_schema" + assert result["format"]["name"] == "person_schema" + assert result["format"]["strict"] is True + assert "schema" in result["format"] + assert result["format"]["schema"]["type"] == "object" + assert "properties" in result["format"]["schema"] + + +def test_transform_response_format_to_text_format_json_object(): + """Test conversion of response_format with json_object to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "json_object" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_object" + + +def test_transform_response_format_to_text_format_text(): + """Test conversion of response_format with text to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "text" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "text" + + +def test_transform_response_format_to_text_format_none(): + """Test that None input returns None""" + handler = LiteLLMResponsesTransformationHandler() + + result = handler._transform_response_format_to_text_format(None) + + assert result is None + + +def test_transform_request_with_response_format(): + """Test that transform_request correctly handles response_format parameter""" + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + {"role": "user", "content": "Extract person info: John Doe, 30 years old"} + ] + + optional_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + } + + litellm_params = {} + headers = {} + + # Mock logging object + class MockLoggingObj: + pass + + litellm_logging_obj = MockLoggingObj() + + result = handler.transform_request( + model="o3-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + litellm_logging_obj=litellm_logging_obj, + ) + + # Verify that text parameter was set with converted format + assert "text" in result + assert result["text"] is not None + assert "format" in result["text"] + assert result["text"]["format"]["type"] == "json_schema" + assert result["text"]["format"]["name"] == "person_schema" + assert "schema" in result["text"]["format"] From 57544f166286bf7e8d1d5166187f0c7d2f53b92a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 20 Nov 2025 12:38:48 -0800 Subject: [PATCH 003/223] [Feat] Adds IAM role assumption support for AWS Secret Manager (#16887) * add AWS fields for KeyManagementSettings * docs IAM roles * use aws iam auth on secret manager v2 * fix: load_aws_secret_manager * test_secret_manager_with_iam_role_settings --- .../secret_managers/aws_secret_manager.md | 54 +++++ litellm/proxy/proxy_server.py | 5 +- .../secret_managers/aws_secret_manager_v2.py | 78 ++++++- litellm/types/secret_managers/main.py | 24 +- .../test_aws_secret_manager.py | 209 ++++++++++++++++++ 5 files changed, 362 insertions(+), 8 deletions(-) diff --git a/docs/my-website/docs/secret_managers/aws_secret_manager.md b/docs/my-website/docs/secret_managers/aws_secret_manager.md index 44fa23a4ae5..5b7ab1e3e7b 100644 --- a/docs/my-website/docs/secret_managers/aws_secret_manager.md +++ b/docs/my-website/docs/secret_managers/aws_secret_manager.md @@ -110,3 +110,57 @@ The `primary_secret_name` allows you to read multiple keys from a single AWS Sec This reduces the number of AWS Secrets you need to manage. +## IAM Role Assumption + +Use IAM roles instead of static AWS credentials for better security. + +### Basic IAM Role + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMSecretManagerRole" + aws_session_name: "litellm-session" +``` + +### Cross-Account Access + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::999999999999:role/CrossAccountRole" + aws_external_id: "unique-external-id" +``` + +### EKS with IRSA + +```yaml +general_settings: + key_management_system: "aws_secret_manager" + key_management_settings: + store_virtual_keys: true + aws_region_name: "us-east-1" + aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMServiceAccountRole" + aws_web_identity_token: "os.environ/AWS_WEB_IDENTITY_TOKEN_FILE" +``` + +### Configuration Parameters + +| Parameter | Description | +|-----------|-------------| +| `aws_region_name` | AWS region | +| `aws_role_name` | IAM role ARN to assume | +| `aws_session_name` | Session name (optional) | +| `aws_external_id` | External ID for cross-account | +| `aws_profile_name` | AWS profile from `~/.aws/credentials` | +| `aws_web_identity_token` | OIDC token path for IRSA | +| `aws_sts_endpoint` | Custom STS endpoint for VPC | + + + diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4040b0aa707..9c98ad215b3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2692,7 +2692,10 @@ class ProxyConfig: AWSSecretsManagerV2, ) - AWSSecretsManagerV2.load_aws_secret_manager(use_aws_secret_manager=True) + AWSSecretsManagerV2.load_aws_secret_manager( + use_aws_secret_manager=True, + key_management_settings=litellm._key_management_settings, + ) elif key_management_system == KeyManagementSystem.AWS_KMS.value: load_aws_kms(use_aws_kms=True) elif ( diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 8f3547fd082..8edfc48336b 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -33,25 +33,73 @@ from .base_secret_manager import BaseSecretManager class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): - def __init__(self, **kwargs): + def __init__( + self, + aws_region_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + aws_session_name: Optional[str] = None, + aws_external_id: Optional[str] = None, + aws_profile_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, + aws_sts_endpoint: Optional[str] = None, + **kwargs + ): BaseSecretManager.__init__(self, **kwargs) BaseAWSLLM.__init__(self, **kwargs) + + # Store AWS authentication settings + self.aws_region_name = aws_region_name + self.aws_role_name = aws_role_name + self.aws_session_name = aws_session_name + self.aws_external_id = aws_external_id + self.aws_profile_name = aws_profile_name + self.aws_web_identity_token = aws_web_identity_token + self.aws_sts_endpoint = aws_sts_endpoint @classmethod def validate_environment(cls): - if "AWS_REGION_NAME" not in os.environ: - raise ValueError("Missing required environment variable - AWS_REGION_NAME") + # AWS_REGION_NAME is only strictly required if not using a profile or role + # When using IAM roles, the region can come from multiple sources + if ( + "AWS_REGION_NAME" not in os.environ + and "AWS_REGION" not in os.environ + and "AWS_DEFAULT_REGION" not in os.environ + ): + verbose_logger.warning( + "No AWS region found in environment. Ensure aws_region_name is set in key_management_settings " + "or AWS_REGION_NAME/AWS_REGION/AWS_DEFAULT_REGION is set in environment." + ) @classmethod - def load_aws_secret_manager(cls, use_aws_secret_manager: Optional[bool]): + def load_aws_secret_manager( + cls, + use_aws_secret_manager: Optional[bool], + key_management_settings: Optional[Any] = None, + ): """ - Initialize AWSSecretsManagerV2 and sets litellm.secret_manager_client = AWSSecretsManagerV2() and litellm._key_management_system = KeyManagementSystem.AWS_SECRET_MANAGER + Initialize AWSSecretsManagerV2 with settings from key_management_settings """ if use_aws_secret_manager is None or use_aws_secret_manager is False: return try: cls.validate_environment() - litellm.secret_manager_client = cls() + + # Extract AWS settings from key_management_settings if provided + aws_kwargs = {} + if key_management_settings is not None: + aws_kwargs = { + "aws_region_name": getattr(key_management_settings, "aws_region_name", None), + "aws_role_name": getattr(key_management_settings, "aws_role_name", None), + "aws_session_name": getattr(key_management_settings, "aws_session_name", None), + "aws_external_id": getattr(key_management_settings, "aws_external_id", None), + "aws_profile_name": getattr(key_management_settings, "aws_profile_name", None), + "aws_web_identity_token": getattr(key_management_settings, "aws_web_identity_token", None), + "aws_sts_endpoint": getattr(key_management_settings, "aws_sts_endpoint", None), + } + # Remove None values + aws_kwargs = {k: v for k, v in aws_kwargs.items() if v is not None} + + litellm.secret_manager_client = cls(**aws_kwargs) litellm._key_management_system = KeyManagementSystem.AWS_SECRET_MANAGER except Exception as e: @@ -327,6 +375,24 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") optional_params = optional_params or {} + + # Build optional_params from instance settings if not provided + # This allows the IAM role settings to be used for Secret Manager calls + if not optional_params.get("aws_role_name") and self.aws_role_name: + optional_params["aws_role_name"] = self.aws_role_name + if not optional_params.get("aws_session_name") and self.aws_session_name: + optional_params["aws_session_name"] = self.aws_session_name + if not optional_params.get("aws_region_name") and self.aws_region_name: + optional_params["aws_region_name"] = self.aws_region_name + if not optional_params.get("aws_external_id") and self.aws_external_id: + optional_params["aws_external_id"] = self.aws_external_id + if not optional_params.get("aws_profile_name") and self.aws_profile_name: + optional_params["aws_profile_name"] = self.aws_profile_name + if not optional_params.get("aws_web_identity_token") and self.aws_web_identity_token: + optional_params["aws_web_identity_token"] = self.aws_web_identity_token + if not optional_params.get("aws_sts_endpoint") and self.aws_sts_endpoint: + optional_params["aws_sts_endpoint"] = self.aws_sts_endpoint + boto3_credentials_info = self._get_boto_credentials_from_optional_params( optional_params ) diff --git a/litellm/types/secret_managers/main.py b/litellm/types/secret_managers/main.py index 8d7aa764d69..e4c7d76573f 100644 --- a/litellm/types/secret_managers/main.py +++ b/litellm/types/secret_managers/main.py @@ -49,4 +49,26 @@ class KeyManagementSettings(LiteLLMPydanticObjectBase): """ Path to custom secret manager class (e.g. "my_secret_manager.InMemorySecretManager") Required when key_management_system is "custom" - """ \ No newline at end of file + """ + + # AWS IAM Role Assumption Settings (for AWS Secret Manager) + aws_region_name: Optional[str] = None + """AWS region for Secret Manager operations (e.g., 'us-east-1')""" + + aws_role_name: Optional[str] = None + """ARN of IAM role to assume for Secret Manager access (e.g., 'arn:aws:iam::123456789012:role/MyRole')""" + + aws_session_name: Optional[str] = None + """Session name for the assumed role session (optional, auto-generated if not provided)""" + + aws_external_id: Optional[str] = None + """External ID for role assumption (required for cross-account access)""" + + aws_profile_name: Optional[str] = None + """AWS profile name to use from ~/.aws/credentials""" + + aws_web_identity_token: Optional[str] = None + """Web identity token for OIDC/IRSA authentication""" + + aws_sts_endpoint: Optional[str] = None + """Custom STS endpoint URL (useful for VPC endpoints or testing)""" \ No newline at end of file diff --git a/tests/litellm_utils_tests/test_aws_secret_manager.py b/tests/litellm_utils_tests/test_aws_secret_manager.py index b6ccd417681..3870d336f0e 100644 --- a/tests/litellm_utils_tests/test_aws_secret_manager.py +++ b/tests/litellm_utils_tests/test_aws_secret_manager.py @@ -31,6 +31,7 @@ import pytest from litellm._uuid import uuid import json from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 +from litellm.types.secret_managers.main import KeyManagementSettings def check_aws_credentials(): @@ -253,3 +254,211 @@ async def test_write_secret_with_description_and_tags(): delete_response = await secret_manager.async_delete_secret(secret_name=test_secret_name) print("Delete Response:", delete_response) assert delete_response is not None + + +def test_secret_manager_with_iam_role_settings(): + """ + Test AWS Secret Manager initialization with IAM role settings + """ + settings = KeyManagementSettings( + aws_region_name="us-east-1", + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_session_name="test-session", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + ) + + # Verify settings are stored + assert secret_manager.aws_role_name == settings.aws_role_name + assert secret_manager.aws_region_name == settings.aws_region_name + assert secret_manager.aws_session_name == settings.aws_session_name + + +def test_secret_manager_with_cross_account_settings(): + """ + Test AWS Secret Manager initialization with cross-account IAM role settings + """ + settings = KeyManagementSettings( + aws_region_name="us-west-2", + aws_role_name="arn:aws:iam::999999999999:role/CrossAccountRole", + aws_session_name="cross-account-session", + aws_external_id="unique-external-id", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + aws_external_id=settings.aws_external_id, + ) + + # Verify settings are stored + assert secret_manager.aws_role_name == settings.aws_role_name + assert secret_manager.aws_region_name == settings.aws_region_name + assert secret_manager.aws_external_id == settings.aws_external_id + + +def test_secret_manager_with_irsa_settings(): + """ + Test AWS Secret Manager initialization with IRSA (EKS) settings + """ + settings = KeyManagementSettings( + aws_region_name="us-east-1", + aws_role_name="arn:aws:iam::123456789012:role/EKSServiceAccountRole", + aws_session_name="eks-session", + aws_web_identity_token="os.environ/AWS_WEB_IDENTITY_TOKEN_FILE", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + aws_web_identity_token=settings.aws_web_identity_token, + ) + + # Verify settings are stored + assert secret_manager.aws_role_name == settings.aws_role_name + assert secret_manager.aws_web_identity_token == settings.aws_web_identity_token + + +def test_secret_manager_with_custom_sts_endpoint(): + """ + Test AWS Secret Manager initialization with custom STS endpoint (VPC endpoint) + """ + settings = KeyManagementSettings( + aws_region_name="us-east-1", + aws_role_name="arn:aws:iam::123456789012:role/VPCRole", + aws_session_name="vpc-session", + aws_sts_endpoint="https://sts.us-east-1.vpce-0123456789abcdef.amazonaws.com", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + aws_sts_endpoint=settings.aws_sts_endpoint, + ) + + # Verify settings are stored + assert secret_manager.aws_role_name == settings.aws_role_name + assert secret_manager.aws_sts_endpoint == settings.aws_sts_endpoint + + +def test_secret_manager_with_aws_profile(): + """ + Test AWS Secret Manager initialization with AWS profile + """ + settings = KeyManagementSettings( + aws_region_name="us-east-1", + aws_profile_name="litellm-dev", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_profile_name=settings.aws_profile_name, + ) + + # Verify settings are stored + assert secret_manager.aws_profile_name == settings.aws_profile_name + + +def test_load_aws_secret_manager_with_settings(): + """ + Test loading AWS Secret Manager with key_management_settings + """ + import litellm + + settings = KeyManagementSettings( + store_virtual_keys=True, + aws_region_name="us-east-1", + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_session_name="test-session", + ) + + # Set environment variable for validation to pass + os.environ["AWS_REGION_NAME"] = "us-east-1" + + try: + AWSSecretsManagerV2.load_aws_secret_manager( + use_aws_secret_manager=True, + key_management_settings=settings, + ) + + # Verify the client was created + assert litellm.secret_manager_client is not None + assert isinstance(litellm.secret_manager_client, AWSSecretsManagerV2) + + # Verify settings were passed through + assert litellm.secret_manager_client.aws_role_name == settings.aws_role_name + assert litellm.secret_manager_client.aws_region_name == settings.aws_region_name + assert litellm.secret_manager_client.aws_session_name == settings.aws_session_name + finally: + # Cleanup + litellm.secret_manager_client = None + + +@pytest.mark.asyncio +async def test_end_to_end_iam_role_secret_write(): + """ + Test writing a secret using IAM role assumption (integration test) + + Requires: + - AWS_REGION_NAME environment variable + - TEST_IAM_ROLE_ARN environment variable with ARN of a role that can be assumed + - Proper AWS credentials configured (via instance profile, IAM role, or environment) + """ + # Skip if TEST_IAM_ROLE_ARN is not set + test_role_arn = os.getenv("TEST_IAM_ROLE_ARN") + if not test_role_arn: + pytest.skip("TEST_IAM_ROLE_ARN environment variable not set") + + aws_region = os.getenv("AWS_REGION_NAME", "us-east-1") + + settings = KeyManagementSettings( + store_virtual_keys=True, + aws_region_name=aws_region, + aws_role_name=test_role_arn, + aws_session_name="integration-test-session", + ) + + secret_manager = AWSSecretsManagerV2( + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + ) + + test_secret_name = f"litellm_test_iam_{uuid.uuid4().hex[:8]}" + test_secret_value = "test_value_iam_role" + + try: + # Test write operation using IAM role + response = await secret_manager.async_write_secret( + secret_name=test_secret_name, + secret_value=test_secret_value, + ) + + print("Write Response with IAM Role:", response) + assert response is not None + assert "ARN" in response + + # Test read operation using IAM role + read_value = await secret_manager.async_read_secret( + secret_name=test_secret_name + ) + + print("Read Value with IAM Role:", read_value) + assert read_value == test_secret_value + + finally: + # Cleanup: Delete the secret + try: + delete_response = await secret_manager.async_delete_secret( + secret_name=test_secret_name + ) + print("Delete Response:", delete_response) + except Exception as e: + print(f"Cleanup failed: {e}") From 0d812f98bc181ccd0673b68371f88829320102fd Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 20 Nov 2025 14:05:27 -0800 Subject: [PATCH 004/223] new u build --- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 .../_next/static/chunks/1116-2d5ec30ef7d86f0e.js | 1 - .../_next/static/chunks/1160-08491effeedbaae3.js | 1 - .../_next/static/chunks/131-d505623ce13e3958.js | 1 - .../_next/static/chunks/1491-80dbf1ebc561e9b6.js | 1 - .../_next/static/chunks/1491-d6d75a98cb22e323.js | 1 + .../_next/static/chunks/1518-708cd3bb943546cf.js | 1 + .../_next/static/chunks/1567-5650c3aa172cd633.js | 1 + ...4dd64df9427ab1c.js => 1598-a69df0ba9a12c4f3.js} | 2 +- ...495570659e854dc.js => 1739-b555a2473b0946bb.js} | 0 ...db32c86493cbcd6.js => 1960-95ee080c178a4e18.js} | 2 +- ...840f6cabd9dbd7e.js => 1994-2289bfd6e7d65533.js} | 2 +- ...acee05add5250f4.js => 2004-a8136543e13a4de3.js} | 0 ...f598b370e2766b0.js => 2012-db5a85de324150e9.js} | 0 ...2887e9caea1e319.js => 2019-2005428b5853a7a6.js} | 0 ...04db85e26800c7a.js => 2202-c4a45b6cf77ca4a9.js} | 0 .../_next/static/chunks/2249-b6e8bcded6cfd197.js | 1 + ...0f31894fc8c0e24.js => 2273-4649f34440d8d399.js} | 0 .../_next/static/chunks/2312-02f166690919a2cc.js | 1 - .../_next/static/chunks/2358-ff0b1979f8905d3f.js | 1 + .../_next/static/chunks/2409-33ed6e753f8afe9b.js | 1 - .../_next/static/chunks/2409-7de3355d049b247b.js | 1 + ...017e0c3b119c340.js => 2837-753bc846fbc41ba6.js} | 2 +- .../_next/static/chunks/2924-98771dd2cd6a8a3d.js | 1 + .../_next/static/chunks/3135-fbf13217ca7d2de9.js | 1 + .../_next/static/chunks/3603-dd19ac8e31e4bc25.js | 1 - ...dee4252a2b35874.js => 3655-03331223540a9ced.js} | 2 +- .../_next/static/chunks/3801-42c33bd07afe4ba2.js | 1 + .../_next/static/chunks/3801-a6c59be67a81d7dc.js | 1 - .../_next/static/chunks/395-9ca940835820f5fa.js | 1 - .../_next/static/chunks/395-e2a4326c962bdd18.js | 1 + .../_next/static/chunks/3978-5fb71317e3cb900e.js | 1 - .../_next/static/chunks/4138-bbc045e58cfe8f02.js | 1 - .../_next/static/chunks/4289-988122838795ecce.js | 1 - .../_next/static/chunks/4289-fcd5e435fdba5a64.js | 1 + ...755f0704f2336bb.js => 4292-4536c43afb350f73.js} | 2 +- .../_next/static/chunks/4819-337db995ee8dbe87.js | 1 + ...4b4dee8acfc3279.js => 5296-577c93bbe6971079.js} | 2 +- .../_next/static/chunks/5402-4a78ee24b996de08.js | 1 + .../_next/static/chunks/5407-9cee54da5c03162b.js | 1 + .../_next/static/chunks/5572-674265ad2317d8c3.js | 1 + .../_next/static/chunks/5869-7be6e0175610c221.js | 1 + .../_next/static/chunks/603-ced069c6ecb41baf.js | 1 - .../_next/static/chunks/6188-003d982a932f88e0.js | 1 + .../_next/static/chunks/6204-5c34c11f5a32fb09.js | 1 - .../_next/static/chunks/6294-f5649a0ee9b63fd2.js | 1 - .../_next/static/chunks/630-57f1c278500ab515.js | 1 + .../_next/static/chunks/6478-f5fd91a406dd265e.js | 1 - ...bc506133f5d9c17.js => 6600-6564d6b9d0d57aaa.js} | 2 +- .../_next/static/chunks/7013-c3a11adde9383db4.js | 1 + ...7549fbb4532c452b.js => 710-56c50cf7f4bc9cca.js} | 2 +- .../_next/static/chunks/7155-5d31a6a1ad834cf0.js | 1 - .../_next/static/chunks/7155-5e95ae8488e27e4a.js | 1 + .../_next/static/chunks/7164-6e382f189a96cd83.js | 1 + .../_next/static/chunks/7164-cc01ff2b7f3ae404.js | 1 - .../_next/static/chunks/7526-2769cf614be0678b.js | 1 + .../_next/static/chunks/7526-c6c315293f6101de.js | 1 - ...0cc38c7be8272a14.js => 773-a2e903ad8cb3a0c8.js} | 0 ...4f3c55775f088dd.js => 7901-fdb65b7b40365397.js} | 2 +- .../_next/static/chunks/795-79942bf68d21d57b.js | 1 + .../_next/static/chunks/8040-dc82c32ea5959cb5.js | 1 + .../_next/static/chunks/8049-db73a2d52dc11ceb.js | 1 - .../_next/static/chunks/8049-fd60df8124c6c5dd.js | 1 + .../_next/static/chunks/8077-dca1854790bc2151.js | 1 - ...da3212991542668.js => 8098-9c93335cc7aa2e14.js} | 2 +- ...d1d42f30f79c699.js => 8143-501ad14e8bdf0c7d.js} | 0 .../_next/static/chunks/8160-c965bc78b5ab7bb4.js | 1 - .../_next/static/chunks/8347-0845abae9a2a5d9e.js | 1 - .../_next/static/chunks/8431-005b8e8885a17d1c.js | 1 + .../_next/static/chunks/8524-a28eab17695cc44f.js | 1 + .../_next/static/chunks/8529-c1b0a14d2f5ce299.js | 1 + ...da64707ff7b4f2e.js => 8542-99458744d213223a.js} | 8 ++++---- .../_next/static/chunks/874-76e69d76bf193ece.js | 1 + .../_next/static/chunks/874-93132d6951f9f945.js | 1 - .../_next/static/chunks/9111-864f745f4eefb1cd.js | 1 + .../_next/static/chunks/9111-ca07479fa2062306.js | 1 - .../_next/static/chunks/9411-c310347bea2a098e.js | 1 - .../_next/static/chunks/9877-6dd6938300a37090.js | 1 + ...4ead20f98f45ddf.js => page-98349ddda3f35fcc.js} | 2 +- ...c781ccfe853de48.js => page-43b2447c9c7717ef.js} | 2 +- ...d258b7776bc3ce7.js => page-ef0e965b0367715f.js} | 2 +- ...45792bdf6b31029.js => page-f9cf760195f957df.js} | 2 +- ...a14fcce61a98cae.js => page-3ddbfbeba5980e8f.js} | 2 +- .../experimental/prompts/page-39f96290454d3782.js | 1 + .../experimental/prompts/page-be80069385d6d581.js | 1 - ...adbdbc7a0844ed6.js => page-2113f596126c0e47.js} | 2 +- .../guardrails/page-734b9d150996ef5b.js | 1 + .../guardrails/page-93dbf37f5765488e.js | 1 - .../app/(dashboard)/layout-bee012ca8e835663.js | 1 + .../app/(dashboard)/layout-e6c01f120876bb3b.js | 1 - ...b6d711a6e6e47a8.js => page-ad205c8a9586cdcc.js} | 2 +- .../(dashboard)/model-hub/page-b9a4a4b9d126a7a8.js | 1 + .../(dashboard)/model-hub/page-f87904b9acc850df.js | 1 - ...1cb8448fd40a510.js => page-899f91bab8ea74ee.js} | 2 +- .../organizations/page-8ca9d4b8cafe0a94.js | 1 + .../organizations/page-930a3a6c479fdc56.js | 1 - .../playground/page-d4c33ed51687ccfb.js | 1 + ...90040b2c5b2d07f.js => page-9be2d59e59f0d5fb.js} | 2 +- .../logging-and-alerts/page-a190d7b38426acea.js | 1 + .../logging-and-alerts/page-a3561c81525bb1c4.js | 1 - .../router-settings/page-e094ec2b3095ba8b.js | 1 + .../router-settings/page-e279c78de9481f8b.js | 1 - ...243061b79b938ba.js => page-48a59f58d38bb62f.js} | 2 +- ...fe5d2c441bb3959.js => page-d74c5df4b3dd7f31.js} | 2 +- .../(dashboard)/test-key/page-142020f9e6dfa205.js | 1 - .../(dashboard)/test-key/page-583efeab91de9dd7.js | 1 + ...ba9e4bed2a45b2c.js => page-e96879c826ebdb2d.js} | 2 +- .../tools/vector-stores/page-63482bd2e2962cff.js | 1 - .../tools/vector-stores/page-eee130405cbbed2b.js | 1 + .../app/(dashboard)/usage/page-62e98a09eac1291f.js | 1 - .../app/(dashboard)/usage/page-a153c162b0685877.js | 1 + ...41741916f7fa54a.js => page-ae7b0cc3203d87d2.js} | 2 +- .../virtual-keys/page-84ea7111221d0f49.js | 1 - .../virtual-keys/page-aa258ee46b094d8f.js | 1 + ...cedabc8a0fb7a.js => layout-8a0b62f2e33934fe.js} | 2 +- .../chunks/app/model_hub/page-a3c078e7643ba9f5.js | 1 - .../chunks/app/model_hub/page-ce980be1c9e07fdc.js | 1 + .../app/model_hub_table/page-24229db8a1eb2a72.js | 1 + .../app/model_hub_table/page-c7afd46ac073b5b1.js | 1 - .../chunks/app/onboarding/page-8236dd58aa6811bf.js | 1 + .../chunks/app/onboarding/page-9d2237541bf5adf9.js | 1 - .../static/chunks/app/page-366f0e8180b3165b.js | 1 - .../static/chunks/app/page-bd3da0b9cf9965a0.js | 1 + ...ec5b2d5e671.js => main-app-77a6ca3c04ee9adf.js} | 2 +- ...93cf31c84027372.js => main-ed1370b6a3ccbff3.js} | 2 +- .../out/_next/static/css/b55bcceaaf7fe863.css | 3 +++ .../out/_next/static/css/d1d84bbc374a5a03.css | 3 --- litellm/proxy/_experimental/out/api-reference.html | 1 + litellm/proxy/_experimental/out/api-reference.txt | 8 ++++---- .../_experimental/out/api-reference/index.html | 1 - .../out/experimental/api-playground.html | 2 +- .../out/experimental/api-playground.txt | 8 ++++---- .../_experimental/out/experimental/budgets.html | 2 +- .../_experimental/out/experimental/budgets.txt | 8 ++++---- .../_experimental/out/experimental/caching.html | 2 +- .../_experimental/out/experimental/caching.txt | 8 ++++---- .../_experimental/out/experimental/old-usage.html | 2 +- .../_experimental/out/experimental/old-usage.txt | 8 ++++---- .../_experimental/out/experimental/prompts.html | 2 +- .../_experimental/out/experimental/prompts.txt | 8 ++++---- .../out/experimental/tag-management.html | 2 +- .../out/experimental/tag-management.txt | 8 ++++---- litellm/proxy/_experimental/out/guardrails.html | 1 + litellm/proxy/_experimental/out/guardrails.txt | 8 ++++---- litellm/proxy/_experimental/out/index.html | 2 +- litellm/proxy/_experimental/out/index.txt | 6 +++--- litellm/proxy/_experimental/out/logs.html | 1 + litellm/proxy/_experimental/out/logs.txt | 8 ++++---- litellm/proxy/_experimental/out/logs/index.html | 5 ----- litellm/proxy/_experimental/out/model-hub.html | 1 + litellm/proxy/_experimental/out/model-hub.txt | 8 ++++---- .../proxy/_experimental/out/model-hub/index.html | 1 - litellm/proxy/_experimental/out/model_hub.txt | 6 +++--- .../proxy/_experimental/out/model_hub_table.html | 1 + .../proxy/_experimental/out/model_hub_table.txt | 6 +++--- .../_experimental/out/model_hub_table/index.html | 1 - .../_experimental/out/models-and-endpoints.html | 1 + .../_experimental/out/models-and-endpoints.txt | 8 ++++---- .../out/models-and-endpoints/index.html | 5 ----- litellm/proxy/_experimental/out/onboarding.html | 1 + litellm/proxy/_experimental/out/onboarding.txt | 6 +++--- litellm/proxy/_experimental/out/organizations.html | 1 + litellm/proxy/_experimental/out/organizations.txt | 8 ++++---- .../_experimental/out/organizations/index.html | 5 ----- litellm/proxy/_experimental/out/playground.html | 1 + litellm/proxy/_experimental/out/playground.txt | 14 ++++++++++++++ .../_experimental/out/settings/admin-settings.html | 2 +- .../_experimental/out/settings/admin-settings.txt | 8 ++++---- .../out/settings/logging-and-alerts.html | 2 +- .../out/settings/logging-and-alerts.txt | 8 ++++---- .../out/settings/router-settings.html | 2 +- .../_experimental/out/settings/router-settings.txt | 8 ++++---- .../proxy/_experimental/out/settings/ui-theme.html | 2 +- .../proxy/_experimental/out/settings/ui-theme.txt | 8 ++++---- .../out/{teams/index.html => teams.html} | 2 +- litellm/proxy/_experimental/out/teams.txt | 8 ++++---- litellm/proxy/_experimental/out/test-key.html | 1 + litellm/proxy/_experimental/out/test-key.txt | 8 ++++---- .../proxy/_experimental/out/test-key/index.html | 1 - .../proxy/_experimental/out/tools/mcp-servers.html | 2 +- .../proxy/_experimental/out/tools/mcp-servers.txt | 8 ++++---- .../_experimental/out/tools/vector-stores.html | 2 +- .../_experimental/out/tools/vector-stores.txt | 8 ++++---- litellm/proxy/_experimental/out/usage.html | 1 + litellm/proxy/_experimental/out/usage.txt | 8 ++++---- litellm/proxy/_experimental/out/usage/index.html | 5 ----- litellm/proxy/_experimental/out/users.html | 1 + litellm/proxy/_experimental/out/users.txt | 8 ++++---- litellm/proxy/_experimental/out/users/index.html | 5 ----- litellm/proxy/_experimental/out/virtual-keys.html | 1 + litellm/proxy/_experimental/out/virtual-keys.txt | 8 ++++---- .../_experimental/out/virtual-keys/index.html | 5 ----- 193 files changed, 226 insertions(+), 227 deletions(-) rename litellm/proxy/_experimental/out/_next/static/{zzKcMfj4Db-ZZ7hcspdhR => RP6qFLO9Sa0mS2DSQFL5i}/_buildManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/{zzKcMfj4Db-ZZ7hcspdhR => RP6qFLO9Sa0mS2DSQFL5i}/_ssgManifest.js (100%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1160-08491effeedbaae3.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/131-d505623ce13e3958.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1491-80dbf1ebc561e9b6.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1491-d6d75a98cb22e323.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1518-708cd3bb943546cf.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/1567-5650c3aa172cd633.js rename litellm/proxy/_experimental/out/_next/static/chunks/{1598-b4dd64df9427ab1c.js => 1598-a69df0ba9a12c4f3.js} (63%) rename litellm/proxy/_experimental/out/_next/static/chunks/{1739-1495570659e854dc.js => 1739-b555a2473b0946bb.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{7220-6db32c86493cbcd6.js => 1960-95ee080c178a4e18.js} (82%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2284-6840f6cabd9dbd7e.js => 1994-2289bfd6e7d65533.js} (54%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2004-2acee05add5250f4.js => 2004-a8136543e13a4de3.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2012-0f598b370e2766b0.js => 2012-db5a85de324150e9.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2019-02887e9caea1e319.js => 2019-2005428b5853a7a6.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2202-404db85e26800c7a.js => 2202-c4a45b6cf77ca4a9.js} (100%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2249-b6e8bcded6cfd197.js rename litellm/proxy/_experimental/out/_next/static/chunks/{2273-40f31894fc8c0e24.js => 2273-4649f34440d8d399.js} (100%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2312-02f166690919a2cc.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2358-ff0b1979f8905d3f.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2409-33ed6e753f8afe9b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2409-7de3355d049b247b.js rename litellm/proxy/_experimental/out/_next/static/chunks/{7519-c017e0c3b119c340.js => 2837-753bc846fbc41ba6.js} (99%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/2924-98771dd2cd6a8a3d.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3135-fbf13217ca7d2de9.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3603-dd19ac8e31e4bc25.js rename litellm/proxy/_experimental/out/_next/static/chunks/{3655-7dee4252a2b35874.js => 3655-03331223540a9ced.js} (59%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3801-42c33bd07afe4ba2.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3801-a6c59be67a81d7dc.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/395-9ca940835820f5fa.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/395-e2a4326c962bdd18.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/3978-5fb71317e3cb900e.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4138-bbc045e58cfe8f02.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4289-988122838795ecce.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4289-fcd5e435fdba5a64.js rename litellm/proxy/_experimental/out/_next/static/chunks/{4292-c755f0704f2336bb.js => 4292-4536c43afb350f73.js} (65%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/4819-337db995ee8dbe87.js rename litellm/proxy/_experimental/out/_next/static/chunks/{8624-54b4dee8acfc3279.js => 5296-577c93bbe6971079.js} (97%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5402-4a78ee24b996de08.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5407-9cee54da5c03162b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5572-674265ad2317d8c3.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/5869-7be6e0175610c221.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/603-ced069c6ecb41baf.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6188-003d982a932f88e0.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6204-5c34c11f5a32fb09.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6294-f5649a0ee9b63fd2.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/630-57f1c278500ab515.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/6478-f5fd91a406dd265e.js rename litellm/proxy/_experimental/out/_next/static/chunks/{6600-bbc506133f5d9c17.js => 6600-6564d6b9d0d57aaa.js} (99%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7013-c3a11adde9383db4.js rename litellm/proxy/_experimental/out/_next/static/chunks/{710-7549fbb4532c452b.js => 710-56c50cf7f4bc9cca.js} (68%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7155-5d31a6a1ad834cf0.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7155-5e95ae8488e27e4a.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7164-6e382f189a96cd83.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7164-cc01ff2b7f3ae404.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7526-2769cf614be0678b.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/7526-c6c315293f6101de.js rename litellm/proxy/_experimental/out/_next/static/chunks/{773-0cc38c7be8272a14.js => 773-a2e903ad8cb3a0c8.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{4886-b4f3c55775f088dd.js => 7901-fdb65b7b40365397.js} (70%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/795-79942bf68d21d57b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8040-dc82c32ea5959cb5.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8049-db73a2d52dc11ceb.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8049-fd60df8124c6c5dd.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8077-dca1854790bc2151.js rename litellm/proxy/_experimental/out/_next/static/chunks/{8098-3da3212991542668.js => 8098-9c93335cc7aa2e14.js} (78%) rename litellm/proxy/_experimental/out/_next/static/chunks/{8143-4d1d42f30f79c699.js => 8143-501ad14e8bdf0c7d.js} (100%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8160-c965bc78b5ab7bb4.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8347-0845abae9a2a5d9e.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8431-005b8e8885a17d1c.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8524-a28eab17695cc44f.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/8529-c1b0a14d2f5ce299.js rename litellm/proxy/_experimental/out/_next/static/chunks/{4388-0da64707ff7b4f2e.js => 8542-99458744d213223a.js} (55%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/874-76e69d76bf193ece.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/874-93132d6951f9f945.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9111-864f745f4eefb1cd.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9111-ca07479fa2062306.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9411-c310347bea2a098e.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/9877-6dd6938300a37090.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/api-reference/{page-a4ead20f98f45ddf.js => page-98349ddda3f35fcc.js} (80%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/api-playground/{page-5c781ccfe853de48.js => page-43b2447c9c7717ef.js} (99%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/budgets/{page-6d258b7776bc3ce7.js => page-ef0e965b0367715f.js} (98%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/caching/{page-345792bdf6b31029.js => page-f9cf760195f957df.js} (96%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/old-usage/{page-aa14fcce61a98cae.js => page-3ddbfbeba5980e8f.js} (97%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/prompts/page-39f96290454d3782.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/prompts/page-be80069385d6d581.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/experimental/tag-management/{page-9adbdbc7a0844ed6.js => page-2113f596126c0e47.js} (67%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/guardrails/page-734b9d150996ef5b.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/guardrails/page-93dbf37f5765488e.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/layout-bee012ca8e835663.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/layout-e6c01f120876bb3b.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/logs/{page-0b6d711a6e6e47a8.js => page-ad205c8a9586cdcc.js} (63%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/model-hub/page-b9a4a4b9d126a7a8.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/model-hub/page-f87904b9acc850df.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/models-and-endpoints/{page-41cb8448fd40a510.js => page-899f91bab8ea74ee.js} (81%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/organizations/page-8ca9d4b8cafe0a94.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/organizations/page-930a3a6c479fdc56.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/playground/page-d4c33ed51687ccfb.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/admin-settings/{page-590040b2c5b2d07f.js => page-9be2d59e59f0d5fb.js} (96%) create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/logging-and-alerts/page-a190d7b38426acea.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/logging-and-alerts/page-a3561c81525bb1c4.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/router-settings/page-e094ec2b3095ba8b.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/router-settings/page-e279c78de9481f8b.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/settings/ui-theme/{page-f243061b79b938ba.js => page-48a59f58d38bb62f.js} (99%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/teams/{page-1fe5d2c441bb3959.js => page-d74c5df4b3dd7f31.js} (59%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/test-key/page-142020f9e6dfa205.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/test-key/page-583efeab91de9dd7.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/tools/mcp-servers/{page-5ba9e4bed2a45b2c.js => page-e96879c826ebdb2d.js} (95%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/tools/vector-stores/page-63482bd2e2962cff.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/tools/vector-stores/page-eee130405cbbed2b.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/usage/page-62e98a09eac1291f.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/usage/page-a153c162b0685877.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/users/{page-041741916f7fa54a.js => page-ae7b0cc3203d87d2.js} (96%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/virtual-keys/page-84ea7111221d0f49.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/(dashboard)/virtual-keys/page-aa258ee46b094d8f.js rename litellm/proxy/_experimental/out/_next/static/chunks/app/{layout-820cedabc8a0fb7a.js => layout-8a0b62f2e33934fe.js} (94%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-a3c078e7643ba9f5.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-ce980be1c9e07fdc.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-24229db8a1eb2a72.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-c7afd46ac073b5b1.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-8236dd58aa6811bf.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-9d2237541bf5adf9.js delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-366f0e8180b3165b.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-bd3da0b9cf9965a0.js rename litellm/proxy/_experimental/out/_next/static/chunks/{main-app-c6945ec5b2d5e671.js => main-app-77a6ca3c04ee9adf.js} (81%) rename litellm/proxy/_experimental/out/_next/static/chunks/{main-a93cf31c84027372.js => main-ed1370b6a3ccbff3.js} (67%) create mode 100644 litellm/proxy/_experimental/out/_next/static/css/b55bcceaaf7fe863.css delete mode 100644 litellm/proxy/_experimental/out/_next/static/css/d1d84bbc374a5a03.css create mode 100644 litellm/proxy/_experimental/out/api-reference.html delete mode 100644 litellm/proxy/_experimental/out/api-reference/index.html create mode 100644 litellm/proxy/_experimental/out/guardrails.html create mode 100644 litellm/proxy/_experimental/out/logs.html delete mode 100644 litellm/proxy/_experimental/out/logs/index.html create mode 100644 litellm/proxy/_experimental/out/model-hub.html delete mode 100644 litellm/proxy/_experimental/out/model-hub/index.html create mode 100644 litellm/proxy/_experimental/out/model_hub_table.html delete mode 100644 litellm/proxy/_experimental/out/model_hub_table/index.html create mode 100644 litellm/proxy/_experimental/out/models-and-endpoints.html delete mode 100644 litellm/proxy/_experimental/out/models-and-endpoints/index.html create mode 100644 litellm/proxy/_experimental/out/onboarding.html create mode 100644 litellm/proxy/_experimental/out/organizations.html delete mode 100644 litellm/proxy/_experimental/out/organizations/index.html create mode 100644 litellm/proxy/_experimental/out/playground.html create mode 100644 litellm/proxy/_experimental/out/playground.txt rename litellm/proxy/_experimental/out/{teams/index.html => teams.html} (80%) create mode 100644 litellm/proxy/_experimental/out/test-key.html delete mode 100644 litellm/proxy/_experimental/out/test-key/index.html create mode 100644 litellm/proxy/_experimental/out/usage.html delete mode 100644 litellm/proxy/_experimental/out/usage/index.html create mode 100644 litellm/proxy/_experimental/out/users.html delete mode 100644 litellm/proxy/_experimental/out/users/index.html create mode 100644 litellm/proxy/_experimental/out/virtual-keys.html delete mode 100644 litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js b/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js deleted file mode 100644 index 7c33ceca243..00000000000 --- a/litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[1116],{69993:function(e,t,r){r.d(t,{Z:function(){return s}});var n=r(1119),o=r(2265),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M300 328a60 60 0 10120 0 60 60 0 10-120 0zM852 64H172c-17.7 0-32 14.3-32 32v660c0 17.7 14.3 32 32 32h680c17.7 0 32-14.3 32-32V96c0-17.7-14.3-32-32-32zm-32 660H204V128h616v596zM604 328a60 60 0 10120 0 60 60 0 10-120 0zm250.2 556H169.8c-16.5 0-29.8 14.3-29.8 32v36c0 4.4 3.3 8 7.4 8h729.1c4.1 0 7.4-3.6 7.4-8v-36c.1-17.7-13.2-32-29.7-32zM664 508H360c-4.4 0-8 3.6-8 8v60c0 4.4 3.6 8 8 8h304c4.4 0 8-3.6 8-8v-60c0-4.4-3.6-8-8-8z"}}]},name:"robot",theme:"outlined"},i=r(55015),s=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,n.Z)({},e,{ref:t,icon:a}))})},92858:function(e,t,r){r.d(t,{Z:function(){return S}});var n=r(5853),o=r(2265),a=r(62963),i=r(90945),s=r(13323),l=r(17684),c=r(80004),u=r(93689),d=r(38198),f=r(47634),m=r(56314),h=r(27847),p=r(64518);let g=(0,o.createContext)(null),v=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-description-".concat(r),...a}=e,i=function e(){let t=(0,o.useContext)(g);if(null===t){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return t}(),s=(0,u.T)(t);(0,p.e)(()=>i.register(n),[n,i.register]);let c={ref:s,...i.props,id:n};return(0,h.sY)({ourProps:c,theirProps:a,slot:i.slot||{},defaultTag:"p",name:i.name||"Description"})}),{});var w=r(37388);let k=(0,o.createContext)(null),b=Object.assign((0,h.yV)(function(e,t){let r=(0,l.M)(),{id:n="headlessui-label-".concat(r),passive:a=!1,...i}=e,s=function e(){let t=(0,o.useContext)(k);if(null===t){let t=Error("You used a