scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge pull request #25562 from BerriAI/litellm_internal_staging_04_11_2026

Browse source 
Litellm internal staging 04 11 2026
This commit is contained in:
yuneng-jiang 2026-04-11 15:55:29 -07:00 • committed by GitHub
commit 99cb3af1a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
56 changed files with 3235 additions and 348 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -197,6 +197,7 @@ router_settings:
| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
| skip_system_message_in_guardrail | boolean | If true, unified guardrails omit `role: system` from scanned input on **chat completions** and **Anthropic `/v1/messages`** only; the LLM still receives full messages. Per-guardrail override: `litellm_params.skip_system_message_in_guardrail` on each guardrail. [Guardrails quick start](./guardrails/quick_start#skip-system-messages-in-guardrail-evaluation) |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |

117
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -9,6 +9,7 @@ Setup Prompt Injection Detection, PII Masking on LiteLLM Proxy (AI Gateway)
## 1. Define guardrails on your LiteLLM config.yaml
Set your guardrails under the `guardrails` section
```yaml
model_list:
- model_name: gpt-3.5-turbo
@ -82,27 +83,58 @@ For generic guardrail APIs you can also set **static headers** (`headers`: key/v
- `during_call` Run **during** LLM call, on **input** Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes
- A list of the above values to run multiple modes, e.g. `mode: [pre_call, post_call]`
### Skip system messages in guardrail evaluation
You can stop **unified** guardrails from scanning `role: system` content while still sending the full `messages` list to the model.
**Global** — in `litellm_settings`:
```yaml
litellm_settings:
skip_system_message_in_guardrail: true
```
**Per guardrail** — under that guardrail’s `litellm_params`: set `skip_system_message_in_guardrail: true` or `false`. If omitted, the global `litellm_settings` value is used; per-guardrail `false` forces system messages to be included even when the global flag is `true`.
**Via LiteLLM UI** — when **creating** or **editing** a guardrail in the LiteLLM Admin Dashboard, set **Skip system messages in guardrail** (under Basic Info on create, or in the edit / guardrail settings flows):
| UI option | Effect |
| ------------------------------------- | -------------------------------------------------------------------------------------- |
| **Use global default** | Uses `litellm_settings.skip_system_message_in_guardrail` from your proxy config |
| **Yes — exclude from guardrail scan** | Sets per-guardrail `skip_system_message_in_guardrail: true` |
| **No — always include in scan** | Sets per-guardrail `skip_system_message_in_guardrail: false` (overrides a global skip) |
<Image
img={require('../../../img/skip_system_message_guardrail_ui.png')}
alt="Create guardrail: Skip system messages in guardrail dropdown with Use global default, Yes exclude from guardrail scan, and No always include in scan"
style={{ width: '100%', maxWidth: '900px', height: 'auto' }}
/>
**Where this applies:** Only the **unified** guardrail path (providers that implement `apply_guardrail` and run through LiteLLM’s message translation layer) on **OpenAI Chat Completions** (`/v1/chat/completions`) and **Anthropic Messages** (`/v1/messages`). Examples include Presidio, Bedrock guardrails, `litellm_content_filter`, OpenAI Moderation, Generic Guardrail API, and custom code guardrails that define `apply_guardrail`.
**Where this does *not* apply:** Guardrails that run only via direct hooks on the raw request (e.g. Lakera v2, Aporia, DynamoAI, Javelin, Lasso, Pangea, Model Armor, Azure Content Safety hooks, Guardrails AI, AIM, tool permission, MCP security). It also does not apply to other routes until those endpoints use the same translation layer (e.g. Responses API, embeddings, speech).
### Load Balancing Guardrails
Need to distribute guardrail requests across multiple accounts or regions? See [Guardrail Load Balancing](./guardrail_load_balancing.md) for details on:
- Load balancing across multiple AWS Bedrock accounts (useful for rate limit management)
- Weighted distribution across guardrail instances
- Multi-region guardrail deployments
## 2. Start LiteLLM Gateway
## 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
## 3. Test request
## 3. Test request
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Unsuccessful call" value = "not-allowed">
Expect this to fail since since `ishaan@berri.ai` in the request is PII
@ -141,9 +173,9 @@ Expected response on failure
```
</TabItem>
<TabItem label="Successful Call " value = "allowed">
```shell
curl -i http://localhost:4000/v1/chat/completions \
@ -158,10 +190,8 @@ curl -i http://localhost:4000/v1/chat/completions \
}'
```
</TabItem>
</Tabs>
## **Default On Guardrails**
@ -183,7 +213,6 @@ guardrails:
In this request, the guardrail `aporia-pre-guard` will run on every request because `default_on: true` is set.
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
@ -207,6 +236,7 @@ x-litellm-applied-guardrails: aporia-pre-guard
### Guardrail Policies
Need more control? Use [Guardrail Policies](./guardrail_policies.md) to:
- Group guardrails into reusable policies
- Enable/disable guardrails for specific teams, keys, or models
- Inherit from existing policies and override specific guardrails
@ -217,7 +247,6 @@ Need more control? Use [Guardrail Policies](./guardrail_policies.md) to:
Pass `guardrails` to your request body to test it
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
@ -239,7 +268,6 @@ Follow this simple workflow to implement and tune guardrails:
First, check what guardrails are available and their parameters:
Call `/guardrails/list` to view available guardrails and the guardrail info (supported parameters, description, etc)
```shell
@ -271,9 +299,12 @@ Expected response
}
```
>
This config will return the `/guardrails/list` response above. The `guardrail_info` field is optional and you can add any fields under info for consumers of your guardrail
>
```yaml
- guardrail_name: "aporia-post-guard"
litellm_params:
@ -291,9 +322,10 @@ This config will return the `/guardrails/list` response above. The `guardrail_in
type: "boolean"
```
### 2. Apply Guardrails
Add selected guardrails to your chat completion request:
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
@ -322,7 +354,6 @@ curl -i http://localhost:4000/v1/chat/completions \
}'
```
### 4. ✨ Pass Dynamic Parameters to Guardrail
:::info
@ -334,9 +365,8 @@ curl -i http://localhost:4000/v1/chat/completions \
Use this to pass additional parameters to the guardrail API call. e.g. things like success threshold. **[See `guardrails` spec for more details](#spec-guardrails-parameter)**
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
Set `guardrails={"aporia-pre-guard": {"extra_body": {"success_threshold": 0.9}}}` to pass additional parameters to the guardrail
@ -371,10 +401,10 @@ response = client.chat.completions.create(
print(response)
```
</TabItem>
<TabItem value="Curl" label="Curl Request">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
@ -396,11 +426,8 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}
}'
```
</TabItem>
</Tabs>
@ -426,9 +453,6 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g
<Image img={require('../../../img/gd_fail.png')} />
### ✨ Control Guardrails per API Key
:::info
@ -438,12 +462,12 @@ Monitor which guardrails were executed and whether they passed or failed. e.g. g
:::
Use this to control what guardrails run per API Key. In this tutorial we only want the following guardrails to run for 1 API Key
- `guardrails`: ["aporia-pre-guard", "aporia-post-guard"]
**Step 1** Create Key with guardrail settings
<Tabs>
<TabItem value="/key/generate" label="/key/generate">
```shell
curl -X POST 'http://0.0.0.0:4000/key/generate' \
@ -454,8 +478,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
}'
```
</TabItem>
<TabItem value="/key/update" label="/key/update">
```shell
curl --location 'http://0.0.0.0:4000/key/update' \
@ -467,8 +490,7 @@ curl --location 'http://0.0.0.0:4000/key/update' \
}'
```
</TabItem>
</Tabs>
**Step 2** Test it with new key
@ -499,8 +521,7 @@ Run guardrails based on the user-agent header. This is useful for running pre-ca
Both `default` and tag values can be a single mode string or a list of modes.
<Tabs>
<TabItem value="single" label="Single Default Mode">
```yaml
model_list:
@ -522,11 +543,10 @@ guardrails:
default_on: true # run on every request
```
</TabItem>
<TabItem value="multi" label="Multiple Default Modes">
```yaml
model_list:
Per guardrailmodel_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
@ -545,8 +565,7 @@ guardrails:
default_on: true
```
</TabItem>
<TabItem value="tag-list" label="Multiple Tag Modes">
```yaml
model_list:
@ -568,8 +587,6 @@ guardrails:
default_on: true
```
</TabItem>
</Tabs>
### ✨ Model-level Guardrails
@ -580,10 +597,8 @@ guardrails:
:::
This is great for cases when you have an on-prem and hosted model, and just want to run prevent sending PII to the hosted model.
```yaml
model_list:
- model_name: claude-sonnet-4
@ -620,8 +635,7 @@ guardrails:
:::
#### 1. Disable team from modifying guardrails
#### 1. Disable team from modifying guardrails
```bash
curl -X POST 'http://0.0.0.0:4000/team/update' \
@ -633,7 +647,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \
}'
```
#### 2. Try to disable guardrails for a call
#### 2. Try to disable guardrails for a call
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
@ -672,8 +686,7 @@ Expect to NOT see `+1 412-612-9992` in your server logs on your callback.
The `pii_masking` guardrail ran on this request because api key=sk-jNm1Zar7XfNdZXp49Z1kSQ has `"permissions": {"pii_masking": true}`
:::
## Specification
## Specification
### `guardrails` Configuration on YAML
@ -723,6 +736,7 @@ The `guardrails` parameter can be passed to any LiteLLM Proxy endpoint (`/chat/c
#### Format Options
1. Simple List Format:
```python
"guardrails": [
"aporia-pre-guard",
@ -730,9 +744,10 @@ The `guardrails` parameter can be passed to any LiteLLM Proxy endpoint (`/chat/c
]
```
2. Advanced Dictionary Format:
1. Advanced Dictionary Format:
In this format the dictionary key is `guardrail_name` you want to run
```python
"guardrails": {
"aporia-pre-guard": {
@ -745,6 +760,7 @@ In this format the dictionary key is `guardrail_name` you want to run
```
#### Type Definition
```python
guardrails: Union[
List[str], # Simple list of guardrail names
@ -754,3 +770,4 @@ guardrails: Union[
class DynamicGuardrailParams:
extra_body: Dict[str, Any] # Additional parameters for the guardrail
```

BIN
docs/my-website/img/skip_system_message_guardrail_ui.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 126 KiB

2
litellm/__init__.py
View file

@ -164,6 +164,7 @@ initialized_langfuse_clients: int = 0
langfuse_default_tags: Optional[List[str]] = None
langsmith_batch_size: Optional[int] = None
prometheus_initialize_budget_metrics: Optional[bool] = False
prometheus_latency_buckets: Optional[List[float]] = None
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
@ -203,6 +204,7 @@ add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
store_audit_logs = False # Enterprise feature, allow users to see audit logs
skip_system_message_in_guardrail: bool = False
### end of callbacks #############
email: Optional[

26
litellm/containers/endpoint_factory.py
View file

@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Type
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.containers.utils import decode_managed_container_id_for_request
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.llms.custom_httpx.container_handler import generic_container_handler
@ -53,7 +54,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
@client
def endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@ -61,6 +62,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
):
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj")
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@ -76,15 +78,27 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
# Get provider config
litellm_params = GenericLiteLLMParams(**kwargs)
# Strip LiteLLM-managed container IDs before calling the provider API
# (OpenAI enforces max length 64 on container_id).
if "container_id" in kwargs and isinstance(kwargs["container_id"], str):
(
kwargs["container_id"],
resolved_custom_llm_provider,
litellm_params,
) = decode_managed_container_id_for_request(
container_id=kwargs["container_id"],
custom_llm_provider=resolved_custom_llm_provider,
litellm_params=litellm_params,
)
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for: {custom_llm_provider}"
f"Container provider config not found for: {resolved_custom_llm_provider}"
)
# Build optional params for logging
@ -96,7 +110,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
model="",
optional_params=optional_params,
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Use generic handler
@ -115,7 +129,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@ -133,7 +147,7 @@ def create_async_endpoint_function(
@client
async def async_endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,

202
litellm/containers/main.py
View file

@ -6,7 +6,10 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overloa
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.containers.utils import ContainerRequestUtils
from litellm.containers.utils import (
ContainerRequestUtils,
decode_managed_container_id_for_request,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.main import base_llm_http_handler
@ -48,7 +51,7 @@ async def acreate_container(
file_ids: Optional[List[str]] = None,
timeout=600, # default to 10 minutes
# LiteLLM specific params,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -122,7 +125,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
acreate_container: Literal[True],
**kwargs,
@ -139,7 +142,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
acreate_container: Literal[False] = False,
**kwargs,
@ -158,7 +161,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -247,7 +250,7 @@ def create_container(
# Set the correct call type for container creation
litellm_logging_obj.call_type = CallTypes.create_container.value
return base_llm_http_handler.container_create_handler(
container_obj = base_llm_http_handler.container_create_handler(
name=name,
container_create_request_params=container_create_request_params,
container_provider_config=container_provider_config,
@ -257,6 +260,17 @@ def create_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
if isinstance(container_obj, ContainerObject):
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return container_obj
except Exception as e:
raise litellm.exception_type(
@ -275,7 +289,7 @@ async def alist_containers(
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -348,7 +362,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_containers: Literal[True],
**kwargs,
@ -365,7 +379,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_containers: Literal[False] = False,
**kwargs,
@ -384,7 +398,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -481,7 +495,7 @@ def list_containers(
async def aretrieve_container(
container_id: str,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -548,7 +562,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aretrieve_container: Literal[True],
**kwargs,
@ -563,7 +577,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aretrieve_container: Literal[False] = False,
**kwargs,
@ -580,7 +594,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -594,6 +608,7 @@ def retrieve_container(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@ -615,16 +630,28 @@ def retrieve_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# True when input was a LiteLLM-managed ID (any length); needed to re-encode output for routing affinity
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@ -635,14 +662,14 @@ def retrieve_container(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.retrieve_container.value
return base_llm_http_handler.container_retrieve_handler(
container_id=container_id,
container_obj = base_llm_http_handler.container_retrieve_handler(
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@ -651,11 +678,33 @@ def retrieve_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(container_obj, ContainerObject):
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return container_obj
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@ -667,7 +716,7 @@ def retrieve_container(
async def adelete_container(
container_id: str,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -734,7 +783,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
adelete_container: Literal[True],
**kwargs,
@ -749,7 +798,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
adelete_container: Literal[False] = False,
**kwargs,
@ -766,7 +815,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@ -780,6 +829,7 @@ def delete_container(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@ -801,16 +851,28 @@ def delete_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# True when input was a LiteLLM-managed ID (any length); needed to re-encode output for routing affinity
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@ -821,14 +883,14 @@ def delete_container(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.delete_container.value
return base_llm_http_handler.container_delete_handler(
container_id=container_id,
delete_result = base_llm_http_handler.container_delete_handler(
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@ -837,11 +899,33 @@ def delete_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id in response with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(delete_result, DeleteContainerResult):
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
delete_result = ContainerRequestUtils.encode_container_id_in_response(
response_obj=delete_result,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return delete_result
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@ -856,7 +940,7 @@ async def alist_container_files(
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@ -930,7 +1014,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_container_files: Literal[True],
**kwargs,
@ -948,7 +1032,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_container_files: Literal[False] = False,
**kwargs,
@ -968,7 +1052,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@ -980,6 +1064,7 @@ def list_container_files(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@ -1001,16 +1086,26 @@ def list_container_files(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@ -1026,14 +1121,14 @@ def list_container_files(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.list_container_files.value
return base_llm_http_handler.container_file_list_handler(
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@ -1049,7 +1144,7 @@ def list_container_files(
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@ -1062,7 +1157,7 @@ async def aupload_container_file(
container_id: str,
file: FileTypes,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@ -1151,7 +1246,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aupload_container_file: Literal[True],
**kwargs,
@ -1167,7 +1262,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aupload_container_file: Literal[False] = False,
**kwargs,
@ -1185,7 +1280,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@ -1226,6 +1321,7 @@ def upload_container_file(
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@ -1247,16 +1343,26 @@ def upload_container_file(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@ -1267,7 +1373,7 @@ def upload_container_file(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
@ -1282,14 +1388,14 @@ def upload_container_file(
extra_query=extra_query,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
file=file,
)
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,

93
litellm/containers/utils.py
View file

@ -1,10 +1,38 @@
from typing import Dict
from typing import Any, Dict, Optional, TypeVar
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerListOptionalRequestParams,
)
from litellm.types.router import GenericLiteLLMParams
def decode_managed_container_id_for_request(
container_id: str,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
) -> tuple[str, str, GenericLiteLLMParams]:
"""Decode a LiteLLM-managed container ID for upstream API calls.
Returns:
(original_container_id, resolved_provider, updated_litellm_params)
"""
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
decoded_model_id = decoded.get("model_id")
if decoded_model_id and not litellm_params.get("model_id"):
litellm_params["model_id"] = decoded_model_id
return original_container_id, custom_llm_provider, litellm_params
T = TypeVar("T")
class ContainerRequestUtils:
@ -68,3 +96,66 @@ class ContainerRequestUtils:
container_list_optional_params[param] = passed_params[param] # type: ignore
return container_list_optional_params
@staticmethod
def encode_container_id_in_response(
response_obj: T,
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
) -> T:
"""
Encode container_id in response object with provider/model metadata for routing.
This mirrors the responses API pattern where response IDs are encoded with
routing metadata so follow-up calls can route to the correct provider.
Encodes when:
1. litellm_metadata contains model_info.id (indicating router/proxy usage), OR
2. extra_body contains target_model_names (indicating model-specific routing)
Direct SDK calls with explicit custom_llm_provider and no routing hints return raw IDs.
Args:
response_obj: Response object with an `id` attribute (ContainerObject, DeleteContainerResult, etc.)
custom_llm_provider: Provider name (e.g., "azure", "openai")
litellm_metadata: Optional litellm_metadata dict that may contain model_info.id
extra_body: Optional extra_body dict that may contain target_model_names
Returns:
The same response object with encoded container_id (if routing metadata present)
"""
# Extract model_id from litellm_metadata
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Check if we should encode based on routing metadata
should_encode = False
# Case 1: Router/proxy usage (model_id from router)
if model_id is not None:
should_encode = True
# Case 2: target_model_names in extra_body (model-specific routing)
if extra_body and "target_model_names" in extra_body:
should_encode = True
# Extract model_id from target_model_names if not already set
if model_id is None:
target_models = extra_body["target_model_names"]
# Use first model as model_id for encoding
if isinstance(target_models, str):
model_id = target_models.split(",")[0].strip()
elif isinstance(target_models, list) and len(target_models) > 0:
model_id = str(target_models[0]).strip()
# Only encode if we have routing metadata
if should_encode and response_obj and hasattr(response_obj, "id"):
encoded_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
container_id=response_obj.id,
)
response_obj.id = encoded_id
return response_obj

17
litellm/integrations/prometheus.py
View file

@ -86,6 +86,11 @@ class PrometheusLogger(CustomLogger):
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
_custom_buckets = litellm.prometheus_latency_buckets
self.latency_buckets = (
tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS
)
# Create metric factory functions
self._counter_factory = self._create_metric_factory(Counter)
self._gauge_factory = self._create_metric_factory(Gauge)
@ -114,14 +119,14 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric(
"litellm_request_total_latency_metric"
),
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
self.litellm_llm_api_latency_metric = self._histogram_factory(
"litellm_llm_api_latency_metric",
"Total latency (seconds) for a models LLM API call",
labelnames=self.get_labels_for_metric("litellm_llm_api_latency_metric"),
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory(
@ -137,7 +142,7 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric(
"litellm_llm_api_time_to_first_token_metric"
),
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
# Counter for spend
@ -314,7 +319,7 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric(
"litellm_overhead_latency_metric"
),
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
# Request queue time metric
@ -324,7 +329,7 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric(
"litellm_request_queue_time_seconds"
),
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
# Guardrail metrics
@ -332,7 +337,7 @@ class PrometheusLogger(CustomLogger):
"litellm_guardrail_latency_seconds",
"Latency (seconds) for guardrail execution",
labelnames=["guardrail_name", "status", "error_type", "hook_type"],
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
self.litellm_guardrail_errors_total = self._counter_factory(

8
litellm/integrations/prometheus_services.py
View file

@ -5,6 +5,7 @@
from typing import Dict, List, Optional, Union
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.types.integrations.prometheus import LATENCY_BUCKETS
from litellm.types.services import (
@ -35,6 +36,11 @@ class PrometheusServicesLogger:
"Missing prometheus_client. Run `pip install prometheus-client`"
)
_custom_buckets = litellm.prometheus_latency_buckets
self.latency_buckets = (
tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS
)
self.Histogram = Histogram
self.Counter = Counter
self.Gauge = Gauge
@ -130,7 +136,7 @@ class PrometheusServicesLogger:
metric_name,
"Latency for {} service".format(service),
labelnames=[service],
buckets=LATENCY_BUCKETS,
buckets=self.latency_buckets,
)
def create_gauge(self, service: str, type_of_request: str):

43
litellm/integrations/s3_v2.py
View file

@ -7,6 +7,7 @@ NOTE 1: S3 does not provide a BATCH PUT API endpoint, so we create tasks to uplo
"""
import asyncio
import time
from datetime import datetime
from typing import List, Optional, cast
@ -403,11 +404,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
# Prepare the signed headers
signed_headers = dict(aws_request.headers.items())
# Make the request
response = await self.async_httpx_client.put(
url, data=json_string, headers=signed_headers
)
response.raise_for_status()
# Make the request with retry for transient S3 errors (500/503)
max_retries = 3
for attempt in range(max_retries):
response = await self.async_httpx_client.put(
url, data=json_string, headers=signed_headers
)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
f"S3 upload returned {response.status_code}, retrying in {wait_time}s "
f"(attempt {attempt + 1}/{max_retries}) "
f"key={batch_logging_element.s3_object_key}"
)
await asyncio.sleep(wait_time)
continue
response.raise_for_status()
break
except Exception as e:
verbose_logger.exception(f"Error uploading to s3: {str(e)}")
self.handle_callback_failure(callback_name="S3Logger")
@ -582,9 +595,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
if self.s3_verify is not None
else None
)
# Make the request
response = httpx_client.put(url, data=json_string, headers=signed_headers)
response.raise_for_status()
# Make the request with retry for transient S3 errors (500/503)
max_retries = 3
for attempt in range(max_retries):
response = httpx_client.put(
url, data=json_string, headers=signed_headers
)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
f"S3 upload returned {response.status_code}, retrying in {wait_time}s "
f"(attempt {attempt + 1}/{max_retries}) "
f"key={batch_logging_element.s3_object_key}"
)
time.sleep(wait_time)
continue
response.raise_for_status()
break
except Exception as e:
verbose_logger.exception(f"Error uploading to s3: {str(e)}")
self.handle_callback_failure(callback_name="S3Logger")

10
litellm/litellm_core_utils/litellm_logging.py
View file

@ -613,7 +613,17 @@ class Logging(LiteLLMLoggingBaseClass):
base_litellm_params["metadata"] = kwargs["litellm_metadata"].copy()
if litellm_params:
# Merge metadata carefully — don't overwrite the merged metadata
# from kwargs/litellm_metadata with the caller's litellm_params metadata.
# e.g. anthropic_messages passes Anthropic's native metadata ({user_id: ...})
# in litellm_params, which would overwrite proxy key-auth fields.
lp_metadata = litellm_params.pop("metadata", None)
base_litellm_params.update(litellm_params)
if lp_metadata and isinstance(lp_metadata, dict):
base_litellm_params.setdefault("metadata", {})
for k, v in lp_metadata.items():
if k not in base_litellm_params["metadata"]:
base_litellm_params["metadata"][k] = v
self.update_environment_variables(
litellm_params=base_litellm_params,

18
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4371,17 +4371,19 @@ class BedrockConverseMessagesProcessor:
# if initial message is assistant message
if messages[0].get("role") is not None and messages[0]["role"] == "assistant":
if user_continue_message is not None:
messages.insert(0, user_continue_message)
elif litellm.modify_params:
messages.insert(0, DEFAULT_USER_CONTINUE_MESSAGE)
if not messages[0].get("prefix"):
if user_continue_message is not None:
messages.insert(0, user_continue_message)
elif litellm.modify_params:
messages.insert(0, DEFAULT_USER_CONTINUE_MESSAGE)
# if final message is assistant message
if messages[-1].get("role") is not None and messages[-1]["role"] == "assistant":
if user_continue_message is not None:
messages.append(user_continue_message)
elif litellm.modify_params:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE)
if not messages[-1].get("prefix"):
if user_continue_message is not None:
messages.append(user_continue_message)
elif litellm.modify_params:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE)
return messages
@staticmethod

7
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -123,10 +123,13 @@ class ChunkProcessor:
finish_reason = "stop"
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
finish_reason = chunk["choices"][0].finish_reason
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
finish_reason = chunk["choices"][0]["finish_reason"]
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
# Initialize the response dictionary
response = ModelResponse(

6
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1134,7 +1134,11 @@ class CustomStreamWrapper:
):
if self.received_finish_reason is not None:
_chunk_has_content = isinstance(chunk, dict) and (
bool(chunk.get("text", "")) or chunk.get("tool_use") is not None
bool(chunk.get("text", ""))
or chunk.get("tool_use") is not None
# Usage-only final chunks are valid and needed to surface
# finish_reason/usage to downstream translators.
or chunk.get("usage") is not None
)
if not _chunk_has_content and (
not isinstance(chunk, dict)

19
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -21,6 +21,10 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
LiteLLMAnthropicMessagesAdapter,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
openai_messages_without_system,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
@ -29,6 +33,7 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
)
@ -75,6 +80,8 @@ class AnthropicMessagesHandler(BaseTranslation):
if messages is None:
return data
skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
(
chat_completion_compatible_request,
_tool_name_mapping,
@ -83,7 +90,12 @@ class AnthropicMessagesHandler(BaseTranslation):
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
)
structured_messages = chat_completion_compatible_request.get("messages", [])
structured_messages = cast(
List[AllMessageValues],
chat_completion_compatible_request.get("messages", []),
)
if skip_system:
structured_messages = openai_messages_without_system(structured_messages)
texts_to_check: List[str] = []
images_to_check: List[str] = []
@ -102,6 +114,7 @@ class AnthropicMessagesHandler(BaseTranslation):
texts_to_check=texts_to_check,
images_to_check=images_to_check,
task_mappings=task_mappings,
skip_system_message=skip_system,
)
# Step 2: Apply guardrail to all texts in batch
@ -165,12 +178,16 @@ class AnthropicMessagesHandler(BaseTranslation):
texts_to_check: List[str],
images_to_check: List[str],
task_mappings: List[Tuple[int, Optional[int]]],
skip_system_message: bool = False,
) -> None:
"""
Extract text content and images from a message.
Override this method to customize text/image extraction logic.
"""
if skip_system_message and str(message.get("role") or "").lower() == "system":
return
content = message.get("content", None)
tools = message.get("tools", None)
if content is None and tools is None:

0
litellm/llms/azure/containers/__init__.py Normal file
View file

48
litellm/llms/azure/containers/transformation.py Normal file
View file

@ -0,0 +1,48 @@
from typing import Optional
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.types.router import GenericLiteLLMParams
class AzureContainerConfig(OpenAIContainerConfig):
"""
Configuration class for Azure OpenAI container API.
Inherits request/response transformations from OpenAIContainerConfig since
Azure's container API is wire-compatible with OpenAI's. Only overrides
authentication (api-key header) and URL construction (openai/v1/containers path).
Azure container API reference:
https://learn.microsoft.com/en-us/azure/foundry/openai/latest#containers
"""
def validate_environment(
self,
headers: dict,
api_key: Optional[str] = None,
) -> dict:
return BaseAzureLLM._base_validate_azure_environment(
headers=headers,
litellm_params=GenericLiteLLMParams(api_key=api_key),
)
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Build the Azure container endpoint URL.
Azure container API uses the path:
{endpoint}/openai/v1/containers
when api_version is 'v1', 'latest', or 'preview'; otherwise:
{endpoint}/openai/containers
"""
return BaseAzureLLM._get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
route="/openai/containers",
default_api_version="v1",
)

24
litellm/llms/base_llm/guardrail_translation/utils.py Normal file
View file

@ -0,0 +1,24 @@
from __future__ import annotations
from typing import Any, List
from litellm.types.llms.openai import AllMessageValues
def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per = getattr(guardrail_to_apply, "skip_system_message_in_guardrail", None)
if per is not None:
return bool(per)
import litellm
return bool(getattr(litellm, "skip_system_message_in_guardrail", False))
def openai_messages_without_system(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
return [
m
for m in messages
if str((m or {}).get("role") or "").lower() != "system"
]

21
litellm/llms/custom_httpx/container_handler.py
View file

@ -61,18 +61,29 @@ def _build_url(
) -> str:
"""Build the full URL by substituting path parameters.
The api_base from get_complete_url already includes /containers,
so we need to strip that prefix from the path_template.
The api_base from get_complete_url already includes /containers and may include
query parameters. We need to parse the URL, append the path, then preserve the
query parameters.
"""
# api_base ends with /containers, path_template starts with /containers
# So we need to strip /containers from the path
if path_template.startswith("/containers"):
path_template = path_template[len("/containers") :]
url = f"{api_base.rstrip('/')}{path_template}"
# Substitute path parameters
for param, value in path_params.items():
url = url.replace(f"{{{param}}}", value)
return url
path_template = path_template.replace(f"{{{param}}}", value)
# Parse the api_base to extract existing query params
parsed_base = httpx.URL(api_base)
# Append the path to the existing path (before query params)
new_path = f"{parsed_base.path.rstrip('/')}{path_template}"
# Rebuild URL with new path, preserving query params
final_url = parsed_base.copy_with(path=new_path)
return str(final_url)
def _build_query_params(

22
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -19,8 +19,12 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
openai_messages_without_system,
)
from litellm.main import stream_chunk_builder
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.utils import (
Choices,
GenericGuardrailAPIInputs,
@ -57,6 +61,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if messages is None:
return data
skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
texts_to_check: List[str] = []
images_to_check: List[str] = []
tool_calls_to_check: List[ChatCompletionToolParam] = []
@ -76,6 +82,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
tool_calls_to_check=tool_calls_to_check,
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
skip_system_message=skip_system,
)
# Step 2: Apply guardrail to all texts and tool calls in batch
@ -86,9 +93,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check # type: ignore
if messages:
inputs[
"structured_messages"
] = messages # pass the openai /chat/completions messages to the guardrail, as-is
msg_list = cast(List[AllMessageValues], messages)
inputs["structured_messages"] = (
openai_messages_without_system(msg_list)
if skip_system
else msg_list
)
# Pass tools (function definitions) to the guardrail
tools = data.get("tools")
if tools:
@ -157,12 +167,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
tool_calls_to_check: List[ChatCompletionToolParam],
text_task_mappings: List[Tuple[int, Optional[int]]],
tool_call_task_mappings: List[Tuple[int, int]],
skip_system_message: bool = False,
) -> None:
"""
Extract text content, images, and tool calls from a message.
Override this method to customize text/image/tool call extraction logic.
"""
if skip_system_message and str(message.get("role") or "").lower() == "system":
return
content = message.get("content", None)
if content is not None:
if isinstance(content, str):

11
litellm/llms/openai/containers/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.types.containers.main import (
from litellm.types.router import GenericLiteLLMParams
from ...base_llm.containers.transformation import BaseContainerConfig
from .utils import join_container_api_base_path
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -197,7 +198,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
) -> Tuple[str, Dict]:
"""Transform the OpenAI container retrieve request."""
# For container retrieve, we just need to construct the URL
url = f"{api_base.rstrip('/')}/{container_id}"
url = join_container_api_base_path(api_base, f"/{container_id}")
# No additional data needed for GET request
data: Dict[str, Any] = {}
@ -229,7 +230,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
- DELETE /v1/containers/{container_id}
"""
# Construct the URL for container delete
url = f"{api_base.rstrip('/')}/{container_id}"
url = join_container_api_base_path(api_base, f"/{container_id}")
# No data needed for DELETE request
data: Dict[str, Any] = {}
@ -266,7 +267,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files
"""
# Construct the URL for container files
url = f"{api_base.rstrip('/')}/{container_id}/files"
url = join_container_api_base_path(api_base, f"/{container_id}/files")
# Prepare query parameters
params: Dict[str, Any] = {}
@ -310,7 +311,9 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files/{file_id}/content
"""
# Construct the URL for container file content
url = f"{api_base.rstrip('/')}/{container_id}/files/{file_id}/content"
url = join_container_api_base_path(
api_base, f"/{container_id}/files/{file_id}/content"
)
# No query parameters needed
params: Dict[str, Any] = {}

18
litellm/llms/openai/containers/utils.py Normal file
View file

@ -0,0 +1,18 @@
"""Shared helpers for OpenAI-compatible container API URL construction."""
import httpx
def join_container_api_base_path(api_base: str, path_suffix: str) -> str:
"""Append ``path_suffix`` to the path of ``api_base``, keeping the query string last.
Azure (and some bases) pass ``api_base`` like
``https://host/openai/v1/containers?api-version=v1``. Naive string concat would
produce ``...?api-version=v1/cntr_...`` which is invalid; this uses ``httpx.URL``
so the result is ``.../containers/cntr_.../files?api-version=v1``.
"""
if not path_suffix.startswith("/"):
path_suffix = f"/{path_suffix}"
parsed = httpx.URL(api_base)
new_path = f"{parsed.path.rstrip('/')}{path_suffix}"
return str(parsed.copy_with(path=new_path))

67
litellm/proxy/common_request_processing.py
View file

@ -9,6 +9,7 @@ from typing import (
Any,
AsyncGenerator,
Callable,
Dict,
Literal,
Optional,
Tuple,
@ -65,6 +66,37 @@ from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
def _serialize_http_exception_detail(
detail: Any,
) -> Tuple[str, Optional[dict]]:
"""
Convert an HTTPException.detail value into (message, structured_fields)
for ProxyException / SSE error frames.
Dict-detail HTTPExceptions raised by guardrails were previously str()-mangled
into a Python repr blob, producing unparseable error responses on both the
streaming and non-streaming proxy surfaces. This helper extracts a clean
human-readable message while preserving the full payload as structured
fields, so the dominant guardrail shapes (`{"error": "..."}` flat and
`{"error": {"message": "..."}}` nested) both round-trip cleanly.
"""
if isinstance(detail, str):
return detail, None
if isinstance(detail, dict):
err = detail.get("error")
if isinstance(err, str):
return err, detail
if isinstance(err, dict):
nested_msg = err.get("message")
if isinstance(nested_msg, str):
return nested_msg, detail
msg = detail.get("message")
if isinstance(msg, str):
return msg, detail
return json.dumps(detail), detail
return str(detail), None
async def _parse_event_data_for_error(event_line: Union[str, bytes]) -> Optional[int]:
"""Parses an event line and returns an error code if present, else None."""
event_line = (
@ -223,12 +255,28 @@ async def create_response(
# Preserve status code from HTTPException (e.g., guardrail blocks)
error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
error_detail = getattr(e, "detail", "Error processing stream start")
if not isinstance(error_detail, str):
error_detail = str(error_detail)
raw_detail = getattr(e, "detail", "Error processing stream start")
message, structured_fields = _serialize_http_exception_detail(raw_detail)
existing_fields = getattr(e, "provider_specific_fields", None) or {}
if structured_fields:
merged_fields: Optional[dict] = {**existing_fields, **structured_fields}
else:
merged_fields = existing_fields or None
# Match ProxyException.to_dict() shape so streaming and non-streaming
# error frames are byte-identical.
error_obj: Dict[str, Any] = {
"message": message,
"type": getattr(e, "type", "None"),
"param": getattr(e, "param", "None"),
"code": str(error_status),
}
if merged_fields:
error_obj["provider_specific_fields"] = merged_fields
async def error_gen_message() -> AsyncGenerator[str, None]:
yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n"
yield f"data: {json.dumps({'error': error_obj})}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(
@ -1593,12 +1641,19 @@ class ProxyBaseLLMRequestProcessing:
pass
if isinstance(e, HTTPException):
raw_detail = getattr(e, "detail", str(e))
message, structured_fields = _serialize_http_exception_detail(raw_detail)
existing_fields = getattr(e, "provider_specific_fields", None) or {}
if structured_fields:
merged_fields: Optional[dict] = {**existing_fields, **structured_fields}
else:
merged_fields = existing_fields or None
raise ProxyException(
message=getattr(e, "detail", str(e)),
message=message,
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
provider_specific_fields=getattr(e, "provider_specific_fields", None),
provider_specific_fields=merged_fields,
headers=headers,
)
elif isinstance(e, httpx.HTTPStatusError):

58
litellm/proxy/container_endpoints/handler_factory.py
View file

@ -19,6 +19,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
def _load_endpoints_config() -> Dict:
@ -40,10 +41,13 @@ def _get_container_provider_config(custom_llm_provider: str):
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
return OpenAIContainerConfig()
else:
raise ValueError(
f"Container API not supported for provider: {custom_llm_provider}"
)
elif custom_llm_provider in ("azure", "azure_text"):
from litellm.llms.azure.containers.transformation import AzureContainerConfig
return AzureContainerConfig()
raise ValueError(
f"Container API not supported for provider: {custom_llm_provider}"
)
def _create_handler_for_path_params(
@ -171,12 +175,21 @@ async def _process_binary_request(
or "openai"
)
# Get the provider config
container_provider_config = _get_container_provider_config(custom_llm_provider)
# Build litellm_params - credentials are resolved by provider config from env
litellm_params = GenericLiteLLMParams()
# Decode container ID and extract provider info
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
# Get the provider config
container_provider_config = _get_container_provider_config(custom_llm_provider)
# Create logging object
logging_obj = Logging(
model="container-file-content",
@ -193,7 +206,7 @@ async def _process_binary_request(
try:
content = await handler.async_container_file_content_handler(
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
file_id=file_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
@ -267,13 +280,22 @@ async def _process_multipart_upload_request(
if isinstance(file_list, list) and len(file_list) > 0:
data["file"] = file_list[0]
data["container_id"] = container_id
custom_llm_provider = (
get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
# Decode container ID and extract provider info
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
data["container_id"] = original_container_id # Use decoded original ID
data["custom_llm_provider"] = custom_llm_provider
processor = ProxyBaseLLMRequestProcessing(data=data)
@ -338,6 +360,22 @@ async def _process_request(
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
# Decode container_id if present in path_params
if "container_id" in path_params:
decoded = ResponsesAPIRequestUtils._decode_container_id(
path_params["container_id"]
)
original_container_id = decoded.get("response_id", path_params["container_id"])
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
# Update path_params with decoded original ID
data["container_id"] = original_container_id
data["custom_llm_provider"] = custom_llm_provider
processor = ProxyBaseLLMRequestProcessing(data=data)

159
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -18,6 +18,7 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Dict,
List,
Literal,
NamedTuple,
@ -636,6 +637,141 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return (status_code, err)
return (status_code, message)
def _extract_blocked_assessments(
self, response: BedrockGuardrailResponse
) -> List[dict]:
"""
Walk the Bedrock guardrail response and emit a structured list of
BLOCKED assessment entries describing exactly which policies fired.
Mirrors the iteration in `_should_raise_guardrail_blocked_exception()`
but produces a list of `{policy, matches}` dicts instead of a bool.
Each `match` carries the originating subcategory, type, action, and
matched term where available, so the client can render a precise
explanation of the violation.
"""
blocked: List[dict] = []
assessments = response.get("assessments", []) or []
for assessment in assessments:
# Topic policy
topic_policy = assessment.get("topicPolicy")
if topic_policy:
topic_matches = [
{
"category": "topics",
"name": t.get("name"),
"type": t.get("type"),
"action": t.get("action"),
}
for t in (topic_policy.get("topics") or [])
if t.get("action") == "BLOCKED"
]
if topic_matches:
blocked.append({"policy": "topicPolicy", "matches": topic_matches})
# Content policy
content_policy = assessment.get("contentPolicy")
if content_policy:
content_matches = [
{
"category": "filters",
"type": f.get("type"),
"confidence": f.get("confidence"),
"filterStrength": f.get("filterStrength"),
"action": f.get("action"),
}
for f in (content_policy.get("filters") or [])
if f.get("action") == "BLOCKED"
]
if content_matches:
blocked.append(
{"policy": "contentPolicy", "matches": content_matches}
)
# Word policy
word_policy = assessment.get("wordPolicy")
if word_policy:
word_matches: List[dict] = []
for w in word_policy.get("customWords") or []:
if w.get("action") == "BLOCKED":
word_matches.append(
{
"category": "customWords",
"match": w.get("match"),
"action": w.get("action"),
}
)
for mw in word_policy.get("managedWordLists") or []:
if mw.get("action") == "BLOCKED":
word_matches.append(
{
"category": "managedWordLists",
"type": mw.get("type"),
"match": mw.get("match"),
"action": mw.get("action"),
}
)
if word_matches:
blocked.append({"policy": "wordPolicy", "matches": word_matches})
# Sensitive information policy (PII)
sensitive_info = assessment.get("sensitiveInformationPolicy")
if sensitive_info:
pii_matches: List[dict] = []
for p in sensitive_info.get("piiEntities") or []:
if p.get("action") == "BLOCKED":
pii_matches.append(
{
"category": "piiEntities",
"type": p.get("type"),
"match": p.get("match"),
"action": p.get("action"),
}
)
for r in sensitive_info.get("regexes") or []:
if r.get("action") == "BLOCKED":
pii_matches.append(
{
"category": "regexes",
"name": r.get("name"),
"regex": r.get("regex"),
"match": r.get("match"),
"action": r.get("action"),
}
)
if pii_matches:
blocked.append(
{
"policy": "sensitiveInformationPolicy",
"matches": pii_matches,
}
)
# Contextual grounding policy
contextual = assessment.get("contextualGroundingPolicy")
if contextual:
grounding_matches = [
{
"category": "filters",
"type": f.get("type"),
"threshold": f.get("threshold"),
"score": f.get("score"),
"action": f.get("action"),
}
for f in (contextual.get("filters") or [])
if f.get("action") == "BLOCKED"
]
if grounding_matches:
blocked.append(
{
"policy": "contextualGroundingPolicy",
"matches": grounding_matches,
}
)
return blocked
def _get_http_exception_for_blocked_guardrail(
self, response: BedrockGuardrailResponse
) -> Union[HTTPException, GuardrailInterventionNormalStringError]:
@ -655,14 +791,21 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return GuardrailInterventionNormalStringError(
message=bedrock_guardrail_output_text
)
else:
return HTTPException(
status_code=400,
detail={
"error": "Violated guardrail policy",
"bedrock_guardrail_response": bedrock_guardrail_output_text,
},
)
detail: Dict[str, Any] = {
"error": "Violated guardrail policy",
"bedrock_guardrail_response": bedrock_guardrail_output_text,
}
if self.guardrailIdentifier:
detail["guardrailIdentifier"] = self.guardrailIdentifier
if self.guardrailVersion:
detail["guardrailVersion"] = self.guardrailVersion
assessments = self._extract_blocked_assessments(response)
if assessments:
detail["assessments"] = assessments
return HTTPException(status_code=400, detail=detail)
def _should_raise_guardrail_blocked_exception(
self, response: BedrockGuardrailResponse

7
litellm/proxy/guardrails/guardrail_registry.py
View file

@ -472,6 +472,13 @@ class InMemoryGuardrailHandler:
else:
raise ValueError(f"Unsupported guardrail: {guardrail_type}")
if custom_guardrail_callback is not None:
setattr(
custom_guardrail_callback,
"skip_system_message_in_guardrail",
getattr(litellm_params, "skip_system_message_in_guardrail", None),
)
parsed_guardrail = Guardrail(
guardrail_id=guardrail.get("guardrail_id"),
guardrail_name=guardrail["guardrail_name"],

124
litellm/proxy/utils.py
View file

@ -15,6 +15,8 @@ from email.mime.text import MIMEText
from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Awaitable,
Dict,
List,
Literal,
@ -300,6 +302,30 @@ def _accepts_litellm_call_info(cb: CustomLogger) -> bool:
return _CALLBACK_ACCEPTS_CALL_INFO[key]
def _enrich_http_exception_with_guardrail_context(
exc: BaseException, callback: Any
) -> None:
"""
If `exc` is an HTTPException with a dict `detail`, mutate it in place to
add `guardrail_name` and `guardrail_mode` taken from the callback instance.
Uses setdefault so guardrails that already populate these fields explicitly
win over the inferred defaults. No-op for non-HTTPException, non-dict-detail,
or callbacks without `guardrail_name`. Never raises.
"""
if not isinstance(exc, HTTPException):
return
detail = getattr(exc, "detail", None)
if not isinstance(detail, dict):
return
guardrail_name = getattr(callback, "guardrail_name", None)
if guardrail_name:
detail.setdefault("guardrail_name", guardrail_name)
event_hook = getattr(callback, "event_hook", None)
if event_hook:
detail.setdefault("guardrail_mode", event_hook)
class ProxyLogging:
"""
Logging/Custom Handlers for proxy.
@ -1063,6 +1089,7 @@ class ProxyLogging:
except Exception as e:
status = "error"
error_type = type(e).__name__
_enrich_http_exception_with_guardrail_context(e, callback)
# Re-raise the exception to maintain existing behavior
raise
finally:
@ -1431,6 +1458,40 @@ class ProxyLogging:
except Exception as e:
raise e
@staticmethod
async def _run_guardrail_task_with_enrichment(
callback: Any, coro: Awaitable[Any]
) -> Any:
"""
Await `coro`; if it raises an HTTPException with dict detail,
enrich the detail with the originating callback's `guardrail_name`
and `guardrail_mode` before re-raising.
"""
try:
return await coro
except Exception as e:
_enrich_http_exception_with_guardrail_context(e, callback)
raise
@staticmethod
async def _wrap_streaming_iterator_with_enrichment(
callback: Any, gen: AsyncGenerator[Any, None]
) -> AsyncGenerator[Any, None]:
"""
Yield from `gen`; if iteration raises an HTTPException with dict detail,
enrich the detail with the originating callback's `guardrail_name` and
`guardrail_mode` before re-raising. Used to wrap each layer of the
async_post_call_streaming_iterator_hook chain so the enrichment is
attributed to the callback that produced the chunk pipeline at that
point in the chain.
"""
try:
async for chunk in gen:
yield chunk
except Exception as e:
_enrich_http_exception_with_guardrail_context(e, callback)
raise
async def during_call_hook(
self,
data: dict,
@ -1481,16 +1542,22 @@ class ProxyLogging:
and user_api_key_dict is not None
):
data["guardrail_to_apply"] = callback
guardrail_task = unified_guardrail.async_moderation_hook(
user_api_key_dict=user_api_key_dict,
data=data,
call_type=call_type,
guardrail_task = self._run_guardrail_task_with_enrichment(
callback,
unified_guardrail.async_moderation_hook(
user_api_key_dict=user_api_key_dict,
data=data,
call_type=call_type,
),
)
else:
guardrail_task = callback.async_moderation_hook(
data=data,
user_api_key_dict=user_api_key_auth_dict, # type: ignore
call_type=call_type, # type: ignore
guardrail_task = self._run_guardrail_task_with_enrichment(
callback,
callback.async_moderation_hook(
data=data,
user_api_key_dict=user_api_key_auth_dict, # type: ignore
call_type=call_type, # type: ignore
),
)
guardrail_tasks.append(guardrail_task)
@ -1985,19 +2052,27 @@ class ProxyLogging:
if "apply_guardrail" in type(callback).__dict__:
data["guardrail_to_apply"] = callback
guardrail_response = (
await unified_guardrail.async_post_call_success_hook(
try:
guardrail_response = (
await unified_guardrail.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
)
)
except Exception as e:
_enrich_http_exception_with_guardrail_context(e, callback)
raise
else:
try:
guardrail_response = await callback.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
)
)
else:
guardrail_response = await callback.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict,
data=data,
response=response,
)
except Exception as e:
_enrich_http_exception_with_guardrail_context(e, callback)
raise
if guardrail_response is not None:
response = guardrail_response
@ -2206,29 +2281,32 @@ class ProxyLogging:
"async_post_call_streaming_iterator_hook"
in type(callback).__dict__
):
current_response = (
current_response = self._wrap_streaming_iterator_with_enrichment(
_callback,
_callback.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=current_response,
request_data=request_data,
)
),
)
elif "apply_guardrail" in type(callback).__dict__:
request_data["guardrail_to_apply"] = callback
current_response = (
current_response = self._wrap_streaming_iterator_with_enrichment(
_callback,
unified_guardrail.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
request_data=request_data,
response=current_response,
)
),
)
else:
current_response = (
current_response = self._wrap_streaming_iterator_with_enrichment(
_callback,
_callback.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=current_response,
request_data=request_data,
)
),
)
# Actually iterate through the chained async generator and yield chunks

10
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -1016,14 +1016,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
):
if src and isinstance(src, dict):
self._merge_provider_specific_fields(src)
# Emit any just-queued output_item event
if self._pending_response_events:
return self._pending_response_events.pop(0)
# Always snapshot before returning any pending events so that
# finish_reason (e.g. content_filter) is captured even when
# _ensure_output_item_for_chunk queues events on the same chunk.
# This mirrors the async path (see __anext__).
self.collected_chat_completion_chunks.append(
self._snapshot_chunk_for_stream_chunk_builder(
cast(ModelResponseStream, chunk)
)
)
# Emit any just-queued output_item event
if self._pending_response_events:
return self._pending_response_events.pop(0)
response_api_chunk = (
self._transform_chat_completion_chunk_to_response_api_chunk(
chunk

4
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1519,7 +1519,7 @@ class LiteLLMCompletionResponsesConfig:
"""
Map chat completion finish_reason to responses API status.
Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call"
Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call", "refusal"
Responses API status values are: "completed", "failed", "in_progress", "cancelled", "queued", "incomplete"
Args:
@ -1534,7 +1534,7 @@ class LiteLLMCompletionResponsesConfig:
# Map finish reasons to status
if finish_reason in ["stop", "tool_calls", "function_call"]:
return "completed"
elif finish_reason in ["length", "content_filter"]:
elif finish_reason in ["length", "content_filter", "refusal"]:
return "incomplete"
else:
# Default to completed for unknown finish reasons

65
litellm/responses/streaming_iterator.py
View file

@ -130,15 +130,64 @@ class BaseResponsesAPIStreamingIterator:
)
)
# if "response" in parsed_chunk, then encode litellm specific information like custom_llm_provider
response_object = getattr(openai_responses_api_chunk, "response", None)
if response_object:
response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id(
responses_api_response=response_object,
litellm_metadata=self.litellm_metadata,
custom_llm_provider=self.custom_llm_provider,
# Only when the SSE JSON carries a response body (delta events do not).
# Using getattr(..., "response") alone is unsafe with Mocks: they synthesize a
# truthy child Mock for any attribute, which breaks tests and is wrong on stream.
if "response" in parsed_chunk:
response_object = getattr(
openai_responses_api_chunk, "response", None
)
setattr(openai_responses_api_chunk, "response", response)
if response_object is not None:
response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id(
responses_api_response=response_object,
litellm_metadata=self.litellm_metadata,
custom_llm_provider=self.custom_llm_provider,
)
setattr(openai_responses_api_chunk, "response", response)
# Encode container_id on streaming events so proxy/UI follow-ups route correctly
_event_type = getattr(openai_responses_api_chunk, "type", None)
_stream_model_id = (
self.litellm_metadata.get("model_info", {}).get("id")
if self.litellm_metadata
else None
)
if _event_type in (
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
):
_item = getattr(openai_responses_api_chunk, "item", None)
if _item is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=_item,
custom_llm_provider=self.custom_llm_provider,
model_id=_stream_model_id,
)
elif _event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED:
_annotation = getattr(
openai_responses_api_chunk, "annotation", None
)
if _annotation is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=_annotation,
custom_llm_provider=self.custom_llm_provider,
model_id=_stream_model_id,
)
elif _event_type == ResponsesAPIStreamEvents.CONTENT_PART_DONE:
_part = getattr(openai_responses_api_chunk, "part", None)
if _part is not None:
if isinstance(_part, dict):
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
_part.get("annotations"),
self.custom_llm_provider,
_stream_model_id,
)
else:
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
getattr(_part, "annotations", None),
self.custom_llm_provider,
_stream_model_id,
)
# Wrap encrypted_content in streaming events (output_item.added, output_item.done)
if self.litellm_metadata and self.litellm_metadata.get(

249
litellm/responses/utils.py
View file

@ -1,4 +1,5 @@
import base64
import re
from typing import (
Any,
Dict,
@ -226,6 +227,15 @@ class ResponsesAPIRequestUtils:
)
)
# Encode container IDs in the response output
responses_api_response = (
ResponsesAPIRequestUtils._update_container_ids_in_response(
responses_api_response=responses_api_response,
custom_llm_provider=custom_llm_provider,
litellm_metadata=litellm_metadata,
)
)
return responses_api_response
@staticmethod
@ -522,6 +532,245 @@ class ResponsesAPIRequestUtils:
)
return decoded_response_id.get("response_id", previous_response_id)
@staticmethod
def _build_container_id(
custom_llm_provider: Optional[str],
model_id: Optional[str],
container_id: str,
) -> str:
"""Build a managed container ID with provider and model info encoded.
Format: cntr_{base64("litellm:custom_llm_provider:{provider};model_id:{model};container_id:{original}")}
"""
# Avoid serializing Python None as the literal string "None" (breaks router affinity).
provider_part = "" if custom_llm_provider is None else custom_llm_provider
model_part = "" if model_id is None else model_id
assembled_id = f"litellm:custom_llm_provider:{provider_part};model_id:{model_part};container_id:{container_id}"
base64_encoded_id = base64.b64encode(assembled_id.encode("utf-8")).decode("utf-8")
return f"cntr_{base64_encoded_id}"
@staticmethod
def _decode_container_id(container_id: str) -> DecodedResponseId:
"""Decode a managed container ID to extract provider, model, and original container ID.
Returns:
DecodedResponseId with custom_llm_provider, model_id, and response_id (original container_id)
"""
try:
# If it doesn't start with cntr_, it's not a managed ID
if not container_id.startswith("cntr_"):
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
# Remove prefix and decode
cleaned_id = container_id.replace("cntr_", "")
decoded_id = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8")
# Parse components using regex to handle semicolons in the container_id
if not decoded_id.startswith("litellm:"):
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
# Use regex to extract the three parts, allowing semicolons in container_id
# Format: litellm:custom_llm_provider:{provider};model_id:{model};container_id:{container}
# * for provider/model allows empty segments (missing router model_id).
pattern = r"^litellm:custom_llm_provider:([^;]*);model_id:([^;]*);container_id:(.+)$"
match = re.match(pattern, decoded_id)
if not match:
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
raw_provider = match.group(1)
raw_model_id = match.group(2)
custom_llm_provider = (
None if raw_provider in ("", "None") else raw_provider
)
model_id = None if raw_model_id in ("", "None") else raw_model_id
original_container_id = match.group(3)
return DecodedResponseId(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
response_id=original_container_id,
)
except Exception as e:
verbose_logger.debug(f"Error decoding container_id '{container_id}': {e}")
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
@staticmethod
def decode_container_id_to_original(container_id: str) -> str:
"""Decode a managed container ID to get the original provider-issued ID.
This is used when making upstream API calls - we need to send the original
container ID that the provider issued, not our encoded version.
"""
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
return decoded.get("response_id", container_id)
@staticmethod
def _encode_container_ids_in_annotations(
annotations: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Encode ``container_id`` on each annotation (e.g. ``container_file_citation``)."""
if not annotations or not isinstance(annotations, list):
return
for ann in annotations:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
ann,
custom_llm_provider,
model_id,
)
@staticmethod
def _encode_container_ids_in_message_content(
content: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Walk message ``content`` parts and encode citation ``container_id`` values."""
if not content:
return
if isinstance(content, list):
for part in content:
if isinstance(part, dict):
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
part.get("annotations"),
custom_llm_provider,
model_id,
)
else:
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
getattr(part, "annotations", None),
custom_llm_provider,
model_id,
)
@staticmethod
def _encode_container_id_on_output_item(
item: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Mutate one output item (dict or object): wrap raw ``container_id`` as LiteLLM-managed.
Handles top-level ``container_id`` and nested ``code_interpreter_call.container_id``
(some wire payloads nest the tool call). Used by non-streaming responses and by
streaming ``response.output_item.*`` events so UIs see managed IDs incrementally.
For ``message`` items, also encodes ``container_id`` inside
``content[].annotations`` (``container_file_citation``), which is what clients use
to fetch generated files.
"""
if item is None:
return
def _maybe_encode(container_id: str) -> Optional[str]:
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
if decoded.get("custom_llm_provider") is not None:
return None
return ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
container_id=container_id,
)
if isinstance(item, dict):
cid = item.get("container_id")
if isinstance(cid, str):
enc = _maybe_encode(cid)
if enc is not None:
item["container_id"] = enc
nested = item.get("code_interpreter_call")
if isinstance(nested, dict):
nc = nested.get("container_id")
if isinstance(nc, str):
enc = _maybe_encode(nc)
if enc is not None:
nested["container_id"] = enc
if item.get("type") == "message":
ResponsesAPIRequestUtils._encode_container_ids_in_message_content(
item.get("content"),
custom_llm_provider,
model_id,
)
return
cid_attr = getattr(item, "container_id", None)
if isinstance(cid_attr, str):
enc = _maybe_encode(cid_attr)
if enc is not None:
try:
setattr(item, "container_id", enc)
except Exception:
verbose_logger.debug(
"Could not set container_id on streaming output item",
exc_info=True,
)
nested_obj = getattr(item, "code_interpreter_call", None)
if nested_obj is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
nested_obj,
custom_llm_provider,
model_id,
)
if getattr(item, "type", None) == "message":
ResponsesAPIRequestUtils._encode_container_ids_in_message_content(
getattr(item, "content", None),
custom_llm_provider,
model_id,
)
@staticmethod
def _update_container_ids_in_response(
responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]],
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Union[ResponsesAPIResponse, Dict[str, Any]]:
"""Encode container IDs in the response output with provider/model info.
This walks through all output items and encodes any container_id fields
so that follow-up container API calls can auto-route to the correct provider.
"""
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Get the output list
if isinstance(responses_api_response, dict):
output = responses_api_response.get("output", [])
else:
output = getattr(responses_api_response, "output", [])
if not output:
return responses_api_response
for item in output:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=item,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
)
return responses_api_response
@staticmethod
def convert_text_format_to_text_param(
text_format: Optional[Union[Type["BaseModel"], dict]],

3
litellm/types/containers/main.py
View file

@ -187,7 +187,8 @@ class DeleteContainerFileResponse(BaseModel):
"""Response object for delete container file request."""
id: str
object: Literal["container_file.deleted"]
# OpenAI / Azure wire format uses dots; keep underscore variant for compatibility.
object: Literal["container.file.deleted", "container_file.deleted"]
deleted: bool
def __contains__(self, key):

10
litellm/types/guardrails.py
View file

@ -607,6 +607,16 @@ class BaseLitellmParams(
description="When True, guardrails only receive the latest message for the relevant role (e.g., newest user input pre-call, newest assistant output post-call)",
)
skip_system_message_in_guardrail: Optional[bool] = Field(
default=None,
description=(
"When True, unified guardrails skip system-role messages when building "
"evaluation inputs (texts and structured_messages). When False, system "
"messages are included even if litellm_settings sets a global skip. When "
"None, use the global litellm.skip_system_message_in_guardrail setting."
),
)
# Lakera specific params
category_thresholds: Optional[LakeraCategoryThresholds] = Field(
default=None,

26
litellm/types/integrations/prometheus.py
View file

@ -122,40 +122,22 @@ STATUS_CODE = "status_code"
EXCEPTION_LABELS = [EXCEPTION_STATUS, EXCEPTION_CLASS]
LATENCY_BUCKETS = (
0.005,
0.00625,
0.0125,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
1.5,
2.0,
2.5,
3.0,
3.5,
4.0,
4.5,
5.0,
5.5,
6.0,
6.5,
7.0,
7.5,
8.0,
8.5,
9.0,
9.5,
10.0,
15.0,
20.0,
25.0,
30.0,
60.0,
120.0,
180.0,
240.0,
300.0,
420.0, # 7 minutes
600.0, # 10 minutes (typical default LLM request timeout)
float("inf"),
)

6
litellm/utils.py
View file

@ -8958,6 +8958,12 @@ class ProviderConfigManager:
)
return OpenAIContainerConfig()
if provider in (LlmProviders.AZURE, LlmProviders.AZURE_TEXT):
from litellm.llms.azure.containers.transformation import (
AzureContainerConfig,
)
return AzureContainerConfig()
return None
@staticmethod

519
tests/test_litellm/containers/test_azure_container_transformation.py Normal file
View file

@ -0,0 +1,519 @@
import os
import sys
from unittest.mock import MagicMock
from urllib.parse import parse_qs, urlparse
import httpx
import pytest
sys.path.insert(0, os.path.abspath("../../../"))
import litellm
from litellm.llms.azure.containers.transformation import AzureContainerConfig
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
class TestAzureContainerConfig:
"""Test suite for Azure container transformation functionality."""
def setup_method(self):
self.config = AzureContainerConfig()
self.logging_obj = LiteLLMLogging(
model="",
messages=[],
stream=False,
call_type="create_container",
start_time=None,
litellm_call_id="test_call_id",
function_id="test_function_id",
)
def test_inherits_base_container_config(self):
assert isinstance(self.config, BaseContainerConfig)
def test_get_supported_openai_params(self):
supported_params = self.config.get_supported_openai_params()
assert "name" in supported_params
assert "expires_after" in supported_params
assert "file_ids" in supported_params
def test_validate_environment_with_api_key(self):
headers = {}
api_key = "test-azure-key"
validated_headers = self.config.validate_environment(
headers=headers, api_key=api_key
)
assert "api-key" in validated_headers
assert validated_headers["api-key"] == api_key
def test_validate_environment_uses_azure_env_var(self, monkeypatch):
monkeypatch.setenv("AZURE_API_KEY", "env-azure-key")
headers = {}
validated_headers = self.config.validate_environment(headers=headers)
assert "api-key" in validated_headers
assert validated_headers["api-key"] == "env-azure-key"
def test_validate_environment_no_bearer_token(self):
"""Azure uses api-key header, not Authorization: Bearer."""
headers = {}
api_key = "azure-test-key"
validated_headers = self.config.validate_environment(
headers=headers, api_key=api_key
)
assert "Authorization" not in validated_headers
assert "api-key" in validated_headers
def test_get_complete_url_default_v1(self):
"""With default_api_version='v1', URL should include /openai/v1/containers."""
api_base = "https://my-resource.openai.azure.com"
litellm_params = {}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "/openai/v1/containers" in url
assert "my-resource.openai.azure.com" in url
def test_get_complete_url_with_explicit_api_version(self):
api_base = "https://my-resource.openai.azure.com"
litellm_params = {"api_version": "2025-01-01"}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "api-version=2025-01-01" in url
assert "/openai/containers" in url
def test_get_complete_url_with_latest_api_version(self):
api_base = "https://my-resource.openai.azure.com"
litellm_params = {"api_version": "latest"}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "/openai/v1/containers" in url
def test_get_complete_url_raises_without_api_base(self, monkeypatch):
monkeypatch.delenv("AZURE_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None)
with pytest.raises(ValueError, match="api_base is required"):
self.config.get_complete_url(api_base=None, litellm_params={})
def test_transform_container_create_request(self):
from litellm.types.router import GenericLiteLLMParams
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
name = "My Azure Container"
optional_params = {
"expires_after": {"anchor": "last_active_at", "minutes": 30},
"file_ids": ["file_abc"],
}
data = self.config.transform_container_create_request(
name=name,
container_create_optional_request_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
assert data["name"] == name
assert data["expires_after"]["minutes"] == 30
assert data["file_ids"] == ["file_abc"]
def test_transform_container_create_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"id": "cntr_azure_123",
"object": "container",
"created_at": 1747857508,
"status": "running",
"expires_after": {"anchor": "last_active_at", "minutes": 30},
"last_active_at": 1747857508,
"name": "My Azure Container",
}
container = self.config.transform_container_create_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(container, ContainerObject)
assert container.id == "cntr_azure_123"
assert container.name == "My Azure Container"
assert container.status == "running"
def test_transform_container_list_request(self):
from litellm.types.router import GenericLiteLLMParams
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_list_request(
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
limit=5,
order="desc",
)
assert url == api_base
assert params["limit"] == "5"
assert params["order"] == "desc"
def test_transform_container_list_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"object": "list",
"data": [
{
"id": "cntr_1",
"object": "container",
"created_at": 1747857508,
"status": "running",
"expires_after": {"anchor": "last_active_at", "minutes": 20},
"last_active_at": 1747857508,
"name": "Container 1",
}
],
"first_id": "cntr_1",
"last_id": "cntr_1",
"has_more": False,
}
container_list = self.config.transform_container_list_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(container_list, ContainerListResponse)
assert len(container_list.data) == 1
assert container_list.first_id == "cntr_1"
def test_transform_container_retrieve_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_abc"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_retrieve_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert url == f"{api_base}/{container_id}"
assert params == {}
def test_transform_container_delete_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_del"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_delete_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert url == f"{api_base}/{container_id}"
assert params == {}
def test_transform_container_delete_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"id": "cntr_azure_del",
"object": "container.deleted",
"deleted": True,
}
delete_result = self.config.transform_container_delete_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(delete_result, DeleteContainerResult)
assert delete_result.id == "cntr_azure_del"
assert delete_result.deleted is True
def test_transform_container_file_list_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_files"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_file_list_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
limit=10,
)
assert url == f"{api_base}/{container_id}/files"
assert params["limit"] == "10"
def test_transform_requests_preserve_query_string_after_path(self):
"""api-version must not appear before /{container_id}/... (Azure bases include ?)."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_r, _ = self.config.transform_container_retrieve_request(
container_id="cntr_x",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert (
url_r
== "https://my-resource.openai.azure.com/openai/v1/containers/cntr_x?api-version=v1"
)
url_fl, _ = self.config.transform_container_file_list_request(
container_id="cntr_x",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert (
url_fl
== "https://my-resource.openai.azure.com/openai/v1/containers/cntr_x/files?api-version=v1"
)
url_fc, _ = self.config.transform_container_file_content_request(
container_id="cntr_x",
file_id="cfile_y",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
expected_fc = (
"https://my-resource.openai.azure.com/openai/v1/containers/"
"cntr_x/files/cfile_y/content?api-version=v1"
)
assert url_fc == expected_fc
assert url_fc.index("/content") < url_fc.index("?")
def test_provider_config_manager_returns_azure_config(self):
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
config = ProviderConfigManager.get_provider_container_config(
provider=LlmProviders.AZURE
)
assert config is not None
assert isinstance(config, AzureContainerConfig)
def test_proxy_handler_factory_returns_azure_config(self):
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
config = _get_container_provider_config("azure")
assert config is not None
assert isinstance(config, AzureContainerConfig)
def test_proxy_handler_factory_raises_for_unsupported_provider(self):
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
with pytest.raises(ValueError, match="Container API not supported"):
_get_container_provider_config("anthropic")
class TestAzureContainerKnownFailureRegressions:
"""Regression tests for real production / proxy failures (Azure containers).
1. **URL / api-version** — ``get_complete_url`` appends ``?api-version=…`` to the
container base. Naïve ``f\"{api_base}/…\"`` put the query *before* path segments,
e.g. ``…/containers?api-version=v1/cntr_…/files``, which Azure rejects
("API version not supported" / 404-style routing).
2. **Bare resource root** — ``AZURE_API_BASE`` is only the host (no ``?``). The
query appears only after LiteLLM builds the full container base; downstream
transforms must still append ``/cntr_…/files/…`` *before* the query string.
3. **File content path** — The worst case in logs was POST/GET logging showing
``…containers?api-version=v1/cntr_…/files/cfile_…/content``; correct wire shape is
``…containers/cntr_…/files/cfile_…/content?api-version=v1``.
"""
def setup_method(self):
self.config = AzureContainerConfig()
def test_regression_query_never_splits_before_container_segment(self):
"""Forbid the broken shape: …/containers?api-version=v1/cntr_…"""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
cid = "cntr_69d4f27de324819082c54f6aeaab6391056f5dbdf1fe2b02"
fid = "cfile_69d4f283bac0819094bfe7805a4f3ce8"
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_fc, _ = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
# Exact substring seen in broken logs
assert "containers?api-version=v1/" + cid not in url_fc
assert "containers?api-version=v1/cntr_" not in url_fc
parsed = urlparse(url_fc)
assert parsed.path == (
f"/openai/v1/containers/{cid}/files/{fid}/content"
)
assert parse_qs(parsed.query).get("api-version") == ["v1"]
assert url_fc.index("/content") < url_fc.index("?")
def test_regression_full_chain_bare_resource_root_like_env(self):
"""Mimics AZURE_API_BASE=https://resource.openai.azure.com — no ? in env."""
from litellm.types.router import GenericLiteLLMParams
resource_root = "https://my-resource.openai.azure.com"
container_base = self.config.get_complete_url(
api_base=resource_root,
litellm_params={},
)
assert "openai.azure.com" in container_base
assert "openai/v1/containers" in container_base or "/openai/containers" in container_base
cid = "cntr_livepath123"
fid = "cfile_live456"
url_fc, params = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=container_base,
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert cid in url_fc
assert fid in url_fc
parsed = urlparse(url_fc)
assert cid in parsed.path
assert "?" not in parsed.path
assert "/content" in parsed.path
assert url_fc.index(cid) < (url_fc.index("?") if "?" in url_fc else len(url_fc))
assert params == {}
def test_regression_all_crud_urls_with_azure_style_api_base(self):
"""Retrieve, delete, list files, and file content all keep ?api-version last."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://iamkankute-5584-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
cid = "cntr_69d4f1c5c6448190930a444af3f84f670b35dc2ee845cd1b"
fid = "cfile_69d4f1c97a1081908d22a9f56268c743"
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_r, _ = self.config.transform_container_retrieve_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_d, _ = self.config.transform_container_delete_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_lf, _ = self.config.transform_container_file_list_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_fc, _ = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
for name, u in (
("retrieve", url_r),
("delete", url_d),
("list_files", url_lf),
("file_content", url_fc),
):
assert f"containers?api-version=v1/{cid}" not in u, name
p = urlparse(u)
assert cid in p.path, name
assert "api-version" in p.query or "api-version=v1" in u, name
assert urlparse(url_fc).path.endswith(f"/{cid}/files/{fid}/content")
def test_regression_api_base_with_extra_query_params(self):
"""Multiple query params must stay at the end after path join."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1&foo=bar"
)
cid = "cntr_x"
url_lf, _ = self.config.transform_container_file_list_request(
container_id=cid,
api_base=api_base,
litellm_params=GenericLiteLLMParams(),
headers={},
)
p = urlparse(url_lf)
assert p.path == f"/openai/v1/containers/{cid}/files"
qs = parse_qs(p.query)
assert qs.get("api-version") == ["v1"]
assert qs.get("foo") == ["bar"]
def test_regression_proxy_resolves_azure_text_same_as_azure(self):
"""Router/proxy treat azure_text like azure for container config."""
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
c1 = _get_container_provider_config("azure")
c2 = _get_container_provider_config("azure_text")
assert type(c1) is type(c2)
assert isinstance(c1, AzureContainerConfig)

71
tests/test_litellm/containers/test_container_api.py
View file

@ -25,6 +25,7 @@ from litellm.containers.main import (
from litellm.main import base_llm_http_handler
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router import Router
from litellm.types.containers.main import (
ContainerListResponse,
@ -220,6 +221,76 @@ class TestContainerAPI:
assert response.expires_after.minutes == 20
assert response.expires_after.anchor == "last_active_at"
def test_retrieve_container_reencodes_short_managed_id_for_routing(self):
"""Short cntr_ IDs must still re-encode output so follow-ups keep router affinity."""
short_managed_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id="router-gpt",
container_id="x",
)
assert short_managed_id.startswith("cntr_")
assert len(short_managed_id) < 100
mock_response = ContainerObject(
id="x",
object="container",
created_at=1747857508,
status="running",
expires_after={"anchor": "last_active_at", "minutes": 20},
last_active_at=1747857508,
name="Tiny",
)
with patch.object(
base_llm_http_handler,
"container_retrieve_handler",
return_value=mock_response,
) as mock_method:
response = retrieve_container(
container_id=short_managed_id,
custom_llm_provider="openai",
)
mock_method.assert_called_once()
assert mock_method.call_args.kwargs["container_id"] == "x"
assert response.id.startswith("cntr_")
decoded = ResponsesAPIRequestUtils._decode_container_id(response.id)
assert decoded.get("response_id") == "x"
assert decoded.get("model_id") == "router-gpt"
assert decoded.get("custom_llm_provider") == "azure"
def test_delete_container_reencodes_short_managed_id_for_routing(self):
"""Same as retrieve: short managed IDs must round-trip encoding on delete result."""
short_managed_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id="router-gpt",
container_id="z",
)
assert len(short_managed_id) < 100
mock_response = DeleteContainerResult(
id="z",
object="container.deleted",
deleted=True,
)
with patch.object(
base_llm_http_handler,
"container_delete_handler",
return_value=mock_response,
) as mock_method:
response = delete_container(
container_id=short_managed_id,
custom_llm_provider="openai",
)
mock_method.assert_called_once()
assert mock_method.call_args.kwargs["container_id"] == "z"
assert response.id.startswith("cntr_")
decoded = ResponsesAPIRequestUtils._decode_container_id(response.id)
assert decoded.get("response_id") == "z"
assert decoded.get("model_id") == "router-gpt"
@pytest.mark.asyncio
async def test_aretrieve_container_basic(self):
"""Test basic async container retrieval functionality."""

47
tests/test_litellm/containers/test_container_utils.py
View file

@ -8,11 +8,17 @@ sys.path.insert(
) # Adds the parent directory to the system path
import litellm
from litellm.containers.utils import ContainerRequestUtils
from litellm.containers.utils import (
ContainerRequestUtils,
decode_managed_container_id_for_request,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.router import GenericLiteLLMParams
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerListOptionalRequestParams
ContainerListOptionalRequestParams,
DeleteContainerFileResponse,
)
@ -228,3 +234,40 @@ class TestContainerRequestUtils:
)
assert result["expires_after"]["minutes"] == 15
def test_decode_managed_container_id_returns_provider_container_id(self):
"""Managed IDs must decode to the short ID sent on upstream requests."""
inner = "cntr_69d4ff00deadbeef"
managed = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="openai",
model_id=None,
container_id=inner,
)
assert len(managed) > len(inner)
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams()
original_id, provider, updated = decode_managed_container_id_for_request(
managed, "openai", litellm_params
)
assert original_id == inner
assert provider == "openai"
assert updated is litellm_params
class TestDeleteContainerFileResponseWireFormat:
"""OpenAI / Azure return ``container.file.deleted`` on DELETE file."""
def test_accepts_openai_dot_notation(self):
m = DeleteContainerFileResponse(
id="cfile_abc",
object="container.file.deleted",
deleted=True,
)
assert m.object == "container.file.deleted"
def test_accepts_legacy_underscore(self):
m = DeleteContainerFileResponse(
id="cfile_abc",
object="container_file.deleted",
deleted=True,
)
assert m.object == "container_file.deleted"

36
tests/test_litellm/integrations/test_prometheus_services.py
View file

@ -104,3 +104,39 @@ def test_update_gauge():
# Verify correct methods were called
mock_labels.assert_called_once_with("test_label")
mock_gauge.set.assert_called_once_with(42.5)
def test_services_logger_default_latency_buckets():
"""PrometheusServicesLogger uses the new reduced default latency buckets."""
from litellm.types.integrations.prometheus import LATENCY_BUCKETS
pl = PrometheusServicesLogger()
assert pl.latency_buckets == LATENCY_BUCKETS
assert 420.0 in pl.latency_buckets
assert 600.0 in pl.latency_buckets
assert 1.5 not in pl.latency_buckets
def test_services_logger_custom_latency_buckets():
"""prometheus_latency_buckets setting is respected by PrometheusServicesLogger."""
import litellm
from prometheus_client import REGISTRY
custom_buckets = [0.1, 0.5, 1.0, 5.0, 10.0]
original = litellm.prometheus_latency_buckets
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
try:
litellm.prometheus_latency_buckets = custom_buckets
pl = PrometheusServicesLogger()
assert pl.latency_buckets == tuple(custom_buckets)
finally:
litellm.prometheus_latency_buckets = original
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass

39
tests/test_litellm/integrations/test_prometheus_user_team_metrics.py
View file

@ -768,3 +768,42 @@ async def test_initialize_org_budget_metrics(prometheus_logger):
prometheus_logger.litellm_org_max_budget_metric.labels().set.assert_called_once_with(
500.0
)
def test_default_latency_buckets(prometheus_logger):
"""PrometheusLogger uses the new reduced default latency buckets."""
from litellm.types.integrations.prometheus import LATENCY_BUCKETS
assert prometheus_logger.latency_buckets == LATENCY_BUCKETS
# 420 and 600 should be present
assert 420.0 in prometheus_logger.latency_buckets
assert 600.0 in prometheus_logger.latency_buckets
# dense half-second buckets from old defaults should be gone
assert 1.5 not in prometheus_logger.latency_buckets
assert 9.5 not in prometheus_logger.latency_buckets
def test_custom_latency_buckets():
"""prometheus_latency_buckets in litellm settings overrides the defaults."""
import litellm
from prometheus_client import REGISTRY
custom_buckets = [0.1, 0.5, 1.0, 5.0, 10.0]
original = litellm.prometheus_latency_buckets
# Clear registry before creating a new PrometheusLogger
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
try:
litellm.prometheus_latency_buckets = custom_buckets
logger = PrometheusLogger()
assert logger.latency_buckets == tuple(custom_buckets)
finally:
litellm.prometheus_latency_buckets = original
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass

203
tests/test_litellm/integrations/test_s3_v2.py
View file

@ -292,6 +292,209 @@ class TestS3V2UnitTests:
assert result == {"downloaded": "data"}
@pytest.mark.asyncio
async def test_async_upload_retries_on_s3_503():
"""
Test that async_upload_data_to_s3 retries on transient S3 503 Slow Down
and succeeds on the second attempt.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="2025-09-14/test-retry.json",
payload={"test": "retry"},
s3_object_download_filename="test-retry.json",
)
# First call returns 503, second call returns 200
response_503 = MagicMock()
response_503.status_code = 503
response_200 = MagicMock()
response_200.status_code = 200
response_200.raise_for_status = MagicMock()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = AsyncMock(side_effect=[response_503, response_200])
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
await logger.async_upload_data_to_s3(test_element)
# Verify PUT was called twice (retry after 503)
assert logger.async_httpx_client.put.call_count == 2
# Verify sleep was called with the backoff delay
mock_sleep.assert_called_once_with(1) # 2**0 = 1s
@pytest.mark.asyncio
async def test_async_upload_retries_on_s3_500():
"""
Test that async_upload_data_to_s3 retries on transient S3 500 errors.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="2025-09-14/test-retry-500.json",
payload={"test": "retry-500"},
s3_object_download_filename="test-retry-500.json",
)
response_500 = MagicMock()
response_500.status_code = 500
response_200 = MagicMock()
response_200.status_code = 200
response_200.raise_for_status = MagicMock()
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = AsyncMock(side_effect=[response_500, response_200])
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
await logger.async_upload_data_to_s3(test_element)
assert logger.async_httpx_client.put.call_count == 2
mock_sleep.assert_called_once_with(1)
@pytest.mark.asyncio
async def test_async_upload_exhausts_retries_on_persistent_503():
"""
Test that async_upload_data_to_s3 raises after exhausting all retries
on persistent S3 503.
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="2025-09-14/test-exhaust.json",
payload={"test": "exhaust"},
s3_object_download_filename="test-exhaust.json",
)
# All 3 attempts return 503
response_503 = MagicMock()
response_503.status_code = 503
response_503.raise_for_status = MagicMock(
side_effect=Exception("503 Service Unavailable")
)
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = AsyncMock(return_value=response_503)
with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
with patch.object(logger, "handle_callback_failure") as mock_failure:
await logger.async_upload_data_to_s3(test_element)
# 3 PUT attempts total
assert logger.async_httpx_client.put.call_count == 3
# 2 sleeps (between attempts 1-2 and 2-3)
assert mock_sleep.call_count == 2
# Callback failure handler called after exhausting retries
mock_failure.assert_called_once_with(callback_name="S3Logger")
@pytest.mark.asyncio
async def test_async_upload_no_retry_on_4xx():
"""
Test that async_upload_data_to_s3 does NOT retry on 4xx errors (client errors).
"""
from unittest.mock import AsyncMock, MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="2025-09-14/test-no-retry.json",
payload={"test": "no-retry"},
s3_object_download_filename="test-no-retry.json",
)
response_403 = MagicMock()
response_403.status_code = 403
response_403.raise_for_status = MagicMock(side_effect=Exception("403 Forbidden"))
logger.async_httpx_client = AsyncMock()
logger.async_httpx_client.put = AsyncMock(return_value=response_403)
with patch.object(logger, "handle_callback_failure") as mock_failure:
await logger.async_upload_data_to_s3(test_element)
# Only 1 attempt — no retry for 4xx
assert logger.async_httpx_client.put.call_count == 1
mock_failure.assert_called_once_with(callback_name="S3Logger")
def test_sync_upload_retries_on_s3_503():
"""
Test that the sync upload_data_to_s3 retries on transient S3 503.
"""
from unittest.mock import MagicMock
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
logger = S3Logger(
s3_bucket_name="test-bucket",
s3_aws_access_key_id="test-key",
s3_aws_secret_access_key="test-secret",
s3_region_name="us-east-1",
)
test_element = s3BatchLoggingElement(
s3_object_key="2025-09-14/test-sync-retry.json",
payload={"test": "sync-retry"},
s3_object_download_filename="test-sync-retry.json",
)
response_503 = MagicMock()
response_503.status_code = 503
response_200 = MagicMock()
response_200.status_code = 200
response_200.raise_for_status = MagicMock()
mock_sync_client = MagicMock()
mock_sync_client.put = MagicMock(side_effect=[response_503, response_200])
with patch(
"litellm.integrations.s3_v2._get_httpx_client",
return_value=mock_sync_client,
):
with patch("time.sleep") as mock_sleep:
logger.upload_data_to_s3(test_element)
assert mock_sync_client.put.call_count == 2
mock_sleep.assert_called_once_with(1)
@pytest.mark.asyncio
async def test_async_log_event_skips_when_standard_logging_object_missing():
"""

74
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -429,7 +429,7 @@ class TestUpdateFromKwargs:
assert logging_obj.litellm_params["litellm_metadata"] == lm_meta
def test_caller_litellm_params_win_over_kwargs(self, logging_obj):
"""Explicit litellm_params from the caller should override auto-extracted values."""
"""Explicit litellm_params metadata merges into kwargs metadata without overwriting."""
kwargs = {"metadata": {"from_kwargs": True}}
logging_obj.update_from_kwargs(
@ -437,7 +437,24 @@ class TestUpdateFromKwargs:
litellm_params={"metadata": {"from_caller": True}, "litellm_call_id": "x"},
)
assert logging_obj.litellm_params["metadata"] == {"from_caller": True}
# kwargs metadata is preserved, caller metadata is merged in
assert logging_obj.litellm_params["metadata"] == {"from_kwargs": True, "from_caller": True}
def test_kwargs_metadata_wins_over_caller_metadata_in_conflict(self, logging_obj):
"""kwargs metadata takes precedence; caller litellm_params metadata is merged without overwriting."""
kwargs = {"metadata": {"from_kwargs": True, "shared_key": "kwargs_value"}}
logging_obj.update_from_kwargs(
kwargs=kwargs,
litellm_params={"metadata": {"from_caller": True, "shared_key": "caller_value"}, "litellm_call_id": "x"},
)
# kwargs metadata is preserved (shared_key keeps the kwargs value), caller-only keys are added
assert logging_obj.litellm_params["metadata"] == {
"from_kwargs": True,
"from_caller": True,
"shared_key": "kwargs_value", # kwargs wins on conflict
}
def test_custom_pricing_detected_via_litellm_metadata(self, logging_obj):
"""Custom pricing in litellm_metadata.model_info should set custom_pricing flag."""
@ -2153,6 +2170,59 @@ def test_function_setup_metadata_takes_precedence_over_litellm_metadata():
assert litellm_metadata.get("user_api_key_hash") == "sk-hashed-xyz"
def test_update_from_kwargs_litellm_params_metadata_does_not_overwrite_proxy_fields():
"""
Test the exact bug: when update_from_kwargs is called with litellm_params
containing a 'metadata' key (e.g. Anthropic's native metadata with user_id),
it must NOT overwrite proxy key-auth fields already merged from litellm_metadata.
This is the anthropic_messages code path where async_anthropic_messages_handler
passes anthropic_messages_optional_request_params (which includes metadata)
as litellm_params to update_from_kwargs.
"""
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "test"}],
stream=False,
call_type="anthropic_messages",
start_time=time.time(),
litellm_call_id="test-overwrite-bug",
function_id="test-function-id",
)
kwargs = {
"litellm_metadata": {
"user_api_key_hash": "sk-hashed-proxy",
"user_api_key_alias": "claude-api",
"user_api_key_team_id": "team-zurich",
},
}
# Simulate what async_anthropic_messages_handler does:
# passes Anthropic's native metadata in litellm_params
logging_obj.update_from_kwargs(
kwargs=kwargs,
litellm_params={
"preset_cache_key": None,
"stream_response": {},
"metadata": {"user_id": "anthropic-device-id"}, # Anthropic native metadata
},
)
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
metadata = litellm_params.get("metadata")
assert metadata is not None
# Proxy key-auth fields must survive the litellm_params.update()
assert metadata.get("user_api_key_hash") == "sk-hashed-proxy"
assert metadata.get("user_api_key_alias") == "claude-api"
assert metadata.get("user_api_key_team_id") == "team-zurich"
# Anthropic native metadata must also be present
assert metadata.get("user_id") == "anthropic-device-id"
def test_function_setup_empty_metadata_falls_back_to_litellm_metadata():
"""
Test that when metadata is explicitly set to {} (empty dict), litellm_metadata

94
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -770,7 +770,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging):
from litellm.llms.vertex_ai.common_utils import VertexAIError
async def _raise_bad_request(**kwargs):
raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None)
raise VertexAIError(
status_code=400, message="invalid maxOutputTokens", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -788,7 +790,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging):
@pytest.mark.asyncio
async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logging):
async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(
logging_obj: Logging,
):
"""Ensure Vertex 429 rate-limit errors raise MidStreamFallbackError, not RateLimitError.
Regression test for https://github.com/BerriAI/litellm/issues/20870
@ -797,7 +801,9 @@ async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_o
from litellm.llms.vertex_ai.common_utils import VertexAIError
async def _raise_rate_limit(**kwargs):
raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None)
raise VertexAIError(
status_code=429, message="Resource exhausted.", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -825,7 +831,9 @@ def test_sync_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logg
from litellm.llms.vertex_ai.common_utils import VertexAIError
def _raise_rate_limit(**kwargs):
raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None)
raise VertexAIError(
status_code=429, message="Resource exhausted.", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -850,7 +858,9 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging):
from litellm.llms.vertex_ai.common_utils import VertexAIError
def _raise_bad_request(**kwargs):
raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None)
raise VertexAIError(
status_code=400, message="invalid maxOutputTokens", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -1363,6 +1373,7 @@ def _build_chunks(pattern: list[str], N: int) -> list[ModelResponseStream]:
chunks.append(_make_chunk(p))
return chunks
_REPETITION_TEST_CASES = [
# Basic cases
pytest.param(
@ -1419,7 +1430,14 @@ _REPETITION_TEST_CASES = [
id="last_chunk_different_no_raise",
),
pytest.param(
["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + ["different_mid"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1),
["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1)
+ ["different_mid"]
+ ["same"]
* (
litellm.REPEATED_STREAMING_CHUNK_LIMIT
- litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2
+ 1
),
False,
id="middle_chunk_different_no_raise",
),
@ -1429,7 +1447,9 @@ _REPETITION_TEST_CASES = [
id="last_two_different_no_raise",
),
pytest.param(
["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["diff"],
["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT
+ ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT
+ ["diff"],
True,
id="in_between_same_and_diff_raise",
),
@ -1455,6 +1475,8 @@ def test_raise_on_model_repetition(
for chunk in chunks:
wrapper.chunks.append(chunk)
wrapper.raise_on_model_repetition()
def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj):
"""
Test that provider-reported usage from a post-finish_reason chunk
@ -1536,12 +1558,13 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj):
last_chunk = collected[-1]
hidden_usage = last_chunk._hidden_params.get("usage")
assert hidden_usage is not None, "Expected usage in _hidden_params"
assert hidden_usage.prompt_tokens == 20, (
f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}"
)
assert hidden_usage.completion_tokens == 135, (
f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}"
)
assert (
hidden_usage.prompt_tokens == 20
), f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}"
assert (
hidden_usage.completion_tokens == 135
), f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}"
@pytest.mark.asyncio
async def test_custom_stream_wrapper_aclose():
@ -1615,9 +1638,9 @@ def test_content_not_dropped_when_finish_reason_already_set(
result = initialized_custom_stream_wrapper.chunk_creator(chunk=content_chunk)
assert result is not None, (
"chunk_creator() returned None — content was dropped (issue #22098)"
)
assert (
result is not None
), "chunk_creator() returned None — content was dropped (issue #22098)"
assert result.choices[0].delta.content == "world!"
@ -1669,18 +1692,45 @@ def test_tool_use_not_dropped_when_finish_reason_already_set(
result = initialized_custom_stream_wrapper.chunk_creator(chunk=tool_chunk)
assert result is not None, (
"chunk_creator() returned None — tool_use data was dropped"
)
assert (
result is not None
), "chunk_creator() returned None — tool_use data was dropped"
tool_calls = result.choices[0].delta.tool_calls
assert tool_calls is not None and len(tool_calls) > 0, (
"tool_calls should contain at least one tool call"
)
assert (
tool_calls is not None and len(tool_calls) > 0
), "tool_calls should contain at least one tool call"
assert tool_calls[0].id == "call_1"
assert tool_calls[0].function.name == "get_weather"
def test_usage_only_chunk_not_dropped_when_finish_reason_already_set(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
Regression test: usage-only chunks must not be dropped once finish_reason
is already set. Dropping these chunks can lose terminal finish_reason in
downstream Responses API streaming translation.
"""
initialized_custom_stream_wrapper.received_finish_reason = "content_filter"
initialized_custom_stream_wrapper.custom_llm_provider = "anthropic"
usage_only_chunk = {
"text": "",
"tool_use": None,
"is_finished": False,
"finish_reason": "",
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
"index": 0,
}
result = initialized_custom_stream_wrapper.chunk_creator(chunk=usage_only_chunk)
assert result is not None, "usage-only chunk should not be dropped"
assert result.choices[0].finish_reason == "content_filter"
assert result.usage is not None
@pytest.mark.asyncio
async def test_custom_stream_wrapper_anext_does_not_block_event_loop_for_sync_iterators(
logging_obj: Logging,

54
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -2418,6 +2418,60 @@ def test_empty_assistant_message_handling():
assert result[1]["content"][0]["text"] == "I'm doing well, thank you!"
def test_bedrock_converse_trailing_prefix_assistant_skips_user_continue():
"""Assistant prefill (prefix: true) must not inject a dummy user 'Please continue.' turn."""
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "user", "content": "Hello, how are you?"},
{
"role": "assistant",
"content": "Good as",
"prefix": True,
},
]
with patch.object(factory_module.litellm, "modify_params", True):
result = _bedrock_converse_messages_pt(
messages=list(messages),
model="anthropic.claude-haiku-4-5-20251001-v1:0",
llm_provider="bedrock_converse",
)
assert len(result) == 2
assert result[0]["role"] == "user"
assert result[1]["role"] == "assistant"
assert result[1]["content"][0]["text"] == "Good as"
def test_bedrock_converse_leading_prefix_assistant_skips_user_continue():
"""Leading assistant with prefix: true should not prepend dummy user."""
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "assistant", "content": "Partial", "prefix": True},
{"role": "user", "content": "Go on"},
]
with patch.object(factory_module.litellm, "modify_params", True):
result = _bedrock_converse_messages_pt(
messages=list(messages),
model="anthropic.claude-haiku-4-5-20251001-v1:0",
llm_provider="bedrock_converse",
)
assert len(result) == 2
assert result[0]["role"] == "assistant"
assert result[0]["content"][0]["text"] == "Partial"
assert result[1]["role"] == "user"
def test_is_nova_2_model():
"""Test the _is_nova_2_model() method for detecting Nova 2 models."""
config = AmazonConverseConfig()

152
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
View file

@ -1186,6 +1186,156 @@ async def test_bedrock_guardrail_blocked_content_with_masking_enabled():
# Verify exception details
assert exc_info.value.status_code == 400
assert "Violated guardrail policy" in str(exc_info.value.detail)
print("✅ BLOCKED content with masking enabled raises exception correctly")
# ---------------------------------------------------------------------------
# L3: _extract_blocked_assessments + _get_http_exception_for_blocked_guardrail
# Regression coverage for case 2026-04-10-internal-bedrock-guardrail-streaming-error.
# ---------------------------------------------------------------------------
def _make_guardrail() -> BedrockGuardrail:
return BedrockGuardrail(
guardrail_name="bedrock-pii-guard",
guardrailIdentifier="amgllac6xf3r",
guardrailVersion="1",
)
def test_extract_blocked_assessments_pii_entity():
"""L3: PII entity match (BLOCKED) is surfaced with category, type, and matched term."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "action": "BLOCKED", "match": "Jack"},
{"type": "EMAIL", "action": "ANONYMIZED", "match": "x@y.z"},
]
}
}
],
}
blocked = g._extract_blocked_assessments(response)
assert len(blocked) == 1
assert blocked[0]["policy"] == "sensitiveInformationPolicy"
matches = blocked[0]["matches"]
assert len(matches) == 1 # only the BLOCKED one is surfaced
assert matches[0]["category"] == "piiEntities"
assert matches[0]["type"] == "NAME"
assert matches[0]["match"] == "Jack"
def test_extract_blocked_assessments_multiple_policies():
"""L3: multiple policies fired in one assessment must all be reported."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
"assessments": [
{
"topicPolicy": {
"topics": [
{"name": "Investment", "type": "DENY", "action": "BLOCKED"}
]
},
"contentPolicy": {
"filters": [
{
"type": "VIOLENCE",
"confidence": "HIGH",
"filterStrength": "HIGH",
"action": "BLOCKED",
}
]
},
"wordPolicy": {
"customWords": [{"match": "forbidden", "action": "BLOCKED"}]
},
}
],
}
blocked = g._extract_blocked_assessments(response)
policies = {entry["policy"] for entry in blocked}
assert policies == {"topicPolicy", "contentPolicy", "wordPolicy"}
def test_extract_blocked_assessments_only_anonymized_returns_empty():
"""L3: if all matches are ANONYMIZED (not BLOCKED), the list is empty."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "action": "ANONYMIZED", "match": "Jack"}
]
}
}
],
}
assert g._extract_blocked_assessments(response) == []
def test_extract_blocked_assessments_no_assessments():
"""L3: response with no assessments returns an empty list, not an error."""
g = _make_guardrail()
assert g._extract_blocked_assessments({"action": "NONE"}) == []
assert g._extract_blocked_assessments({"assessments": None}) == []
def test_get_http_exception_includes_assessments_and_identifier():
"""L3: end-to-end — _get_http_exception_for_blocked_guardrail emits the new fields."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
"outputs": [{"text": "Sorry, the model cannot answer this question."}],
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "action": "BLOCKED", "match": "Jack"}
]
}
}
],
}
exc = g._get_http_exception_for_blocked_guardrail(response)
assert isinstance(exc, HTTPException)
assert exc.status_code == 400
assert exc.detail["error"] == "Violated guardrail policy"
assert (
exc.detail["bedrock_guardrail_response"]
== "Sorry, the model cannot answer this question."
)
assert exc.detail["guardrailIdentifier"] == "amgllac6xf3r"
assert exc.detail["guardrailVersion"] == "1"
assert exc.detail["assessments"][0]["policy"] == "sensitiveInformationPolicy"
assert exc.detail["assessments"][0]["matches"][0]["type"] == "NAME"
def test_get_http_exception_no_blocked_assessments_omits_field():
"""L3: when no assessments are blocked, the `assessments` key is omitted entirely."""
g = _make_guardrail()
response = {
"action": "GUARDRAIL_INTERVENED",
"outputs": [{"text": "blocked"}],
"assessments": [
{
"sensitiveInformationPolicy": {
"piiEntities": [
{"type": "NAME", "action": "ANONYMIZED", "match": "Jack"}
]
}
}
],
}
exc = g._get_http_exception_for_blocked_guardrail(response)
assert isinstance(exc, HTTPException)
assert "assessments" not in exc.detail
assert exc.detail["guardrailIdentifier"] == "amgllac6xf3r"

111
tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py
View file

@ -2,9 +2,17 @@
import pytest
import litellm
from litellm.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
openai_messages_without_system,
)
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse
from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler
from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import (
@ -68,6 +76,109 @@ def _inject_mcp_handler_mapping():
class TestUnifiedLLMGuardrails:
class TestSkipSystemMessageForChatCompletions:
def test_openai_messages_without_system(self):
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": "hi"},
]
out = openai_messages_without_system(msgs)
assert len(out) == 1
assert out[0]["role"] == "user"
assert msgs[0]["content"] == "sys"
def test_effective_skip_respects_per_guardrail_over_global(self, monkeypatch):
monkeypatch.setattr(
litellm, "skip_system_message_in_guardrail", True, raising=False
)
class G:
skip_system_message_in_guardrail = False
assert effective_skip_system_message_for_guardrail(G()) is False
class G2:
skip_system_message_in_guardrail = None
assert effective_skip_system_message_for_guardrail(G2()) is True
@pytest.mark.asyncio
async def test_openai_handler_skips_system_in_guardrail_inputs(
self, monkeypatch
):
monkeypatch.setattr(
litellm, "skip_system_message_in_guardrail", True, raising=False
)
captured = {}
class MockGuardrail:
skip_system_message_in_guardrail = None
async def apply_guardrail(
self, inputs, request_data, input_type, logging_obj=None
):
captured["inputs"] = inputs
return inputs
data = {
"messages": [
{"role": "system", "content": "secret system"},
{"role": "user", "content": "hello"},
],
"model": "gpt-4o",
}
handler = OpenAIChatCompletionsHandler()
await handler.process_input_messages(
data=data,
guardrail_to_apply=MockGuardrail(),
litellm_logging_obj=None,
)
assert captured["inputs"]["texts"] == ["hello"]
sm = captured["inputs"].get("structured_messages") or []
assert all(m.get("role") != "system" for m in sm)
assert data["messages"][0]["content"] == "secret system"
@pytest.mark.asyncio
async def test_openai_handler_per_guardrail_skip_false_overrides_global(
self, monkeypatch
):
monkeypatch.setattr(
litellm, "skip_system_message_in_guardrail", True, raising=False
)
captured = {}
class MockGuardrail:
skip_system_message_in_guardrail = False
async def apply_guardrail(
self, inputs, request_data, input_type, logging_obj=None
):
captured["inputs"] = inputs
return inputs
data = {
"messages": [
{"role": "system", "content": "sys"},
{"role": "user", "content": "u"},
],
}
await OpenAIChatCompletionsHandler().process_input_messages(
data=data,
guardrail_to_apply=MockGuardrail(),
litellm_logging_obj=None,
)
assert "sys" in captured["inputs"]["texts"]
roles = {
m.get("role") for m in (captured["inputs"].get("structured_messages") or [])
}
assert "system" in roles
class TestAsyncPreCallHook:
@pytest.mark.asyncio
async def test_uses_mcp_event_type(self):

179
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -886,14 +886,17 @@ class TestCommonRequestProcessingHelpers:
response = await create_response(mock_gen, "text/event-stream", {})
assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
content = await self.consume_stream(response)
# Streaming SSE error frame now mirrors ProxyException.to_dict() shape
# so streaming and non-streaming surfaces emit byte-identical errors.
expected_error_data = {
"error": {
"message": "Error processing stream start",
"code": status.HTTP_500_INTERNAL_SERVER_ERROR,
"type": "None",
"param": "None",
"code": str(status.HTTP_500_INTERNAL_SERVER_ERROR),
}
}
assert len(content) == 2
# Use json.dumps to match the formatting in create_streaming_response's exception handler
import json
assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n"
@ -919,13 +922,130 @@ class TestCommonRequestProcessingHelpers:
expected_error_data = {
"error": {
"message": "Content blocked by guardrail",
"code": 400,
"type": "None",
"param": "None",
"code": "400",
}
}
assert len(content) == 2
assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n"
assert content[1] == "data: [DONE]\n\n"
async def test_create_streaming_response_http_exception_dict_detail_bedrock_shape(
self,
):
"""
Bedrock-style dict detail (with the post-L3 shape) must be preserved as
structured `provider_specific_fields` in the SSE error frame, not stringified
into a Python-repr blob inside `error.message`. Regression for case
2026-04-10-internal-bedrock-guardrail-streaming-error.
"""
import json
mock_gen = AsyncMock()
mock_gen.__anext__.side_effect = HTTPException(
status_code=400,
detail={
"error": "Violated guardrail policy",
"bedrock_guardrail_response": "Sorry, the model cannot answer this question. Prompt is blocked",
"guardrailIdentifier": "amgllac6xf3r",
"guardrailVersion": "1",
"assessments": [
{
"policy": "sensitiveInformationPolicy",
"matches": [
{
"category": "piiEntities",
"type": "NAME",
"action": "BLOCKED",
"match": "Jack",
}
],
}
],
"guardrail_name": "bedrock-pii-guard",
"guardrail_mode": "post_call",
},
)
response = await create_response(mock_gen, "text/event-stream", {})
assert response.status_code == 400
content = await self.consume_stream(response)
assert len(content) == 2
assert content[1] == "data: [DONE]\n\n"
payload = json.loads(content[0][len("data: ") :].strip())
assert payload["error"]["message"] == "Violated guardrail policy"
assert payload["error"]["code"] == "400"
psf = payload["error"]["provider_specific_fields"]
assert psf["guardrail_name"] == "bedrock-pii-guard"
assert psf["guardrail_mode"] == "post_call"
assert psf["guardrailIdentifier"] == "amgllac6xf3r"
assert psf["assessments"][0]["policy"] == "sensitiveInformationPolicy"
assert psf["assessments"][0]["matches"][0]["type"] == "NAME"
async def test_create_streaming_response_http_exception_dict_detail_nested_error_shape(
self,
):
"""PANW Prisma AIRS-style nested `{"error": {"message": ...}}` detail must
extract `error.message` as the human-readable summary while preserving the
full payload."""
import json
mock_gen = AsyncMock()
mock_gen.__anext__.side_effect = HTTPException(
status_code=400,
detail={
"error": {
"message": "MCP request blocked: no rewritable argument field present",
"type": "guardrail_violation",
"code": "panw_prisma_airs_blocked",
}
},
)
response = await create_response(mock_gen, "text/event-stream", {})
content = await self.consume_stream(response)
payload = json.loads(content[0][len("data: ") :].strip())
assert (
payload["error"]["message"]
== "MCP request blocked: no rewritable argument field present"
)
assert (
payload["error"]["provider_specific_fields"]["error"]["code"]
== "panw_prisma_airs_blocked"
)
async def test_serialize_http_exception_detail_helper(self):
"""Direct unit coverage for the L1 helper across all branches."""
from litellm.proxy.common_request_processing import (
_serialize_http_exception_detail,
)
import json as _json
assert _serialize_http_exception_detail("plain") == ("plain", None)
msg, fields = _serialize_http_exception_detail(
{"error": "Violated", "extra": "x"}
)
assert msg == "Violated"
assert fields == {"error": "Violated", "extra": "x"}
msg, fields = _serialize_http_exception_detail(
{"error": {"message": "blocked", "code": "x"}}
)
assert msg == "blocked"
assert fields == {"error": {"message": "blocked", "code": "x"}}
msg, fields = _serialize_http_exception_detail({"message": "top-level"})
assert msg == "top-level"
assert fields == {"message": "top-level"}
msg, fields = _serialize_http_exception_detail({"weird": ["a", "b"]})
assert msg == _json.dumps({"weird": ["a", "b"]})
assert fields == {"weird": ["a", "b"]}
assert _serialize_http_exception_detail(42) == ("42", None)
async def test_create_streaming_response_first_chunk_error_string_code(self):
"""
Test that when the first chunk contains a string error code, a JSON error response is returned
@ -1853,3 +1973,56 @@ class TestHasAttributeErrorInChain:
exc_a.__context__ = exc_b
exc_b.__context__ = exc_a # circular
assert _has_attribute_error_in_chain(exc_a) is False
@pytest.mark.asyncio
class TestHandleLLMApiExceptionDictDetail:
"""
Coverage for `_handle_llm_api_exception` HTTPException branch (Site 2).
Regression for case 2026-04-10-internal-bedrock-guardrail-streaming-error:
dict-detail HTTPExceptions raised by guardrails must round-trip cleanly
through ProxyException instead of being str()-mangled into a Python repr.
"""
async def _invoke(self, exc: Exception):
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
processor = ProxyBaseLLMRequestProcessing(data={})
user_api_key_dict = UserAPIKeyAuth(api_key="sk-test")
proxy_logging_obj = MagicMock()
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
try:
await processor._handle_llm_api_exception(
e=exc,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
)
except ProxyException as raised:
return raised
raise AssertionError("ProxyException was not raised")
async def test_dict_detail_bedrock_shape_preserved(self):
exc = HTTPException(
status_code=400,
detail={
"error": "Violated guardrail policy",
"bedrock_guardrail_response": "...",
"guardrail_name": "bedrock-pii-guard",
},
)
proxy_exc = await self._invoke(exc)
assert proxy_exc.message == "Violated guardrail policy"
assert (
proxy_exc.provider_specific_fields["guardrail_name"]
== "bedrock-pii-guard"
)
# No Python repr leakage of the dict into the message field.
assert "{'error':" not in proxy_exc.message
async def test_string_detail_unchanged(self):
exc = HTTPException(status_code=400, detail="Content blocked by guardrail")
proxy_exc = await self._invoke(exc)
assert proxy_exc.message == "Content blocked by guardrail"
assert proxy_exc.provider_specific_fields is None

76
tests/test_litellm/proxy/test_proxy_utils.py
View file

@ -190,3 +190,79 @@ def test_get_projected_spend_over_limit_includes_current_spend(monkeypatch):
projected_spend, projected_exceeded_date = result
assert projected_spend == 290.0
assert projected_exceeded_date == real_datetime.date(2026, 4, 21)
# ---------------------------------------------------------------------------
# L2: _enrich_http_exception_with_guardrail_context
# Regression coverage for case 2026-04-10-internal-bedrock-guardrail-streaming-error.
# ---------------------------------------------------------------------------
def test_enrich_http_exception_with_guardrail_context_dict_detail():
"""L2: dict-detail HTTPException is enriched with guardrail_name and mode."""
from litellm.proxy.utils import _enrich_http_exception_with_guardrail_context
class StubCallback:
guardrail_name = "bedrock-pii-guard"
event_hook = "post_call"
exc = HTTPException(
status_code=400, detail={"error": "Violated guardrail policy"}
)
_enrich_http_exception_with_guardrail_context(exc, StubCallback())
assert exc.detail["guardrail_name"] == "bedrock-pii-guard"
assert exc.detail["guardrail_mode"] == "post_call"
def test_enrich_http_exception_string_detail_noop():
"""L2: string-detail HTTPException is not mutated (can't add fields to a str)."""
from litellm.proxy.utils import _enrich_http_exception_with_guardrail_context
class StubCallback:
guardrail_name = "x"
event_hook = "pre_call"
exc = HTTPException(status_code=400, detail="Content blocked")
_enrich_http_exception_with_guardrail_context(exc, StubCallback())
assert exc.detail == "Content blocked"
def test_enrich_http_exception_setdefault_does_not_overwrite():
"""L2: a guardrail that already populates guardrail_name explicitly wins."""
from litellm.proxy.utils import _enrich_http_exception_with_guardrail_context
class StubCallback:
guardrail_name = "inferred-name"
event_hook = "pre_call"
exc = HTTPException(
status_code=400,
detail={"error": "x", "guardrail_name": "explicit-name"},
)
_enrich_http_exception_with_guardrail_context(exc, StubCallback())
assert exc.detail["guardrail_name"] == "explicit-name"
def test_enrich_http_exception_non_http_exception_noop():
"""L2: non-HTTPException is left alone and the helper does not raise."""
from litellm.proxy.utils import _enrich_http_exception_with_guardrail_context
class StubCallback:
guardrail_name = "x"
event_hook = "pre_call"
exc = ValueError("not an HTTPException")
_enrich_http_exception_with_guardrail_context(exc, StubCallback())
assert str(exc) == "not an HTTPException"
def test_enrich_http_exception_callback_without_guardrail_name_noop():
"""L2: callback without guardrail_name attribute leaves detail alone."""
from litellm.proxy.utils import _enrich_http_exception_with_guardrail_context
class StubCallback:
pass
exc = HTTPException(status_code=400, detail={"error": "x"})
_enrich_http_exception_with_guardrail_context(exc, StubCallback())
assert exc.detail == {"error": "x"}

202
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -504,6 +504,35 @@ class TestLiteLLMCompletionResponsesConfig:
]
assert item.status != "stop"
def test_transform_chat_completion_response_status_with_refusal(self):
"""
`finish_reason=refusal` should map to `status=incomplete` in Responses API.
"""
chat_completion_response = ModelResponse(
id="test-response-id",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion",
choices=[
Choices(
finish_reason="refusal",
index=0,
message=Message(
content="",
role="assistant",
),
)
],
)
responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input="this is a test",
responses_api_request={},
chat_completion_response=chat_completion_response,
)
assert responses_api_response.status == "incomplete"
def test_transform_chat_completion_response_preserves_hidden_params(self):
"""Test that _hidden_params from chat completion response are preserved in responses API response"""
# Setup
@ -976,10 +1005,11 @@ class TestToolTransformation:
tools = [vertex_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -999,10 +1029,11 @@ class TestToolTransformation:
tools = [mcp_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1022,10 +1053,11 @@ class TestToolTransformation:
tools = [computer_use_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1045,10 +1077,11 @@ class TestToolTransformation:
tools = [web_search_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1077,10 +1110,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1108,10 +1142,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1135,10 +1170,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1162,10 +1198,11 @@ class TestToolTransformation:
tools = [code_execution_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1187,10 +1224,11 @@ class TestToolTransformation:
tools = [tool_search_regex, tool_search_bm25]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1220,10 +1258,11 @@ class TestToolTransformation:
]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1256,10 +1295,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1280,10 +1320,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1302,10 +1343,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1325,10 +1367,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -2055,6 +2098,53 @@ class TestEnsureOutputItemContentPartAdded:
assert events[1].part.type == "output_text"
assert iterator.sent_content_part_added_event is True
def test_emit_response_completed_uses_stream_finish_reason(self):
"""
When the assembled model response carries finish_reason="content_filter"
(snapshotted from the underlying stream before any pending events fire),
_emit_response_completed_event must produce status="incomplete".
"""
from unittest.mock import Mock
import litellm
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
mock_stream_wrapper = Mock(spec=litellm.CustomStreamWrapper)
mock_stream_wrapper.logging_obj = Mock()
iterator = LiteLLMCompletionStreamingIterator(
model="anthropic/claude-sonnet-4-6",
litellm_custom_stream_wrapper=mock_stream_wrapper,
request_input="test",
responses_api_request={},
custom_llm_provider="anthropic",
)
litellm_model_response = ModelResponse(
id="chatcmpl-test",
created=1234567890,
model="anthropic/claude-sonnet-4-6",
object="chat.completion",
choices=[
Choices(
finish_reason="content_filter",
index=0,
message=Message(content="", role="assistant"),
)
],
usage=Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11),
)
completed_event = iterator._emit_response_completed_event(
litellm_model_response
)
assert completed_event is not None
assert completed_event.response.status == "incomplete"
assert completed_event.response.output[0].status == "incomplete"
def test_reasoning_item_does_not_emit_content_part_added(self):
"""Reasoning items should not get a content_part.added event."""
from litellm.types.llms.openai import OutputItemAddedEvent

29
tests/test_litellm/responses/test_responses_utils.py
View file

@ -138,6 +138,35 @@ class TestResponsesAPIRequestUtils:
assert decoded.get("model_id") == "gpt-4o"
assert decoded.get("custom_llm_provider") == "openai"
def test_build_decode_container_id_omits_none_model_id(self):
"""model_id=None must not round-trip as the truthy string 'None'."""
encoded = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id=None,
container_id="cntr_upstream_abc",
)
assert "None" not in base64.b64decode(
encoded.replace("cntr_", "").encode("utf-8")
).decode("utf-8")
decoded = ResponsesAPIRequestUtils._decode_container_id(encoded)
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("model_id") is None
assert decoded.get("response_id") == "cntr_upstream_abc"
def test_decode_container_id_legacy_literal_none_model_id(self):
"""IDs encoded before the None fix should decode without a bogus model_id."""
legacy_inner = (
"litellm:custom_llm_provider:azure;model_id:None;container_id:cntr_x"
)
legacy_id = (
"cntr_"
+ base64.b64encode(legacy_inner.encode("utf-8")).decode("utf-8")
)
decoded = ResponsesAPIRequestUtils._decode_container_id(legacy_id)
assert decoded.get("model_id") is None
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("response_id") == "cntr_x"
class TestResponseAPILoggingUtils:
def test_is_response_api_usage_true(self):

20
ui/litellm-dashboard/src/components/guardrails/add_guardrail_form.tsx
View file

@ -4,6 +4,7 @@ import NotificationsManager from "../molecules/notifications_manager";
import { createGuardrailCall, getGuardrailProviderSpecificParams, getGuardrailUISettings } from "../networking";
import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration";
import {
choiceToSkipSystemForCreate,
getGuardrailProviders,
guardrail_provider_map,
guardrailLogoMap,
@ -179,6 +180,7 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
guardrail_name: preset.guardrailNameSuggestion,
mode: preset.mode,
default_on: preset.defaultOn,
skip_system_message_choice: "inherit",
};
if (preset.provider === "BlockCodeExecution") {
baseValues.confidence_threshold = 0.5;
@ -414,6 +416,11 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
guardrail_info: {},
};
const skipForCreate = choiceToSkipSystemForCreate(values.skip_system_message_choice);
if (skipForCreate !== undefined) {
guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate;
}
// For Presidio PII, add the entity and action configurations
if (values.provider === "PresidioPII" && selectedEntities.length > 0) {
const piiEntitiesConfig: { [key: string]: string } = {};
@ -749,6 +756,18 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
</Select>
</Form.Item>
<Form.Item
name="skip_system_message_choice"
label="Skip system messages in guardrail"
tooltip="Unified guardrails only: omit role: system from guardrail evaluation input (OpenAI chat + Anthropic messages). The model still receives full messages. Use global default follows litellm_settings.skip_system_message_in_guardrail."
>
<Select>
<Select.Option value="inherit">Use global default</Select.Option>
<Select.Option value="yes">Yes — exclude from guardrail scan</Select.Option>
<Select.Option value="no">No — always include in scan</Select.Option>
</Select>
</Form.Item>
{/* Use the GuardrailProviderFields component to render provider-specific fields */}
{!isToolPermissionProvider && !shouldRenderContentFilterConfigSettings(selectedProvider) && (
<GuardrailProviderFields
@ -1096,6 +1115,7 @@ const AddGuardrailForm: React.FC<AddGuardrailFormProps> = ({ visible, onClose, a
initialValues={{
mode: "pre_call",
default_on: false,
skip_system_message_choice: "inherit",
}}
>
{stepConfigs.map((step, index) => {

89
ui/litellm-dashboard/src/components/guardrails/edit_guardrail_form.tsx
View file

@ -1,7 +1,12 @@
import React, { useState, useEffect } from "react";
import { Form, Typography, Select, Input, Switch, Modal } from "antd";
import { Button, TextInput } from "@tremor/react";
import { guardrail_provider_map, guardrailLogoMap, getGuardrailProviders } from "./guardrail_info_helpers";
import {
guardrail_provider_map,
guardrailLogoMap,
getGuardrailProviders,
type SkipSystemMessageChoice,
} from "./guardrail_info_helpers";
import { getGuardrailUISettings, getGlobalLitellmHeaderName } from "../networking";
import PiiConfiguration from "./pii_configuration";
import NotificationsManager from "../molecules/notifications_manager";
@ -15,12 +20,15 @@ interface EditGuardrailFormProps {
accessToken: string | null;
onSuccess: () => void;
guardrailId: string;
/** Full stored params merged into PUT so optional fields (e.g. content filter) are preserved. */
fullLitellmParams?: Record<string, any> | null;
initialValues: {
guardrail_name: string;
provider: string;
mode: string;
default_on: boolean;
pii_entities_config?: { [key: string]: string };
skip_system_message_choice?: SkipSystemMessageChoice;
[key: string]: any;
};
}
@ -41,6 +49,7 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
accessToken,
onSuccess,
guardrailId,
fullLitellmParams,
initialValues,
}) => {
const [form] = Form.useForm();
@ -113,31 +122,23 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
// Get the guardrail provider value from the map
const guardrailProvider = guardrail_provider_map[values.provider];
// Prepare the guardrail data with proper types for litellm_params
const guardrailData: {
guardrail_id: string;
guardrail: {
guardrail_name: string;
litellm_params: {
guardrail: string;
mode: string;
default_on: boolean;
[key: string]: any; // Allow dynamic properties
};
guardrail_info: any;
};
} = {
guardrail_id: guardrailId,
guardrail: {
guardrail_name: values.guardrail_name,
litellm_params: {
guardrail: guardrailProvider,
mode: values.mode,
default_on: values.default_on,
},
guardrail_info: {},
},
};
const litellm_params: Record<string, any> =
fullLitellmParams && typeof fullLitellmParams === "object" ? { ...fullLitellmParams } : {};
litellm_params.guardrail = guardrailProvider;
litellm_params.mode = values.mode;
litellm_params.default_on = values.default_on;
const skipChoice = values.skip_system_message_choice as SkipSystemMessageChoice | undefined;
if (skipChoice === "yes") {
litellm_params.skip_system_message_in_guardrail = true;
} else if (skipChoice === "no") {
litellm_params.skip_system_message_in_guardrail = false;
} else {
delete litellm_params.skip_system_message_in_guardrail;
}
let guardrail_info: any = {};
// For Presidio PII, add the entity and action configurations
if (values.provider === "PresidioPII" && selectedEntities.length > 0) {
@ -146,7 +147,7 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
piiEntitiesConfig[entity] = selectedActions[entity] || "MASK"; // Default to MASK if no action selected
});
guardrailData.guardrail.litellm_params.pii_entities_config = piiEntitiesConfig;
litellm_params.pii_entities_config = piiEntitiesConfig;
}
// Add config values to the guardrail_info if provided
else if (values.config) {
@ -156,14 +157,14 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
// Especially for providers like Bedrock that need guardrailIdentifier and guardrailVersion
if (values.provider === "Bedrock" && configObj) {
if (configObj.guardrail_id) {
guardrailData.guardrail.litellm_params.guardrailIdentifier = configObj.guardrail_id;
litellm_params.guardrailIdentifier = configObj.guardrail_id;
}
if (configObj.guardrail_version) {
guardrailData.guardrail.litellm_params.guardrailVersion = configObj.guardrail_version;
litellm_params.guardrailVersion = configObj.guardrail_version;
}
} else {
// For other providers, add the config to guardrail_info
guardrailData.guardrail.guardrail_info = configObj;
guardrail_info = configObj;
}
} catch (error) {
NotificationsManager.fromBackend("Invalid JSON in configuration");
@ -172,6 +173,22 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
}
}
const guardrailData: {
guardrail_id: string;
guardrail: {
guardrail_name: string;
litellm_params: Record<string, any>;
guardrail_info: any;
};
} = {
guardrail_id: guardrailId,
guardrail: {
guardrail_name: values.guardrail_name,
litellm_params,
guardrail_info,
},
};
if (!accessToken) {
throw new Error("No access token available");
}
@ -403,6 +420,18 @@ const EditGuardrailForm: React.FC<EditGuardrailFormProps> = ({
<Switch />
</Form.Item>
<Form.Item
name="skip_system_message_choice"
label="Skip system messages in guardrail"
tooltip="Unified guardrails only: whether role: system content is omitted from guardrail input (LLM still receives full messages). Use global default follows litellm_settings.skip_system_message_in_guardrail."
>
<Select>
<Option value="inherit">Use global default</Option>
<Option value="yes">Yes — exclude from guardrail scan</Option>
<Option value="no">No — always include in scan</Option>
</Select>
</Form.Item>
{renderProviderSpecificFields()}
<div className="flex justify-end space-x-2 mt-4">

49
ui/litellm-dashboard/src/components/guardrails/guardrail_info.tsx
View file

@ -25,7 +25,12 @@ import React, { useCallback, useEffect, useState } from "react";
import NotificationsManager from "../molecules/notifications_manager";
import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager";
import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal";
import { getGuardrailLogoAndName, guardrail_provider_map } from "./guardrail_info_helpers";
import {
getGuardrailLogoAndName,
guardrail_provider_map,
skipSystemMessageToChoice,
type SkipSystemMessageChoice,
} from "./guardrail_info_helpers";
import GuardrailOptionalParams from "./guardrail_optional_params";
import GuardrailProviderFields from "./guardrail_provider_fields";
import PiiConfiguration from "./pii_configuration";
@ -207,9 +212,14 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
// Reset form when guardrail data or provider params change
useEffect(() => {
if (guardrailData && form) {
const lp = { ...(guardrailData.litellm_params || {}) };
delete lp.skip_system_message_in_guardrail;
form.setFieldsValue({
guardrail_name: guardrailData.guardrail_name,
...guardrailData.litellm_params,
...lp,
skip_system_message_choice: skipSystemMessageToChoice(
guardrailData.litellm_params?.skip_system_message_in_guardrail,
),
guardrail_info: guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "",
// Include any optional_params if they exist
...(guardrailData.litellm_params?.optional_params && {
@ -278,6 +288,20 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
updateData.litellm_params.default_on = values.default_on;
}
const prevSkipChoice = skipSystemMessageToChoice(
guardrailData.litellm_params?.skip_system_message_in_guardrail,
);
const nextSkipChoice = values.skip_system_message_choice as SkipSystemMessageChoice | undefined;
if (nextSkipChoice !== undefined && nextSkipChoice !== prevSkipChoice) {
if (nextSkipChoice === "inherit") {
updateData.litellm_params.skip_system_message_in_guardrail = null;
} else if (nextSkipChoice === "yes") {
updateData.litellm_params.skip_system_message_in_guardrail = true;
} else {
updateData.litellm_params.skip_system_message_in_guardrail = false;
}
}
// Only include guardrail_info if it has changed
const originalGuardrailInfo = guardrailData.guardrail_info;
const newGuardrailInfo = values.guardrail_info ? JSON.parse(values.guardrail_info) : undefined;
@ -647,7 +671,14 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
onFinish={handleGuardrailUpdate}
initialValues={{
guardrail_name: guardrailData.guardrail_name,
...guardrailData.litellm_params,
...(() => {
const lp = { ...(guardrailData.litellm_params || {}) };
delete lp.skip_system_message_in_guardrail;
return lp;
})(),
skip_system_message_choice: skipSystemMessageToChoice(
guardrailData.litellm_params?.skip_system_message_in_guardrail,
),
guardrail_info: guardrailData.guardrail_info
? JSON.stringify(guardrailData.guardrail_info, null, 2)
: "",
@ -673,6 +704,18 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
</Select>
</Form.Item>
<Form.Item
label="Skip system messages in guardrail"
name="skip_system_message_choice"
tooltip="Unified guardrails: omit role: system from guardrail input (LLM still gets full messages). Use global default follows litellm_settings.skip_system_message_in_guardrail."
>
<Select>
<Select.Option value="inherit">Use global default</Select.Option>
<Select.Option value="yes">Yes — exclude from guardrail scan</Select.Option>
<Select.Option value="no">No — always include in scan</Select.Option>
</Select>
</Form.Item>
{guardrailData.litellm_params?.guardrail === "presidio" && (
<>
<Divider orientation="left">PII Protection</Divider>

16
ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.test.tsx
View file

@ -10,6 +10,8 @@ import {
DynamicGuardrailProviders,
guardrail_provider_map,
GuardrailProviders,
skipSystemMessageToChoice,
choiceToSkipSystemForCreate,
} from "./guardrail_info_helpers";
describe("guardrail_info_helpers", () => {
@ -199,4 +201,18 @@ describe("guardrail_info_helpers", () => {
expect(result.logo).toContain("noma_security.png");
});
});
describe("skipSystemMessageToChoice / choiceToSkipSystemForCreate", () => {
it("maps API values to form choices and back for create", () => {
expect(skipSystemMessageToChoice(undefined)).toBe("inherit");
expect(skipSystemMessageToChoice(null)).toBe("inherit");
expect(skipSystemMessageToChoice(true)).toBe("yes");
expect(skipSystemMessageToChoice(false)).toBe("no");
expect(choiceToSkipSystemForCreate("inherit")).toBeUndefined();
expect(choiceToSkipSystemForCreate(undefined)).toBeUndefined();
expect(choiceToSkipSystemForCreate("yes")).toBe(true);
expect(choiceToSkipSystemForCreate("no")).toBe(false);
});
});
});

16
ui/litellm-dashboard/src/components/guardrails/guardrail_info_helpers.tsx
View file

@ -149,3 +149,19 @@ export const getGuardrailLogoAndName = (guardrailValue: string): { logo: string;
return { logo: logo || "", displayName: displayName || guardrailValue };
};
/** Tri-state UI value for `litellm_params.skip_system_message_in_guardrail` (inherit = use global). */
export type SkipSystemMessageChoice = "inherit" | "yes" | "no";
export function skipSystemMessageToChoice(v: boolean | null | undefined): SkipSystemMessageChoice {
if (v === true) return "yes";
if (v === false) return "no";
return "inherit";
}
/** Create flow: omit key when inheriting global default. */
export function choiceToSkipSystemForCreate(choice: SkipSystemMessageChoice | undefined): boolean | undefined {
if (choice === "yes") return true;
if (choice === "no") return false;
return undefined;
}

6
ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx
View file

@ -11,7 +11,7 @@ import {
SortingState,
useReactTable,
} from "@tanstack/react-table";
import { getGuardrailLogoAndName, guardrail_provider_map } from "./guardrail_info_helpers";
import { getGuardrailLogoAndName, guardrail_provider_map, skipSystemMessageToChoice } from "./guardrail_info_helpers";
import EditGuardrailForm from "./edit_guardrail_form";
import { Guardrail, GuardrailDefinitionLocation } from "./types";
@ -291,6 +291,7 @@ const GuardrailTable: React.FC<GuardrailTableProps> = ({
accessToken={accessToken}
onSuccess={handleEditSuccess}
guardrailId={selectedGuardrail.guardrail_id || ""}
fullLitellmParams={selectedGuardrail.litellm_params}
initialValues={{
guardrail_name: selectedGuardrail.guardrail_name || "",
provider:
@ -300,6 +301,9 @@ const GuardrailTable: React.FC<GuardrailTableProps> = ({
mode: selectedGuardrail.litellm_params.mode,
default_on: selectedGuardrail.litellm_params.default_on,
pii_entities_config: selectedGuardrail.litellm_params.pii_entities_config,
skip_system_message_choice: skipSystemMessageToChoice(
selectedGuardrail.litellm_params?.skip_system_message_in_guardrail,
),
...selectedGuardrail.guardrail_info,
}}
/>