scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Merge remote-tracking branch 'origin/main' into feat/github-copilot-thinking-reasoning-support

Browse source
This commit is contained in:
Tim Elfrink 2025-08-19 10:11:59 +02:00
commit b5fa2ee73f
67 changed files with 2151 additions and 477 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.circleci/config.yml
View file

@ -95,7 +95,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -220,7 +220,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -327,7 +327,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -602,7 +602,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -1522,7 +1522,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -1679,7 +1679,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
# Run pytest and generate JUnit XML report
- run:
name: Install dockerize
@ -1819,7 +1819,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -2399,7 +2399,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
@ -2790,7 +2790,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"

2
.circleci/requirements.txt
View file

@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.99.5
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata

9
.github/workflows/test-linting.yml vendored
View file

@ -22,11 +22,8 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.99.5
poetry install --with dev
pip install openai==1.99.5
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@ -40,6 +37,10 @@ jobs:
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.4
version: 0.4.5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -73,6 +73,10 @@ spec:
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}

4
deploy/charts/litellm-helm/values.yaml
View file

@ -206,6 +206,10 @@ migrationJob:
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
ttlSecondsAfterFinished: 120
resources: {}
# requests:
# cpu: 100m
# memory: 100Mi
extraContainers: []
# Hook configuration

1
docker/build_from_pip/requirements.txt
View file

@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
ddtrace==2.19.0 # for advanced DD tracing / profiling

17
docs/my-website/docs/providers/litellm_proxy.md
View file

@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
| Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. |
| Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) |
| Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` |
@ -111,6 +111,21 @@ response = litellm.image_generation(
)
```
## Image Edit
```python
import litellm
with open("your-image.png", "rb") as f:
response = litellm.image_edit(
model="litellm_proxy/gpt-image-1",
prompt="Make this image a watercolor painting",
image=[f],
api_base="your-litellm-proxy-url",
api_key="your-litellm-proxy-api-key",
)
```
## Audio Transcription
```python

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -349,6 +349,7 @@ router_settings:
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow

11
docs/my-website/docs/response_api.md
View file

@ -803,11 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
1. Enable storing request / response content in the database
Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database.
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
```yaml
litellm_settings:
callbacks: ["s3_v2"]
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
s3_region_name: us-west-2
general_settings:
store_prompts_in_spend_logs: true
cold_storage_custom_logger: s3_v2
store_prompts_in_cold_storage: true
```
2. Make request 1 with no `previous_response_id` (new session)

BIN
docs/my-website/img/release_notes/faster_caching_calls.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 270 KiB

BIN
docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 472 KiB

47
docs/my-website/release_notes/v1.75.5-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.75.5-stable"
title: "v1.75.5-stable - Redis latency improvements"
slug: "v1-75-5"
date: 2025-08-10T10:00:00
authors:
@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.5.rc.1
ghcr.io/berriai/litellm:v1.75.5-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.5.post1
pip install litellm==1.75.5.post2
```
</TabItem>
@ -43,8 +43,49 @@ pip install litellm==1.75.5.post1
---
## Key Highlights
- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled.
- **Responses API Session Management** - Support for managing responses API sessions with images.
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
### Risk of Upgrade
If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step.
Users of our Docker image, are **not** affected by this change.
---
## Redis Latency Improvements
<Image
img={require('../../img/release_notes/faster_caching_calls.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits.
---
## Responses API Session Management w/ Images
<Image
img={require('../../img/release_notes/responses_api_session_mgt_images.jpg')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify.
---
## New Models / Updated Models
#### New Model Support

8
litellm/caching/s3_cache.py
View file

@ -55,13 +55,17 @@ class S3Cache(BaseCache):
**kwargs,
)
def _to_s3_key(self, key: str) -> str:
"""Convert cache key to S3 key"""
return self.key_prefix + key.replace(":", "/")
def set_cache(self, key, value, **kwargs):
try:
print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}")
ttl = kwargs.get("ttl", None)
# Convert value to JSON before storing in S3
serialized_value = json.dumps(value)
key = self.key_prefix + key
key = self._to_s3_key(key)
if ttl is not None:
cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
@ -104,7 +108,7 @@ class S3Cache(BaseCache):
import botocore
try:
key = self.key_prefix + key
key = self._to_s3_key(key)
print_verbose(f"Get S3 Cache: key: {key}")
# Download the data from S3

3
litellm/constants.py
View file

@ -1,6 +1,9 @@
import os
from typing import List, Literal
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))

2
litellm/images/main.py
View file

@ -369,6 +369,7 @@ def image_generation( # noqa: PLR0915
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
@ -444,7 +445,6 @@ def image_generation( # noqa: PLR0915
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")

100
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -27,7 +27,12 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
@ -102,6 +107,24 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
verbose_logger.exception(
f"DataDogLLMObs: Error logging success event - {str(e)}"
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
verbose_logger.debug(
f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
)
payload = self.create_llm_obs_payload(
kwargs, start_time, end_time
)
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
self.log_queue.append(payload)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"DataDogLLMObs: Error logging failure event - {str(e)}"
)
async def async_send_batch(self):
try:
@ -174,11 +197,14 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
call_type=standard_logging_payload.get("call_type")
))
error_info = self._assemble_error_info(standard_logging_payload)
meta = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
error=error_info,
)
# Calculate metrics (you may need to adjust these based on available data)
@ -199,11 +225,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
start_ns=int(start_time.timestamp() * 1e9),
duration=int((end_time - start_time).total_seconds() * 1e9),
metrics=metrics,
status="error" if error_info else "ok",
tags=[
self._get_datadog_tags(standard_logging_object=standard_logging_payload)
],
)
def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
"""
Assemble error information for failure cases according to DD LLM Obs API spec
"""
# Handle error information for failure cases according to DD LLM Obs API spec
error_info: Optional[DDLLMObsError] = None
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
if error_information:
error_info = DDLLMObsError(
message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
type=error_information.get("error_class"),
stack=error_information.get("traceback")
)
return error_info
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
"""
Get the time to first token in seconds
@ -232,8 +278,20 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
for now this handles logging /chat/completions responses
"""
if response_obj is None:
return []
if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
return [response_obj["choices"][0]["message"]]
try:
# Safely extract message from response_obj, handle failure cases
if isinstance(response_obj, dict) and "choices" in response_obj:
choices = response_obj["choices"]
if choices and len(choices) > 0 and "message" in choices[0]:
return [choices[0]["message"]]
return []
except (KeyError, IndexError, TypeError):
# In case of any error accessing the response structure, return empty list
return []
return []
def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
@ -350,11 +408,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
def _get_dd_llm_obs_payload_metadata(
self, standard_logging_payload: StandardLoggingPayload
) -> Dict:
) -> Dict[str, Any]:
"""
Fields to track in DD LLM Observability metadata from litellm standard logging payload
"""
_metadata = {
_metadata: Dict[str, Any] = {
"model_name": standard_logging_payload.get("model", "unknown"),
"model_provider": standard_logging_payload.get(
"custom_llm_provider", "unknown"
@ -365,8 +423,42 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
}
#########################################################
# Add latency metrics to metadata
#########################################################
latency_metrics = self._get_latency_metrics(standard_logging_payload)
_metadata.update({"latency_metrics": dict(latency_metrics)})
_standard_logging_metadata: dict = (
dict(standard_logging_payload.get("metadata", {})) or {}
)
_metadata.update(_standard_logging_metadata)
return _metadata
def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
"""
Get the latency metrics from the standard logging payload
"""
latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
# Add latency metrics to metadata
# Time to first token (convert from seconds to milliseconds for consistency)
time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
if time_to_first_token_seconds > 0:
latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
# LiteLLM overhead time
hidden_params = standard_logging_payload.get("hidden_params", {})
litellm_overhead_ms = hidden_params.get("litellm_overhead_time_ms")
if litellm_overhead_ms is not None:
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
if guardrail_info is not None:
_guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
if _guardrail_duration_seconds is not None:
# Convert from seconds to milliseconds for consistency
latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
return latency_metrics

5
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -18,6 +18,7 @@ from typing import (
cast,
)
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -453,6 +454,10 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, file_headers = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
file_content = file_data
content_type = file_data.content_type
else:
file_content = file_data
# Convert content to bytes

7
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -33,7 +33,12 @@ class SensitiveDataMasker:
value_str = str(value)
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
# Handle the case where visible_suffix is 0 to avoid showing the entire string
if self.visible_suffix == 0:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}"
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str) -> bool:
key_lower = str(key).lower()

69
litellm/llms/azure/common_utils.py
View file

@ -365,14 +365,16 @@ def get_azure_ad_token(
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
except ValueError:
verbose_logger.debug("Azure AD Token Provider could not be used.")
#########################################################
# If litellm.enable_azure_ad_token_refresh is True and no other token provider is available,
# try to get DefaultAzureCredential provider
#########################################################
if azure_ad_token_provider is None and azure_ad_token is None:
azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
azure_ad_token_provider = (
BaseAzureLLM._try_get_default_azure_credential_provider(
scope=scope,
)
)
# Execute the token provider to get the token if available
@ -403,27 +405,27 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> Optional[Callable[[], str]]:
"""
Try to get DefaultAzureCredential provider
Args:
scope: Azure scope for the token
Returns:
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
"""
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
verbose_logger.debug(
"Attempting to use DefaultAzureCredential for Azure Auth"
)
verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth")
try:
azure_ad_token_provider = get_azure_ad_token_provider(
azure_scope=scope,
azure_credential=AzureCredentialType.DefaultAzureCredential,
)
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
verbose_logger.debug(
"Successfully obtained Azure AD token provider using DefaultAzureCredential"
)
return azure_ad_token_provider
except Exception as e:
verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}")
@ -656,17 +658,17 @@ class BaseAzureLLM(BaseOpenAILLM):
else:
client = AzureOpenAI(**azure_client_params) # type: ignore
return client
@staticmethod
def _base_validate_azure_environment(
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
headers: dict, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
litellm_params = litellm_params or GenericLiteLLMParams()
# If api-key is already in headers, preserve it
if "api-key" in headers:
return headers
api_key = (
litellm_params.api_key
or litellm.api_key
@ -686,13 +688,24 @@ class BaseAzureLLM(BaseOpenAILLM):
headers["Authorization"] = f"Bearer {azure_ad_token}"
return headers
@staticmethod
def _get_base_azure_url(
api_base: Optional[str],
litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]],
route: Literal["/openai/responses", "/openai/vector_stores"]
route: Literal["/openai/responses", "/openai/vector_stores"],
default_api_version: Optional[Union[str, Literal["latest", "preview"]]] = None,
) -> str:
"""
Get the base Azure URL for the given route and API version.
Args:
api_base: The base URL of the Azure API.
litellm_params: The litellm parameters.
route: The route to the API.
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
@ -702,7 +715,10 @@ class BaseAzureLLM(BaseOpenAILLM):
# Extract api_version or use default
litellm_params = litellm_params or {}
api_version = cast(Optional[str], litellm_params.get("api_version"))
api_version = (
cast(Optional[str], litellm_params.get("api_version"))
or default_api_version
)
# Create a new dictionary with existing params
query_params = dict(original_url.params)
@ -710,27 +726,28 @@ class BaseAzureLLM(BaseOpenAILLM):
# Add api_version if needed
if "api-version" not in query_params and api_version:
query_params["api-version"] = api_version
# Add the path to the base URL
if route not in api_base:
new_url = _add_path_to_api_base(
api_base=api_base, ending_path=route
)
new_url = _add_path_to_api_base(api_base=api_base, ending_path=route)
else:
new_url = api_base
if BaseAzureLLM._is_azure_v1_api_version(api_version):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(
parsed_url.copy_with(
path=parsed_url.path.replace("/openai", "/openai/v1")
)
)
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
@staticmethod
def _is_azure_v1_api_version(api_version: Optional[str]) -> bool:
if api_version is None:

7
litellm/llms/azure/responses/transformation.py
View file

@ -70,8 +70,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
return BaseAzureLLM._get_base_azure_url(
api_base=api_base, litellm_params=litellm_params, route="/openai/responses"
api_base=api_base,
litellm_params=litellm_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
#########################################################

26
litellm/llms/litellm_proxy/image_edit/transformation.py Normal file
View file

@ -0,0 +1,26 @@
from typing import Optional
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig):
"""Configuration for image edit requests routed through LiteLLM Proxy."""
def validate_environment(
self, headers: dict, model: str, api_key: Optional[str] = None
) -> dict:
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
headers.update({"Authorization": f"Bearer {api_key}"})
return headers
def get_complete_url(
self, model: str, api_base: Optional[str], litellm_params: dict
) -> str:
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
if api_base is None:
raise ValueError(
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
)
api_base = api_base.rstrip("/")
return f"{api_base}/images/edits"

40
litellm/llms/litellm_proxy/image_generation/transformation.py Normal file
View file

@ -0,0 +1,40 @@
from typing import Optional
from litellm.llms.openai.image_generation.gpt_transformation import (
GPTImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig):
"""Configuration for image generation requests routed through LiteLLM Proxy."""
def validate_environment(
self,
headers: dict,
model: str,
messages,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
headers.update({"Authorization": f"Bearer {api_key}"})
return headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
if api_base is None:
raise ValueError(
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
)
api_base = api_base.rstrip("/")
return f"{api_base}/images/generations"

18
litellm/llms/openai_like/chat/handler.py
View file

@ -49,8 +49,15 @@ async def make_call(
model_response = ModelResponse(**response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
# Use aiter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
async def utf8_aiter_lines():
async for line in response.aiter_text(encoding='utf-8'):
for line_part in line.splitlines(keepends=True):
if line_part.strip():
yield line_part.rstrip('\r\n')
completion_stream = ModelResponseIterator(
streaming_response=response.aiter_lines(), sync_stream=False
streaming_response=utf8_aiter_lines(), sync_stream=False
)
# LOGGING
logging_obj.post_call(
@ -93,8 +100,15 @@ def make_sync_call(
model_response = ModelResponse(**response.json())
completion_stream = MockResponseIterator(model_response=model_response)
else:
# Use iter_text with explicit UTF-8 encoding to avoid ASCII encoding errors
def utf8_iter_lines():
for line in response.iter_text(encoding='utf-8'):
for line_part in line.splitlines(keepends=True):
if line_part.strip():
yield line_part.rstrip('\r\n')
completion_stream = ModelResponseIterator(
streaming_response=response.iter_lines(), sync_stream=True
streaming_response=utf8_iter_lines(), sync_stream=True
)
# LOGGING

55
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -305,9 +305,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return None
for tool in value:
openai_function_object: Optional[
ChatCompletionToolParamFunctionChunk
] = None
openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = (
None
)
if "function" in tool: # tools list
_openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore
**tool["function"]
@ -597,14 +597,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif param == "seed":
optional_params["seed"] = value
elif param == "reasoning_effort" and isinstance(value, str):
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
)
elif param == "thinking":
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value)
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value)
)
)
elif param == "modalities" and isinstance(value, list):
response_modalities = self.map_response_modalities(value)
@ -1000,6 +1000,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
GenerateContentResponseBody, BidiGenerateContentServerMessage
],
) -> Usage:
if (
completion_response is not None
and "usageMetadata" not in completion_response
@ -1038,6 +1039,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
text_tokens = detail.get("tokenCount", 0)
if "thoughtsTokenCount" in usage_metadata:
reasoning_tokens = usage_metadata["thoughtsTokenCount"]
## adjust 'text_tokens' to subtract cached tokens
if (
(audio_tokens is None or audio_tokens == 0)
and text_tokens is not None
and text_tokens > 0
and cached_tokens is not None
):
text_tokens = text_tokens - cached_tokens
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
audio_tokens=audio_tokens,
@ -1344,28 +1355,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
## ADD METADATA TO RESPONSE ##
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
model_response._hidden_params[
"vertex_ai_grounding_metadata"
] = grounding_metadata
model_response._hidden_params["vertex_ai_grounding_metadata"] = (
grounding_metadata
)
setattr(
model_response, "vertex_ai_url_context_metadata", url_context_metadata
)
model_response._hidden_params[
"vertex_ai_url_context_metadata"
] = url_context_metadata
model_response._hidden_params["vertex_ai_url_context_metadata"] = (
url_context_metadata
)
setattr(model_response, "vertex_ai_safety_results", safety_ratings)
model_response._hidden_params[
"vertex_ai_safety_results"
] = safety_ratings # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_safety_results"] = (
safety_ratings # older approach - maintaining to prevent regressions
)
## ADD CITATION METADATA ##
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
model_response._hidden_params[
"vertex_ai_citation_metadata"
] = citation_metadata # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_citation_metadata"] = (
citation_metadata # older approach - maintaining to prevent regressions
)
except Exception as e:
raise VertexAIError(

18
litellm/model_prices_and_context_window_backup.json
View file

@ -722,7 +722,7 @@
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"supported_modalities": [
"text",
"image"
],
@ -730,13 +730,13 @@
"text"
],
"supports_pdf_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -762,13 +762,13 @@
"text"
],
"supports_pdf_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -11476,9 +11476,9 @@
},
"openrouter/anthropic/claude-sonnet-4": {
"supports_computer_use": true,
"max_tokens": 8192,
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_image": 0.0048,

0
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View file

1
litellm/proxy/_experimental/out/onboarding.html
View file

File diff suppressed because one or more lines are too long

15
litellm/proxy/_new_secret_config.yaml
View file

@ -3,7 +3,16 @@ model_list:
litellm_params:
model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
api_base: https://webhook.site/4feb0d46-4b23-468c-bf55-7008b5deb36d
- model_name: gpt-5-mini
litellm_params:
model: azure/gpt-5-mini
api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
stream_timeout: 60
merge_reasoning_content_in_choices: true
model_info:
mode: chat
litellm_settings:
cache: true
@ -11,3 +20,7 @@ litellm_settings:
type: redis
ttl: 600
supported_call_types: ["acompletion", "completion"]
model_group_settings:
forward_client_headers_to_llm_api:
- fake-openai-endpoint

25
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -16,6 +16,7 @@ import json
import sys
from typing import Any, AsyncGenerator, List, Literal, Optional, Tuple, Union
import httpx
from fastapi import HTTPException
import litellm
@ -284,6 +285,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
response: Optional[Union[Any, litellm.ModelResponse]] = None,
request_data: Optional[dict] = None
) -> BedrockGuardrailResponse:
from datetime import datetime
start_time = datetime.now()
credentials, aws_region_name = self._load_credentials()
bedrock_request_data: dict = dict(
self.convert_to_bedrock_format(
@ -317,6 +320,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
data=prepared_request.body, # type: ignore
headers=prepared_request.headers, # type: ignore
)
#########################################################
# Add guardrail information to request trace
#########################################################
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=response.json(),
request_data=request_data or {},
guardrail_status=self._get_bedrock_guardrail_response_status(response=response),
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
)
#########################################################
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
@ -338,6 +353,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return bedrock_guardrail_response
def _get_bedrock_guardrail_response_status(self, response: httpx.Response) -> Literal["success", "failure"]:
"""
Get the status of the bedrock guardrail response.
"""
if response.status_code == 200:
return "success"
return "failure"
def _get_http_exception_for_blocked_guardrail(self, response: BedrockGuardrailResponse) -> HTTPException:
"""
@ -501,10 +523,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
add_guardrail_to_applied_guardrails_header(
request_data=data, guardrail_name=self.guardrail_name
)
return data
@log_guardrail_information
async def async_moderation_hook(
self,
data: dict,
@ -561,7 +581,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
return data
@log_guardrail_information
async def async_post_call_success_hook(
self,
data: dict,

34
litellm/proxy/litellm_pre_call_utils.py
View file

@ -384,6 +384,29 @@ class LiteLLMProxyRequestSetup:
return returned_headers
@staticmethod
def add_headers_to_llm_call_by_model_group(
data: dict, headers: dict, user_api_key_dict: UserAPIKeyAuth
) -> dict:
"""
Add headers to the LLM call by model group
"""
data_model = data.get("model")
if (
data_model is not None
and litellm.model_group_settings is not None
and litellm.model_group_settings.forward_client_headers_to_llm_api
is not None
and data_model
in litellm.model_group_settings.forward_client_headers_to_llm_api
):
_headers = LiteLLMProxyRequestSetup.add_headers_to_llm_call(
headers, user_api_key_dict
)
if _headers != {}:
data["headers"] = _headers
return data
@staticmethod
def add_litellm_data_for_backend_llm_call(
*,
@ -439,7 +462,7 @@ class LiteLLMProxyRequestSetup:
user_api_key_request_route=user_api_key_dict.request_route,
)
return user_api_key_logged_metadata
@staticmethod
def add_user_api_key_auth_to_request_metadata(
data: dict,
@ -457,9 +480,7 @@ class LiteLLMProxyRequestSetup:
data[_metadata_variable_name].update(user_api_key_logged_metadata)
data[_metadata_variable_name][
"user_api_key"
] = (
user_api_key_dict.api_key
) # this is just the hashed token
] = user_api_key_dict.api_key # this is just the hashed token
data[_metadata_variable_name]["user_api_end_user_max_budget"] = getattr(
user_api_key_dict, "end_user_max_budget", None
@ -624,6 +645,11 @@ async def add_litellm_data_to_request( # noqa: PLR0915
)
)
# check for forwardable headers
data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
data=data, headers=_headers, user_api_key_dict=user_api_key_dict
)
# Parse user info from headers
user = LiteLLMProxyRequestSetup.get_user_from_headers(_headers, general_settings)
if user is not None:

10
litellm/proxy/proxy_config.yaml
View file

@ -5,3 +5,13 @@ model_list:
- model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
litellm_settings:
callbacks: ["datadog_llm_observability"]
guardrails:
- guardrail_name: "bedrock-pre-guard"
litellm_params:
guardrail: bedrock # supported values: "aporia", "bedrock", "lakera"
mode: "during_call"
guardrailIdentifier: ff6ujrregl1q
guardrailVersion: "DRAFT"

3
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -33,7 +33,6 @@ from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponseTextConfig,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
@ -659,7 +658,7 @@ class LiteLLMCompletionResponsesConfig:
),
reasoning=Reasoning(),
status=getattr(chat_completion_response, "status", "completed"),
text=ResponseTextConfig(),
text={},
truncation=getattr(chat_completion_response, "truncation", None),
usage=LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(
chat_completion_response=chat_completion_response

8
litellm/responses/main.py
View file

@ -21,7 +21,7 @@ from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponseTextConfigParam,
ResponseText,
ToolChoice,
ToolParam,
)
@ -104,7 +104,7 @@ async def aresponses_api_with_mcp(
background: Optional[bool] = None,
stream: Optional[bool] = None,
temperature: Optional[float] = None,
text: Optional[ResponseTextConfigParam] = None,
text: Optional["ResponseText"] = None,
tool_choice: Optional[ToolChoice] = None,
tools: Optional[Iterable[ToolParam]] = None,
top_p: Optional[float] = None,
@ -237,7 +237,7 @@ async def aresponses(
background: Optional[bool] = None,
stream: Optional[bool] = None,
temperature: Optional[float] = None,
text: Optional[ResponseTextConfigParam] = None,
text: Optional["ResponseText"] = None,
tool_choice: Optional[ToolChoice] = None,
tools: Optional[Iterable[ToolParam]] = None,
top_p: Optional[float] = None,
@ -348,7 +348,7 @@ def responses(
background: Optional[bool] = None,
stream: Optional[bool] = None,
temperature: Optional[float] = None,
text: Optional[ResponseTextConfigParam] = None,
text: Optional["ResponseText"] = None,
tool_choice: Optional[ToolChoice] = None,
tools: Optional[Iterable[ToolParam]] = None,
top_p: Optional[float] = None,

13
litellm/router.py
View file

@ -93,9 +93,6 @@ from litellm.router_utils.fallback_event_handlers import (
get_fallback_model_group,
run_async_fallback,
)
from litellm.router_utils.forward_clientside_headers_by_model_group import (
ForwardClientSideHeadersByModelGroup,
)
from litellm.router_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy,
)
@ -624,9 +621,7 @@ class Router:
Apply the default settings to the router.
"""
default_pre_call_checks: OptionalPreCallChecks = [
"forward_client_headers_by_model_group",
]
default_pre_call_checks: OptionalPreCallChecks = []
self.add_optional_pre_call_checks(default_pre_call_checks)
return None
@ -892,8 +887,6 @@ class Router:
)
elif pre_call_check == "responses_api_deployment_check":
_callback = ResponsesApiDeploymentCheck()
elif pre_call_check == "forward_client_headers_by_model_group":
_callback = ForwardClientSideHeadersByModelGroup()
if _callback is not None:
if self.optional_callbacks is None:
self.optional_callbacks = []
@ -4323,7 +4316,9 @@ class Router:
"deployment", None
) # stable name - works for wildcard routes as well
# Get model_group and id from kwargs like the sync version does
model_group = kwargs["litellm_params"]["metadata"].get("model_group", None)
model_group = kwargs["litellm_params"]["metadata"].get(
"model_group", None
)
model_info = kwargs["litellm_params"].get("model_info", {}) or {}
id = model_info.get("id", None)
if model_group is None or id is None:

5
litellm/router_utils/batch_utils.py
View file

@ -7,9 +7,10 @@ from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose
class InMemoryFile(io.BytesIO):
def __init__(self, content: bytes, name: str):
def __init__(self, content: bytes, name: str, content_type: str = "application/jsonl"):
super().__init__(content)
self.name = name
self.content_type = content_type
def should_replace_model_in_jsonl(
@ -63,7 +64,7 @@ def replace_model_in_jsonl(file_content: FileTypes, new_model_name: str) -> File
# Reassemble the modified lines and return as bytes
modified_file_content = "\n".join(modified_lines).encode("utf-8")
return InMemoryFile(modified_file_content, name="modified_file.jsonl") # type: ignore
return InMemoryFile(modified_file_content, name="modified_file.jsonl", content_type="application/jsonl") # type: ignore
except (json.JSONDecodeError, UnicodeDecodeError, TypeError):
# return the original file content if there is an error replacing the model name

11
litellm/router_utils/cooldown_cache.py
View file

@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, Union
from litellm import verbose_logger
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -29,6 +30,12 @@ class CooldownCache:
self.cache = cache
self.default_cooldown_time = default_cooldown_time
self.in_memory_cache = InMemoryCache()
# Initialize the masker with custom settings for exception strings
self.exception_masker = SensitiveDataMasker(
visible_prefix=50, # Show first 50 characters
visible_suffix=0, # Show last 0 characters
mask_char="*", # Use * for masking
)
def _common_add_cooldown_logic(
self, model_id: str, original_exception, exception_status, cooldown_time: float
@ -39,7 +46,9 @@ class CooldownCache:
# Store the cooldown information for the deployment separately
cooldown_data = CooldownCacheValue(
exception_received=str(original_exception),
exception_received=self.exception_masker._mask_value(
str(original_exception)
),
status_code=str(exception_status),
timestamp=current_time,
cooldown_time=cooldown_time,

20
litellm/router_utils/cooldown_handlers.py
View file

@ -118,16 +118,16 @@ def _should_run_cooldown_logic(
"Should Not Run Cooldown Logic: deployment id is none or model group can't be found."
)
return False
#########################################################
# If time_to_cooldown is 0 or 0.0000000, don't run cooldown logic
#########################################################
if time_to_cooldown is not None and math.isclose(
a=time_to_cooldown,
b=0.0,
abs_tol=1e-9
a=time_to_cooldown, b=0.0, abs_tol=1e-9
):
verbose_router_logger.debug("Should Not Run Cooldown Logic: time_to_cooldown is effectively 0")
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: time_to_cooldown is effectively 0"
)
return False
if litellm_router_instance.disable_cooldowns:
@ -275,8 +275,8 @@ def _set_cooldown_deployments(
if (
_should_run_cooldown_logic(
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
time_to_cooldown=time_to_cooldown,
)
@ -290,9 +290,9 @@ def _set_cooldown_deployments(
verbose_router_logger.debug(f"Attempting to add {deployment} to cooldown list")
if _should_cooldown_deployment(
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
):
litellm_router_instance.cooldown_cache.add_deployment_to_cooldown(

84
litellm/router_utils/forward_clientside_headers_by_model_group.py
View file

@ -1,84 +0,0 @@
from typing import Any, Dict, Optional, TypedDict
from litellm.types.utils import CallTypes
from ..integrations.custom_logger import CustomLogger
class PotentialModelGroups(TypedDict):
deployment_model_name: Optional[str]
model_group_alias: Optional[str]
class ForwardClientSideHeadersByModelGroup(CustomLogger):
def get_potential_model_groups_from_kwargs(
self, kwargs: Dict[str, Any]
) -> Optional[PotentialModelGroups]:
"""
Get the model group from the kwargs.
Returns the potential model groups from the kwargs.
- deployment_model_name (useful for wildcard model names)
- model_group_alias (if the model is an alias)
"""
metadata = kwargs.get("litellm_metadata") or kwargs.get("metadata")
if metadata is None:
return None
deployment_model_name = metadata.get("deployment_model_name", None)
model_group_alias = metadata.get("model_group_alias", None)
return {
"deployment_model_name": deployment_model_name,
"model_group_alias": model_group_alias,
}
def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]:
"""
Filter the headers to only include the headers that are forwarded to the LLM API.
E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side.
"""
return {
k: v
for k, v in headers.items()
if k.lower() not in ["connection", "content-length"]
}
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
if kwargs["proxy_server_request"]["headers"] is not None:
and kwargs["forward_client_headers_to_llm_api"] is not None:
add the headers to the request
kwargs["headers"].update(kwargs["proxy_server_request"]["headers"])
"""
import litellm
if litellm.model_group_settings is None:
return None
potential_model_groups = self.get_potential_model_groups_from_kwargs(kwargs)
if potential_model_groups is None:
return None
if (
"secret_fields" in kwargs
and kwargs["secret_fields"]["raw_headers"] is not None
and isinstance(kwargs["secret_fields"]["raw_headers"], dict)
):
for model_group in potential_model_groups.values():
if model_group is None:
continue
if (
litellm.model_group_settings.forward_client_headers_to_llm_api
is not None
and model_group
in litellm.model_group_settings.forward_client_headers_to_llm_api
):
kwargs.setdefault("headers", {}).update(
self.filter_headers(kwargs["secret_fields"]["raw_headers"])
)
return kwargs

7
litellm/router_utils/handle_error.py
View file

@ -82,9 +82,14 @@ async def async_raise_no_deployment_exception(
litellm_router_instance=litellm_router_instance,
parent_otel_span=parent_otel_span,
)
verbose_router_logger.info(
f"No deployment found for model: {model}, cooldown_list with debug info: {_cooldown_list}"
)
cooldown_list_ids = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])]
return RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
cooldown_list=_cooldown_list,
cooldown_list=cooldown_list_ids,
)

25
litellm/types/integrations/datadog_llm_obs.py
View file

@ -18,12 +18,20 @@ class OutputMeta(TypedDict):
messages: List[Any]
class Meta(TypedDict):
class DDLLMObsError(TypedDict, total=False):
"""Error information on the span according to DD LLM Obs API spec"""
message: str # The error message
stack: Optional[str] # The stack trace
type: Optional[str] # The error type
class Meta(TypedDict, total=False):
# The span kind: "agent", "workflow", "llm", "tool", "task", "embedding", or "retrieval".
kind: Literal["llm", "tool", "task", "embedding", "retrieval"]
input: InputMeta # The span’s input information.
output: OutputMeta # The span’s output information.
input: InputMeta # The span's input information.
output: OutputMeta # The span's output information.
metadata: Dict[str, Any]
error: Optional[DDLLMObsError] # Error information on the span
class LLMMetrics(TypedDict, total=False):
@ -35,7 +43,7 @@ class LLMMetrics(TypedDict, total=False):
total_cost: float
class LLMObsPayload(TypedDict):
class LLMObsPayload(TypedDict, total=False):
parent_id: str
trace_id: str
span_id: str
@ -45,6 +53,7 @@ class LLMObsPayload(TypedDict):
duration: int
metrics: LLMMetrics
tags: List
status: Literal["ok", "error"] # Error status ("ok" or "error"). Defaults to "ok".
class DDSpanAttributes(TypedDict):
@ -62,4 +71,10 @@ class DatadogLLMObsInitParams(StandardCustomLoggerInitParams):
"""
Params for initializing a DatadogLLMObs logger on litellm
"""
pass
pass
class DDLLMObsLatencyMetrics(TypedDict, total=False):
time_to_first_token_ms: float
litellm_overhead_time_ms: float
guardrail_overhead_time_ms: float

14
litellm/types/llms/openai.py
View file

@ -37,15 +37,21 @@ from openai.types.responses.response import (
IncompleteDetails,
Response,
ResponseOutputItem,
ResponseTextConfig,
Tool,
ToolChoice,
)
# Handle OpenAI SDK version compatibility for Text type
try:
from openai.types.responses.response_create_params import Text as ResponseText
except (ImportError, AttributeError):
# Fall back to the concrete config type available in all SDK versions
from openai.types.responses.response_text_config_param import ResponseTextConfigParam as ResponseText
from openai.types.responses.response_create_params import (
Reasoning,
ResponseIncludable,
ResponseInputParam,
ResponseTextConfigParam,
ToolChoice,
ToolParam,
)
@ -959,7 +965,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
background: Optional[bool]
stream: Optional[bool]
temperature: Optional[float]
text: Optional[ResponseTextConfigParam]
text: Optional["ResponseText"]
tool_choice: Optional[ToolChoice]
tools: Optional[List[ALL_RESPONSES_API_TOOL_PARAMS]]
top_p: Optional[float]
@ -1034,7 +1040,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
previous_response_id: Optional[str]
reasoning: Optional[Reasoning]
status: Optional[str]
text: Optional[ResponseTextConfig]
text: Optional[Union["ResponseText", Dict[str, Any]]]
truncation: Optional[Literal["auto", "disabled"]]
usage: Optional[ResponseAPIUsage]
user: Optional[str]

12
litellm/utils.py
View file

@ -7305,6 +7305,12 @@ class ProviderConfigManager:
)
return get_gemini_image_generation_config(model)
elif LlmProviders.LITELLM_PROXY == provider:
from litellm.llms.litellm_proxy.image_generation.transformation import (
LiteLLMProxyImageGenerationConfig,
)
return LiteLLMProxyImageGenerationConfig()
return None
@staticmethod
@ -7341,6 +7347,12 @@ class ProviderConfigManager:
)
return RecraftImageEditConfig()
elif LlmProviders.LITELLM_PROXY == provider:
from litellm.llms.litellm_proxy.image_edit.transformation import (
LiteLLMProxyImageEditConfig,
)
return LiteLLMProxyImageEditConfig()
return None
@staticmethod

18
model_prices_and_context_window.json
View file

@ -722,7 +722,7 @@
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"supported_modalities": [
"text",
"image"
],
@ -730,13 +730,13 @@
"text"
],
"supports_pdf_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -762,13 +762,13 @@
"text"
],
"supports_pdf_input": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_function_calling": false,
"supports_parallel_function_calling": false,
"supports_response_schema": true,
"supports_vision": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_choice": false,
"supports_native_streaming": true,
"supports_reasoning": true
},
@ -11476,9 +11476,9 @@
},
"openrouter/anthropic/claude-sonnet-4": {
"supports_computer_use": true,
"max_tokens": 8192,
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"input_cost_per_image": 0.0048,

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.75.8"
version = "1.75.9"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@ -155,7 +155,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.75.8"
version = "1.75.9"
version_files = [
"pyproject.toml:^version"
]

54
scripts/test_groq_streaming_issue.py Normal file
View file

@ -0,0 +1,54 @@
"""
Test script to reproduce the Groq streaming ASCII encoding issue.
This reproduces the issue described in #12660 where streaming responses
containing non-ASCII characters like µ cause encoding errors.
"""
import asyncio
import os
import traceback
from litellm import acompletion
async def test_groq_streaming_with_special_chars():
"""Test that reproduces the ASCII encoding issue with Groq streaming."""
try:
print("Testing acompletion + streaming with Groq...")
# Test message that should trigger the µ character or similar non-ASCII content
test_messages = [
{"content": "What is the symbol for micro? Please include the µ symbol in your response.", "role": "user"}
]
# This should trigger the ASCII encoding error described in the issue
response = await acompletion(
model="groq/llama-3.3-70b-versatile",
messages=test_messages,
stream=True
)
print(f"Response type: {type(response)}")
# Try to iterate through the stream
async for chunk in response:
print(f"Chunk: {chunk}")
print("✅ Test completed successfully - no encoding errors!")
except Exception as e:
print(f"❌ Error occurred: {e}")
print(f"Error type: {type(e)}")
print(f"Traceback:\n{traceback.format_exc()}")
return False
return True
if __name__ == "__main__":
# Note: This requires GROQ_API_KEY to be set
if not os.getenv("GROQ_API_KEY"):
print("⚠️ GROQ_API_KEY not set. Skipping test.")
else:
success = asyncio.run(test_groq_streaming_with_special_chars())
if success:
print("🎉 All tests passed!")
else:
print("💥 Test failed!")

3
tests/llm_responses_api_testing/base_responses_api.py
View file

@ -21,7 +21,6 @@ from litellm.types.utils import StandardLoggingPayload
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponseTextConfig,
ResponseAPIUsage,
IncompleteDetails,
)
@ -78,7 +77,7 @@ def validate_responses_api_response(response, final_chunk: bool = False):
"previous_response_id": (str, type(None)),
"reasoning": dict,
"status": str,
"text": ResponseTextConfig,
"text": dict,
"truncation": (str, type(None)),
"usage": ResponseAPIUsage,
"user": (str, type(None)),

1
tests/llm_responses_api_testing/test_anthropic_responses_api.py
View file

@ -17,7 +17,6 @@ from litellm.types.utils import StandardLoggingPayload
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponseTextConfig,
ResponseAPIUsage,
IncompleteDetails,
)

1
tests/llm_responses_api_testing/test_azure_responses_api.py
View file

@ -13,7 +13,6 @@ from litellm.types.utils import StandardLoggingPayload
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponseTextConfig,
ResponseAPIUsage,
IncompleteDetails,
)

1
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -18,7 +18,6 @@ from litellm.types.utils import StandardLoggingPayload
from litellm.types.llms.openai import (
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponseTextConfig,
ResponseAPIUsage,
IncompleteDetails,
)

14
tests/llm_translation/test_azure_openai.py
View file

@ -630,3 +630,17 @@ def test_azure_openai_responses_bridge():
== "test-azure-computer-use-preview"
)
assert mock_responses.call_args.kwargs["custom_llm_provider"] == "azure"
def test_azure_openai_gpt_5_responses_api():
from litellm import responses
litellm._turn_on_debug()
response = responses(
model="azure/gpt-5",
input="Hello world",
api_key=os.getenv("AZURE_SWEDEN_API_KEY"),
api_base=os.getenv("AZURE_SWEDEN_API_BASE"),
)
print(f"response: {response}")

122
tests/llm_translation/test_litellm_proxy_provider.py
View file

@ -2,6 +2,7 @@ import json
import os
import sys
from datetime import datetime
from io import BytesIO
from unittest.mock import AsyncMock
sys.path.insert(
@ -184,6 +185,127 @@ async def test_litellm_gateway_from_sdk_image_generation(is_async):
assert "dall-e-3" == mock_method.call_args.kwargs["model"]
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_litellm_gateway_image_generation_direct(is_async):
"""Test image generation using the litellm_proxy provider directly."""
litellm._turn_on_debug()
# Create mock response that matches OpenAI's response structure
mock_openai_response = MagicMock()
mock_openai_response.model_dump.return_value = {
"created": 1,
"data": [{"url": "https://example.com/image.png"}],
}
if is_async:
# Mock the AsyncOpenAI client that gets created inside _get_openai_client
mock_async_client = AsyncMock()
mock_async_client.images.generate = AsyncMock(return_value=mock_openai_response)
with patch("litellm.llms.openai.openai.AsyncOpenAI", return_value=mock_async_client) as mock_async_constructor:
response = await litellm.aimage_generation(
model="litellm_proxy/dall-e-3",
prompt="A beautiful sunset over mountains",
api_base="http://my-proxy",
api_key="sk-1234",
)
# Verify the AsyncOpenAI client constructor was called with correct parameters
mock_async_constructor.assert_called_once()
constructor_kwargs = mock_async_constructor.call_args.kwargs
print("KWARGS to Async OpenAI constructor=", constructor_kwargs)
assert constructor_kwargs["api_key"] == "sk-1234"
assert constructor_kwargs["base_url"] == "http://my-proxy"
# Verify the AsyncOpenAI client was called correctly
mock_async_client.images.generate.assert_awaited_once()
call_kwargs = mock_async_client.images.generate.call_args.kwargs
assert call_kwargs["model"] == "dall-e-3"
assert call_kwargs["prompt"] == "A beautiful sunset over mountains"
else:
# Mock the sync OpenAI client that gets created inside _get_openai_client
mock_sync_client = MagicMock()
mock_sync_client.images.generate.return_value = mock_openai_response
with patch("litellm.llms.openai.openai.OpenAI", return_value=mock_sync_client) as mock_sync_constructor:
response = litellm.image_generation(
model="litellm_proxy/dall-e-3",
prompt="A beautiful sunset over mountains",
api_base="http://my-proxy",
api_key="sk-1234",
)
# Verify the OpenAI client constructor was called with correct parameters
mock_sync_constructor.assert_called_once()
constructor_kwargs = mock_sync_constructor.call_args.kwargs
assert constructor_kwargs["api_key"] == "sk-1234"
assert constructor_kwargs["base_url"] == "http://my-proxy"
# Verify the OpenAI client was called correctly
mock_sync_client.images.generate.assert_called_once()
call_kwargs = mock_sync_client.images.generate.call_args.kwargs
assert call_kwargs["model"] == "dall-e-3"
assert call_kwargs["prompt"] == "A beautiful sunset over mountains"
# Verify the response structure
assert response is not None
assert hasattr(response, 'data') or isinstance(response, dict)
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_litellm_gateway_from_sdk_image_edit(is_async):
litellm._turn_on_debug()
mock_response = {
"created": 1,
"data": [{"b64_json": ""}],
}
class MockResponse:
def __init__(self, json_data, status_code):
self._json_data = json_data
self.status_code = status_code
self.text = json.dumps(json_data)
def json(self):
return self._json_data
image_file = BytesIO(b"fake-image")
if is_async:
mock_post = AsyncMock(return_value=MockResponse(mock_response, 200))
patch_target = "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
else:
mock_post = MagicMock(return_value=MockResponse(mock_response, 200))
patch_target = "litellm.llms.custom_httpx.http_handler.HTTPHandler.post"
with patch(patch_target, new=mock_post):
if is_async:
await litellm.aimage_edit(
model="litellm_proxy/gpt-image-1",
prompt="A test prompt",
image=[image_file],
api_base="http://my-proxy",
api_key="sk-1234",
)
mock_post.assert_awaited_once()
else:
litellm.image_edit(
model="litellm_proxy/gpt-image-1",
prompt="A test prompt",
image=[image_file],
api_base="http://my-proxy",
api_key="sk-1234",
)
mock_post.assert_called_once()
called_kwargs = mock_post.call_args.kwargs
assert called_kwargs["url"] == "http://my-proxy/images/edits"
assert called_kwargs["headers"]["Authorization"] == "Bearer sk-1234"
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.asyncio
async def test_litellm_gateway_from_sdk_transcription(is_async):

11
tests/router_unit_tests/test_router_batch_utils.py
View file

@ -18,7 +18,7 @@ from io import BytesIO
from typing import Dict, List
from litellm.router_utils.batch_utils import (
replace_model_in_jsonl,
_get_router_metadata_variable_name,
_get_router_metadata_variable_name, InMemoryFile,
)
@ -57,6 +57,9 @@ def test_bytes_input(sample_jsonl_bytes):
result = replace_model_in_jsonl(sample_jsonl_bytes, new_model)
assert result is not None
assert isinstance(result, InMemoryFile)
assert result.name == "modified_file.jsonl"
assert result.content_type == "application/jsonl"
def test_tuple_input(sample_jsonl_bytes):
@ -66,6 +69,9 @@ def test_tuple_input(sample_jsonl_bytes):
result = replace_model_in_jsonl(test_tuple, new_model)
assert result is not None
assert isinstance(result, InMemoryFile)
assert result.name == "modified_file.jsonl"
assert result.content_type == "application/jsonl"
def test_file_like_object(sample_file_like):
@ -74,6 +80,9 @@ def test_file_like_object(sample_file_like):
result = replace_model_in_jsonl(sample_file_like, new_model)
assert result is not None
assert isinstance(result, InMemoryFile)
assert result.name == "modified_file.jsonl"
assert result.content_type == "application/jsonl"
def test_router_metadata_variable_name():

145
tests/router_unit_tests/test_router_handle_error.py
View file

@ -1,6 +1,7 @@
import sys, os, time
import traceback, asyncio
import pytest
from typing import List
sys.path.insert(
0, os.path.abspath("../..")
@ -111,3 +112,147 @@ async def test_send_llm_exception_alert_when_proxy_server_request_in_kwargs():
# Assert that no exception was raised and the function completed successfully
mock_router.slack_alerting_logger.send_alert.assert_not_called()
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception():
"""
Test that async_raise_no_deployment_exception returns a RouterRateLimitError
with cooldown_list containing just IDs (not tuples with debug info).
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"]
mock_router.cooldown_cache.get_min_cooldown.return_value = 30.0
mock_router.enable_pre_call_checks = True
# Mock the _async_get_cooldown_deployments_with_debug_info function
# It should return a list of tuples where each tuple contains (model_id, debug_info)
mock_cooldown_list = [
("deployment-1", {"error": "rate_limit", "time": "2024-01-01"}),
("deployment-2", {"error": "server_error", "time": "2024-01-01"}),
("deployment-3", {"error": "timeout", "time": "2024-01-01"}),
]
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# Call the function
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="gpt-3.5-turbo",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "gpt-3.5-turbo"
assert result.cooldown_time == 30.0
assert result.enable_pre_call_checks is True
# Assert that cooldown_list contains only IDs (extracted from tuples)
expected_cooldown_list = ["deployment-1", "deployment-2", "deployment-3"]
assert result.cooldown_list == expected_cooldown_list
# Verify that cooldown_list contains only strings (IDs), not tuples
for item in result.cooldown_list:
assert isinstance(item, str), f"Expected string ID, got {type(item)}: {item}"
# Verify mock calls
mock_router.get_model_ids.assert_called_once_with(model_name="gpt-3.5-turbo")
mock_router.cooldown_cache.get_min_cooldown.assert_called_once_with(
model_ids=["deployment-1", "deployment-2"], parent_otel_span=None
)
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception_empty_cooldown_list():
"""
Test that async_raise_no_deployment_exception handles empty cooldown list correctly.
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"]
mock_router.cooldown_cache.get_min_cooldown.return_value = 15.0
mock_router.enable_pre_call_checks = False
# Mock empty cooldown list
mock_cooldown_list: List = []
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# Call the function
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="claude-3-sonnet",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "claude-3-sonnet"
assert result.cooldown_time == 15.0
assert result.enable_pre_call_checks is False
# Assert that cooldown_list is an empty list when no cooldowns exist
assert result.cooldown_list == []
assert isinstance(result.cooldown_list, list)
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception_none_cooldown_list():
"""
Test that async_raise_no_deployment_exception handles None cooldown list correctly.
Note: In practice, _async_get_cooldown_deployments_with_debug_info should never return None
based on the implementation, but this tests defensive programming.
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = []
mock_router.cooldown_cache.get_min_cooldown.return_value = 45.0
mock_router.enable_pre_call_checks = True
# Mock None cooldown list (though this shouldn't happen in practice)
mock_cooldown_list = None
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# After the defensive fix, this should handle None gracefully and return empty list
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="gpt-4",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "gpt-4"
assert result.cooldown_time == 45.0
assert result.enable_pre_call_checks is True
# Assert that cooldown_list is an empty list when cooldown_list is None
assert result.cooldown_list == []
assert isinstance(result.cooldown_list, list)

126
tests/test_litellm/caching/test_s3_cache.py Normal file
View file

@ -0,0 +1,126 @@
import os
import sys
from unittest.mock import MagicMock, patch
import json
import datetime
import pytest
sys.path.insert(
0, os.path.abspath("../../..")
) # Adds the parent directory to the system path
from litellm.caching.s3_cache import S3Cache
@pytest.fixture
def mock_s3_dependencies():
mock_s3_client = MagicMock()
with patch("boto3.client", return_value=mock_s3_client):
yield {"s3_client": mock_s3_client}
def test_s3_cache_set_cache(mock_s3_dependencies):
"""Test basic set_cache functionality"""
cache = S3Cache("test-bucket")
test_value = {"key": "value", "number": 42}
cache.set_cache("test_key", test_value)
cache.s3_client.put_object.assert_called_once()
call_args = cache.s3_client.put_object.call_args
assert call_args[1]["Bucket"] == "test-bucket"
assert call_args[1]["Key"] == "test_key"
assert call_args[1]["Body"] == json.dumps(test_value)
assert call_args[1]["ContentType"] == "application/json"
assert call_args[1]["ContentLanguage"] == "en"
assert call_args[1]["ContentDisposition"] == 'inline; filename="test_key.json"'
def test_s3_cache_set_cache_with_ttl(mock_s3_dependencies):
"""Test set_cache with TTL functionality"""
cache = S3Cache("test-bucket")
test_value = {"key": "value"}
ttl = datetime.timedelta(seconds=3600) # 1 hour
cache.set_cache("test_key", test_value, ttl=ttl)
cache.s3_client.put_object.assert_called_once()
call_args = cache.s3_client.put_object.call_args
assert "Expires" in call_args[1]
assert "CacheControl" in call_args[1]
assert "max-age=1:00:00" in call_args[1]["CacheControl"]
def test_s3_cache_get_cache(mock_s3_dependencies):
"""Test basic get_cache functionality"""
cache = S3Cache("test-bucket")
mock_response = {
"Body": MagicMock()
}
mock_response["Body"].read.return_value = b'{"key": "value", "number": 42}'
cache.s3_client.get_object.return_value = mock_response
result = cache.get_cache("test_key")
cache.s3_client.get_object.assert_called_once_with(
Bucket="test-bucket",
Key="test_key"
)
assert result == {"key": "value", "number": 42}
def test_s3_cache_get_cache_not_found(mock_s3_dependencies):
"""Test get_cache when key is not found"""
import botocore.exceptions
cache = S3Cache("test-bucket")
error_response = {"Error": {"Code": "NoSuchKey"}}
cache.s3_client.get_object.side_effect = botocore.exceptions.ClientError(
error_response, "GetObject"
)
result = cache.get_cache("nonexistent_key")
cache.s3_client.get_object.assert_called_once_with(
Bucket="test-bucket",
Key="nonexistent_key"
)
assert result is None
def test_s3_key_transformation():
"""Test the _to_s3_key method for key transformation"""
cache = S3Cache("test-bucket")
# Test basic key transformation (colon to slash)
result = cache._to_s3_key("user:123:session:456")
assert result == "user/123/session/456"
# Test with s3_path prefix
cache_with_prefix = S3Cache("test-bucket", s3_path="cache/data")
result = cache_with_prefix._to_s3_key("namespace:key")
assert result == "cache/data/namespace/key"
# Test with s3_path that has trailing slash
cache_with_slash = S3Cache("test-bucket", s3_path="cache/data/")
result = cache_with_slash._to_s3_key("namespace:key")
assert result == "cache/data/namespace/key"
def test_s3_cache_initialization():
"""Test S3Cache initialization with various parameters"""
# Test basic initialization
cache = S3Cache("test-bucket")
assert cache.bucket_name == "test-bucket"
assert cache.key_prefix == ""
# Test with s3_path
cache_with_path = S3Cache("test-bucket", s3_path="my/cache/path")
assert cache_with_path.key_prefix == "my/cache/path/"

260
tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py
View file

@ -20,10 +20,12 @@ from litellm.types.integrations.datadog_llm_obs import (
LLMObsPayload,
)
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
StandardLoggingHiddenParams,
StandardLoggingMetadata,
StandardLoggingModelInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
@ -81,6 +83,67 @@ def create_standard_logging_payload_with_cache() -> StandardLoggingPayload:
)
def create_standard_logging_payload_with_failure() -> StandardLoggingPayload:
"""Create a StandardLoggingPayload object for failure testing"""
return StandardLoggingPayload(
id="test-request-id-failure-789",
call_type="completion",
response_cost=0.0,
response_cost_failure_debug_info=None,
status="failure",
total_tokens=0,
prompt_tokens=10,
completion_tokens=0,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=1234567890.5,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-4", model_map_value=None
),
model="gpt-4",
model_id="model-123",
model_group="openai-gpt",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_org_id=None,
user_api_key_alias="test_alias",
user_api_key_team_id="test_team",
user_api_key_user_id="test_user",
user_api_key_team_alias="test_team_alias",
spend_logs_metadata=None,
requester_ip_address="127.0.0.1",
requester_metadata=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address="127.0.0.1",
messages=[{"role": "user", "content": "Hello, world!"}],
response=None,
error_str="RateLimitError: You exceeded your current quota",
error_information=StandardLoggingPayloadErrorInformation(
error_code="rate_limit_exceeded",
error_class="RateLimitError",
llm_provider="openai",
traceback="Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota",
error_message="RateLimitError: You exceeded your current quota"
),
model_parameters={"stream": False},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.0",
additional_headers=None,
),
trace_id="test-trace-id-failure-456",
custom_llm_provider="openai",
)
class TestDataDogLLMObsLogger:
"""Test suite for DataDog LLM Observability Logger"""
@ -118,7 +181,7 @@ class TestDataDogLLMObsLogger:
start_time = datetime.now()
end_time = datetime.now()
payload = logger.create_llm_obs_payload(kwargs, mock_response_obj, start_time, end_time)
payload = logger.create_llm_obs_payload(kwargs, start_time, end_time)
# Test 1: Verify total_cost is correctly extracted from response_cost
assert payload["metrics"].get("total_cost") == 0.05
@ -148,7 +211,7 @@ class TestDataDogLLMObsLogger:
start_time = datetime.now()
end_time = datetime.now()
payload = logger.create_llm_obs_payload(kwargs, mock_response_obj, start_time, end_time)
payload = logger.create_llm_obs_payload(kwargs, start_time, end_time)
# Test the _get_dd_llm_obs_payload_metadata method directly
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
@ -217,9 +280,56 @@ class TestDataDogLLMObsLogger:
assert logger._get_datadog_span_kind("unknown_call_type") == "llm"
assert logger._get_datadog_span_kind(None) == "llm"
@pytest.mark.asyncio
async def test_async_log_failure_event(self, mock_env_vars):
"""Test that async_log_failure_event correctly processes failure payloads according to DD LLM Obs API spec"""
with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \
patch('asyncio.create_task'):
logger = DataDogLLMObsLogger()
# Ensure log_queue starts empty
logger.log_queue = []
standard_failure_payload = create_standard_logging_payload_with_failure()
kwargs = {
"standard_logging_object": standard_failure_payload,
"model": "gpt-4",
"litellm_params": {"metadata": {}}
}
start_time = datetime.now()
end_time = datetime.now() + timedelta(seconds=2)
# Mock async_send_batch to prevent actual network calls
with patch.object(logger, 'async_send_batch') as mock_send_batch:
# Call the method under test
await logger.async_log_failure_event(kwargs, None, start_time, end_time)
# Verify payload was added to queue
assert len(logger.log_queue) == 1
# Verify the payload has correct failure characteristics according to DD LLM Obs API spec
payload = logger.log_queue[0]
assert payload["trace_id"] == "test-trace-id-failure-456"
assert payload["meta"]["metadata"]["id"] == "test-request-id-failure-789"
assert payload["status"] == "error"
# Verify error information follows DD LLM Obs API spec
assert payload["meta"]["error"]["message"] == "RateLimitError: You exceeded your current quota"
assert payload["meta"]["error"]["type"] == "RateLimitError"
assert payload["meta"]["error"]["stack"] == "Traceback (most recent call last):\n File test.py, line 1\n RateLimitError: You exceeded your current quota"
assert payload["metrics"]["total_cost"] == 0.0
assert payload["metrics"]["total_tokens"] == 0
assert payload["metrics"]["output_tokens"] == 0
# Verify batch sending not triggered (queue size < batch_size)
mock_send_batch.assert_not_called()
class TestDataDogLLMObsLogger(DataDogLLMObsLogger):
class TestDataDogLLMObsLoggerForRedaction(DataDogLLMObsLogger):
"""Test suite for DataDog LLM Observability Logger"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
@ -245,7 +355,7 @@ async def test_dd_llms_obs_redaction(mock_env_vars):
litellm._turn_on_debug()
from litellm.types.utils import LiteLLMCommonStrings
litellm.datadog_llm_observability_params = DatadogLLMObsInitParams(turn_off_message_logging=True)
dd_llms_obs_logger = TestDataDogLLMObsLogger()
dd_llms_obs_logger = TestDataDogLLMObsLoggerForRedaction()
test_s3_logger = TestS3Logger()
litellm.callbacks = [
dd_llms_obs_logger,
@ -315,3 +425,145 @@ async def test_create_llm_obs_payload(mock_env_vars):
assert payload["metrics"]["input_tokens"] == 10
assert payload["metrics"]["output_tokens"] == 20
assert payload["metrics"]["total_tokens"] == 30
def create_standard_logging_payload_with_latency_metrics() -> StandardLoggingPayload:
"""Create a StandardLoggingPayload object with latency metrics for testing"""
guardrail_info = StandardLoggingGuardrailInformation(
guardrail_name="test_guardrail",
guardrail_status="success",
start_time=1234567890.0,
end_time=1234567890.5,
duration=0.5, # 500ms
)
hidden_params = StandardLoggingHiddenParams(
model_id="model-123",
cache_key="test-cache-key",
api_base="https://api.openai.com",
response_cost="0.05",
litellm_overhead_time_ms=150.0, # 150ms
additional_headers=None,
)
return StandardLoggingPayload(
id="test-request-id-latency",
call_type="completion",
response_cost=0.05,
response_cost_failure_debug_info=None,
status="success",
total_tokens=30,
prompt_tokens=10,
completion_tokens=20,
startTime=1234567890.0,
endTime=1234567892.0,
completionStartTime=1234567890.8, # 800ms after start
response_time=2.0,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-4", model_map_value=None
),
model="gpt-4",
model_id="model-123",
model_group="openai-gpt",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_org_id=None,
user_api_key_alias="test_alias",
user_api_key_team_id="test_team",
user_api_key_user_id="test_user",
user_api_key_team_alias="test_team_alias",
spend_logs_metadata=None,
requester_ip_address="127.0.0.1",
requester_metadata=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address="127.0.0.1",
messages=[{"role": "user", "content": "Hello, world!"}],
response={"choices": [{"message": {"content": "Hi there!"}}]},
error_str=None,
error_information=None,
model_parameters={"stream": True},
hidden_params=hidden_params,
guardrail_information=guardrail_info,
trace_id="test-trace-id-latency",
custom_llm_provider="openai",
)
def test_latency_metrics_in_metadata(mock_env_vars):
"""Test that time to first token, litellm overhead, and guardrail overhead are included in metadata"""
with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \
patch('asyncio.create_task'):
logger = DataDogLLMObsLogger()
standard_payload = create_standard_logging_payload_with_latency_metrics()
kwargs = {
"standard_logging_object": standard_payload,
"litellm_params": {"metadata": {}}
}
start_time = datetime.now()
end_time = datetime.now()
# Test the metadata generation directly
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
latency_metadata = metadata.get("latency_metrics", {})
# Verify time to first token is included (800ms)
assert "time_to_first_token_ms" in latency_metadata
assert abs(latency_metadata["time_to_first_token_ms"] - 800.0) < 0.001 # 0.8 seconds * 1000 with tolerance for floating-point precision
# Verify litellm overhead is included (150ms)
assert "litellm_overhead_time_ms" in latency_metadata
assert latency_metadata["litellm_overhead_time_ms"] == 150.0
# Verify guardrail overhead is included (500ms)
assert "guardrail_overhead_time_ms" in latency_metadata
assert latency_metadata["guardrail_overhead_time_ms"] == 500.0 # 0.5 seconds * 1000
# Verify these metrics are also included in the full payload
payload = logger.create_llm_obs_payload(kwargs, start_time, end_time)
payload_metadata_latency = payload["meta"]["metadata"]["latency_metrics"]
assert abs(payload_metadata_latency["time_to_first_token_ms"] - 800.0) < 0.001
assert payload_metadata_latency["litellm_overhead_time_ms"] == 150.0
assert payload_metadata_latency["guardrail_overhead_time_ms"] == 500.0
def test_latency_metrics_edge_cases(mock_env_vars):
"""Test latency metrics with edge cases (missing fields, zero values, etc.)"""
with patch('litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client'), \
patch('asyncio.create_task'):
logger = DataDogLLMObsLogger()
# Test case 1: No latency metrics present
standard_payload = create_standard_logging_payload_with_cache()
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
# Should not have latency fields if data is missing/zero
assert "time_to_first_token_ms" not in metadata # Will be 0, so not included
assert "litellm_overhead_time_ms" not in metadata # Not present in hidden_params
assert "guardrail_overhead_time_ms" not in metadata # No guardrail_information
# Test case 2: Zero time to first token should not be included
standard_payload = create_standard_logging_payload_with_cache()
standard_payload["startTime"] = 1000.0
standard_payload["completionStartTime"] = 1000.0 # Same time = 0 difference
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
assert "time_to_first_token_ms" not in metadata
# Test case 3: Missing guardrail duration should not crash
standard_payload = create_standard_logging_payload_with_cache()
standard_payload["guardrail_information"] = StandardLoggingGuardrailInformation(
guardrail_name="test",
guardrail_status="success",
# duration is missing
)
metadata = logger._get_dd_llm_obs_payload_metadata(standard_payload)
assert "guardrail_overhead_time_ms" not in metadata

140
tests/test_litellm/llms/azure/response/test_azure_transformation.py
View file

@ -8,10 +8,14 @@ sys.path.insert(
0, os.path.abspath("../../../../..")
) # Adds the parent directory to the system path
from unittest.mock import MagicMock
from litellm.llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig
from litellm.types.router import GenericLiteLLMParams
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
@pytest.mark.serial
@ -27,6 +31,7 @@ def test_validate_environment_api_key_within_litellm_params():
assert result == expected
@pytest.mark.serial
def test_validate_environment_api_key_within_litellm():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -41,6 +46,7 @@ def test_validate_environment_api_key_within_litellm():
assert result == expected
@pytest.mark.serial
def test_validate_environment_azure_key_within_litellm():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -55,6 +61,7 @@ def test_validate_environment_azure_key_within_litellm():
assert result == expected
@pytest.mark.serial
def test_validate_environment_azure_key_within_headers():
azure_openai_responses_apiconfig = AzureOpenAIResponsesAPIConfig()
@ -93,10 +100,10 @@ def test_azure_o_series_responses_api_supported_params():
"""Test that Azure OpenAI O-series responses API excludes temperature from supported parameters."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
supported_params = config.get_supported_openai_params("o_series/gpt-o1")
# Temperature should not be in supported params for O-series models
assert "temperature" not in supported_params
# Other parameters should still be supported
assert "input" in supported_params
assert "max_output_tokens" in supported_params
@ -108,35 +115,32 @@ def test_azure_o_series_responses_api_supported_params():
def test_azure_o_series_responses_api_drop_temperature_param():
"""Test that temperature parameter is dropped when drop_params is True for O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Create request params with temperature
request_params = ResponsesAPIOptionalRequestParams(
temperature=0.7,
max_output_tokens=1000,
stream=False,
top_p=0.9
temperature=0.7, max_output_tokens=1000, stream=False, top_p=0.9
)
# Test with drop_params=True
mapped_params_with_drop = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=True
drop_params=True,
)
# Temperature should be dropped
assert "temperature" not in mapped_params_with_drop
# Other params should remain
assert mapped_params_with_drop["max_output_tokens"] == 1000
assert mapped_params_with_drop["top_p"] == 0.9
# Test with drop_params=False
mapped_params_without_drop = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=False
drop_params=False,
)
# Temperature should still be present when drop_params=False
assert mapped_params_without_drop["temperature"] == 0.7
assert mapped_params_without_drop["max_output_tokens"] == 1000
@ -147,21 +151,19 @@ def test_azure_o_series_responses_api_drop_temperature_param():
def test_azure_o_series_responses_api_drop_params_no_temperature():
"""Test that map_openai_params works correctly when temperature is not present for O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Create request params without temperature
request_params = ResponsesAPIOptionalRequestParams(
max_output_tokens=1000,
stream=False,
top_p=0.9
max_output_tokens=1000, stream=False, top_p=0.9
)
# Should work fine even with drop_params=True
mapped_params = config.map_openai_params(
response_api_optional_params=request_params,
model="o_series/gpt-o1",
drop_params=True
drop_params=True,
)
assert "temperature" not in mapped_params
assert mapped_params["max_output_tokens"] == 1000
assert mapped_params["top_p"] == 0.9
@ -172,10 +174,10 @@ def test_azure_regular_responses_api_supports_temperature():
"""Test that regular Azure OpenAI responses API (non-O-series) supports temperature parameter."""
config = AzureOpenAIResponsesAPIConfig()
supported_params = config.get_supported_openai_params("gpt-4o")
# Regular Azure models should support temperature
assert "temperature" in supported_params
# Other parameters should still be supported
assert "input" in supported_params
assert "max_output_tokens" in supported_params
@ -187,11 +189,11 @@ def test_azure_regular_responses_api_supports_temperature():
def test_o_series_model_detection():
"""Test that the O-series configuration correctly identifies O-series models."""
config = AzureOpenAIOSeriesResponsesAPIConfig()
# Test explicit o_series naming
assert config.is_o_series_model("o_series/gpt-o1") == True
assert config.is_o_series_model("azure/o_series/gpt-o3") == True
# Test regular models
assert config.is_o_series_model("gpt-4o") == False
assert config.is_o_series_model("gpt-3.5-turbo") == False
@ -200,28 +202,94 @@ def test_o_series_model_detection():
@pytest.mark.serial
def test_provider_config_manager_o_series_selection():
"""Test that ProviderConfigManager returns the correct config for O-series vs regular models."""
from litellm.utils import ProviderConfigManager
import litellm
from litellm.utils import ProviderConfigManager
# Test O-series model selection
o_series_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model="o_series/gpt-o1"
provider=litellm.LlmProviders.AZURE, model="o_series/gpt-o1"
)
assert isinstance(o_series_config, AzureOpenAIOSeriesResponsesAPIConfig)
# Test regular model selection
regular_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model="gpt-4o"
provider=litellm.LlmProviders.AZURE, model="gpt-4o"
)
assert isinstance(regular_config, AzureOpenAIResponsesAPIConfig)
assert not isinstance(regular_config, AzureOpenAIOSeriesResponsesAPIConfig)
# Test with no model specified (should default to regular)
default_config = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders.AZURE,
model=None
provider=litellm.LlmProviders.AZURE, model=None
)
assert isinstance(default_config, AzureOpenAIResponsesAPIConfig)
assert not isinstance(default_config, AzureOpenAIOSeriesResponsesAPIConfig)
class TestAzureResponsesAPIConfig:
def setup_method(self):
self.config = AzureOpenAIResponsesAPIConfig()
self.model = "gpt-4o"
self.logging_obj = MagicMock()
def test_azure_get_complete_url_with_version_types(self):
"""Test Azure get_complete_url with different API version types"""
base_url = "https://litellm8397336933.openai.azure.com"
# Test with preview version - should use openai/v1/responses
result_preview = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "preview"},
)
assert (
result_preview
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
)
# Test with latest version - should use openai/v1/responses
result_latest = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "latest"},
)
assert (
result_latest
== "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
)
# Test with date-based version - should use openai/responses
result_date = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "2025-01-01"},
)
assert (
result_date
== "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
)
def test_azure_get_complete_url_with_default_api_version(self):
"""Test Azure get_complete_url uses default API version when none is provided"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
base_url = "https://litellm8397336933.openai.azure.com"
# Test with no api_version provided - should use default
result_no_version = self.config.get_complete_url(
api_base=base_url,
litellm_params={},
)
expected_url = f"https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version={AZURE_DEFAULT_RESPONSES_API_VERSION}"
assert result_no_version == expected_url
# Test with empty litellm_params - should use default
result_empty_params = self.config.get_complete_url(
api_base=base_url,
litellm_params={},
)
assert result_empty_params == expected_url
# Test with None api_version - should use default
result_none_version = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": None},
)
assert result_none_version == expected_url

34
tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py
View file

@ -147,7 +147,7 @@ class TestOpenAIResponsesAPIConfig:
assert result.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
assert result.response.id == "resp_123"
@pytest.mark.serial
def test_validate_environment(self):
"""Test that validate_environment correctly sets the Authorization header"""
@ -283,38 +283,6 @@ class TestOpenAIResponsesAPIConfig:
assert result.type == "test"
class TestAzureResponsesAPIConfig:
def setup_method(self):
self.config = AzureOpenAIResponsesAPIConfig()
self.model = "gpt-4o"
self.logging_obj = MagicMock()
def test_azure_get_complete_url_with_version_types(self):
"""Test Azure get_complete_url with different API version types"""
base_url = "https://litellm8397336933.openai.azure.com"
# Test with preview version - should use openai/v1/responses
result_preview = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "preview"},
)
assert result_preview == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=preview"
# Test with latest version - should use openai/v1/responses
result_latest = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "latest"},
)
assert result_latest == "https://litellm8397336933.openai.azure.com/openai/v1/responses?api-version=latest"
# Test with date-based version - should use openai/responses
result_date = self.config.get_complete_url(
api_base=base_url,
litellm_params={"api_version": "2025-01-01"},
)
assert result_date == "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2025-01-01"
class TestTransformListInputItemsRequest:
"""Test suite for transform_list_input_items_request function"""

237
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -606,7 +606,6 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging():
assert result.callback_vars["arize_space_id"] == "test_arize_space_id"
def test_get_num_retries_from_request():
"""
Test LiteLLMProxyRequestSetup._get_num_retries_from_request method
@ -668,6 +667,7 @@ def test_get_num_retries_from_request():
)
assert result == -1
def test_add_user_api_key_auth_to_request_metadata():
"""
Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata
@ -676,9 +676,9 @@ def test_add_user_api_key_auth_to_request_metadata():
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"litellm_metadata": {} # This will be the metadata variable name
"litellm_metadata": {}, # This will be the metadata variable name
}
user_api_key_dict = UserAPIKeyAuth(
api_key="hashed-test-key-123",
user_id="test-user-123",
@ -689,21 +689,21 @@ def test_add_user_api_key_auth_to_request_metadata():
team_alias="test-team-alias",
end_user_id="test-end-user-123",
request_route="/chat/completions",
end_user_max_budget=500.0
end_user_max_budget=500.0,
)
metadata_variable_name = "litellm_metadata"
# Call the function
result = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
data=data,
user_api_key_dict=user_api_key_dict,
_metadata_variable_name=metadata_variable_name
_metadata_variable_name=metadata_variable_name,
)
# Verify the metadata was properly added
metadata = result[metadata_variable_name]
# Check that user API key information was added
assert metadata["user_api_key_hash"] == "hashed-test-key-123"
assert metadata["user_api_key_alias"] == "test-key-alias"
@ -714,13 +714,224 @@ def test_add_user_api_key_auth_to_request_metadata():
assert metadata["user_api_key_end_user_id"] == "test-end-user-123"
assert metadata["user_api_key_user_email"] == "test@example.com"
assert metadata["user_api_key_request_route"] == "/chat/completions"
# Check that the hashed API key was added
assert metadata["user_api_key"] == "hashed-test-key-123"
# Check that end user max budget was added
assert metadata["user_api_end_user_max_budget"] == 500.0
# Verify original data is preserved
assert result["model"] == "gpt-3.5-turbo"
assert result["messages"] == [{"role": "user", "content": "Hello"}]
assert result["messages"] == [{"role": "user", "content": "Hello"}]
@pytest.mark.parametrize(
"data, model_group_settings, expected_headers_added",
[
# Test case 1: Model is in forward_client_headers_to_llm_api list
(
{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]},
MagicMock(forward_client_headers_to_llm_api=["gpt-4"]),
True,
),
# Test case 2: Model is not in forward_client_headers_to_llm_api list
(
{"model": "claude-3", "messages": [{"role": "user", "content": "Hello"}]},
MagicMock(forward_client_headers_to_llm_api=["gpt-4"]),
False,
),
# Test case 3: Model group settings is None
(
{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]},
None,
False,
),
# Test case 4: forward_client_headers_to_llm_api is None
(
{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]},
MagicMock(forward_client_headers_to_llm_api=None),
False,
),
# Test case 5: Data has no model
(
{"messages": [{"role": "user", "content": "Hello"}]},
MagicMock(forward_client_headers_to_llm_api=["gpt-4"]),
False,
),
# Test case 6: Model is None
(
{"model": None, "messages": [{"role": "user", "content": "Hello"}]},
MagicMock(forward_client_headers_to_llm_api=["gpt-4"]),
False,
),
],
)
def test_add_headers_to_llm_call_by_model_group(
data, model_group_settings, expected_headers_added
):
"""
Test LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group method
This tests various scenarios:
1. When model is in the forward_client_headers_to_llm_api list
2. When model is not in the list
3. When model_group_settings is None
4. When forward_client_headers_to_llm_api is None
5. When data has no model
6. When model is None
"""
import litellm
# Setup test headers and user API key
headers = {
"Authorization": "Bearer token123",
"User-Agent": "test-client/1.0",
"X-Custom-Header": "custom-value",
}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key", user_id="test-user", org_id="test-org"
)
# Mock the model_group_settings
original_model_group_settings = getattr(litellm, "model_group_settings", None)
litellm.model_group_settings = model_group_settings
try:
# Mock the add_headers_to_llm_call method to return expected headers
expected_returned_headers = {
"X-LiteLLM-User": "test-user",
"X-LiteLLM-Org": "test-org",
}
with patch.object(
LiteLLMProxyRequestSetup,
"add_headers_to_llm_call",
return_value=expected_returned_headers if expected_headers_added else {},
) as mock_add_headers:
# Make a copy of original data to verify it's not mutated unexpectedly
original_data = copy.deepcopy(data)
# Call the method under test
result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
data=data, headers=headers, user_api_key_dict=user_api_key_dict
)
# Verify the result
assert result is not None
assert isinstance(result, dict)
if expected_headers_added:
# Verify that add_headers_to_llm_call was called
mock_add_headers.assert_called_once_with(headers, user_api_key_dict)
# Verify that headers were added to the data
assert "headers" in result
assert result["headers"] == expected_returned_headers
else:
# Verify that add_headers_to_llm_call was not called
mock_add_headers.assert_not_called()
# Verify that no headers were added
assert "headers" not in result or result.get("headers") is None
# Verify that original data fields are preserved
for key, value in original_data.items():
if key != "headers": # headers might be added
assert result[key] == value
finally:
# Restore original model_group_settings
litellm.model_group_settings = original_model_group_settings
def test_add_headers_to_llm_call_by_model_group_empty_headers_returned():
"""
Test that when add_headers_to_llm_call returns empty dict, no headers are added to data
"""
import litellm
# Setup test data
data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}
headers = {"Authorization": "Bearer token123"}
user_api_key_dict = UserAPIKeyAuth(api_key="test-key")
# Mock model_group_settings with model in the list
mock_settings = MagicMock(forward_client_headers_to_llm_api=["gpt-4"])
original_model_group_settings = getattr(litellm, "model_group_settings", None)
litellm.model_group_settings = mock_settings
try:
with patch.object(
LiteLLMProxyRequestSetup,
"add_headers_to_llm_call",
return_value={}, # Return empty dict
) as mock_add_headers:
result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
data=data, headers=headers, user_api_key_dict=user_api_key_dict
)
# Verify that add_headers_to_llm_call was called
mock_add_headers.assert_called_once_with(headers, user_api_key_dict)
# Verify that no headers were added since returned headers were empty
assert "headers" not in result
# Verify original data is preserved
assert result["model"] == "gpt-4"
assert result["messages"] == [{"role": "user", "content": "Hello"}]
finally:
# Restore original model_group_settings
litellm.model_group_settings = original_model_group_settings
def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data():
"""
Test that existing headers in data are overwritten when new headers are added
"""
import litellm
# Setup test data with existing headers
data = {
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"headers": {"Existing-Header": "existing-value"},
}
headers = {"Authorization": "Bearer token123"}
user_api_key_dict = UserAPIKeyAuth(api_key="test-key")
# Mock model_group_settings with model in the list
mock_settings = MagicMock(forward_client_headers_to_llm_api=["gpt-4"])
original_model_group_settings = getattr(litellm, "model_group_settings", None)
litellm.model_group_settings = mock_settings
try:
new_headers = {"X-LiteLLM-User": "test-user"}
with patch.object(
LiteLLMProxyRequestSetup,
"add_headers_to_llm_call",
return_value=new_headers,
) as mock_add_headers:
result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
data=data, headers=headers, user_api_key_dict=user_api_key_dict
)
# Verify that add_headers_to_llm_call was called
mock_add_headers.assert_called_once_with(headers, user_api_key_dict)
# Verify that headers were overwritten
assert "headers" in result
assert result["headers"] == new_headers
assert result["headers"] != {"Existing-Header": "existing-value"}
# Verify original data is preserved
assert result["model"] == "gpt-4"
assert result["messages"] == [{"role": "user", "content": "Hello"}]
finally:
# Restore original model_group_settings
litellm.model_group_settings = original_model_group_settings

1
tests/test_litellm/router_utils/pre_call_checks/test_responses_api_deployment_check.py
View file

@ -17,7 +17,6 @@ from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponseTextConfig,
)
from litellm.types.utils import StandardLoggingPayload

257
tests/test_litellm/router_utils/test_cooldown_cache.py Normal file
View file

@ -0,0 +1,257 @@
"""
Unit tests for CooldownCache exception masking functionality
"""
import os
import sys
from unittest.mock import MagicMock
import pytest
# Add the parent directory to the system path
sys.path.insert(0, os.path.abspath("../../.."))
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue
class TestCooldownCacheExceptionMasking:
"""Test suite for CooldownCache exception masking functionality"""
@pytest.fixture
def cooldown_cache(self):
"""Create a CooldownCache instance for testing"""
mock_dual_cache = MagicMock(spec=DualCache)
return CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0)
def test_exception_masker_initialization(self, cooldown_cache):
"""Test that the exception masker is properly initialized"""
assert isinstance(cooldown_cache.exception_masker, SensitiveDataMasker)
assert cooldown_cache.exception_masker.visible_prefix == 50
assert cooldown_cache.exception_masker.visible_suffix == 0
assert cooldown_cache.exception_masker.mask_char == "*"
def test_short_exception_string_not_masked(self, cooldown_cache):
"""Test that short exception strings are not masked"""
short_exception = "Short error"
model_id = "test-model"
exception_status = 500
cooldown_time = 30.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(short_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
# Short exception should not be masked
assert cooldown_data["exception_received"] == short_exception
assert cooldown_key == f"deployment:{model_id}:cooldown"
def test_long_exception_string_masked(self, cooldown_cache):
"""Test that long exception strings are properly masked"""
# Create a long exception string that simulates prompt leakage
long_exception = (
"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occurred - "
"No deployments available for selected model, Try again in 5 seconds. "
"Passed model=anthropic_claude_sonnet_4_v1_0. pre-call-checks=False, "
"cooldown_list=[('deepseek_r1-eastus', {'exception_received': "
"'litellm.RateLimitError: RateLimitError: Azure_aiException - "
'{"error":{"code":"Invalid input","status":422,"message":"invalid input error",'
'"details":[{"type":"model_attributes_type","loc":["body"],'
'"msg":"Tell me a story about a dragon and a princess in a magical kingdom '
"where the dragon is actually protecting the princess from an evil wizard "
'who wants to steal her magical powers and use them to conquer the world"}]}'
)
model_id = "test-model"
exception_status = 429
cooldown_time = 60.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(long_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should start with first 50 characters
assert masked_exception.startswith(long_exception[:50])
# Should contain masking characters
assert "*" in masked_exception
# Should be same length (prefix + asterisks)
assert len(masked_exception) == len(long_exception)
# Should not contain the sensitive prompt content
assert "Tell me a story about a dragon" not in masked_exception
assert "magical kingdom" not in masked_exception
# Should preserve the error type information at the beginning (first 50 chars)
assert masked_exception.startswith(
"litellm.proxy.proxy_server._handle_llm_api_excepti"
)
def test_exception_with_api_keys_masked(self, cooldown_cache):
"""Test that API keys in exceptions are properly masked"""
exception_with_key = (
"Authentication failed with api_key=sk-1234567890abcdefghijklmnopqrstuvwxyz "
"and token=bearer_token_123456789 for model gpt-4"
)
model_id = "test-model"
exception_status = 401
cooldown_time = 30.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(exception_with_key),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should mask the sensitive content while preserving structure
assert masked_exception.startswith(
"Authentication failed with api_key=sk-12345678"
)
assert "*" in masked_exception
assert len(masked_exception) == len(exception_with_key)
def test_cooldown_data_structure(self, cooldown_cache):
"""Test that the cooldown data structure is correctly formed"""
exception_msg = "Test exception for structure validation"
model_id = "test-model"
exception_status = 500
cooldown_time = 45.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(exception_msg),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
# Verify cooldown data structure
assert isinstance(cooldown_data, dict)
assert "exception_received" in cooldown_data
assert "status_code" in cooldown_data
assert "timestamp" in cooldown_data
assert "cooldown_time" in cooldown_data
# Verify data types
assert isinstance(cooldown_data["exception_received"], str)
assert isinstance(cooldown_data["status_code"], str)
assert isinstance(cooldown_data["timestamp"], float)
assert isinstance(cooldown_data["cooldown_time"], float)
# Verify values
assert cooldown_data["status_code"] == str(exception_status)
assert cooldown_data["cooldown_time"] == cooldown_time
assert cooldown_data["exception_received"] == exception_msg
def test_exception_object_conversion(self, cooldown_cache):
"""Test that different exception types are properly converted to strings"""
# Test with different exception types
exceptions = [
ValueError("Invalid value provided"),
KeyError("Missing required key"),
RuntimeError("Runtime error occurred"),
Exception("Generic exception"),
]
for exc in exceptions:
model_id = f"test-model-{exc.__class__.__name__}"
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=exc,
exception_status=500,
cooldown_time=30.0,
)
# Should successfully convert exception to string
assert isinstance(cooldown_data["exception_received"], str)
assert (
str(exc) == cooldown_data["exception_received"]
) # Short exceptions not masked
def test_masking_preserves_error_debugging_info(self, cooldown_cache):
"""Test that masking preserves essential debugging information"""
debugging_exception = (
"RateLimitError: Rate limit exceeded for model gpt-4. "
"Current usage: 1000 tokens/minute. Limit: 500 tokens/minute. "
"Request details: model=gpt-4, user_id=user123, "
"prompt='Write a comprehensive analysis of the economic implications "
"of artificial intelligence adoption in the healthcare sector, including "
"potential cost savings, job displacement, and regulatory challenges'"
)
model_id = "gpt-4-deployment"
exception_status = 429
cooldown_time = 120.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(debugging_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should preserve error type and initial debugging info (first 50 chars)
assert masked_exception.startswith(
"RateLimitError: Rate limit exceeded for model gpt-"
)
# Should mask the prompt content
assert "Write a comprehensive analysis" not in masked_exception
assert "healthcare sector" not in masked_exception
# Should contain masking indicator
assert "*" in masked_exception
def test_error_handling_in_common_add_cooldown_logic(self, cooldown_cache):
"""Test error handling in the _common_add_cooldown_logic method"""
# This test ensures that edge cases are properly handled
model_id = "test-model"
# Test with None exception (edge case) - should be handled gracefully
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=None,
exception_status=500,
cooldown_time=30.0,
)
# Should handle None by converting to string
assert cooldown_data["exception_received"] == "None"
assert cooldown_key == f"deployment:{model_id}:cooldown"
def test_custom_masker_settings(self):
"""Test that custom masker settings work correctly"""
mock_dual_cache = MagicMock(spec=DualCache)
# Create cooldown cache and verify default settings
cache = CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0)
# Test that we can access and verify the masker configuration
assert cache.exception_masker.visible_prefix == 50
assert cache.exception_masker.visible_suffix == 0
assert cache.exception_masker.mask_char == "*"
# Test masking behavior with these settings
long_string = "A" * 100 # 100 character string
masked = cache.exception_masker._mask_value(long_string)
# Should show first 50 characters, then all asterisks
expected = "A" * 50 + "*" * 50
assert masked == expected

66
tests/test_litellm/test_cost_calculator.py
View file

@ -480,3 +480,69 @@ def test_gemini_25_implicit_caching_cost():
), f"Expected cost {expected_cost}, but got {result}"
print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}")
def test_gemini_25_explicit_caching_cost_direct_usage():
"""
Test that Gemini 2.5 models correctly calculate costs with explicit caching.
This test reproduces the issue from #11156 where cached tokens should receive
a 75% discount.
"""
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
Usage,
)
from litellm.utils import get_model_info
model_info = get_model_info(model="gemini-2.5-pro", custom_llm_provider="gemini")
usage = Usage(
completion_tokens=2522,
prompt_tokens=42001,
total_tokens=44523,
completion_tokens_details=CompletionTokensDetailsWrapper(
accepted_prediction_tokens=None,
audio_tokens=None,
reasoning_tokens=1908,
rejected_prediction_tokens=None,
text_tokens=614,
),
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=None, cached_tokens=40938, text_tokens=1063, image_tokens=None
),
)
input_cost, output_cost = generic_cost_per_token(
model="gemini/gemini-2.5-pro",
usage=usage,
custom_llm_provider="gemini",
)
total_cost = input_cost + output_cost
expected_higher_than_actual_cost = (
model_info["input_cost_per_token"] * usage.prompt_tokens
+ model_info["output_cost_per_token"] * usage.completion_tokens
)
print(f"expected_higher_than_actual_cost: {expected_higher_than_actual_cost}")
assert expected_higher_than_actual_cost > total_cost
expected_actual_cost = (
model_info["input_cost_per_token"] * usage.prompt_tokens_details.text_tokens
+ model_info["cache_read_input_token_cost"]
* usage.prompt_tokens_details.cached_tokens
+ model_info["output_cost_per_token"] * usage.completion_tokens
)
print(
f"model_info['input_cost_per_token']: {model_info['input_cost_per_token']}, usage.prompt_tokens_details.text_tokens: {usage.prompt_tokens_details.text_tokens}, model_info['cache_read_input_token_cost']: {model_info['cache_read_input_token_cost']}, model_info['output_cost_per_token']: {model_info['output_cost_per_token']}"
)
print(f"Expected actual cost: {expected_actual_cost}")
assert expected_actual_cost == total_cost

129
tests/test_litellm/test_groq_streaming_encoding.py Normal file
View file

@ -0,0 +1,129 @@
"""
Test for Groq streaming ASCII encoding issue fix.
This test verifies that the OpenAI-like handler correctly handles
UTF-8 encoded content in streaming responses, specifically fixing
the ASCII encoding error described in issue #12660.
"""
import pytest
import asyncio
from unittest.mock import Mock, AsyncMock
from litellm.llms.openai_like.chat.handler import make_call, make_sync_call
class MockResponse:
"""Mock httpx response for testing UTF-8 handling."""
def __init__(self, test_content: str):
self.test_content = test_content
self.status_code = 200
def iter_text(self, encoding='utf-8'):
"""Mock iter_text that yields content with the specified encoding."""
yield self.test_content
async def aiter_text(self, encoding='utf-8'):
"""Mock aiter_text that yields content with the specified encoding."""
yield self.test_content
def json(self):
return {"choices": [{"delta": {"content": "test"}}]}
class MockSyncClient:
"""Mock synchronous HTTP client for testing."""
def __init__(self, response_content: str):
self.response_content = response_content
def post(self, *args, **kwargs):
return MockResponse(self.response_content)
class MockAsyncClient:
"""Mock asynchronous HTTP client for testing."""
def __init__(self, response_content: str):
self.response_content = response_content
async def post(self, *args, **kwargs):
return MockResponse(self.response_content)
def test_utf8_streaming_sync():
"""Test that synchronous streaming handles UTF-8 characters correctly."""
# Content with the µ character that was causing issues
test_content = "data: {\"choices\":[{\"delta\":{\"content\":\"The symbol µ represents micro\"}}]}\n\n"
mock_client = MockSyncClient(test_content)
mock_logging = Mock()
# This should not raise an ASCII encoding error
completion_stream = make_sync_call(
client=mock_client,
api_base="https://test.com/v1/chat/completions",
headers={"Authorization": "Bearer test"},
data='{"model": "test", "messages": []}',
model="test-model",
messages=[],
logging_obj=mock_logging
)
# Verify we can iterate through the stream without encoding errors
assert completion_stream is not None
@pytest.mark.asyncio
async def test_utf8_streaming_async():
"""Test that asynchronous streaming handles UTF-8 characters correctly."""
# Content with the µ character that was causing issues
test_content = "data: {\"choices\":[{\"delta\":{\"content\":\"The symbol µ represents micro\"}}]}\n\n"
mock_client = MockAsyncClient(test_content)
mock_logging = Mock()
# This should not raise an ASCII encoding error
completion_stream = await make_call(
client=mock_client,
api_base="https://test.com/v1/chat/completions",
headers={"Authorization": "Bearer test"},
data='{"model": "test", "messages": []}',
model="test-model",
messages=[],
logging_obj=mock_logging
)
# Verify we can iterate through the stream without encoding errors
assert completion_stream is not None
def test_various_unicode_characters():
"""Test streaming with various Unicode characters that could cause issues."""
unicode_test_cases = [
"µ", # Micro symbol (the original issue)
"©", # Copyright symbol
"™", # Trademark symbol
"€", # Euro symbol
"北京", # Chinese characters
"🚀", # Emoji
"Ñoño", # Spanish characters with tildes
]
for unicode_char in unicode_test_cases:
test_content = f"data: {{\"choices\":[{{\"delta\":{{\"content\":\"Testing {unicode_char} character\"}}}}]}}\n\n"
mock_client = MockSyncClient(test_content)
mock_logging = Mock()
# This should not raise an ASCII encoding error for any Unicode character
completion_stream = make_sync_call(
client=mock_client,
api_base="https://test.com/v1/chat/completions",
headers={"Authorization": "Bearer test"},
data='{"model": "test", "messages": []}',
model="test-model",
messages=[],
logging_obj=mock_logging
)
assert completion_stream is not None, f"Failed to handle Unicode character: {unicode_char}"
if __name__ == "__main__":
test_utf8_streaming_sync()
asyncio.run(test_utf8_streaming_async())
test_various_unicode_characters()
print("All UTF-8 streaming tests passed!")

142
tests/test_litellm/test_router.py
View file

@ -896,148 +896,6 @@ async def test_router_ageneric_api_call_with_fallbacks_helper():
assert router.fail_calls["gpt-3.5-turbo"] == initial_fail_count + 1
@pytest.mark.asyncio
async def test_router_forward_client_headers_by_model_group():
"""
Test that router.forward_client_headers_by_model_group returns the correct response
"""
from unittest.mock import MagicMock, patch
from litellm.types.router import ModelGroupSettings
litellm.model_group_settings = ModelGroupSettings(
forward_client_headers_to_llm_api=[
"gpt-3.5-turbo-allow",
"openai/*",
"gpt-3.5-turbo-custom",
]
)
router = litellm.Router(
model_list=[
{
"model_name": "gpt-3.5-turbo-allow",
"litellm_params": {
"model": "gpt-3.5-turbo",
},
},
{
"model_name": "gpt-3.5-turbo-disallow",
"litellm_params": {
"model": "gpt-3.5-turbo",
},
},
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
},
},
{
"model_name": "openai/gpt-4o-mini",
"litellm_params": {
"model": "openai/gpt-4o-mini",
},
},
],
model_group_alias={
"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow",
},
)
## Scenario 1: Direct model name
with patch.object(
litellm.main, "completion", return_value=MagicMock()
) as mock_completion:
await router.acompletion(
model="gpt-3.5-turbo-allow",
messages=[{"role": "user", "content": "Hello, world!"}],
mock_response="Hello, world!",
secret_fields={"raw_headers": {"test": "test"}},
)
mock_completion.assert_called_once()
print(mock_completion.call_args.kwargs["headers"])
## Scenario 2: Wildcard model name
with patch.object(
litellm.main, "completion", return_value=MagicMock()
) as mock_completion:
await router.acompletion(
model="openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello, world!"}],
mock_response="Hello, world!",
secret_fields={"raw_headers": {"test": "test"}},
)
mock_completion.assert_called_once()
print(mock_completion.call_args.kwargs["headers"])
## Scenario 3: Not in model_group_settings
with patch.object(
litellm.main, "completion", return_value=MagicMock()
) as mock_completion:
await router.acompletion(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "Hello, world!"}],
mock_response="Hello, world!",
secret_fields={"raw_headers": {"test": "test"}},
)
mock_completion.assert_called_once()
assert mock_completion.call_args.kwargs.get("headers") is None
## Scenario 4: Model group alias
with patch.object(
litellm.main, "completion", return_value=MagicMock()
) as mock_completion:
await router.acompletion(
model="gpt-3.5-turbo-custom",
messages=[{"role": "user", "content": "Hello, world!"}],
mock_response="Hello, world!",
secret_fields={"raw_headers": {"test": "test"}},
)
mock_completion.assert_called_once()
print(mock_completion.call_args.kwargs["headers"])
def test_router_apply_default_settings():
"""
Test that Router.apply_default_settings() adds the expected default pre-call checks
"""
router = litellm.Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "gpt-3.5-turbo"},
}
],
)
# Apply default settings
result = router.apply_default_settings()
# Verify the method returns None
assert result is None
# Verify that the forward_client_headers_by_model_group pre-call check was added
# Check if any callback is of the ForwardClientHeadersByModelGroupCheck type
has_forward_headers_check = False
for callback in litellm.callbacks:
print(callback)
print(f"callback.__class__: {callback.__class__}")
if hasattr(
callback, "__class__"
) and "ForwardClientSideHeadersByModelGroup" in str(callback.__class__):
has_forward_headers_check = True
break
assert (
has_forward_headers_check
), "Expected ForwardClientSideHeadersByModelGroup to be added to callbacks"
def test_router_get_model_access_groups_team_only_models():
"""
Test that Router.get_model_access_groups returns the correct response for team-only models

28
ui/litellm-dashboard/src/components/add_model/conditional_public_model_name.tsx
View file

@ -2,6 +2,7 @@ import React, { useEffect, useState } from "react";
import { Form, Table } from "antd";
import { TextInput } from "@tremor/react";
import { Tooltip } from "../atoms/index";
import { Providers } from "../provider_info_helpers";
const ConditionalPublicModelName: React.FC = () => {
const form = Form.useFormInstance();
@ -12,13 +13,19 @@ const ConditionalPublicModelName: React.FC = () => {
const selectedModels = Array.isArray(modelValue) ? modelValue : [modelValue];
const customModelName = Form.useWatch('custom_model_name', form);
const showPublicModelName = !selectedModels.includes('all-wildcard');
const selectedProvider = Form.useWatch('custom_llm_provider', form);
// Force table to re-render when custom model name changes
useEffect(() => {
if (customModelName && selectedModels.includes('custom')) {
const currentMappings = form.getFieldValue('model_mappings') || [];
const updatedMappings = currentMappings.map((mapping: any) => {
if (mapping.public_name === 'custom' || mapping.litellm_model === 'custom') {
if (selectedProvider === Providers.Azure) {
return {
public_name: customModelName,
litellm_model: `azure/${customModelName}`
};
}
return {
public_name: customModelName,
litellm_model: customModelName
@ -29,7 +36,7 @@ const ConditionalPublicModelName: React.FC = () => {
form.setFieldValue('model_mappings', updatedMappings);
setTableKey(prev => prev + 1); // Force table re-render
}
}, [customModelName, selectedModels, form]);
}, [customModelName, selectedModels, selectedProvider, form]);
// Initial setup of model mappings when models are selected
useEffect(() => {
@ -44,17 +51,32 @@ const ConditionalPublicModelName: React.FC = () => {
if (model === 'custom') {
return mapping.litellm_model === 'custom' || mapping.litellm_model === customModelName;
}
if (selectedProvider === Providers.Azure) {
return mapping.litellm_model === `azure/${model}`;
}
return mapping.litellm_model === model;
}));
if (shouldUpdateMappings) {
const mappings = selectedModels.map((model: string) => {
if (model === 'custom' && customModelName) {
if (selectedProvider === Providers.Azure) {
return {
public_name: customModelName,
litellm_model: `azure/${customModelName}`
};
}
return {
public_name: customModelName,
litellm_model: customModelName
};
}
if (selectedProvider === Providers.Azure) {
return {
public_name: model,
litellm_model: `azure/${model}`
};
}
return {
public_name: model,
litellm_model: model
@ -65,7 +87,7 @@ const ConditionalPublicModelName: React.FC = () => {
setTableKey(prev => prev + 1); // Force table re-render
}
}
}, [selectedModels, customModelName, form]);
}, [selectedModels, customModelName, selectedProvider,form]);
if (!showPublicModelName) return null;

48
ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx
View file

@ -32,10 +32,18 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
if (JSON.stringify(currentModel) !== JSON.stringify(values)) {
// Create mappings first
const mappings = values.map(model => ({
public_name: model,
litellm_model: model
}));
const mappings = values.map(model => {
if (selectedProvider === Providers.Azure) {
return {
public_name: model,
litellm_model: `azure/${model}`
};
}
return {
public_name: model,
litellm_model: model
};
});
// Update both fields in one call to reduce re-renders
form.setFieldsValue({
@ -47,6 +55,22 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
}
};
const handleAzureDeploymentNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const deploymentName = e.target.value;
// Create mapping with Azure-specific format
const mappings = deploymentName ? [{
public_name: deploymentName,
litellm_model: `azure/${deploymentName}`
}] : [];
// Update both fields
form.setFieldsValue({
model: deploymentName,
model_mappings: mappings
});
};
// Handle custom model name changes
const handleCustomModelNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const customName = e.target.value;
@ -55,6 +79,12 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
const currentMappings = form.getFieldValue('model_mappings') || [];
const updatedMappings = currentMappings.map((mapping: any) => {
if (mapping.public_name === 'custom' || mapping.litellm_model === 'custom') {
if (selectedProvider === Providers.Azure) {
return {
public_name: customName,
litellm_model: `azure/${customName}`
};
}
return {
public_name: customName,
litellm_model: customName
@ -75,7 +105,7 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
>
<Form.Item
name="model"
rules={[{ required: true, message: "Please select at least one model." }]}
rules={[{ required: true, message: `Please enter ${selectedProvider === Providers.Azure ? 'a deployment name' : 'at least one model'}.` }]}
noStyle
>
{(selectedProvider === Providers.Azure) ||
@ -84,6 +114,7 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
<>
<TextInput
placeholder={getPlaceholder(selectedProvider)}
onChange={selectedProvider === Providers.Azure ? handleAzureDeploymentNameChange : undefined}
/>
</>
) : providerModels.length > 0 ? (
@ -135,7 +166,7 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
className="mt-2"
>
<TextInput
placeholder="Enter custom model name"
placeholder={selectedProvider === Providers.Azure ? "Enter Azure deployment name" : "Enter custom model name"}
onChange={handleCustomModelNameChange}
/>
</Form.Item>
@ -147,7 +178,10 @@ const LiteLLMModelNameField: React.FC<LiteLLMModelNameFieldProps> = ({
<Col span={10}></Col>
<Col span={14}>
<Text className="mb-3 mt-1">
The model name LiteLLM will send to the LLM API
{selectedProvider === Providers.Azure
? "Your deployment name will be saved as the public model name, and LiteLLM will use 'azure/deployment-name' internally"
: "The model name LiteLLM will send to the LLM API"
}
</Text>
</Col>
</Row>

25
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -13,6 +13,7 @@ import {
Grid,
Col,
DateRangePicker,
TextInput,
} from "@tremor/react";
import {
CredentialItem,
@ -220,6 +221,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] =
useState<string | null>(null);
const [modelNameSearch, setModelNameSearch] = useState<string>("");
// Add new state for current team and model view mode
const [currentTeam, setCurrentTeam] = useState<string>("personal"); // 'personal' or team_id
const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">(
@ -1272,7 +1275,18 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
{/* Other Filters */}
<div className="flex items-center justify-between">
<div className="flex items-center gap-3">
<div className="flex items-center gap-2">
{/* Model Name Search */}
<div className="flex items-center gap-2">
<Text>Search Public Model Name:</Text>
<TextInput
className="w-64"
placeholder="Search model names..."
value={modelNameSearch}
onValueChange={setModelNameSearch}
/>
</div>
{/* Model Name Filter */}
<div className="flex items-center gap-2">
<Text>Filter by Public Model Name:</Text>
@ -1336,6 +1350,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
Showing{" "}
{modelData && modelData.data.length > 0
? modelData.data.filter((model: any) => {
const searchMatch = modelNameSearch === "" ||
model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase());
const modelNameMatch =
selectedModelGroup === "all" ||
model.model_name ===
@ -1365,6 +1382,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
}
return (
searchMatch &&
modelNameMatch &&
accessGroupMatch &&
teamAccessMatch
@ -1393,6 +1411,10 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
)}
data={modelData.data.filter(
(model: any) => {
// Model name search filter
const searchMatch = modelNameSearch === "" ||
model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase());
// Model name filter
const modelNameMatch = selectedModelGroup === "all" ||
model.model_name === selectedModelGroup ||
@ -1422,6 +1444,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
// For 'all' mode, show all models (teamAccessMatch remains true)
return (
searchMatch &&
modelNameMatch &&
accessGroupMatch &&
teamAccessMatch