scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_/search-tools-sync-issue-e522a2

Browse source
This commit is contained in:
Yuneng Jiang 2026-08-26 12:11:06 -07:00
commit 291484a5e2
No known key found for this signature in database
121 changed files with 4781 additions and 878 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/mutmut-coverage.rc vendored Normal file
View file

@ -0,0 +1,5 @@
# mutmut's gather_coverage() looks covered lines up by absolute path, so the
# repo's `relative_files = true` makes every lookup miss and mutmut generates
# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
[run]
relative_files = false

10
.github/workflows/mutation-test.yml vendored
View file

@ -87,11 +87,20 @@ jobs:
run: |
uv pip uninstall pytest-retry || true
# Ends before the job's own deadline so a run that outlasts the budget is
# still followed by the report and upload steps. mutmut saves after every
# mutant result, to mutants/<source path>.meta, so an interrupted run
# still scores the mutants it finished and export-cicd-stats can read
# them; a cancelled job skips those steps and publishes nothing at all.
- name: Run mutmut
timeout-minutes: 300
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
# Without this mutmut finds no covered lines and generates 0 mutants.
# See the file itself for why.
COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
run: |
set -o pipefail
mkdir -p mutants
@ -130,6 +139,7 @@ jobs:
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/**/*.meta
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

2
.gitignore vendored
View file

@ -3,6 +3,8 @@
tests/e2e/.fixtures/
.venv-typecheck
.venv_policy_test
.venv-mutmut
mutants/
.env
.claude
CLAUDE.local.md

2
CLAUDE.md
View file

@ -66,6 +66,8 @@ Commit and push your work when you're done without asking
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch, and work built on a stale base lands on top of code that has already moved. Run `git fetch origin` first, then update the branch you're on with `git pull --no-rebase`, which fast-forwards when the branch hasn't diverged and merges the remote tip in when it has. When working a feature branch, bring it up to date with a freshly fetched `origin/litellm_internal_staging` before touching it: rebase onto it while the branch is still unpushed, merge it in once it has been pushed, and never rewrite pushed history
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

12
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 18505
"limit": 18483
},
"reportArgumentType": {
"limit": 2564
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 5976
"limit": 5960
},
"reportFunctionMemberAccess": {
"limit": 7
@ -57,7 +57,7 @@
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15504
"limit": 15484
},
"reportMissingTypeStubs": {
"limit": 40
@ -105,13 +105,13 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38828
"limit": 38808
},
"reportUnknownParameterType": {
"limit": 19847
"limit": 19829
},
"reportUnknownVariableType": {
"limit": 30386
"limit": 30356
},
"reportUnnecessaryCast": {
"limit": 117

130
litellm/_redis.py
View file

@ -12,8 +12,9 @@ import json
# s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
import os
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
from urllib.parse import urlsplit, urlunsplit
import redis
import redis.asyncio as async_redis
@ -50,6 +51,7 @@ def _get_redis_kwargs():
include_args: Final = {
"url",
"redis_connect_func",
"credential_provider",
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
@ -155,7 +157,8 @@ def _get_redis_cluster_kwargs(client=None):
def _get_redis_env_kwarg_mapping():
PREFIX: Final = "REDIS_"
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs()}
exclude_from_environment: Final = frozenset({"credential_provider"})
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment}
def _redis_kwargs_from_environment():
@ -353,6 +356,12 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _url_without_userinfo(url: str) -> str:
parts: Final = urlsplit(url)
netloc: Final = parts.netloc.rsplit("@", 1)[-1]
return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
def _get_redis_client_logic(**env_overrides):
"""
Common functionality across sync + async redis client implementations
@ -410,54 +419,58 @@ def _get_redis_client_logic(**env_overrides):
if _service_name is not None:
redis_kwargs["service_name"] = _service_name
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
if redis_kwargs.get("credential_provider") is None:
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str(
"REDIS_GCP_SERVICE_ACCOUNT"
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str("AZURE_CLIENT_SECRET")
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str(
"AZURE_CLIENT_SECRET"
)
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
@ -465,6 +478,13 @@ def _get_redis_client_logic(**env_overrides):
redis_kwargs.pop("azure_tenant_id", None)
redis_kwargs.pop("azure_client_secret", None)
if redis_kwargs.get("credential_provider") is not None:
redis_kwargs.pop("redis_connect_func", None)
redis_kwargs.pop("username", None)
redis_kwargs.pop("password", None)
if redis_kwargs.get("url") is not None:
redis_kwargs["url"] = _url_without_userinfo(redis_kwargs["url"])
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
# Only strip host/port/db/password when not routing to a cluster.
# When startup_nodes is also present the cluster path takes priority and
@ -532,8 +552,7 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
service_name: Final = redis_kwargs.get("service_name")
connection_kwargs: Final = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs: Final = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
sentinel_kwargs: Final = _sentinel_auth_kwargs(connection_kwargs, sentinel_password)
if not sentinel_nodes or not service_name:
raise ValueError("Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.")
@ -605,7 +624,12 @@ def _async_credential_provider(redis_connect_func: object | None) -> CredentialP
def _async_auth_kwargs(redis_kwargs: dict) -> dict:
"""Swaps a connect func an async path cannot run for the equivalent credential provider,
which supersedes any static username or password redis-py would otherwise reject it with."""
credential_provider: Final = _async_credential_provider(redis_kwargs.get("redis_connect_func"))
explicit_provider: Final = redis_kwargs.get("credential_provider")
credential_provider: Final = (
explicit_provider
if explicit_provider is not None
else _async_credential_provider(redis_kwargs.get("redis_connect_func"))
)
if credential_provider is None:
return redis_kwargs
@ -738,8 +762,20 @@ def get_redis_connection_pool(
return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)
def _redis_kwargs_for_logging(redis_kwargs: Mapping[str, object]) -> Mapping[str, object]:
return {
key: "<credential provider>"
if key == "credential_provider" and value is not None
else "<redis connect function>"
if key == "redis_connect_func" and value is not None
else value
for key, value in redis_kwargs.items()
}
def _pretty_print_redis_config(redis_kwargs: dict) -> None:
"""Pretty print the Redis configuration using rich with sensitive data masking"""
redis_kwargs_for_logging: Final = _redis_kwargs_for_logging(redis_kwargs)
try:
import logging
@ -757,7 +793,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
masker = SensitiveDataMasker()
# Mask sensitive data in redis_kwargs
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
# Create main panel title
title: Final = Text("Redis Configuration", style="bold blue")
@ -820,7 +856,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
except ImportError:
# Fallback to simple logging if rich is not available
masker = SensitiveDataMasker()
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
verbose_logger.info("Redis configuration: %s", masked_redis_kwargs)
except Exception as e:
verbose_logger.error("Error pretty printing Redis configuration: %s", e)

4
litellm/batches/batch_utils.py
View file

@ -551,7 +551,7 @@ def _get_batch_job_usage_from_response_body(
return usage
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict:
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
"""
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
@ -563,7 +563,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
def _get_response_from_batch_job_output_file(
batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
) -> Any:
) -> Mapping[str, Any]:
"""
Get the response from the batch job output file
"""

11
litellm/caching/redis_cache.py
View file

@ -175,6 +175,10 @@ _RedisCallResult = TypeVar("_RedisCallResult")
_swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0)
def _opaque_kwarg_key(value: object) -> str:
return f"{type(value).__name__}-{id(value)}"
@functools.lru_cache(maxsize=1)
def _redis_health_error_types() -> tuple[type, ...]:
"""Exception types that mean the Redis backend itself is unhealthy.
@ -399,10 +403,9 @@ class RedisCache(BaseCache):
Generate a cache key for the async Redis client based on connection parameters.
This ensures different Redis configurations use different cached clients.
"""
# Create a stable representation of redis_kwargs for hashing
# Sort keys to ensure consistent hash regardless of parameter order
sorted_kwargs: Final = sorted(self.redis_kwargs.items())
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True)
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True, default=_opaque_kwarg_key)
kwargs_hash: Final = hashlib.sha256(kwargs_str.encode()).hexdigest()[:16]
return f"async-redis-client-{kwargs_hash}"
@ -1384,10 +1387,10 @@ class RedisCache(BaseCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from .._redis import get_redis_async_client
# Create a fresh Redis client with current settings
redis_client: Final = redis_async.Redis(**self.redis_kwargs)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

17
litellm/caching/redis_cluster_cache.py
View file

@ -64,22 +64,9 @@ class RedisClusterCache(RedisCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from redis.cluster import ClusterNode
from .._redis import get_redis_async_client
# Create ClusterNode objects from startup_nodes
cluster_kwargs: Final = self.redis_kwargs.copy()
startup_nodes: Final = cluster_kwargs.pop("startup_nodes", [])
new_startup_nodes: Final[list[ClusterNode]] = []
for item in startup_nodes:
new_startup_nodes.append(ClusterNode(**item))
# Create a fresh Redis Cluster client with current settings
redis_client: Final = redis_async.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

2
litellm/integrations/custom_logger.py
View file

@ -295,7 +295,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_post_call_failure_deployment_hook(
self,
request_data: Mapping[str, Any],
request_data: Mapping[str, object],
exception: Exception,
call_type: CallTypes | None,
fallback_depth: int | None = None,

5
litellm/integrations/otel/plumbing/routing.py
View file

@ -15,7 +15,7 @@ from collections import OrderedDict
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, TypeAlias
from typing import Final, TypeAlias
from urllib.parse import quote
from opentelemetry.sdk.trace import TracerProvider
@ -32,6 +32,7 @@ from litellm.integrations.otel.presets import (
dynamic_otlp_headers,
project_routing_headers,
)
from litellm.types.utils import StandardCallbackDynamicParams
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
_NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
@ -166,7 +167,7 @@ class TenantTracerCache:
def route_for(
self,
default: Tracer,
dynamic_params: Any,
dynamic_params: StandardCallbackDynamicParams | None,
auth_metadata: Mapping[str, str] | None = None,
) -> TenantRoute:
"""Return the tracer (and trace-detachment flag) for this request.

16
litellm/integrations/prometheus.py
View file

@ -2495,12 +2495,12 @@ class PrometheusLogger(CustomLogger):
return None
def _get_user_email() -> str | None:
val = _metadata.get("user_api_key_user_email")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_user_email")
if val is not None:
return val
from_metadata: Final = _metadata.get("user_api_key_user_email")
if from_metadata is not None:
return from_metadata
from_params: Final = _litellm_params_metadata.get("user_api_key_user_email")
if from_params is not None:
return from_params
if user_api_key_auth is not None:
return self._safe_get(user_api_key_auth, "user_email")
return None
@ -3576,7 +3576,9 @@ class PrometheusLogger(CustomLogger):
except Exception as e:
verbose_logger.exception("Error initializing user/team count metrics: %s", e)
async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]):
async def _set_key_list_budget_metrics(
self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]
) -> None:
"""Helper function to set budget metrics for a list of keys"""
for key in keys:
if isinstance(key, UserAPIKeyAuth):

138
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -550,6 +550,13 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"AnthropicException - {error_str}",
llm_provider="anthropic",
model=model,
response=original_exception.response,
)
elif original_exception.status_code == 400 or original_exception.status_code == 413:
raise BadRequestError(
message=f"AnthropicException - {error_str}",
@ -755,12 +762,19 @@ def _map_openai_like_exception(
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 401 or original_exception.status_code == 403:
elif original_exception.status_code == 401:
raise AuthenticationError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=403),
)
elif original_exception.status_code == 400:
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
@ -2187,6 +2201,120 @@ def _map_openrouter_exception(
)
def _response_or_stub(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
response: Final = original_exception.response if hasattr(original_exception, "response") else None
if response is not None:
return response
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs")
)
def _map_exception_by_status(
*,
model: str,
original_exception: _ProviderHTTPException,
custom_llm_provider: str,
error_str: str,
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:
case 401:
raise AuthenticationError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 403:
raise PermissionDeniedError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=status_code),
litellm_debug_info=extra_information,
)
case 404:
raise NotFoundError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 408:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
case 429:
raise RateLimitError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 500:
raise InternalServerError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 502:
raise BadGatewayError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 503:
raise ServiceUnavailableError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 504:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=status_code,
)
case _ if status_code < 500:
raise BadRequestError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case _:
raise APIError(
status_code=status_code,
message=message,
llm_provider=custom_llm_provider,
model=model,
request=original_exception.request if hasattr(original_exception, "request") else None,
litellm_debug_info=extra_information,
)
def exception_type(
model,
original_exception,
@ -2501,6 +2629,14 @@ def exception_type(
For unmapped exceptions - raise the exception with traceback - https://github.com/BerriAI/litellm/issues/4201
"""
exception_mapping_worked = True
_map_exception_by_status(
model=model,
original_exception=mappable_exception,
custom_llm_provider=custom_llm_provider,
error_str=error_str,
exception_provider=exception_provider,
extra_information=extra_information,
)
if hasattr(original_exception, "request"):
raise APIConnectionError(
message=f"{exception_provider} - {error_str}",

15
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -2,6 +2,7 @@
Helper functions for health check calls.
"""
import base64
from collections.abc import Callable
from typing import TYPE_CHECKING, Final, Literal
@ -13,6 +14,14 @@ if TYPE_CHECKING:
# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
# Minimal image for health checks - base64 encoded 512x512 solid-gray PNG
TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAFlklEQVR42u3VMQEAAAzCMKQjHQ97l0jo0xSAlyIBgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAUgAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGACAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgBgAAAYAAAGAIABAGAAABgAAAYAgAEAYAAAGAAABgCAAQBgAAAYAAAGAIABAGAAABgAADcDrctaAb6XeXAAAAAASUVORK5CYII="
def get_image_file_for_health_check() -> bytes:
"""Return the image used for health checks."""
return base64.b64decode(TEST_IMAGE_BASE64)
class HealthCheckHelpers:
@staticmethod
@ -127,6 +136,7 @@ class HealthCheckHelpers:
"audio_speech",
"audio_transcription",
"image_generation",
"image_edit",
"video_generation",
"rerank",
"realtime",
@ -185,6 +195,11 @@ class HealthCheckHelpers:
**_filter_model_params(model_params=model_params),
prompt=prompt,
),
"image_edit": lambda: litellm.aimage_edit(
**_filter_model_params(model_params=model_params),
image=get_image_file_for_health_check(),
prompt=prompt or "test",
),
"video_generation": lambda: litellm.avideo_generation(
**_filter_model_params(model_params=model_params),
prompt=prompt or "test video generation",

37
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,6 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from typing import Any, Final
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
@ -39,7 +39,7 @@ class TranscriptionUsageObjectTransformation:
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
_INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
@ -59,7 +59,7 @@ def _token_count(value: object) -> int:
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
@ -69,10 +69,13 @@ def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, i
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
entries: Final = usage_object.get("grounding_tool_count")
if not isinstance(entries, Sequence):
return 0
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
for entry in entries
if isinstance(entry, Mapping) and entry.get("type") == "google_search"
)
@ -112,30 +115,30 @@ class InteractionsUsageObjectTransformation:
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
input_sums: Final = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
reasoning_tokens: Final = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
prompt_tokens: Final = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
completion_tokens: Final = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens: Final = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
web_search_requests: Final = _google_search_query_count(usage_object)
prompt_tokens_details: Final = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
@ -144,7 +147,7 @@ class InteractionsUsageObjectTransformation:
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
completion_tokens_details: Final = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -511,9 +511,6 @@ def update_messages_with_model_file_ids(
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
@ -588,9 +585,6 @@ def update_responses_input_with_model_file_ids(
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)

3
litellm/litellm_core_utils/redact_messages.py
View file

@ -10,6 +10,7 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import litellm
@ -191,7 +192,7 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_tool_calls_dict(message: dict) -> None:
def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
"""Redact tool call / function_call arguments in a dict-form message or delta."""
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list):

23
litellm/llms/anthropic/common_utils.py
View file

@ -4,7 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import re
from collections.abc import Mapping, Sequence
from collections.abc import Mapping, MutableMapping, Sequence
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Any, Final, Literal
@ -93,8 +93,8 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
"""
Handle Anthropic OAuth token detection and header setup.
If an OAuth token is detected in the Authorization header, extracts it
and sets the required OAuth headers.
If an OAuth token is detected in the Authorization header (any casing),
extracts it and sets the required OAuth headers.
Args:
headers: Request headers dict
@ -104,16 +104,21 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
Tuple of (updated headers, api_key)
"""
# Check Authorization header (passthrough / forwarded requests)
auth_header: Final = headers.get("authorization", "")
if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.replace("Bearer ", "")
headers.pop("x-api-key", None)
auth_header: Final = next((value for name, value in headers.items() if name.lower() == "authorization"), "")
if auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.removeprefix("Bearer ")
for name in tuple(
header_name for header_name in headers if header_name.lower() in ("x-api-key", "authorization")
):
headers.pop(name)
headers["authorization"] = auth_header
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
# Check api_key directly (standard chat/completion flow)
if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX):
headers.pop("x-api-key", None)
for name in tuple(header_name for header_name in headers if header_name.lower() == "x-api-key"):
headers.pop(name)
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
@ -468,7 +473,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
@staticmethod
def maybe_drop_disabled_thinking(
model: str,
optional_params: dict, # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param, as in _maybe_drop_speed_param
custom_llm_provider: str,
) -> None:
"""Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models

4
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -352,8 +352,8 @@ async def _check_summary_model_budget(
)
return False
user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
user_id: Final = getattr(user_api_key_auth, "user_id", None)
user_model_max_budget: Final = user_api_key_auth.user_model_max_budget
user_id: Final = user_api_key_auth.user_id
if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
try:
await model_max_budget_limiter.is_user_within_model_budget(

21
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,7 @@ from functools import partial
from typing import Any, Final, cast
import litellm
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
flatten_unencrypted_web_search_results_in_anthropic_messages,
@ -21,6 +22,7 @@ from litellm.llms.anthropic.common_utils import (
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -382,13 +384,18 @@ async def anthropic_messages(
)
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
try:
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
return await init_response
return init_response
except BaseLLMException as e:
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
extra_kwargs=kwargs,
)
def validate_anthropic_api_metadata(metadata: dict | None = None) -> dict | None:

17
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -307,10 +308,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# Check for Anthropic OAuth token in Authorization header
headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
if "x-api-key" not in headers and "authorization" not in headers:
header_names: Final = frozenset(name.lower() for name in headers)
if "x-api-key" not in header_names and "authorization" not in header_names:
auth_header: Final = AnthropicModelInfo.get_auth_header(api_key)
if auth_header is not None:
headers.update(auth_header)
if auth_header is None:
raise AuthenticationError(
message=(
"Missing Anthropic API Key - A call is being made to anthropic but no key is set "
"either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` "
"or `ANTHROPIC_AUTH_TOKEN` in your environment vars"
),
llm_provider=self._resolved_provider,
model=model,
)
headers.update(auth_header)
if "anthropic-version" not in headers:
headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
if "content-type" not in headers:

2
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -582,7 +582,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": True,
"strict": output_format.get("strict", False),
}
}

23
litellm/llms/azure/realtime/handler.py
View file

@ -4,6 +4,8 @@ This file contains the calling Azure OpenAI's `/openai/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, cast
from litellm._logging import _redact_string, verbose_proxy_logger
@ -30,6 +32,21 @@ async def forward_messages(client_ws: Any, backend_ws: Any):
class AzureOpenAIRealtime(AzureChatCompletion):
@staticmethod
def get_auth_headers(api_key: str | None, azure_ad_token: str | None) -> Mapping[str, str]:
"""
Build the websocket handshake auth headers, preferring a static api-key and falling back to
an Azure AD (Entra ID) bearer token. Never sends both.
"""
if api_key:
return MappingProxyType({"api-key": api_key})
if azure_ad_token:
return MappingProxyType({"Authorization": f"Bearer {azure_ad_token}"})
raise ValueError(
"Missing Azure credentials for the realtime endpoint. Set an api_key, or configure Azure AD auth "
"(azure_ad_token, tenant_id/client_id/client_secret, or a managed identity)"
)
def _construct_url(
self,
api_base: str,
@ -117,13 +134,13 @@ class AzureOpenAIRealtime(AzureChatCompletion):
query_params=query_params,
)
auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
try:
ssl_context: Final = get_shared_realtime_ssl_context()
async with websockets.connect(
url,
additional_headers={
"api-key": api_key,
},
additional_headers=auth_headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
) as backend_ws:

8
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -418,12 +418,16 @@ class AmazonConverseConfig(BaseConfig):
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS models: passed through unchanged via additionalModelRequestFields.
- OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model:
optional_params["reasoning_effort"] = reasoning_effort
elif "openai.gpt-5" in model:
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
optional_params["reasoning"] = reasoning
elif self._is_nova_2_model(model):
reasoning_config: Final = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
optional_params.update(reasoning_config)
@ -555,7 +559,7 @@ class AmazonConverseConfig(BaseConfig):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice")
if "gpt-oss" in model:
if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model:
supported_params.append("reasoning_effort")
elif self._is_nova_2_model(model):
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
@ -903,7 +907,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["_parallel_tool_use_config"] = {
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking":
if param == "thinking" and "openai.gpt-5" not in model:
if (
isinstance(value, dict)
and value.get("type") == "adaptive"

8
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -243,7 +243,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
@staticmethod
def _agent_message_text(item: "Mapping[str, Any]") -> str:
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
@ -254,7 +254,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
)
@classmethod
def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = cls._agent_message_text(item)
if not text:
return None
@ -266,7 +266,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
@ -274,7 +274,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None

6
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5620,10 +5620,9 @@ class BaseLLMHTTPHandler:
kwargs=hook_kwargs,
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_agentic_loop [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)
@ -5645,10 +5644,9 @@ class BaseLLMHTTPHandler:
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)

81
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,8 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Container, Coroutine
from types import MappingProxyType
from collections.abc import Callable, Container, Coroutine
from typing import (
Final,
Literal,
@ -17,26 +16,42 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling
from litellm.utils import supports_function_calling, supports_response_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
LITELLM_INTERNAL_ASSISTANT_FIELDS: Final = frozenset({"thinking_blocks", "provider_specific_fields"})
PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs"
def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None:
try:
if check(model):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get(flag) is False:
return False
return None
def _function_calling_verdict(model: str) -> bool | None:
try:
if supports_function_calling(model, custom_llm_provider="together_ai"):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
return False
return None
return _registry_verdict(
model,
"supports_function_calling",
lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"),
)
def _response_schema_verdict(model: str) -> bool | None:
return _registry_verdict(
model,
"supports_response_schema",
lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"),
)
def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
@ -68,6 +83,32 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
)
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
if "response_format" not in passed_params:
return False
verdict: Final = _response_schema_verdict(model)
if verdict is True:
return False
if verdict is None:
verbose_logger.warning(
"together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return False
if drop_params or litellm.drop_params:
verbose_logger.warning(
"together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return True
raise UnsupportedParamsError(
status_code=500,
message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
)
def _without_litellm_internal_fields(message: AllMessageValues) -> AllMessageValues:
if message["role"] != "assistant" or LITELLM_INTERNAL_ASSISTANT_FIELDS.isdisjoint(message):
return message
@ -112,18 +153,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
return super()._transform_messages(stripped, model, is_async=True)
return super()._transform_messages(stripped, model, is_async=False)
def get_supported_openai_params(self, model: str) -> list:
supports_fc: Final = _function_calling_verdict(model)
supported_params: Final = super().get_supported_openai_params(model)
if supports_fc is True:
return supported_params
verbose_logger.debug(
"Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
)
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
param for param in supported_params if param != "response_format"
]
def map_openai_params(
self,
non_default_params: dict,
@ -134,6 +163,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
mapped_openai_params.pop(param)
if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
if _drop_response_format(mapped_openai_params, model, drop_params):
mapped_openai_params.pop("response_format")
return mapped_openai_params

29
litellm/model_prices_and_context_window_backup.json
View file

@ -12282,7 +12282,7 @@
},
"claude-3-haiku-20240307": {
"cache_creation_input_token_cost": 3e-07,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr": 5e-07,
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-04-20",
"input_cost_per_token": 2.5e-07,
@ -12301,7 +12301,7 @@
},
"claude-3-opus-20240229": {
"cache_creation_input_token_cost": 1.875e-05,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr": 3e-05,
"cache_read_input_token_cost": 1.5e-06,
"deprecation_date": "2026-01-05",
"input_cost_per_token": 1.5e-05,
@ -12515,7 +12515,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 1024,
"provider_specific_entry": {
"us": 1.1
}
},
"claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
@ -49473,6 +49476,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-sol": {
@ -49498,6 +49502,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"us.openai.gpt-5.6-terra": {
@ -49523,6 +49528,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-terra": {
@ -49548,6 +49554,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"us.openai.gpt-5.6-luna": {
@ -49573,6 +49580,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-luna": {
@ -49598,6 +49606,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"bedrock_mantle/openai.gpt-5.5": {
@ -49605,7 +49614,7 @@
"cache_read_input_token_cost": 5.5e-07,
"output_cost_per_token": 3.3e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -49632,7 +49641,7 @@
"cache_read_input_token_cost": 2.75e-07,
"output_cost_per_token": 1.65e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -50825,7 +50834,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true
"supports_native_structured_output": true,
"provider_specific_entry": {
"us": 1.1
}
},
"claude-mythos-preview": {
"cache_creation_input_token_cost": 1.25e-05,
@ -50860,7 +50872,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true
"supports_native_structured_output": true,
"provider_specific_entry": {
"us": 1.1
}
},
"gemini/gemini-robotics-er-2-streaming-preview": {
"input_cost_per_audio_token": 2e-06,

117
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,6 +1,8 @@
import re
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
from fastapi import HTTPException
@ -13,6 +15,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.oauth_utils import (
get_passthrough_resource_metadata_url,
get_passthrough_www_authenticate,
get_request_base_url,
well_known_root_suffix,
)
@ -298,6 +301,16 @@ def _admission_failure_fallback(
raise exc
@dataclass(frozen=True, slots=True)
class DcrBridgeTarget:
"""The single DCR-bridge server a request targets, paired with the exact name the caller
used to reach it (alias or server_name, whichever they typed), which is the spelling an
``invalid_token`` challenge must echo back."""
requested_name: str
server: MCPServer
class MCPRequestHandler:
"""
Class to handle MCP request processing, including:
@ -437,27 +450,33 @@ class MCPRequestHandler:
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
) or (
MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
is not None
and not oauth2_headers
and not mcp_server_auth_headers
and not mcp_auth_header
):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
(
bridge_delegate_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
bridge_delegate_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
is not None
and oauth2_headers
and is_bridge_envelope_shaped(oauth2_headers["Authorization"])
):
# A single DCR-bridge oauth_delegate target carrying an envelope-shaped
# Authorization: open the envelope, admit under its recovered identity, and
# inject the inner upstream token for egress. A non-envelope bearer on the same
# server is NOT admitted here — it falls through to the oauth2 arm, which 401s.
validated_user_api_key_auth, mcp_server_auth_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=bridge_delegate_target,
) is not None and oauth2_headers:
(
validated_user_api_key_auth,
mcp_server_auth_headers,
) = await MCPRequestHandler._admit_dcr_bridge_authorization(
server=bridge_delegate_target.server,
requested_name=bridge_delegate_target.requested_name,
authorization_value=oauth2_headers["Authorization"],
litellm_api_key=litellm_api_key,
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=request_route,
@ -723,10 +742,10 @@ class MCPRequestHandler:
@staticmethod
def _single_dcr_bridge_delegate_target(
path: str, mcp_servers: list[str] | None, client_ip: str | None
) -> MCPServer | None:
) -> DcrBridgeTarget | None:
"""The one DCR-bridge ``oauth_delegate`` server this request targets, or ``None``.
Returns the server only when EXACTLY ONE target resolves and it is both
Returns the target only when EXACTLY ONE name resolves and its server is both
``is_oauth_delegate`` and ``is_dcr_bridge``. Fails closed (``None``) on a
multi-target request, an unresolved target, or a non-matching server, so the
envelope admission arm never fires for an aggregate scope or a server that did not
@ -740,17 +759,21 @@ class MCPRequestHandler:
if len(target_names) != 1:
return None
server: Final = global_mcp_server_manager.get_mcp_server_by_name(target_names[0], client_ip=client_ip)
if server is None or not server.is_oauth_delegate or not server.is_dcr_bridge:
# Both flags are security-sensitive opt-ins. Require literal booleans so
# partially populated objects and truthy proxy values cannot enable bridge
# admission accidentally.
if server is None or server.is_oauth_delegate is not True or server.is_dcr_bridge is not True:
return None
# Egress resolves the injected per-server token only by alias / server_name; a server with
# neither cannot receive the forwarded token, so fail closed rather than admit-and-drop.
if not (server.server_name or server.alias):
return None
return server
return DcrBridgeTarget(requested_name=target_names[0], server=server)
@staticmethod
async def _admit_dcr_bridge_delegate(
server: MCPServer,
requested_name: str,
authorization_value: str,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
request: Request,
@ -798,10 +821,62 @@ class MCPRequestHandler:
new_headers: Final = {**(mcp_server_auth_headers or {}), **injected}
return admitted, new_headers
case BridgeEnvelopeInvalid() | NotBridgeEnvelope():
raise HTTPException(status_code=401, detail="Invalid or expired credential")
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
requested_name=requested_name, request=request
)
case _:
assert_never(result)
@staticmethod
async def _admit_dcr_bridge_authorization(
server: MCPServer,
requested_name: str,
authorization_value: str,
litellm_api_key: str,
mcp_server_auth_headers: dict[str, dict[str, str]] | None, # mutable-ok: existing MCP sink shape
request: Request,
route: str,
) -> tuple[UserAPIKeyAuth, dict[str, dict[str, str]] | None]: # mutable-ok: existing MCP sink shape
if is_bridge_envelope_shaped(authorization_value):
return await MCPRequestHandler._admit_dcr_bridge_delegate(
server=server,
requested_name=requested_name,
authorization_value=authorization_value,
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=route,
)
try:
admitted: Final = await user_api_key_auth(api_key=litellm_api_key, request=request)
except (HTTPException, ProxyException) as exc:
if not _is_litellm_auth_admission_error(exc):
raise
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
requested_name=requested_name, request=request
) from exc
return admitted, mcp_server_auth_headers
@staticmethod
def _dcr_bridge_invalid_token_challenge(requested_name: str, request: Request) -> HTTPException:
"""The RFC 6750 ``invalid_token`` challenge for a failed bridge admission.
Named by the exact spelling the caller requested, matching the per-server well-known
document and the other challenge emitters, so ``resource_metadata`` always points at the
resource the client actually asked for even when alias and server_name differ."""
return HTTPException(
status_code=401,
detail="Invalid or expired credential",
headers=MappingProxyType(
{
"www-authenticate": get_passthrough_www_authenticate(
scope=request.scope,
server_name=requested_name,
invalid_token=True,
)
}
),
)
@staticmethod
async def _admit_gateway_session(
authorization_value: str,

24
litellm/proxy/_experimental/mcp_server/bridge_token_flow.py
View file

@ -306,15 +306,15 @@ _UpstreamGrantRejection = Literal["no_access_token", "expired_lifetime"]
- ``expired_lifetime``: the response reports a parseable, non-positive ``expires_in``, i.e. an upstream
token that is already dead, so sealing it would forward a bearer the edge cannot use
An absent or unparseable ``expires_in`` is NOT a rejection; the lifetime is merely unknown and the
envelope caps it, the by-design behaviour for an upstream that omits the field."""
envelope uses its fallback lifetime, the by-design behaviour for an upstream that omits the field."""
def _classify_upstream_lifetime(raw_expires_in: object) -> "int | Literal['unspecified', 'expired']":
"""Classify an upstream ``expires_in`` into a positive number of seconds, ``"unspecified"`` (absent
or unparseable, so the envelope caps it), or ``"expired"`` (a non-positive value the upstream reports
or unparseable, so the envelope uses its fallback), or ``"expired"`` (a non-positive value the upstream reports
as already elapsed). Telling "we do not know the lifetime" apart from "the upstream says it is
already dead" is what stops an explicitly-expired token from silently receiving the envelope's 1h
cap. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
already dead" is what stops an explicitly-expired token from silently receiving the envelope's
one-hour fallback. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
positive sub-second lifetime in ``(0, 1)`` is not truncated to ``0`` and misread as elapsed; the
envelope works in whole seconds, so such a lifetime clamps up to its 1s floor. ``bool`` is excluded
(an ``int`` subclass but never a real lifetime), and the conversions can raise on ``NaN`` /
@ -335,7 +335,7 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
"""Validate an upstream OAuth token response into a typed grant, or say why it cannot back an
envelope. Each field is isinstance-checked so nothing untyped from ``response.json()`` reaches the
grant. ``expires_in`` is read three ways (see :func:`_classify_upstream_lifetime`): an unknown
lifetime leaves the grant ``expires_in`` ``None`` for the envelope to cap, a positive value is
lifetime leaves the grant ``expires_in`` ``None`` for the envelope fallback, a positive value is
honoured, and an explicit already-elapsed value is a rejection rather than a silent fall-through to
the cap."""
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import
@ -357,8 +357,8 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
token_type=token_type if isinstance(token_type, str) and token_type else "Bearer",
# The upstream refresh_token is deliberately NOT sealed: the edge never consumes it (it forwards
# only token_type + access_token), so it would be dead weight embedding a long-lived upstream
# credential in the client-held bearer, and it enlarges the envelope. Refresh support is a
# follow-up (a dedicated refresh-envelope); the client re-runs authorization_code at the cap.
# credential in the client-held bearer, and it enlarges the envelope. The dedicated refresh
# envelope carries that credential separately.
refresh_token=None,
scope=scope if isinstance(scope, str) and scope else None,
expires_in=lifetime if isinstance(lifetime, int) else None,
@ -387,6 +387,7 @@ _BridgeMintError = Literal[
"not_configured",
"no_upstream_token",
"upstream_token_expired",
"upstream_lifetime_unrepresentable",
"too_large",
]
@ -456,6 +457,12 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
"server_error",
"the upstream token response reports an already-expired lifetime",
)
case "upstream_lifetime_unrepresentable":
status, code, desc = (
502,
"server_error",
"the upstream token response reports an unrepresentable lifetime",
)
case "too_large":
status, code, desc = (
502,
@ -619,6 +626,7 @@ def _finish_bridge_mint(
build_bridge_token_response,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import
EnvelopeLifetimeUnrepresentable,
SealedEnvelope,
UpstreamTokenGrant,
)
@ -627,6 +635,8 @@ def _finish_bridge_mint(
if not isinstance(grant, UpstreamTokenGrant):
return _upstream_rejection_to_mint_error(grant)
sealed: Final = build_bridge_token_response(ready.identity, grant, ready.keys, now)
if isinstance(sealed, EnvelopeLifetimeUnrepresentable):
return "upstream_lifetime_unrepresentable"
if not isinstance(sealed, SealedEnvelope):
return "too_large"
# Report expires_in from the JWT's own second-truncated exp, rounding the elapsed portion up, so the

2
litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py
View file

@ -92,7 +92,7 @@ def build_bridge_token_response(
The producer mirror of :func:`resolve_bridge_envelope`: a thin, pure wrapper over
:func:`mint_envelope` that returns the sealed envelope, or the mint error as a value
(an oversized grant) for the caller to map onto an OAuth error response.
for the caller to map onto an OAuth error response.
"""
return mint_envelope(identity, grant, keys, now)

44
litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py
View file

@ -19,17 +19,16 @@ in plaintext anywhere in the envelope.
Failures are values: :func:`open_envelope` returns one of the frozen
``EnvelopeOpenError`` variants (discriminated on ``tag``) for invalid, expired,
tampered, or undecryptable input, and :func:`mint_envelope` returns
``EnvelopeTooLarge`` for oversized grants. Error values carry tags and sizes only,
never token material.
tampered, or undecryptable input, and :func:`mint_envelope` returns a typed error
for oversized grants or an unrepresentable provider lifetime. Error values carry
tags and metadata only, never token material.
The pydantic input models reject programmer errors at construction (e.g. a
non-positive ``expires_in`` or an empty required field). :func:`open_envelope` is
additionally total over hostile, attacker-controlled input: it never raises, only
returns an ``EnvelopeOpenError``. :func:`mint_envelope` operates on a
gateway-supplied grant (an upstream IdP's UTF-8 JSON token response), so it does not
defend against non-UTF-8 field content that cannot survive JSON parsing; its only
value-typed failure is ``EnvelopeTooLarge``.
defend against non-UTF-8 field content that cannot survive JSON parsing.
"""
from __future__ import annotations
@ -57,10 +56,11 @@ ENVELOPE_ISSUER: Final = "litellm-mcp-bridge"
"""``iss`` claim stamped into every envelope and required back on open."""
MAX_ENVELOPE_TTL_SECONDS: Final = 3600
"""Hard ceiling on ACCESS envelope lifetime. ``exp`` is ``min(upstream expires_in, this cap)``
(the cap alone when the upstream omits ``expires_in``), matching the 1h lifetime of the
BYOK session bearer this module's signing approach is borrowed from: a client-held
credential should never outlive a bounded window even when the upstream token does."""
"""Fallback ACCESS envelope lifetime when the upstream omits ``expires_in``.
The historical exported name is retained for import compatibility. When the upstream
reports a positive lifetime, the envelope matches it so a renewal does not consume a
still-valid provider refresh grant."""
MAX_REFRESH_ENVELOPE_TTL_SECONDS: Final = 1209600
"""Hard ceiling on REFRESH envelope lifetime (14 days). A refresh envelope only renews the short-lived
@ -202,7 +202,15 @@ class EnvelopeTooLarge(BaseModel):
max_bytes: int
EnvelopeMintError: TypeAlias = EnvelopeTooLarge
class EnvelopeLifetimeUnrepresentable(BaseModel):
"""A positive provider lifetime cannot be represented as a Python datetime."""
model_config = ConfigDict(frozen=True)
tag: Literal["envelope_lifetime_unrepresentable"] = "envelope_lifetime_unrepresentable"
expires_in: int
EnvelopeMintError: TypeAlias = EnvelopeTooLarge | EnvelopeLifetimeUnrepresentable
class NotAnEnvelope(BaseModel):
@ -307,11 +315,17 @@ def mint_envelope(
) -> SealedEnvelope | EnvelopeMintError:
"""Seal ``grant`` for ``identity`` into a client-held envelope.
``exp`` is ``min(grant.expires_in, MAX_ENVELOPE_TTL_SECONDS)`` seconds from ``now``
(the cap alone when ``expires_in`` is absent). Returns ``EnvelopeTooLarge`` when the
serialized envelope exceeds ``MAX_ENVELOPE_BYTES``.
``exp`` is ``grant.expires_in`` seconds from ``now`` when the upstream reports a
lifetime, or ``MAX_ENVELOPE_TTL_SECONDS`` when it does not. Returns
``EnvelopeLifetimeUnrepresentable`` when that positive lifetime cannot be represented
as a Python datetime, or ``EnvelopeTooLarge`` when the serialized envelope exceeds
``MAX_ENVELOPE_BYTES``.
"""
expires_at: Final = now + timedelta(seconds=_envelope_ttl_seconds(grant.expires_in))
ttl_seconds: Final = _envelope_ttl_seconds(grant.expires_in)
try:
expires_at: Final = now + timedelta(seconds=ttl_seconds)
except OverflowError:
return EnvelopeLifetimeUnrepresentable(expires_in=ttl_seconds)
return _seal(
kind="access",
prefix=ENVELOPE_PREFIX,
@ -457,7 +471,7 @@ def _open_claims(
def _envelope_ttl_seconds(upstream_expires_in: int | None) -> int:
if upstream_expires_in is None:
return MAX_ENVELOPE_TTL_SECONDS
return min(upstream_expires_in, MAX_ENVELOPE_TTL_SECONDS)
return upstream_expires_in
def _refresh_ttl_seconds(upstream_refresh_expires_in: int | None) -> int:

6
litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py
View file

@ -54,9 +54,9 @@ the envelope issuer so a token of one family can never validate in the other eve
hypothetical shared signing key."""
SESSION_TTL_SECONDS: Final = 3600
"""Session ACCESS token lifetime (1h), matching the access-envelope and BYOK session bearer
windows: a client-held credential never outlives a bounded window, and each refresh
re-validates the live user before re-minting."""
"""Session ACCESS token lifetime (1h), matching the BYOK session bearer window: a
client-held credential never outlives a bounded window, and each refresh re-validates
the live user before re-minting."""
SESSION_REFRESH_TTL_SECONDS: Final = 1209600
"""Session REFRESH token lifetime (14 days), matching the refresh-envelope bound. Each

6
litellm/proxy/_types.py
View file

@ -2819,7 +2819,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
# Values stay `object` rather than BudgetConfig: this is the raw JSON column,
# and validating it here would make one malformed row fail auth outright.
# resolve_model_budget validates the single entry a request actually needs.
user_model_max_budget: dict[str, object] | None = None
user_model_max_budget: Mapping[str, object] | None = None
request_route: str | None = None
is_session_token: bool = False
# Server-only marker set exclusively by the MCP gateway admission path
@ -2997,8 +2997,8 @@ class UserInfoV2Response(LiteLLMPydanticObjectBase):
sso_user_id: str | None = None
teams: list[str] = [] # Just team IDs, not full team objects
object_permission: LiteLLM_ObjectPermissionTable | None = None
model_max_budget: dict | None = None
model_max_budget_usage: dict | None = None
model_max_budget: Mapping[str, object] | None = None
model_max_budget_usage: Mapping[str, Mapping[str, object]] | None = None
from litellm.models.config import LiteLLM_Config as LiteLLM_Config # noqa: E402

11
litellm/proxy/auth/user_api_key_auth.py
View file

@ -212,9 +212,9 @@ async def _read_user_model_max_budget(
user_id: str | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
parent_otel_span: object,
parent_otel_span: Span | None,
proxy_logging_obj: ProxyLogging,
) -> dict | None:
) -> Mapping[str, object] | None:
"""The user row's `model_max_budget`, or None when the row cannot be read.
A user whose row is missing must not be refused: this is a budget lookup,
@ -228,13 +228,13 @@ async def _read_user_model_max_budget(
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
parent_otel_span=parent_otel_span, # pyright: ignore[reportArgumentType] # Span is a runtime union, not usable in an annotation here
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except Exception as e: # noqa: BLE001 # mirrors the main path's tolerance
verbose_logger.debug("Unable to read user for the per-model budget check: %s", e)
return None
return getattr(user_obj, "model_max_budget", None)
return user_obj.model_max_budget if user_obj is not None else None
async def _check_user_model_budget(
@ -3267,8 +3267,7 @@ async def _run_post_custom_auth_checks(
# loaded the user row yet. The attach is unconditional because the post-call
# spend hook reads this field off the token: gating it on the same condition
# as enforcement would leave the user's counter uncharged whenever this
# request was not itself enforceable, which is the untracked-spend bug this
# PR exists to fix.
# request was not itself enforceable, so its spend would go untracked.
user_budget: Final = await _read_user_model_max_budget(
user_id=valid_token.user_id,
prisma_client=prisma_client,

9
litellm/proxy/common_request_processing.py
View file

@ -4,7 +4,7 @@ import json
import logging
import math
import traceback
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
@ -284,7 +284,7 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool:
)
def _assembled_model_came_from_a_later_chunk(chunks: list, assembled_model: object) -> bool:
def _assembled_model_came_from_a_later_chunk(chunks: Sequence[object], assembled_model: object) -> bool:
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
Azure Model Router puts the routed model on the chunks after the first one, and the
@ -306,7 +306,10 @@ def _assembled_model_came_from_a_later_chunk(chunks: list, assembled_model: obje
)
def _assembled_model_is_the_name_the_client_asked_for(request_data: dict, assembled_model: object) -> bool:
def _assembled_model_is_the_name_the_client_asked_for(
request_data: Mapping[str, object],
assembled_model: object,
) -> bool:
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
That stamp is what leaves an unpriced alias on the partial response, so the deployment's

2
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -1182,7 +1182,7 @@ class ResetBudgetJob:
if not raw:
continue
row_id: str = row[source.id_column]
windows: list = raw if isinstance(raw, list) else json.loads(raw)
windows: list[dict[str, object]] = raw if isinstance(raw, list) else json.loads(raw)
changed = False
for window in windows:
counter_key = f"{source.counter_prefix}:{row_id}:window:{window['budget_duration']}"

11
litellm/proxy/common_utils/sse_keepalive.py
View file

@ -91,6 +91,17 @@ def is_sse_content_type(content_type: str | None) -> bool:
return content_type is not None and content_type.split(";", 1)[0].strip().lower() == _SSE_MEDIA_TYPE
def split_complete_sse_frames(pending: bytes) -> tuple[bytes, bytes]:
"""Split buffered SSE bytes into ``(complete_frames, unterminated_tail)``."""
boundary_end: Final = max(
(pending.rfind(delimiter) + len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS if delimiter in pending),
default=0,
)
if boundary_end == 0:
return b"", pending
return pending[:boundary_end], pending[boundary_end:]
def wrap_passthrough_sse_bytes_with_keepalive_pings(
stream: AsyncGenerator[bytes, None],
ping_interval_seconds: float | str | None,

17
litellm/proxy/db/db_url_settings.py
View file

@ -60,6 +60,11 @@ AzureTokenAuthFlag = Annotated[
bool, BeforeValidator(partial(token_auth_flag_enabled, env_var=AZURE_POSTGRESQL_AUTH_ENV_VAR))
]
DISABLE_PREPARED_STATEMENTS_ENV_VAR: Final = "DATABASE_DISABLE_PREPARED_STATEMENTS"
DisablePreparedStatementsFlag = Annotated[
bool, BeforeValidator(partial(token_auth_flag_enabled, env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR))
]
# schema.prisma pins `provider = "postgresql"`, so these are the only schemes
# Prisma can actually connect with.
SUPPORTED_DB_SCHEMES: Final[frozenset[str]] = frozenset({"postgresql", "postgres"})
@ -153,6 +158,9 @@ class DatabaseURLSettings(BaseSettings):
iam_token_db_auth: IamTokenAuthFlag = Field(default=False, validation_alias=IAM_TOKEN_DB_AUTH_ENV_VAR)
azure_postgresql_auth: AzureTokenAuthFlag = Field(default=False, validation_alias=AZURE_POSTGRESQL_AUTH_ENV_VAR)
disable_prepared_statements: DisablePreparedStatementsFlag = Field(
default=False, validation_alias=DISABLE_PREPARED_STATEMENTS_ENV_VAR
)
# Writer
database_url: str | None = Field(default=None, validation_alias="DATABASE_URL")
@ -375,6 +383,15 @@ class DatabaseURLSettings(BaseSettings):
self._raise_for_unsupported_scheme()
wrote_writer: Final = self.apply_writer_url_to_env()
# DATABASE_DISABLE_PREPARED_STATEMENTS maps to Prisma's `pgbouncer=true`
# URL param, same as the CLI's `database_disable_prepared_statements`
# config key. An explicit `pgbouncer` value already on the URL wins.
if self.disable_prepared_statements:
for env_var in ("DATABASE_URL", "DIRECT_URL"):
url = os.environ.get(env_var)
if url:
os.environ[env_var] = add_missing_query_params(url, MappingProxyType({"pgbouncer": "true"}))
# The reader inherits the writer's connection params (pool size, timeouts,
# pgbouncer mode). Without this the reader pool ignores the configured cap
# and falls back to Prisma's `num_physical_cpus * 2 + 1` default.

2
litellm/proxy/db/token_auth.py
View file

@ -62,7 +62,7 @@ def token_auth_flag_enabled(value: str | bool | None, *, env_var: str) -> bool:
return False
raise ValueError(
f"{env_var}={value!r} is not a recognized boolean. Set it to one of "
f"{', '.join(sorted(TRUTHY_TOKEN_AUTH_VALUES))} to turn token auth on, or to one of "
f"{', '.join(sorted(TRUTHY_TOKEN_AUTH_VALUES))} to turn it on, or to one of "
f"{', '.join(sorted(v for v in FALSY_TOKEN_AUTH_VALUES if v))} to turn it off."
)

2
litellm/proxy/health_check.py
View file

@ -183,7 +183,7 @@ async def run_with_timeout(task, timeout):
return {"error": "Timeout exceeded", "exception": timeout_exception}
def _is_strategy_router_deployment(litellm_params: dict) -> bool:
def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool:
"""True for strategy-router deployments."""
model: Final[object] = litellm_params.get("model", "")
return isinstance(model, str) and classify_strategy_router_model(model) is not None

1
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -1796,6 +1796,7 @@ async def test_model_connection(
"audio_speech",
"audio_transcription",
"image_generation",
"image_edit",
"video_generation",
"batch",
"rerank",

18
litellm/proxy/hooks/responses_id_security.py
View file

@ -28,6 +28,21 @@ if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
_RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
def _is_responses_api_create_route(request_route: str | None) -> bool:
if request_route is None:
return False
canonical: Final = (
request_route[len(_RESPONSES_API_PROVIDER_PREFIX) :]
if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/")
else request_route
)
return canonical in _RESPONSES_API_CREATE_ROUTES
class ResponsesIDSecurity(CustomLogger):
def __init__(self):
pass
@ -267,8 +282,7 @@ class ResponsesIDSecurity(CustomLogger):
async for chunk in response:
if (
isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
and user_api_key_dict.request_route
== "/v1/responses" # only encrypt the response id for the responses api
and _is_responses_api_create_route(user_api_key_dict.request_route)
and not general_settings.get("disable_responses_id_security", False)
):
chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)

4
litellm/proxy/litellm_pre_call_utils.py
View file

@ -4,7 +4,7 @@ import json
import re
import time
from collections import OrderedDict
from collections.abc import Mapping
from collections.abc import Mapping, MutableMapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
@ -1629,7 +1629,7 @@ class LiteLLMProxyRequestSetup:
def refresh_proxy_server_request_body_snapshot(
data: dict, # mutable-ok: mutates proxy_server_request.body in place on the shared request dict
data: MutableMapping[str, object],
) -> None:
"""
Re-snapshot ``data["proxy_server_request"]["body"]`` from the current state of ``data``.

3
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -2294,10 +2294,9 @@ async def _process_single_key_update(
prisma_client=prisma_client,
)
_existing_row_metadata: Final = getattr(existing_key_row, "metadata", None)
enforce_batch_enqueued_token_limit_is_admin_only(
data=update_key_request,
existing_metadata=_existing_row_metadata if isinstance(_existing_row_metadata, dict) else None,
existing_metadata=existing_key_row.metadata,
user_api_key_dict=user_api_key_dict,
entity="key",
)

9
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -1354,11 +1354,12 @@ def _completed_batch_safe_to_retire(response: "LiteLLMBatch") -> bool:
reports no successful request lines. When counts are unknown, stay eligible so
the next poller pass revisits it. (#37713)
"""
if getattr(response, "output_file_id", None) is not None:
if response.output_file_id is not None:
return True
request_counts = getattr(response, "request_counts", None)
completed = getattr(request_counts, "completed", None)
return completed == 0
request_counts = response.request_counts
if request_counts is None:
return False
return request_counts.completed == 0
async def update_batch_in_database(

127
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -6,6 +6,7 @@ Provider-specific Pass-Through Endpoints
Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc.
"""
import hmac
import json
import os
import re
@ -28,6 +29,7 @@ from litellm.constants import (
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._types import *
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import (
_get_bearer_token,
@ -1791,29 +1793,11 @@ _HEADERS_NEVER_FORWARDED_TO_VERTEX: Final = frozenset({"content-length", "host"}
)
_VERTEX_CALLER_KEY_HEADER_PRECEDENCE: Final = (
SpecialHeaders.custom_litellm_api_key.value.lower(),
SpecialHeaders.openai_authorization.value.lower(),
SpecialHeaders.azure_authorization.value.lower(),
SpecialHeaders.anthropic_authorization.value.lower(),
SpecialHeaders.google_ai_studio_authorization.value.lower(),
SpecialHeaders.azure_apim_authorization.value.lower(),
)
_MAPPED_ROUTE_CALLER_KEY_HEADER: Final = "litellm_user_api_key"
def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tuple[str, ...]]:
"""Operator-configured caller-key header names, as (override, pass_through).
``user_api_key_auth`` accepts the caller's key from two runtime-configured
header sources beyond the built-in ones, at opposite ends of its precedence.
``general_settings.litellm_key_header_name`` overrides every built-in source
(it replaces the resolved key after ``get_api_key`` runs), so it is highest
precedence. Each ``general_settings.pass_through_endpoints`` entry's
``headers.litellm_user_api_key`` is checked last inside ``get_api_key``, so it
is lowest. Google never consumes either, so both are also dropped by name.
"""
def _operator_configured_caller_key_header_names() -> tuple[str, ...]:
"""Operator-configured header names ``user_api_key_auth`` reads the caller's key from."""
from litellm.proxy.proxy_server import general_settings
custom_key_header: Final = general_settings.get("litellm_key_header_name")
@ -1829,80 +1813,53 @@ def _operator_configured_caller_key_header_names() -> tuple[tuple[str, ...], tup
if isinstance(headers, dict) and isinstance(headers.get("litellm_user_api_key"), str)
)
)
return override, pass_through
return override + pass_through
def _authenticated_caller_key_values(request: Request) -> frozenset[str]:
"""The value ``user_api_key_auth`` would accept as this caller's LiteLLM key.
The Vertex route authenticates through ``Depends(user_api_key_auth)``, which
resolves the key by precedence, matched here exactly. The ``/vertex_ai`` route
is a mapped pass-through route, so a header literally named
``litellm_user_api_key`` overrides every other source (``user_api_key_auth``
applies it last), making it highest precedence. Then an operator
``litellm_key_header_name``, then the built-in headers in ``get_api_key`` order,
then a ``pass_through_endpoints`` ``litellm_user_api_key`` header which
``get_api_key`` checks last. Some of those headers (``Authorization``,
``x-goog-api-key``) are also kept as genuine bring-your-own Google credentials,
so returning only the value that actually authenticated lets the filter strip
that value wherever it appears while leaving a real Google credential in place.
An empty set means no caller key was found, so nothing is value-stripped.
"""
incoming: Final = _safe_get_request_headers(request)
override_headers, pass_through_headers = _operator_configured_caller_key_header_names()
ordered_names: Final = (
(_MAPPED_ROUTE_CALLER_KEY_HEADER,)
+ override_headers
+ _VERTEX_CALLER_KEY_HEADER_PRECEDENCE
+ pass_through_headers
def _is_authenticated_caller_jwt(value: str, jwt_claims: Mapping[str, object]) -> bool:
"""Whether a header value is the JWT whose claims ``user_api_key_auth`` stored as ``jwt_claims``."""
presented_claims: Final = JWTHandler.get_unverified_claims(value)
if presented_claims is None:
return False
return all(
presented_claims.get(name) == claim
for name, claim in jwt_claims.items()
if name not in JWTHandler.LITELLM_INTERNAL_CLAIMS
)
present_values: Final = (incoming[name] for name in ordered_names if incoming.get(name))
authenticated_key: Final = next(
(stripped for value in present_values if (stripped := _normalize_credential_value(value))),
"",
)
return frozenset({authenticated_key}) if authenticated_key else frozenset()
def _forwarded_headers_for_credentialless_vertex_passthrough(request: Request) -> Mapping[str, str]:
"""
Header set to forward on the bring-your-own-credentials Vertex passthrough
branch, used when the proxy has no Vertex credential configured.
def _is_authenticated_caller_secret(value: str, user_api_key_dict: UserAPIKeyAuth) -> bool:
"""Whether a header value is the master key, the JWT that authenticated, or the key stored as ``api_key``."""
from litellm.proxy.proxy_server import master_key
No credential the proxy accepts for caller authentication is forwarded to
Google. ``user_api_key_auth`` reads the caller's key from every header in
``SpecialHeaders.litellm_credential_header_names()``, and Vertex only ever
authenticates with an OAuth token in ``Authorization`` or an API key in
``x-goog-api-key``. So the proxy-only auth headers Google never consumes
(everything in that set except those two, e.g. ``x-litellm-api-key`` /
``api-key`` / ``x-api-key`` / ``Ocp-Apim-Subscription-Key``, plus the mapped
pass-through ``litellm_user_api_key`` header and any operator-configured
``litellm_key_header_name`` / ``pass_through_endpoints`` key header) are dropped
by name. ``Authorization`` and ``x-goog-api-key`` may
instead carry a genuine bring-your-own Google credential, so they are kept
unless their value is the caller's authenticated LiteLLM key, which is dropped
by value (normalizing any ``Bearer`` / ``Basic`` / ``AWS4`` auth-scheme prefix
the same way authentication does). Because the value that authenticated is
resolved by the same precedence ``user_api_key_auth`` uses, a virtual key sent
only in ``x-goog-api-key`` (or in an operator-configured key header) is dropped
too, while a real Google key in ``x-goog-api-key`` alongside a virtual key in a
higher-precedence header is preserved. When neither a surviving
``Authorization`` nor ``x-goog-api-key`` remains the request is rejected so the
virtual key cannot leak upstream.
"""
normalized: Final = _normalize_credential_value(value)
if master_key is not None and hmac.compare_digest(normalized.encode(), master_key.encode()):
return True
jwt_claims: Final = user_api_key_dict.jwt_claims
if jwt_claims and _is_authenticated_caller_jwt(normalized, jwt_claims):
return True
authenticated_key: Final = user_api_key_dict.api_key
if authenticated_key is None:
return False
if master_key is None and not normalized.startswith("sk-"):
return False
stored_representation: Final = UserAPIKeyAuth._safe_hash_litellm_api_key(normalized) # pyright: ignore[reportPrivateUsage] # the exact transform auth applied when it stored api_key
return hmac.compare_digest(stored_representation.encode(), authenticated_key.encode())
def _forwarded_headers_for_credentialless_vertex_passthrough(
request: Request, user_api_key_dict: UserAPIKeyAuth
) -> Mapping[str, str]:
"""Caller headers to forward on the bring-your-own-credentials Vertex branch, minus LiteLLM secrets."""
incoming: Final = _safe_get_request_headers(request)
caller_key_values: Final = _authenticated_caller_key_values(request)
override_headers, pass_through_headers = _operator_configured_caller_key_header_names()
never_forwarded: Final = (
_HEADERS_NEVER_FORWARDED_TO_VERTEX.union((_MAPPED_ROUTE_CALLER_KEY_HEADER,))
.union(override_headers)
.union(pass_through_headers)
never_forwarded: Final = _HEADERS_NEVER_FORWARDED_TO_VERTEX.union(
(_MAPPED_ROUTE_CALLER_KEY_HEADER, *_operator_configured_caller_key_header_names())
)
forwarded: Final = MappingProxyType(
{
name: value
for name, value in incoming.items()
if name not in never_forwarded and _normalize_credential_value(value) not in caller_key_values
if name not in never_forwarded and not _is_authenticated_caller_secret(value, user_api_key_dict)
}
)
if "authorization" not in forwarded and "x-goog-api-key" not in forwarded:
@ -1918,6 +1875,7 @@ async def _prepare_vertex_auth_headers(
vertex_location: str | None,
base_target_url: str | None,
get_vertex_pass_through_handler: BaseVertexAIPassThroughHandler,
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[Mapping[str, str], str | None, bool, str | None, str | None]:
"""
Prepare authentication headers for Vertex AI pass-through requests.
@ -1930,6 +1888,8 @@ async def _prepare_vertex_auth_headers(
vertex_location: Vertex location
base_target_url: Base URL for the Vertex AI service
get_vertex_pass_through_handler: Handler for the specific Vertex AI service
user_api_key_dict: The caller's resolved authentication, so only the secret that
authenticated them is stripped on the credential-less branch
Returns:
Tuple containing:
@ -1944,7 +1904,7 @@ async def _prepare_vertex_auth_headers(
# Use headers from the incoming request if no vertex credentials are found
if (vertex_credentials is None or vertex_credentials.vertex_project is None) and router_credentials is None:
headers = _forwarded_headers_for_credentialless_vertex_passthrough(request)
headers = _forwarded_headers_for_credentialless_vertex_passthrough(request, user_api_key_dict)
headers_passed_through = True
verbose_proxy_logger.debug(
"default_vertex_config not set, forwarding caller-provided headers %s", tuple(headers.keys())
@ -2104,6 +2064,7 @@ async def _base_vertex_proxy_route(
vertex_location=vertex_location,
base_target_url=base_target_url,
get_vertex_pass_through_handler=get_vertex_pass_through_handler,
user_api_key_dict=user_api_key_dict,
)
if base_target_url is None:

120
litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
View file

@ -32,7 +32,7 @@ from __future__ import annotations
import json
import re
from collections.abc import Callable, Mapping, Sequence
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
from typing import (
TYPE_CHECKING,
Final,
@ -43,7 +43,7 @@ from typing import (
from urllib.parse import quote, unquote
from fastapi import HTTPException
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.managed_resources.isolation import (
@ -52,6 +52,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.repositories.table_repositories import (
ManagedFileRepository,
ManagedObjectRepository,
@ -820,6 +821,121 @@ async def rewrite_response_ids(
return mutated if changed else body
_RESPONSE_ID_PREFIX: Final = "resp_"
_STREAMED_RESPONSE_ID_SPEC: Final[_FieldSpec] = ("id", _RESPONSE_ID_PREFIX)
_SSE_DATA_PREFIX: Final = "data:"
_SSE_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue])
def _first_streamed_response(frames: bytes) -> tuple[str, Mapping[str, JsonValue]] | None:
for line in frames.decode("utf-8", errors="replace").splitlines():
if not line.startswith(_SSE_DATA_PREFIX):
continue
try:
event = _SSE_EVENT_ADAPTER.validate_json(line[len(_SSE_DATA_PREFIX) :])
except ValidationError:
continue
response = event.get("response")
if not isinstance(response, dict):
continue
raw_id = response.get("id")
if isinstance(raw_id, str) and raw_id.startswith(_RESPONSE_ID_PREFIX):
return raw_id, response
return None
class _StreamedResponseIdRewriter:
__slots__ = ("_is_create_route", "_pending", "_prisma_client", "_provider", "_replacement", "_user_api_key_dict")
def __init__(
self,
provider: str,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
is_create_route: bool,
) -> None:
self._provider: Final = provider
self._user_api_key_dict: Final = user_api_key_dict
self._prisma_client: Final = prisma_client
self._is_create_route: Final = is_create_route
self._pending = b""
self._replacement: tuple[bytes, bytes] | None = None
async def feed(self, chunk: bytes) -> bytes:
complete_frames, self._pending = split_complete_sse_frames(self._pending + chunk)
if not complete_frames:
return b""
if self._replacement is None:
self._replacement = await self._mint(complete_frames)
return self._rewrite(complete_frames)
def flush(self) -> bytes:
tail: Final = self._pending
self._pending = b""
return self._rewrite(tail)
async def _mint(self, frames: bytes) -> tuple[bytes, bytes] | None:
first: Final = _first_streamed_response(frames)
if first is None:
return None
raw_id, snapshot = first
managed_id: Final = await _mint_or_reuse_object(
raw_id,
self._provider,
"response",
snapshot,
self._user_api_key_dict,
self._prisma_client,
self._is_create_route,
)
return raw_id.encode(), managed_id.encode()
def _rewrite(self, frames: bytes) -> bytes:
if self._replacement is None:
return frames
raw_id, managed_id = self._replacement
return frames.replace(raw_id, managed_id)
async def rewrite_streamed_response_ids(
stream: AsyncGenerator[bytes, None],
provider: str,
method: str,
route: str,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
) -> AsyncGenerator[bytes, None]:
"""
Record ownership of the response object streamed back by a Responses API
passthrough and swap its managed id into every SSE frame, so a streamed
response is owned and resolved exactly like a non-streamed one.
Streams for any other ``(provider, method, route)`` are relayed untouched.
"""
from litellm.proxy.auth.auth_utils import normalize_request_route
canonical: Final = normalize_request_route(_canonical_path(route))
field_specs: Final = BUILTIN_OUTPUT_ID_FIELD_MAP.get((provider, method, canonical), ())
if _STREAMED_RESPONSE_ID_SPEC not in field_specs:
async for chunk in stream:
yield chunk
return
rewriter: Final = _StreamedResponseIdRewriter(
provider=provider,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
is_create_route="{" not in canonical,
)
async for chunk in stream:
rewritten_frames = await rewriter.feed(chunk)
if rewritten_frames:
yield rewritten_frames
tail: Final = rewriter.flush()
if tail:
yield tail
# ---------------------------------------------------------------------------
# List-route interception — serve listing entirely from DB
# ---------------------------------------------------------------------------

72
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -1209,14 +1209,19 @@ async def pass_through_request(
return StreamingResponse(
wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
stream=_own_streamed_managed_ids(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
),
managed_id_provider=_managed_id_provider,
request=request,
user_api_key_dict=user_api_key_dict,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
upstream_headers=response.headers,
@ -1285,14 +1290,19 @@ async def pass_through_request(
return StreamingResponse(
wrap_passthrough_sse_bytes_with_keepalive_pings(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
stream=_own_streamed_managed_ids(
stream=PassThroughStreamingHandler.chunk_processor(
response=response,
request_body=_parsed_body,
litellm_logging_obj=logging_obj,
endpoint_type=endpoint_type,
start_time=start_time,
passthrough_success_handler_obj=pass_through_endpoint_logging,
url_route=str(url),
),
managed_id_provider=_managed_id_provider,
request=request,
user_api_key_dict=user_api_key_dict,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
upstream_headers=response.headers,
@ -2441,6 +2451,36 @@ def _is_streaming_response(response: httpx.Response) -> bool:
return False
def _own_streamed_managed_ids(
stream: AsyncGenerator[bytes, None],
managed_id_provider: str | None,
request: Request,
user_api_key_dict: UserAPIKeyAuth,
) -> AsyncGenerator[bytes, None]:
from litellm.proxy.proxy_server import general_settings, prisma_client, proxy_logging_obj
if (
managed_id_provider is None
or not general_settings.get("passthrough_managed_object_ids", False)
or prisma_client is None
or proxy_logging_obj.get_proxy_hook("managed_files") is None
):
return stream
from litellm.proxy.auth.auth_utils import get_request_route
from litellm.proxy.pass_through_endpoints.managed_id_rewriter import (
rewrite_streamed_response_ids,
)
return rewrite_streamed_response_ids(
stream=stream,
provider=managed_id_provider,
method=request.method,
route=get_request_route(request),
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
def _should_buffer_passthrough_response(response: httpx.Response) -> bool:
"""
Decide from the response headers whether the body must be read into memory.

14
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -10,6 +10,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy._types import PassThroughEndpointLoggingResultValues
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.utils import StandardPassThroughResponseObject
@ -101,7 +102,7 @@ class PassThroughStreamingHandler:
async for chunk in response.aiter_bytes():
raw_bytes.append(chunk)
PassThroughStreamingHandler._stamp_first_chunk_if_needed(litellm_logging_obj)
complete_frames, pending = PassThroughStreamingHandler._split_complete_sse_frames(
complete_frames, pending = split_complete_sse_frames(
pending + chunk
) # rebind-ok: SSE frame reassembly buffer across transport chunks
if complete_frames:
@ -139,17 +140,6 @@ class PassThroughStreamingHandler:
except Exception as e:
verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e)
@staticmethod
def _split_complete_sse_frames(pending: bytes) -> tuple[bytes, bytes]:
lf_boundary_end: Final = pending.rfind(b"\n\n") + 2
crlf_boundary_end: Final = pending.rfind(b"\r\n\r\n") + 4
boundary_end: Final = max(
lf_boundary_end if lf_boundary_end >= 2 else 0, crlf_boundary_end if crlf_boundary_end >= 4 else 0
)
if boundary_end == 0:
return b"", pending
return pending[:boundary_end], pending[boundary_end:]
@staticmethod
async def _route_streaming_logging_to_handler(
litellm_logging_obj: LiteLLMLoggingObj,

6
litellm/proxy/proxy_server.py
View file

@ -4101,7 +4101,7 @@ def resolve_complexity_router_plugins(
complexity_router_config["classifier_plugin"] = resolved_classifier # rebind-ok: out-param, resolved in place
def validate_deployment_max_agentic_loops(model: Mapping[str, Any]) -> None:
def validate_deployment_max_agentic_loops(model: Mapping[str, object]) -> None:
"""
Reject a per-deployment `max_agentic_loops` the agentic loop cannot honor.
@ -4111,7 +4111,9 @@ def validate_deployment_max_agentic_loops(model: Mapping[str, Any]) -> None:
start. Left unchecked entirely, a `0` used to read as the default ceiling
of 3 and a non-integer failed every request to that model instead.
"""
litellm_params: Final = model.get("litellm_params") or {}
litellm_params: Final = model.get("litellm_params")
if not isinstance(litellm_params, Mapping):
return
if "max_agentic_loops" not in litellm_params:
return

2
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -1267,7 +1267,7 @@ def _count_input_tokens_for_models(
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
def _approximate_input_size(request_body: dict) -> int:
def _approximate_input_size(request_body: Mapping[str, object]) -> int:
"""Length of the request's input text, a cheap stand-in for tokenizing cost.
Every field _count_input_tokens hands the tokenizer is sized here, and

29
litellm/realtime_api/main.py
View file

@ -2,6 +2,8 @@
import asyncio
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, Literal, cast
import litellm
@ -29,6 +31,7 @@ from litellm.utils import ProviderConfigManager
from ..litellm_core_utils.get_litellm_params import get_litellm_params
from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ..llms.azure.common_utils import get_azure_ad_token
from ..llms.azure.realtime.handler import AzureOpenAIRealtime
from ..llms.bedrock.realtime.handler import BedrockRealtime
from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
@ -44,6 +47,7 @@ bedrock_realtime: Final = BedrockRealtime()
xai_realtime: Final = XAIRealtime()
vertex_llm_base: Final = VertexBase()
base_llm_http_handler = BaseLLMHTTPHandler()
_EMPTY_MODEL_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({})
def _with_resolved_session_model(session: dict[str, Any], model_name: str) -> dict[str, Any]:
@ -411,13 +415,16 @@ async def _arealtime(
if realtime_protocol is None and (query_params or {}).get("intent") == "transcription":
realtime_protocol = "GA"
realtime_protocol = realtime_protocol or "beta"
resolved_azure_ad_token: Final = (
None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token))
)
await azure_realtime.async_realtime(
model=model,
websocket=websocket,
api_base=api_base,
api_key=api_key,
api_version=api_version,
azure_ad_token=None,
azure_ad_token=resolved_azure_ad_token,
client=None,
timeout=timeout,
logging_obj=litellm_logging_obj,
@ -550,6 +557,17 @@ async def _arealtime(
raise ValueError(f"Unsupported model: {model}")
def _realtime_health_check_auth_headers(
custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, Any]
) -> Mapping[str, str | None]:
if custom_llm_provider != "azure":
return MappingProxyType({"api-key": api_key})
return azure_realtime.get_auth_headers(
api_key=api_key,
azure_ad_token=(None if api_key else get_azure_ad_token(GenericLiteLLMParams(**model_params))),
)
async def _realtime_health_check(
model: str,
custom_llm_provider: str,
@ -578,6 +596,11 @@ async def _realtime_health_check(
import websockets
url: str | None = None
auth_headers: Final = _realtime_health_check_auth_headers(
custom_llm_provider=custom_llm_provider,
api_key=api_key,
model_params=model_params or _EMPTY_MODEL_PARAMS,
)
if custom_llm_provider == "azure":
url = azure_realtime._construct_url(
api_base=api_base or "",
@ -627,9 +650,7 @@ async def _realtime_health_check(
ssl_context = get_shared_realtime_ssl_context()
async with websockets.connect(
url,
additional_headers={
"api-key": api_key,
},
additional_headers=auth_headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
):

2
litellm/repositories/model_repository.py
View file

@ -32,7 +32,7 @@ class _PrismaClientView(Protocol):
class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
"""Repository for proxy model database operations with encryption support."""
def __init__(self, prisma_client: object, encryption_key: str | None = None):
def __init__(self, prisma_client: object, encryption_key: str | None = None) -> None:
super().__init__(prisma_client)
self._encryption_key = encryption_key

30
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1300,16 +1300,14 @@ class LiteLLMCompletionResponsesConfig:
if isinstance(content, str) and content.strip():
return content
if isinstance(content, list):
text_parts: Final[list[str]] = [] # mutable-ok: text accumulator
for block in content:
if not isinstance(block, Mapping):
continue
block_type = block.get("type")
if block_type in ("encrypted_content", "redacted_thinking"):
continue
text = block.get("text")
if isinstance(text, str) and text.strip():
text_parts.append(text.strip())
text_parts: Final = tuple(
text.strip()
for block in content
if isinstance(block, Mapping)
and block.get("type") not in ("encrypted_content", "redacted_thinking")
and isinstance(text := block.get("text"), str)
and text.strip()
)
if text_parts:
return "\n".join(text_parts)
return None
@ -1325,13 +1323,11 @@ class LiteLLMCompletionResponsesConfig:
summary: Final[object] = input_item.get("summary")
if not isinstance(summary, list):
return None
text_parts: Final[list[str]] = [] # mutable-ok: text accumulator
for block in summary:
if not isinstance(block, Mapping):
continue
text = block.get("text")
if isinstance(text, str) and text.strip():
text_parts.append(text.strip())
text_parts: Final = tuple(
text.strip()
for block in summary
if isinstance(block, Mapping) and isinstance(text := block.get("text"), str) and text.strip()
)
return "\n".join(text_parts) if text_parts else None
@staticmethod

7
litellm/router.py
View file

@ -7344,7 +7344,7 @@ class Router:
):
raise error # then raise the error
if isinstance(error, openai.AuthenticationError):
if isinstance(error, (openai.AuthenticationError, openai.PermissionDeniedError)):
"""
- if other deployments available -> retry
- else -> raise error
@ -10697,10 +10697,9 @@ class Router:
_router_model_name: str = model_value
elif isinstance(model_value, dict):
_model_value = RouterModelGroupAliasItem(**model_value)
if _model_value["hidden"] is True:
if _model_value["hidden"] is True and model_name is None:
continue
else:
_router_model_name = _model_value["model"]
_router_model_name = _model_value["model"]
else:
continue

16
litellm/types/integrations/prometheus.py
View file

@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import MISSING, dataclass, field, fields
from enum import Enum
from types import MappingProxyType
from typing import Any, ClassVar, Final, Literal
from typing import Any, ClassVar, Final, Literal, cast
import litellm
@ -326,21 +326,25 @@ def validate_prometheus_deployment_and_latency_caller_identity() -> str:
)
def validate_caller_identity_settings(litellm_settings: Mapping[str, Any]) -> None:
def validate_caller_identity_settings(litellm_settings: Mapping[str, object]) -> None:
"""Store the caller-identity mode from litellm_settings and validate it together
with prometheus_metrics_config, raising on an invalid value or on include_labels
that request a label the selected mode removes."""
if "prometheus_deployment_and_latency_caller_identity" not in litellm_settings:
return
litellm.prometheus_deployment_and_latency_caller_identity = litellm_settings[
"prometheus_deployment_and_latency_caller_identity"
]
litellm.prometheus_deployment_and_latency_caller_identity = (
cast( # cast-ok: validated on the next line, which raises on an invalid value
'Literal["api_key_alias", "user_email", "both"]',
litellm_settings["prometheus_deployment_and_latency_caller_identity"],
)
)
caller_identity_mode: Final = validate_prometheus_deployment_and_latency_caller_identity()
if caller_identity_mode != "user_email":
return
raw_metrics_config: Final = litellm_settings.get("prometheus_metrics_config")
conflicting_metrics: Final = tuple(
metric_name
for metric_config in (litellm_settings.get("prometheus_metrics_config") or ())
for metric_config in (raw_metrics_config if isinstance(raw_metrics_config, list) else ())
if isinstance(metric_config, dict) and "api_key_alias" in (metric_config.get("include_labels") or ())
for metric_name in (metric_config.get("metrics") or ())
if metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS

1
litellm/types/llms/anthropic.py
View file

@ -36,6 +36,7 @@ AnthropicInputSchema = TypedDict(
class AnthropicOutputSchema(TypedDict, total=False):
type: Required[Literal["json_schema"]]
schema: Required[dict]
strict: ReadOnly[bool]
class AnthropicOutputConfig(TypedDict, total=False):

6
litellm/types/llms/bedrock.py
View file

@ -3,7 +3,7 @@ from collections.abc import Sequence
from enum import Enum
from typing import TYPE_CHECKING, Any, Final, Literal
from typing_extensions import Required, TypedDict, override
from typing_extensions import ReadOnly, Required, TypedDict, override
from .openai import ChatCompletionToolCallChunk
@ -97,6 +97,10 @@ class BedrockConverseReasoningContentBlockDelta(TypedDict, total=False):
text: str
class BedrockConverseGptReasoningEffortBlock(TypedDict):
effort: ReadOnly[str]
class GuardrailConverseTextBlock(TypedDict, total=False):
text: str

2
litellm/utils.py
View file

@ -1292,7 +1292,7 @@ async def async_post_call_success_deployment_hook(
async def async_post_call_failure_deployment_hook(
request_data: Mapping[str, Any], exception: Exception, call_type: str
request_data: Mapping[str, object], exception: Exception, call_type: str
) -> None:
"""
Notify CustomLogger callbacks that a deployment attempt failed.

29
model_prices_and_context_window.json
View file

@ -12282,7 +12282,7 @@
},
"claude-3-haiku-20240307": {
"cache_creation_input_token_cost": 3e-07,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr": 5e-07,
"cache_read_input_token_cost": 3e-08,
"deprecation_date": "2026-04-20",
"input_cost_per_token": 2.5e-07,
@ -12301,7 +12301,7 @@
},
"claude-3-opus-20240229": {
"cache_creation_input_token_cost": 1.875e-05,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_creation_input_token_cost_above_1hr": 3e-05,
"cache_read_input_token_cost": 1.5e-06,
"deprecation_date": "2026-01-05",
"input_cost_per_token": 1.5e-05,
@ -12515,7 +12515,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_output_config": true,
"prompt_cache_min_tokens": 1024
"prompt_cache_min_tokens": 1024,
"provider_specific_entry": {
"us": 1.1
}
},
"claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 3.75e-06,
@ -49473,6 +49476,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-sol": {
@ -49498,6 +49502,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"us.openai.gpt-5.6-terra": {
@ -49523,6 +49528,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-terra": {
@ -49548,6 +49554,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"us.openai.gpt-5.6-luna": {
@ -49573,6 +49580,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"global.openai.gpt-5.6-luna": {
@ -49598,6 +49606,7 @@
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_vision": true
},
"bedrock_mantle/openai.gpt-5.5": {
@ -49605,7 +49614,7 @@
"cache_read_input_token_cost": 5.5e-07,
"output_cost_per_token": 3.3e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -49632,7 +49641,7 @@
"cache_read_input_token_cost": 2.75e-07,
"output_cost_per_token": 1.65e-05,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 272000,
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
@ -50825,7 +50834,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true
"supports_native_structured_output": true,
"provider_specific_entry": {
"us": 1.1
}
},
"claude-mythos-preview": {
"cache_creation_input_token_cost": 1.25e-05,
@ -50860,7 +50872,10 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true
"supports_native_structured_output": true,
"provider_specific_entry": {
"us": 1.1
}
},
"gemini/gemini-robotics-er-2-streaming-preview": {
"input_cost_per_audio_token": 2e-06,

6
ruff-strict-budget.json
View file

@ -12,10 +12,10 @@
"limit": 2012
},
"ANN202": {
"limit": 852
"limit": 847
},
"ANN204": {
"limit": 711
"limit": 706
},
"ANN205": {
"limit": 112
@ -24,7 +24,7 @@
"limit": 133
},
"ANN401": {
"limit": 1157
"limit": 1153
},
"ASYNC230": {
"limit": 11

18
tests/e2e/coverage_registry/README.md
View file

@ -77,6 +77,24 @@ Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checke
the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest
collection errors.
## Provider x feature matrix: customer-run Bedrock combinations
The provider and feature combinations customers actually run get explicit cells, expanded
here as incidents surface new ones. The current Bedrock set, seeded from a customer's
production shape (regional `us.anthropic.*` inference-profile ids over both chat routes,
provider response headers for AWS-side correlation, and the Test Connection probe for a
responses-mode Bedrock Mantle deployment):
| Cell | Feature | Covering test |
|------|---------|---------------|
| `llm.chat_completions.bedrock_converse.basic.nonstream.works` | regional `us.` id, Converse | `llm_translation/test_chat_completions_regression_e2e.py` |
| `llm.chat_completions.bedrock_converse.basic.stream.works` | regional `us.` id, Converse stream | `llm_translation/test_chat_completions_regression_e2e.py` |
| `llm.chat_completions.bedrock_invoke.basic.nonstream.works` | regional `us.` id, Invoke | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
| `llm.chat_completions.bedrock_invoke.basic.stream.works` | regional `us.` id, Invoke stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
| `llm.chat_completions.bedrock_converse.response_headers.nonstream.works` | `llm_provider-*` headers | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
| `llm.chat_completions.bedrock_converse.response_headers.stream.works` | `llm_provider-*` headers, stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
| `mgmt.model.test_connection.happy_path` | Test Connection, Bedrock Mantle | `management/test_model_test_connection_e2e.py` |
## Status: this is a draft for review
The cells were enumerated from the codebase and the tiers are a first proposal. Known

4
tests/e2e/coverage_registry/llm_conversational.yaml
View file

@ -29,6 +29,10 @@
- {id: llm.chat_completions.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Bedrock vision (Anthropic/Nova)"}
- {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"}
- {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"}
- {id: llm.chat_completions.bedrock_converse.response_headers.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: nonstream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:248", rationale: "Bedrock request ids must surface as llm_provider-* response headers on /chat/completions so callers can correlate calls with AWS-side logs (#37003)", fail_before_fix: proven}
- {id: llm.chat_completions.bedrock_converse.response_headers.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: stream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:154", rationale: "The llm_provider-* headers must also surface on streaming /chat/completions, where CustomStreamWrapper carries them instead of the nonstream setter"}
- {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"}
- {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"}
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}

1
tests/e2e/coverage_registry/mgmt.yaml
View file

@ -75,3 +75,4 @@
- {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
- {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
- {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}
- {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven}

1
tests/e2e/coverage_registry/schema.py
View file

@ -71,6 +71,7 @@ LlmCapability = Literal[
"pdf_input",
"prompt_cache_1h",
"prompt_cache_5m",
"response_headers",
"service_tier",
"structured_output",
"thinking",

157
tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py Normal file
View file

@ -0,0 +1,157 @@
"""Live e2e for the Bedrock cells of the provider-feature matrix: provider
response headers on /chat/completions and regional inference-profile model ids
(us.anthropic.*) over the invoke route.
Header forwarding is the #37003 contract: the proxy surfaces Bedrock's response
headers prefixed llm_provider- (llm_provider-x-amzn-requestid above all) so a
caller can hand AWS support the request id behind a completion. Regional
inference-profile ids are the deployment shape most Bedrock customers run; a
v1.90.0 regression timed them out, and the Converse route keeps them covered in
test_chat_completions_regression_e2e.py, so the invoke route carries its own
rows here.
"""
from __future__ import annotations
import pytest
from pydantic import BaseModel
from e2e_config import unique_marker
from e2e_http import StreamingResponse, unwrap
from lifecycle import ResourceManager
from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
CONVERSE_REGIONAL_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
INVOKE_REGIONAL_BACKEND = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
PROVIDER_HEADER_PREFIX = "llm_provider-"
BEDROCK_REQUEST_ID_HEADER = "llm_provider-x-amzn-requestid"
class _StreamDelta(BaseModel):
content: str | None = None
class _StreamChoice(BaseModel):
delta: _StreamDelta = _StreamDelta()
class _StreamChunk(BaseModel):
choices: list[_StreamChoice] = []
def _streamed_text(events: list[str]) -> str:
chunks = [_StreamChunk.model_validate_json(event) for event in events]
return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
def _assert_streamed_completion(result: StreamingResponse) -> None:
assert result.ok and result.is_streaming, f"stream was not established: {result}"
assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
assert _streamed_text(result.stream_events).strip(), (
f"stream completed with no content deltas: {result.stream_events[:3]}"
)
def _assert_request_id_header(result: StreamingResponse) -> None:
forwarded = [name for name in result.headers if name.startswith(PROVIDER_HEADER_PREFIX)]
assert result.headers.get(BEDROCK_REQUEST_ID_HEADER), (
f"missing {BEDROCK_REQUEST_ID_HEADER}; forwarded provider headers: {forwarded}"
)
def _assert_completion(response: ChatResponse) -> None:
assert response.choices, f"completion returned no choices: {response}"
message = response.choices[0].message
content = (message.content if message else None) or ""
assert content.strip(), f"completion carried no content: {response}"
def _register_bedrock_model(
client: PassthroughClient, resources: ResourceManager, prefix: str, backend: str
) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
def _prompt() -> list[ChatMessage]:
return [ChatMessage(role="user", content="reply with one word")]
class TestBedrockResponseHeaders:
@pytest.mark.covers(
"llm.chat_completions.bedrock_converse.response_headers.nonstream.works",
exercised_on=[],
)
def test_bedrock_request_id_header_surfaces(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = _register_bedrock_model(client, resources, "e2e-bedrock-headers", CONVERSE_REGIONAL_BACKEND)
key = resources.key()
result = client.proxy.transport.send(
"/chat/completions",
headers=client.proxy.transport.bearer(key),
json=ChatBody(model=model, messages=_prompt(), max_tokens=64),
)
assert result.ok, f"chat call failed: {result.status_code} {result.body[:300]}"
_assert_request_id_header(result)
@pytest.mark.covers(
"llm.chat_completions.bedrock_converse.response_headers.stream.works",
exercised_on=[],
)
def test_bedrock_request_id_header_surfaces_on_stream(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = _register_bedrock_model(
client, resources, "e2e-bedrock-headers-stream", CONVERSE_REGIONAL_BACKEND
)
key = resources.key()
result = client.proxy.chat_stream(
key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
)
_assert_streamed_completion(result)
_assert_request_id_header(result)
class TestBedrockInvokeRegionalModelIds:
@pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.nonstream.works", exercised_on=[])
def test_invoke_regional_id_completes(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke", INVOKE_REGIONAL_BACKEND)
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_prompt(), max_tokens=64)))
_assert_completion(response)
@pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.stream.works", exercised_on=[])
def test_invoke_regional_id_streams(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke-stream", INVOKE_REGIONAL_BACKEND)
key = resources.key()
result = client.proxy.chat_stream(
key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
)
_assert_streamed_completion(result)

52
tests/e2e/llm_translation/test_together_ai_e2e.py
View file

@ -5,14 +5,19 @@ in the proxy's own cost map that carries both capability flags. Two backends are
pinned because the registry has no flag for what they prove: ``enable_thinking`` is a
Qwen chat-template contract, and MiniMax-M3 is the serverless model whose template
renders a replayed ``reasoning_content`` back into the prompt (Qwen and DeepSeek
silently drop it). Requires TOGETHER_API_KEY on the proxy; no skip gate.
silently drop it). MiniMax-M3 honors that replayed field on nearly every call, not
every call (one miss in dozens of otherwise identical calls), so the replay case asks
up to ``REPLAY_ATTEMPTS`` times and fails only when no answer carries the secret, which
a proxy that strips the field guarantees. Requires TOGETHER_API_KEY on the proxy; no
skip gate.
"""
from __future__ import annotations
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from datetime import date
from typing import Final
import pytest
from e2e_config import unique_marker
@ -51,6 +56,7 @@ REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3"
SECRET_PROMPT = "Remember this for later and reply with just OK."
SECRET_REASONING = "The user told me their favorite color is chartreuse. I must remember it."
SECRET_QUESTION = "What is my favorite color? Answer with one word."
REPLAY_ATTEMPTS: Final = 3
ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number."
WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
@ -179,6 +185,18 @@ def _message(response: ChatResponse) -> OutMessage:
return message
def _carries_secret(answer: OutMessage) -> bool:
return answer.content is not None and "chartreuse" in answer.content.lower()
def _answers_until_secret(client: PassthroughClient, key: str, body: ChatBody) -> Iterator[OutMessage]:
answers: Final = (_message(unwrap(client.proxy.chat(key, body))) for _ in range(REPLAY_ATTEMPTS))
for answer in answers:
yield answer
if _carries_secret(answer):
return
def _deltas(result: StreamingResponse) -> list[_StreamDelta]:
require_successful_call(result)
assert result.is_streaming, f"response was not streamed: {result.headers}"
@ -376,25 +394,19 @@ class TestTogetherChatCompletions:
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model, key = _register(client, resources, REASONING_REPLAY_BACKEND)
answer = _message(
unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(role="user", content=SECRET_PROMPT),
ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING),
ChatMessage(role="user", content=SECRET_QUESTION),
],
max_tokens=512,
),
)
)
body: Final = ChatBody(
model=model,
messages=[
ChatMessage(role="user", content=SECRET_PROMPT),
ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING),
ChatMessage(role="user", content=SECRET_QUESTION),
],
max_tokens=512,
)
assert answer.content and "chartreuse" in answer.content.lower(), (
f"the replayed reasoning_content never reached Together: {answer}"
answers: Final = tuple(_answers_until_secret(client, key, body))
assert any(_carries_secret(answer) for answer in answers), (
f"the replayed reasoning_content never reached Together in {len(answers)} attempts: {answers}"
)
@pytest.mark.covers("llm.chat_completions.together_ai.basic.nonstream.cost_logged")

58
tests/e2e/logging/test_otel_trace_e2e.py
View file

@ -743,3 +743,61 @@ class TestOtelTraceCompleteness:
)
genai = next(span for span in hits[0].spans if span.operation_name == genai_span)
_assert_error_span_contract(genai)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["messages"])
def test_failed_messages_error_span_attributes(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
) -> None:
"""A failed `/v1/messages` request must carry the same error-span
contract as a failed `/chat/completions` request (LIT-6164). The
async messages entrypoint used to surface the provider handler's raw
BaseLLMException to the failure logger, so the model-call span came
out with error.type=BaseLLMException and no
litellm.provider.error.llm_provider attribute.
Same setup as the chat sibling: a deployment with an invalid upstream
API key passes proxy auth and fails at the provider with a real 401,
and failed requests are not billed, so no cost-write span."""
route = "/v1/messages"
_assert_otel_destination_configured(client)
model_name = f"otel-err-{unique_marker()}"
model_id = client.create_model(
model_name,
LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY),
)
resources.defer(lambda: client.delete_model(model_id))
key = client.key_with_alias(f"otel-err-{unique_marker()}", models=[model_name])
resources.defer(lambda: client.delete_key(key))
deadline = time.monotonic() + client.proxy.poll_timeout
while True:
outcome = client.messages_raw(key, model_name, "trigger an upstream auth failure", max_tokens=16)
assert not outcome.ok, "the call must fail; the deployment's upstream key is invalid"
if "AnthropicException" in outcome.body or time.monotonic() >= deadline:
break
time.sleep(client.proxy.poll_interval)
assert "AnthropicException" in outcome.body, (
"never saw the mapped upstream provider failure before the deadline; either the key is "
"still propagating or the messages route surfaced the raw unmapped provider error - "
f"last outcome {outcome.status_code}: {outcome.body[:200]}"
)
assert outcome.status_code == 401, (
f"an upstream auth failure must map to 401, got {outcome.status_code}: {outcome.body[:200]}"
)
assert outcome.call_id is not None, "failed responses must still carry x-litellm-call-id"
genai_span = f"chat {model_name}"
hits = otel_reader.poll_traces_for_call(
call_id=outcome.call_id,
settled_names=_settled_names(route=route, genai_span=genai_span, require_cost_span=False),
settled_prefixes={DB_SPAN_PREFIX},
)
_assert_complete_trace(hits, route=route, genai_span=genai_span, require_cost_span=False)
root = next(span for span in hits[0].spans if not span.references)
assert str(_tag(root, "http.status_code")) == "401", (
f"the SERVER span must record the 401 the client received, got {_tag(root, 'http.status_code')!r}"
)
genai = next(span for span in hits[0].spans if span.operation_name == genai_span)
_assert_error_span_contract(genai)

13
tests/e2e/management/management_client.py
View file

@ -14,6 +14,8 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un
from models import (
ChatBody,
ChatMessage,
ConnectionTestBody,
ConnectionTestResponse,
CustomerDeleteBody,
CustomerInfoParams,
CustomerNewBody,
@ -118,6 +120,17 @@ class ManagementClient:
)
)
def connection_test(self, body: ConnectionTestBody) -> Result[ConnectionTestResponse]:
"""POST /health/test_connection, the call behind the Admin UI's Test
Connection button, probing the live provider with the supplied params."""
return self.proxy.transport.post(
"/health/test_connection",
headers=self.proxy.transport.master,
json=body,
response_type=ConnectionTestResponse,
timeout=120.0,
)
def block_key(self, key: str) -> None:
_ = unwrap(
self.proxy.transport.post(

67
tests/e2e/management/test_model_test_connection_e2e.py Normal file
View file

@ -0,0 +1,67 @@
"""Live e2e for POST /health/test_connection, the API behind the Admin UI's
Test Connection button on the add-model form.
The covered cell is a responses-mode Bedrock Mantle deployment: exactly this
shape 500ed on a functools.partial acompletion conflict before v1.91.0 while
every chat-mode probe stayed green, so the happy path asserts a real success
verdict from the live provider rather than just a 200 envelope. The region is a
literal because the endpoint rejects request-supplied os.environ/ references;
credentials fall through to the proxy's own environment (bearer token locally,
pod identity in CI).
The endpoint caps every probe at HEALTH_CHECK_TIMEOUT_SECONDS and answers a
timed-out probe with HTTP 200 and an in-body "Timeout exceeded", which the
harness's status-code retry policy cannot see. A Mantle probe can hit that cap
transiently while the rest of the suite saturates the same AWS account, so only
that exact error is retried here; any other error verdict fails immediately.
"""
from __future__ import annotations
import time
import pytest
from e2e_http import unwrap
from management_client import ManagementClient
from models import ConnectionTestBody, ConnectionTestResponse, LiteLLMParamsBody
pytestmark = pytest.mark.e2e
MANTLE_RESPONSES_BACKEND = "bedrock_mantle/openai.gpt-5.6-luna"
MANTLE_REGION = "us-east-1"
PROBE_TIMEOUT_ERROR = "Timeout exceeded"
PROBE_ATTEMPTS = 3
PROBE_RETRY_SLEEP_SECONDS = 30
def _probe_mantle(client: ManagementClient) -> ConnectionTestResponse:
return unwrap(
client.connection_test(
ConnectionTestBody(
litellm_params=LiteLLMParamsBody(
model=MANTLE_RESPONSES_BACKEND, aws_region_name=MANTLE_REGION
),
mode="responses",
)
)
)
class TestModelTestConnection:
@pytest.mark.covers("mgmt.model.test_connection.happy_path")
def test_bedrock_mantle_responses_connection_succeeds(self, client: ManagementClient) -> None:
for attempt in range(1, PROBE_ATTEMPTS + 1):
response = _probe_mantle(client)
if response.status == "success":
return
error = response.result.error if response.result else None
assert error == PROBE_TIMEOUT_ERROR, f"test_connection reported an error: {error}"
if attempt < PROBE_ATTEMPTS:
print(
f"test_connection probe timed out; retry {attempt}/{PROBE_ATTEMPTS - 1}"
f" in {PROBE_RETRY_SLEEP_SECONDS}s",
flush=True,
)
time.sleep(PROBE_RETRY_SLEEP_SECONDS)
pytest.fail(f"test_connection timed out on all {PROBE_ATTEMPTS} attempts")

20
tests/e2e/models.py
View file

@ -857,6 +857,26 @@ class ModelDeleteBody(BaseModel):
id: str
class ConnectionTestBody(BaseModel):
"""POST /health/test_connection body, the API behind the Admin UI's Test
Connection button: the deployment params as typed into the add-model form and
the health-check mode picking which endpoint the probe calls. The endpoint
rejects `os.environ/` references, so credentials are either literal values or
omitted to fall through to the proxy's own environment."""
litellm_params: LiteLLMParamsBody
mode: Literal["chat", "completion", "embedding", "responses"]
class ConnectionTestResult(BaseModel):
error: str | None = None
class ConnectionTestResponse(BaseModel):
status: Literal["success", "error"]
result: ConnectionTestResult | None = None
class CredentialCreateBody(BaseModel):
credential_name: str
credential_values: dict[str, str]

310
tests/test_litellm/integrations/test_prometheus_caller_identity.py
View file

@ -16,9 +16,12 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.integrations.prometheus import (
DEFINED_PROMETHEUS_METRICS,
PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS,
LabelValidationError,
PrometheusMetricLabels,
UserAPIKeyLabelNames,
UserAPIKeyLabelValues,
validate_caller_identity_settings,
validate_prometheus_deployment_and_latency_caller_identity,
)
from litellm.types.utils import StandardLoggingPayload
@ -379,23 +382,12 @@ def test_deployment_failure_email_fallbacks_reach_both_real_counters(
async def test_proxy_config_loads_caller_identity_before_initializing_callbacks(tmp_path: Path):
from litellm.proxy.proxy_server import ProxyConfig
config_path = tmp_path / "config.yaml"
config_path.write_text(
yaml.safe_dump(
{
"model_list": [
{
"model_name": "test-model",
"litellm_params": {"model": "openai/gpt-4", "api_key": "test-key"},
}
],
"litellm_settings": {
"callbacks": ["prometheus"],
"prometheus_deployment_and_latency_caller_identity": "both",
},
},
sort_keys=False,
)
config_path = _write_proxy_config(
tmp_path,
{
"callbacks": ["prometheus"],
"prometheus_deployment_and_latency_caller_identity": "both",
},
)
observed_modes: list[str] = []
@ -409,3 +401,287 @@ async def test_proxy_config_loads_caller_identity_before_initializing_callbacks(
assert observed_modes == ["both"]
assert litellm.prometheus_deployment_and_latency_caller_identity == "both"
def _identity_settings(mode: object, metrics_config: object = None) -> dict[str, object]:
settings: dict[str, object] = {"prometheus_deployment_and_latency_caller_identity": mode}
if metrics_config is not None:
settings["prometheus_metrics_config"] = metrics_config
return settings
def test_validate_mode_returns_each_accepted_value_and_defaults_to_api_key_alias(
monkeypatch: pytest.MonkeyPatch,
):
for mode in IDENTITY_MODES:
_set_caller_identity(monkeypatch, mode)
assert validate_prometheus_deployment_and_latency_caller_identity() == mode
monkeypatch.delattr(litellm, "prometheus_deployment_and_latency_caller_identity")
assert validate_prometheus_deployment_and_latency_caller_identity() == "api_key_alias"
def test_accepted_values_constant_matches_parametrized_modes():
from litellm.types.integrations.prometheus import (
PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES,
)
assert PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES == IDENTITY_MODES
assert len(TARGET_METRICS) == 9
@pytest.mark.parametrize(
"invalid_mode",
("user-email", "USER_EMAIL", "", None, True, 1, ["user_email"], {"mode": "user_email"}),
)
def test_validate_mode_rejects_invalid_values_and_names_accepted_ones(
monkeypatch: pytest.MonkeyPatch,
invalid_mode: object,
):
monkeypatch.setattr(litellm, "prometheus_deployment_and_latency_caller_identity", invalid_mode)
with pytest.raises(ValueError, match="prometheus_deployment_and_latency_caller_identity") as exc_info:
validate_prometheus_deployment_and_latency_caller_identity()
message = str(exc_info.value)
assert repr(invalid_mode) in message
for accepted_value in IDENTITY_MODES:
assert accepted_value in message
def test_validate_caller_identity_settings_without_key_leaves_mode_untouched(
monkeypatch: pytest.MonkeyPatch,
):
_set_caller_identity(monkeypatch, "both")
validate_caller_identity_settings({"prometheus_metrics_config": []})
assert litellm.prometheus_deployment_and_latency_caller_identity == "both"
@pytest.mark.parametrize("mode", IDENTITY_MODES)
def test_validate_caller_identity_settings_stores_each_valid_mode(mode: str):
validate_caller_identity_settings(_identity_settings(mode))
assert litellm.prometheus_deployment_and_latency_caller_identity == mode
@pytest.mark.parametrize("invalid_mode", ("user-email", None))
def test_validate_caller_identity_settings_rejects_invalid_and_null_modes(invalid_mode: object):
with pytest.raises(ValueError, match="prometheus_deployment_and_latency_caller_identity"):
validate_caller_identity_settings(_identity_settings(invalid_mode))
def test_user_email_mode_conflict_error_names_every_conflicting_metric_and_only_those():
metrics_config = [
{
"group": "non_target",
"metrics": ["litellm_overhead_with_guardrails_latency_metric"],
"include_labels": ["api_key_alias"],
},
{
"group": "target_pair",
"metrics": ["litellm_deployment_total_requests", "litellm_llm_api_latency_metric"],
"include_labels": ["api_key_alias"],
},
{
"group": "target_single",
"metrics": ["litellm_request_queue_time_seconds"],
"include_labels": ["api_key_alias"],
},
]
with pytest.raises(ValueError, match="prometheus_deployment_and_latency_caller_identity") as exc_info:
validate_caller_identity_settings(_identity_settings("user_email", metrics_config))
message = str(exc_info.value)
for conflicting_metric in (
"litellm_deployment_total_requests",
"litellm_llm_api_latency_metric",
"litellm_request_queue_time_seconds",
):
assert conflicting_metric in message
assert "litellm_overhead_with_guardrails_latency_metric" not in message
assert "prometheus_deployment_and_latency_caller_identity" in message
assert "user_email" in message
@pytest.mark.parametrize(
("mode", "metrics_config"),
(
(
"user_email",
[
{
"group": "g",
"metrics": ["litellm_deployment_total_requests"],
"include_labels": ["user_email"],
}
],
),
(
"user_email",
[
{
"group": "g",
"metrics": ["litellm_overhead_with_guardrails_latency_metric"],
"include_labels": ["api_key_alias"],
}
],
),
(
"api_key_alias",
[
{
"group": "g",
"metrics": ["litellm_deployment_total_requests"],
"include_labels": ["api_key_alias"],
}
],
),
(
"both",
[
{
"group": "g",
"metrics": ["litellm_deployment_total_requests"],
"include_labels": ["api_key_alias"],
}
],
),
("user_email", None),
("user_email", ["not-a-dict"]),
(
"user_email",
[{"group": "g", "metrics": ["litellm_deployment_total_requests"], "include_labels": None}],
),
("user_email", [{"group": "g", "metrics": None, "include_labels": ["api_key_alias"]}]),
),
)
def test_validate_caller_identity_settings_accepts_non_conflicting_configs(
mode: str,
metrics_config: object,
):
settings = _identity_settings(mode)
settings["prometheus_metrics_config"] = metrics_config
validate_caller_identity_settings(settings)
assert litellm.prometheus_deployment_and_latency_caller_identity == mode
def _write_proxy_config(tmp_path: Path, litellm_settings: dict[str, object]) -> Path:
config_path = tmp_path / "config.yaml"
config_path.write_text(
yaml.safe_dump(
{
"model_list": [
{
"model_name": "test-model",
"litellm_params": {"model": "openai/gpt-4", "api_key": "test-key"},
}
],
"litellm_settings": litellm_settings,
},
sort_keys=False,
)
)
return config_path
@pytest.mark.asyncio
@pytest.mark.parametrize(
"litellm_settings",
(
{
"callbacks": ["prometheus"],
"prometheus_deployment_and_latency_caller_identity": "user-email",
},
{
"callbacks": ["prometheus"],
"prometheus_deployment_and_latency_caller_identity": None,
},
{
"callbacks": ["prometheus"],
"prometheus_deployment_and_latency_caller_identity": "user_email",
"prometheus_metrics_config": [
{
"group": "g",
"metrics": ["litellm_deployment_total_requests"],
"include_labels": ["api_key_alias"],
}
],
},
),
ids=("typo-mode", "null-mode", "include-labels-conflict"),
)
async def test_proxy_config_fails_boot_before_callbacks_on_invalid_caller_identity_config(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
litellm_settings: dict[str, object],
):
from litellm.proxy.proxy_server import ProxyConfig
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
config_path = _write_proxy_config(tmp_path, litellm_settings)
with patch( # test-quality-ok: asserts boot fails before any callback initialization
"litellm.proxy.proxy_server.initialize_callbacks_on_proxy"
) as callback_init:
with pytest.raises(ValueError, match="prometheus_deployment_and_latency_caller_identity"):
await ProxyConfig().load_config(router=None, config_file_path=str(config_path))
callback_init.assert_not_called()
def test_failed_init_leaves_registry_clean_so_a_corrected_retry_succeeds(
monkeypatch: pytest.MonkeyPatch,
):
_set_caller_identity(monkeypatch, "user-email")
with pytest.raises(ValueError, match="prometheus_deployment_and_latency_caller_identity"):
PrometheusLogger()
assert list(REGISTRY._collector_to_names) == [] # pyright: ignore[reportPrivateUsage]
_set_caller_identity(monkeypatch, "user_email")
logger = PrometheusLogger()
assert "user_email" in logger.get_labels_for_metric("litellm_deployment_total_requests")
@pytest.mark.parametrize("invalid_label", ("api_key_alias", "user_email"))
def test_label_validation_error_names_mode_setting_for_identity_labels_on_target_metric(
monkeypatch: pytest.MonkeyPatch,
invalid_label: str,
):
_set_caller_identity(monkeypatch, "user_email")
error = LabelValidationError(
metric_name="litellm_deployment_total_requests",
invalid_labels=[invalid_label],
valid_labels=["user_email"],
)
assert "prometheus_deployment_and_latency_caller_identity='user_email'" in error.message
assert invalid_label in error.message
def test_label_validation_error_keeps_base_message_for_non_identity_cases(
monkeypatch: pytest.MonkeyPatch,
):
_set_caller_identity(monkeypatch, "user_email")
non_target_metric = LabelValidationError(
metric_name="litellm_overhead_with_guardrails_latency_metric",
invalid_labels=["api_key_alias"],
valid_labels=[],
)
non_identity_label = LabelValidationError(
metric_name="litellm_deployment_total_requests",
invalid_labels=["bogus_label"],
valid_labels=[],
)
for error in (non_target_metric, non_identity_label):
assert "caller-identity" not in error.message
assert error.message.startswith("Invalid labels for metric")

167
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -531,6 +531,43 @@ def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(_local_model_c
)
@pytest.mark.parametrize(
"model",
[
"bedrock_mantle/openai.gpt-5.5",
"bedrock_mantle/openai.gpt-5.4",
],
)
def test_generic_cost_per_token_bedrock_mantle_gpt55_gpt54_long_context_flat_rate(_local_model_cost_map, model):
"""Bedrock serves gpt-5.5 and gpt-5.4 up to its enforced 1,050,000-token prompt maximum and documents
no long-context tier for them, so a prompt past 272K is billed at the flat per-token rates."""
model_cost_map = litellm.model_cost[model]
assert model_cost_map["max_input_tokens"] == 1050000
assert [key for key in model_cost_map if "above_272k" in key] == []
served_prompt_tokens = 1030590
cached_tokens = 100000
completion_tokens = 1000
usage = Usage(
prompt_tokens=served_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=served_prompt_tokens + completion_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider="bedrock_mantle",
)
assert round(prompt_cost, 10) == round(
model_cost_map["input_cost_per_token"] * (served_prompt_tokens - cached_tokens)
+ model_cost_map["cache_read_input_token_cost"] * cached_tokens,
10,
)
assert round(completion_cost, 10) == round(model_cost_map["output_cost_per_token"] * completion_tokens, 10)
def test_generic_cost_per_token_honors_non_standard_above_threshold():
"""Regression for #30344: get_model_info must keep arbitrary
input/output_cost_per_token_above_<N>_tokens thresholds, not only the hard-coded
@ -716,6 +753,136 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate
litellm.model_cost.pop(model, None)
def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier_input_rate():
model = "litellm-test-tiered-no-cache-rates"
custom_llm_provider = "openrouter"
litellm.register_model(
{
model: {
"litellm_provider": custom_llm_provider,
"mode": "chat",
"cache_read_input_token_cost": 9e-09,
"cache_creation_input_token_cost": 9e-06,
"tiered_pricing": [
{
"range": [0, 32000],
"input_cost_per_token": 4.6e-07,
"output_cost_per_token": 2.3e-06,
},
{
"range": [32000, 128000],
"input_cost_per_token": 7e-07,
"output_cost_per_token": 3.5e-06,
},
],
}
}
)
try:
uncached = Usage(prompt_tokens=40000, completion_tokens=100, total_tokens=40100)
cached = Usage(
prompt_tokens=40000,
completion_tokens=100,
total_tokens=40100,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=5000, cache_creation_tokens=15000
),
)
uncached_prompt_cost, _ = generic_cost_per_token(
model=model,
usage=uncached,
custom_llm_provider=custom_llm_provider,
)
cached_prompt_cost, cached_completion_cost = generic_cost_per_token(
model=model,
usage=cached,
custom_llm_provider=custom_llm_provider,
)
tier_input_rate = 7e-07
assert round(cached_prompt_cost, 12) == round(40000 * tier_input_rate, 12)
assert round(cached_prompt_cost, 12) == round(uncached_prompt_cost, 12)
assert round(cached_completion_cost, 12) == round(100 * 3.5e-06, 12)
finally:
litellm.model_cost.pop(model, None)
def test_generic_cost_per_token_tier_without_a_1hr_cache_rate_bills_the_tier_cache_creation_rate():
model = "litellm-test-tiered-no-1hr-cache-rate"
custom_llm_provider = "openrouter"
litellm.register_model(
{
model: {
"litellm_provider": custom_llm_provider,
"mode": "chat",
"cache_creation_input_token_cost_above_1hr": 9e-05,
"tiered_pricing": [
{
"range": [0, 128000],
"input_cost_per_token": 7e-07,
"output_cost_per_token": 3.5e-06,
"cache_creation_input_token_cost": 8.75e-07,
}
],
}
}
)
try:
usage = Usage(
prompt_tokens=1000,
completion_tokens=10,
total_tokens=1010,
prompt_tokens_details=PromptTokensDetailsWrapper(
cache_creation_tokens=800,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=300, ephemeral_1h_input_tokens=500
),
),
)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider,
)
tier_cache_creation_rate = 8.75e-07
expected_prompt = (200 * 7e-07) + (800 * tier_cache_creation_rate)
assert round(prompt_cost, 12) == round(expected_prompt, 12)
assert round(completion_cost, 12) == round(10 * 3.5e-06, 12)
finally:
litellm.model_cost.pop(model, None)
def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier():
model = "litellm-test-tiered-no-input-rate"
custom_llm_provider = "openrouter"
litellm.register_model(
{
model: {
"litellm_provider": custom_llm_provider,
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"tiered_pricing": [{"range": [0, 128000], "output_cost_per_token": 3.5e-06}],
}
}
)
try:
usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider,
)
assert round(prompt_cost, 12) == round(1000 * 1e-06, 12)
assert round(completion_cost, 12) == round(100 * 2e-06, 12)
finally:
litellm.model_cost.pop(model, None)
def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backend_rate():
"""Regression: the router registers a deployment's custom pricing as a standalone
model_cost entry holding only the supplied fields, so an input-only tier table left

124
tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -13,6 +13,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import (
extract_and_raise_litellm_exception,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.types.utils import LlmProviders
# Test cases for is_error_str_context_window_exceeded
# Tuple format: (error_message, expected_result)
@ -785,33 +786,24 @@ OPENAI_SHAPED = {
503: (litellm.ServiceUnavailableError, 503),
}
UPSTREAM_STATUS_DISCARDED = (litellm.APIConnectionError, 500)
PERMISSION_DENIED = (litellm.PermissionDeniedError, 403)
PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS = ("cloudflare", "ollama", "vllm")
STATUS_KEYED = {**OPENAI_SHAPED, 403: PERMISSION_DENIED}
DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
"anthropic": {403: UPSTREAM_STATUS_DISCARDED, 422: UPSTREAM_STATUS_DISCARDED},
"anthropic": {403: PERMISSION_DENIED},
"azure": {500: (litellm.APIError, 500)},
"bedrock": {
403: UPSTREAM_STATUS_DISCARDED,
403: PERMISSION_DENIED,
500: (litellm.ServiceUnavailableError, 503),
},
"cohere": {
401: UPSTREAM_STATUS_DISCARDED,
403: UPSTREAM_STATUS_DISCARDED,
404: UPSTREAM_STATUS_DISCARDED,
422: UPSTREAM_STATUS_DISCARDED,
429: UPSTREAM_STATUS_DISCARDED,
503: UPSTREAM_STATUS_DISCARDED,
},
"cloudflare": {403: PERMISSION_DENIED},
"cohere": {403: PERMISSION_DENIED},
"databricks": {
403: (litellm.AuthenticationError, 401),
403: PERMISSION_DENIED,
422: (litellm.BadRequestError, 400),
},
"gemini": {
403: (litellm.PermissionDeniedError, 403),
422: UPSTREAM_STATUS_DISCARDED,
},
"gemini": {403: PERMISSION_DENIED},
"huggingface": {
404: (litellm.APIError, 404),
422: (litellm.APIError, 422),
@ -824,6 +816,7 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
500: (litellm.APIError, 500),
503: (litellm.APIError, 503),
},
"ollama": {403: PERMISSION_DENIED},
"openrouter": {500: (litellm.APIError, 500)},
"replicate": {
403: (litellm.APIError, 500),
@ -833,17 +826,11 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
503: (litellm.APIError, 500),
},
"sagemaker": {
403: UPSTREAM_STATUS_DISCARDED,
403: PERMISSION_DENIED,
500: (litellm.ServiceUnavailableError, 503),
},
"vertex_ai": {
403: (litellm.PermissionDeniedError, 403),
422: UPSTREAM_STATUS_DISCARDED,
},
**{
provider: dict.fromkeys(UPSTREAM_STATUS_CODES, UPSTREAM_STATUS_DISCARDED)
for provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS
},
"vertex_ai": {403: PERMISSION_DENIED},
"vllm": {403: PERMISSION_DENIED},
}
PROVIDERS_WITH_A_HANDLER = (
@ -875,6 +862,38 @@ PROVIDERS_WITH_A_HANDLER = (
"xai",
)
PROVIDER_ALIASES_WITH_A_HANDLER = (
"aleph_alpha",
"anthropic_text",
"azure_text",
"bedrock_mantle",
"cohere_chat",
"custom_openai",
"lemonade",
"litellm_proxy",
"ollama_chat",
"predibase",
"sagemaker_chat",
"text-completion-openai",
"vertex_ai_beta",
"watsonx",
)
PROVIDERS_WITHOUT_A_HANDLER = tuple(
sorted(
frozenset(provider.value for provider in LlmProviders)
- frozenset(PROVIDERS_WITH_A_HANDLER)
- frozenset(PROVIDER_ALIASES_WITH_A_HANDLER)
- frozenset(litellm.openai_compatible_providers)
)
)
MINIMAX_401_BODY = (
'{"type":"error","error":{"type":"authorized_error","message":"login fail: Please carry the API secret key '
"in the 'Authorization' field of the request header (1004)\",\"http_code\":\"401\"},"
'"request_id":"06ddc9ba97ee6340e38f10e09787f547"}'
)
def _expected_for(provider: str, status_code: int) -> tuple[type[Exception], int]:
return DEVIATIONS_FROM_THE_OPENAI_SHAPE.get(provider, {}).get(
@ -938,6 +957,51 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(
assert returned is already_mapped
@pytest.mark.parametrize("status_code", UPSTREAM_STATUS_CODES)
@pytest.mark.parametrize("provider", PROVIDERS_WITHOUT_A_HANDLER)
def test_a_provider_without_a_handler_maps_by_the_upstream_status(
provider, status_code, quiet_exception_mapping
):
expected_class, expected_status = STATUS_KEYED[status_code]
with pytest.raises(openai.APIError) as raised:
exception_type(
model="test-model",
original_exception=_UpstreamHTTPError(status_code=status_code),
custom_llm_provider=provider,
)
assert type(raised.value) is expected_class
assert raised.value.status_code == expected_status
assert raised.value.llm_provider == provider
assert raised.value.model == "test-model"
def test_a_minimax_bad_key_is_an_authentication_error(quiet_exception_mapping):
from litellm.llms.base_llm.chat.transformation import BaseLLMException
with pytest.raises(litellm.AuthenticationError) as raised:
exception_type(
model="MiniMax-M2.5",
original_exception=BaseLLMException(status_code=401, message=MINIMAX_401_BODY),
custom_llm_provider="minimax",
)
assert raised.value.status_code == 401
assert raised.value.llm_provider == "minimax"
assert raised.value.message.startswith("litellm.AuthenticationError: MinimaxException - ")
assert "login fail" in raised.value.message
def test_an_exception_without_a_status_is_still_a_connection_error(quiet_exception_mapping):
with pytest.raises(litellm.APIConnectionError):
exception_type(
model="MiniMax-M2.5",
original_exception=RuntimeError("socket hung up"),
custom_llm_provider="minimax",
)
CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens."
CONTENT_POLICY_MESSAGE = (
'{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}'
@ -993,9 +1057,7 @@ class _UpstreamErrorWithMessage(_UpstreamHTTPError):
def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(
provider, quiet_exception_mapping
):
if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS:
expected_class, expected_status = UPSTREAM_STATUS_DISCARDED
elif provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW:
if provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW:
expected_class, expected_status = litellm.ContextWindowExceededError, 400
else:
expected_class, expected_status = litellm.BadRequestError, 400
@ -1015,9 +1077,7 @@ def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(
def test_a_content_policy_block_reaches_the_caller_as_the_router_needs_it(
provider, quiet_exception_mapping
):
if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS:
expected_class, expected_status = UPSTREAM_STATUS_DISCARDED
elif provider in PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK:
if provider in PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK:
expected_class, expected_status = litellm.ContentPolicyViolationError, 400
else:
expected_class, expected_status = litellm.BadRequestError, 400

49
tests/test_litellm/litellm_core_utils/test_health_check_helpers.py
View file

@ -12,6 +12,55 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
@pytest.mark.asyncio
async def test_image_edit_health_check_handler_uses_png_and_prompt():
model_params = {"model": "openai/gpt-image-1", "api_key": "sk-test"}
mode_handlers = HealthCheckHelpers.get_mode_handlers(
model="gpt-image-1",
custom_llm_provider="openai",
model_params=model_params,
)
assert "image_edit" in mode_handlers
with patch( # test-quality-ok: the public health-check path has no dependency injection seam
"litellm.aimage_edit", new_callable=AsyncMock, return_value={}
) as mock_aimage_edit:
await mode_handlers["image_edit"]()
await HealthCheckHelpers.get_mode_handlers(
model="gpt-image-1",
custom_llm_provider="openai",
model_params=model_params,
prompt="edit this image",
)["image_edit"]()
assert mock_aimage_edit.call_count == 2
default_call = mock_aimage_edit.call_args_list[0].kwargs
explicit_call = mock_aimage_edit.call_args_list[1].kwargs
assert default_call["model"] == "openai/gpt-image-1"
assert default_call["prompt"] == "test"
assert explicit_call["prompt"] == "edit this image"
image = default_call["image"]
assert isinstance(image, bytes)
assert image.startswith(b"\x89PNG")
assert int.from_bytes(image[16:20], "big") == 512
assert int.from_bytes(image[20:24], "big") == 512
@pytest.mark.asyncio
async def test_ahealth_check_supports_image_edit_mode():
with patch( # test-quality-ok: the public health-check path has no dependency injection seam
"litellm.aimage_edit", new_callable=AsyncMock, return_value={}
):
result = await ahealth_check(
{"model": "gpt-image-1", "api_key": "sk-test"},
mode="image_edit",
)
assert "error" not in result
assert "Mode image_edit not supported" not in str(result)
def test_update_model_params_with_health_check_tracking_information():
"""Test _update_model_params_with_health_check_tracking_information adds required tracking info."""
initial_model_params = {"model": "gpt-3.5-turbo", "api_key": "test_key"}

1
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -1050,6 +1050,7 @@ def test_anthropic_messages_validate_adds_beta_header():
messages=[{"role": "user", "content": [{"type": "text", "text": "Hi"}]}],
optional_params={"context_management": _sample_context_management_payload()},
litellm_params={},
api_key="fake-anthropic-key",
)
assert headers["anthropic-beta"] == "context-management-2025-06-27"

2
tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
View file

@ -679,7 +679,7 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
def _translate_with_metadata(
model: str, metadata: dict[str, Any], custom_llm_provider: str | None
model: str, metadata: dict[str, str], custom_llm_provider: str | None
) -> dict[str, Any]:
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request={

94
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -7,6 +7,7 @@ from typing import Any, Dict, List
import httpx
import pytest
from fastapi.testclient import TestClient
from pydantic import ValidationError
from unittest.mock import AsyncMock, MagicMock, patch
@ -1286,3 +1287,96 @@ class TestMessagesStreamingSuccessLogging:
assert payload["call_type"] == "acompletion"
assert payload["total_tokens"] > 0
assert payload["response_cost"] > 0
class _FailureCapture(CustomLogger):
def __init__(self):
super().__init__()
self.error_information: List[Dict[str, Any]] = []
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
payload = kwargs.get("standard_logging_object") or {}
self.error_information.append(payload.get("error_information") or {})
@pytest.mark.asyncio
@pytest.mark.parametrize(
"upstream_status, upstream_error_type, expected_exception",
[
(401, "authentication_error", litellm.AuthenticationError),
(403, "permission_error", litellm.PermissionDeniedError),
],
)
async def test_anthropic_messages_maps_provider_exception_before_failure_logging(
monkeypatch, upstream_status, upstream_error_type, expected_exception
):
"""Regression test for LIT-6164. The async /v1/messages entrypoint awaited the
provider handler without exception_type mapping, so the @client failure
handler (and every logger behind it, e.g. OTel error spans) saw the raw
BaseLLMException: error.type=BaseLLMException and no llm_provider.
The 403 row pins the upstream status on the way through the mapper: Anthropic's
documented permission_error must reach the caller as a 403, never as the mapper's
APIConnectionError 500 fallthrough."""
from litellm.llms.anthropic.experimental_pass_through.messages import handler
capture = _FailureCapture()
monkeypatch.setattr(litellm, "callbacks", [capture])
def upstream_rejects_the_request(request: httpx.Request) -> httpx.Response:
return httpx.Response(
upstream_status,
json={"type": "error", "error": {"type": upstream_error_type, "message": "rejected upstream"}},
request=request,
)
upstream = AsyncHTTPHandler()
upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_rejects_the_request))
with pytest.raises(expected_exception) as excinfo:
await handler.anthropic_messages(
max_tokens=16,
messages=[{"role": "user", "content": "hi"}],
model="anthropic/claude-haiku-4-5",
custom_llm_provider="anthropic",
api_key="sk-invalid",
client=upstream,
)
assert excinfo.value.status_code == upstream_status
assert excinfo.value.llm_provider == "anthropic"
assert "AnthropicException" in excinfo.value.message
assert f'"{upstream_error_type}"' in excinfo.value.message
assert capture.error_information, "the failure handler must have logged the mapped exception"
error_information = capture.error_information[0]
assert error_information.get("error_class") == expected_exception.__name__
assert error_information.get("llm_provider") == "anthropic"
assert error_information.get("error_code") == str(upstream_status)
@pytest.mark.asyncio
async def test_anthropic_messages_leaves_non_provider_failures_unmapped():
"""The mapping boundary is for provider failures only. A request rejected before
the provider call (here invalid metadata) must surface as the original exception,
not as the mapper's APIConnectionError, whose message embeds a server traceback."""
from litellm.llms.anthropic.experimental_pass_through.messages import handler
def upstream_must_not_be_called(request: httpx.Request) -> httpx.Response:
raise AssertionError("the provider must not be called for a request rejected locally")
upstream = AsyncHTTPHandler()
upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_must_not_be_called))
with pytest.raises(ValidationError) as excinfo:
await handler.anthropic_messages(
max_tokens=16,
messages=[{"role": "user", "content": "hi"}],
model="anthropic/claude-haiku-4-5",
custom_llm_provider="anthropic",
api_key="sk-invalid",
client=upstream,
metadata={"user_id": 123},
)
assert "Traceback" not in str(excinfo.value)

2
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_speed.py
View file

@ -28,6 +28,7 @@ def test_messages_drop_params_strips_speed_for_unsupported_models():
messages=[{"role": "user", "content": "Hello"}],
optional_params=dict(optional_params),
litellm_params={},
api_key="fake-anthropic-key",
)
result = config.transform_anthropic_messages_request(
model="claude-sonnet-4-6",
@ -60,6 +61,7 @@ def test_messages_drop_params_keeps_speed_for_supporting_models():
messages=[{"role": "user", "content": "Hello"}],
optional_params=dict(optional_params),
litellm_params={},
api_key="fake-anthropic-key",
)
result = config.transform_anthropic_messages_request(
model="claude-opus-4-6",

64
tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
View file

@ -135,16 +135,24 @@ class TestOutputConfigStructuredOutput:
}
def test_output_config_format_json_schema_converted(self):
"""output_config.format.json_schema is converted to OpenAI text.format."""
"""output_config.format.json_schema is converted to OpenAI text.format, defaulting strict to False."""
req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA}})
kwargs = _ADAPTER.translate_request(req)
assert "text" in kwargs
fmt = kwargs["text"]["format"]
assert fmt["type"] == "json_schema"
assert fmt["schema"] == self._SCHEMA
assert fmt["strict"] is True
assert fmt["strict"] is False
assert fmt["name"] == "structured_output"
def test_output_config_format_explicit_strict_true_is_preserved(self):
"""Nested output_config.format with explicit strict=True is preserved."""
req = _make_request(
output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}}
)
kwargs = _ADAPTER.translate_request(req)
assert kwargs["text"]["format"]["strict"] is True
def test_output_config_without_format_does_not_set_text(self):
"""output_config with only non-format keys doesn't produce text.format."""
req = _make_request(output_config={"effort": "high"})
@ -152,21 +160,65 @@ class TestOutputConfigStructuredOutput:
assert "text" not in kwargs
def test_output_format_still_works(self):
"""The original output_format field still takes precedence when present."""
"""The original output_format field still takes precedence when present, defaulting strict to False."""
req = _make_request(output_format={"type": "json_schema", "schema": self._SCHEMA})
kwargs = _ADAPTER.translate_request(req)
assert "text" in kwargs
assert kwargs["text"]["format"]["type"] == "json_schema"
assert kwargs["text"]["format"]["strict"] is False
def test_output_format_explicit_strict_false_is_preserved(self):
"""output_format with an explicit strict=False is preserved as False."""
req = _make_request(output_format={"type": "json_schema", "schema": self._SCHEMA, "strict": False})
kwargs = _ADAPTER.translate_request(req)
assert kwargs["text"]["format"]["strict"] is False
def test_output_format_explicit_strict_true_is_preserved(self):
"""output_format with an explicit strict=True is preserved as True."""
req = _make_request(output_format={"type": "json_schema", "schema": self._SCHEMA, "strict": True})
kwargs = _ADAPTER.translate_request(req)
assert kwargs["text"]["format"]["strict"] is True
def test_output_format_takes_precedence_over_output_config(self):
"""output_format takes precedence over output_config.format."""
"""output_format takes precedence over output_config.format, for both schema and strict."""
other_schema = {"type": "object", "properties": {"id": {"type": "integer"}}}
req = _make_request(
output_format={"type": "json_schema", "schema": self._SCHEMA},
output_config={"format": {"type": "json_schema", "schema": other_schema}},
output_format={"type": "json_schema", "schema": self._SCHEMA, "strict": False},
output_config={"format": {"type": "json_schema", "schema": other_schema, "strict": True}},
)
kwargs = _ADAPTER.translate_request(req)
assert kwargs["text"]["format"]["schema"] == self._SCHEMA
assert kwargs["text"]["format"]["strict"] is False
def test_optional_property_stays_out_of_required_list(self):
"""A property absent from required must stay absent from required in the translated schema."""
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"nickname": {"type": "string"},
},
"required": ["name"],
"additionalProperties": False,
}
req = _make_request(output_format={"type": "json_schema", "schema": schema})
kwargs = _ADAPTER.translate_request(req)
fmt_schema = kwargs["text"]["format"]["schema"]
assert fmt_schema["required"] == ["name"]
assert "nickname" not in fmt_schema["required"]
assert fmt_schema["additionalProperties"] is False
def test_translate_request_does_not_mutate_input_schema(self):
"""translate_request must not mutate the caller's output_format or schema dicts."""
schema = {"type": "object", "properties": {"x": {"type": "number"}}, "required": ["x"]}
output_format = {"type": "json_schema", "schema": schema, "strict": False}
req = _make_request(output_format=output_format)
snapshot = json.loads(json.dumps(output_format))
_ADAPTER.translate_request(req)
assert output_format == snapshot
assert req["output_format"] == snapshot
# ---------------------------------------------------------------------------

219
tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py
View file

@ -18,9 +18,7 @@ from unittest.mock import patch
import pytest
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
)
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")))
# Fake tokens for testing (not real secrets)
FAKE_OAUTH_TOKEN = "sk-ant-oat01-fake-token-for-testing-123456789abcdef"
@ -31,21 +29,37 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789"
class TestOptionallyHandleAnthropicOAuth:
"""Tests for optionally_handle_anthropic_oauth function."""
def test_oauth_token_in_authorization_header(self):
@pytest.mark.parametrize("header_name", ["authorization", "Authorization", "AUTHORIZATION"])
def test_oauth_token_in_authorization_header(self, header_name):
"""OAuth token in Authorization header should be detected and headers set correctly."""
from litellm.llms.anthropic.common_utils import (
optionally_handle_anthropic_oauth,
)
headers = {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}"}
updated_headers, extracted_api_key = optionally_handle_anthropic_oauth(
headers, None
)
headers = {header_name: f"Bearer {FAKE_OAUTH_TOKEN}"}
updated_headers, extracted_api_key = optionally_handle_anthropic_oauth(headers, None)
assert extracted_api_key == FAKE_OAUTH_TOKEN
assert updated_headers["anthropic-beta"] == "oauth-2025-04-20"
assert updated_headers["anthropic-dangerous-direct-browser-access"] == "true"
assert "x-api-key" not in updated_headers
assert [name for name in updated_headers if name.lower() == "authorization"] == ["authorization"]
assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
@pytest.mark.parametrize("api_key_header_name", ["x-api-key", "X-Api-Key"])
def test_oauth_removes_x_api_key_any_casing(self, api_key_header_name):
"""When OAuth wins, a client x-api-key header is removed whatever its casing."""
from litellm.llms.anthropic.common_utils import (
optionally_handle_anthropic_oauth,
)
headers = {api_key_header_name: FAKE_REGULAR_KEY, "Authorization": f"Bearer {FAKE_OAUTH_TOKEN}"}
updated_headers, extracted_api_key = optionally_handle_anthropic_oauth(headers, None)
assert extracted_api_key == FAKE_OAUTH_TOKEN
assert [name for name in updated_headers if name.lower() == "x-api-key"] == []
assert [name for name in updated_headers if name.lower() == "authorization"] == ["authorization"]
assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
def test_oauth_token_in_api_key_directly(self):
"""OAuth token passed as api_key should set Authorization: Bearer header."""
@ -54,9 +68,7 @@ class TestOptionallyHandleAnthropicOAuth:
)
headers = {}
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(
headers, FAKE_OAUTH_TOKEN
)
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(headers, FAKE_OAUTH_TOKEN)
assert returned_api_key == FAKE_OAUTH_TOKEN
assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
@ -71,9 +83,7 @@ class TestOptionallyHandleAnthropicOAuth:
)
headers = {"x-api-key": FAKE_OAUTH_TOKEN}
updated_headers, _ = optionally_handle_anthropic_oauth(
headers, FAKE_OAUTH_TOKEN
)
updated_headers, _ = optionally_handle_anthropic_oauth(headers, FAKE_OAUTH_TOKEN)
assert "x-api-key" not in updated_headers
assert updated_headers["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
@ -85,9 +95,7 @@ class TestOptionallyHandleAnthropicOAuth:
)
headers = {}
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(
headers, FAKE_REGULAR_KEY
)
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(headers, FAKE_REGULAR_KEY)
assert returned_api_key == FAKE_REGULAR_KEY
assert "authorization" not in updated_headers
@ -101,9 +109,7 @@ class TestOptionallyHandleAnthropicOAuth:
)
headers = {"authorization": f"Bearer {FAKE_REGULAR_KEY}"}
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(
headers, FAKE_REGULAR_KEY
)
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(headers, FAKE_REGULAR_KEY)
assert returned_api_key == FAKE_REGULAR_KEY
assert "anthropic-dangerous-direct-browser-access" not in updated_headers
@ -115,9 +121,7 @@ class TestOptionallyHandleAnthropicOAuth:
)
headers = {}
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(
headers, None
)
updated_headers, returned_api_key = optionally_handle_anthropic_oauth(headers, None)
assert returned_api_key is None
assert "authorization" not in updated_headers
@ -539,16 +543,12 @@ class TestProxyOAuthHeaderForwarding:
)
# Should preserve OAuth even with flag=False
cleaned_without_flag = clean_headers(
raw_headers, forward_llm_provider_auth_headers=False
)
cleaned_without_flag = clean_headers(raw_headers, forward_llm_provider_auth_headers=False)
assert "authorization" in cleaned_without_flag
assert cleaned_without_flag["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
# Should also preserve OAuth with flag=True
cleaned_with_flag = clean_headers(
raw_headers, forward_llm_provider_auth_headers=True
)
cleaned_with_flag = clean_headers(raw_headers, forward_llm_provider_auth_headers=True)
assert "authorization" in cleaned_with_flag
assert cleaned_with_flag["authorization"] == f"Bearer {FAKE_OAUTH_TOKEN}"
@ -932,9 +932,7 @@ class TestValidateEnvironmentAuthToken:
config = AnthropicModelInfo()
with mock_patch.dict("os.environ", {}, clear=True):
with pytest.raises(
Exception, match=r"ANTHROPIC_API_KEY.*ANTHROPIC_AUTH_TOKEN"
):
with pytest.raises(Exception, match=r"ANTHROPIC_API_KEY.*ANTHROPIC_AUTH_TOKEN"):
config.validate_environment(
headers={},
model="claude-sonnet-4-5-20250929",
@ -980,9 +978,7 @@ class TestGetAuthToken:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
with mock_patch.dict(
"os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True
):
with mock_patch.dict("os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True):
assert AnthropicModelInfo.get_auth_token() == FAKE_AUTH_TOKEN
def test_returns_none_when_not_set(self):
@ -1106,7 +1102,9 @@ class TestGetAuthHeader:
"""Non-standard API key and custom api_base returns Bearer when use_bearer_for_custom_base=True."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
result = AnthropicModelInfo.get_auth_header(api_key="my-custom-key", api_base="https://custom-gateway.com", use_bearer_for_custom_base=True)
result = AnthropicModelInfo.get_auth_header(
api_key="my-custom-key", api_base="https://custom-gateway.com", use_bearer_for_custom_base=True
)
assert result == {"authorization": "Bearer my-custom-key"}
def test_custom_api_base_get_auth_header_uses_x_api_key_when_standard(self):
@ -1124,10 +1122,7 @@ class TestGetApiBaseFallbackChain:
"""Explicit api_base param takes precedence over all env vars."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert (
AnthropicModelInfo.get_api_base("https://explicit.example.com")
== "https://explicit.example.com"
)
assert AnthropicModelInfo.get_api_base("https://explicit.example.com") == "https://explicit.example.com"
def test_defaults_to_anthropic_api(self):
"""get_api_base returns the default Anthropic API base when no env vars are set."""
@ -1180,9 +1175,7 @@ class TestPassthroughAuthToken:
)
config = AnthropicMessagesConfig()
with mock_patch.dict(
"os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True
):
with mock_patch.dict("os.environ", {"ANTHROPIC_AUTH_TOKEN": FAKE_AUTH_TOKEN}, clear=True):
updated_headers, _ = config.validate_anthropic_messages_environment(
headers={},
model="claude-sonnet-4-5-20250929",
@ -1227,6 +1220,52 @@ class TestPassthroughAuthToken:
assert updated_headers["x-api-key"] == FAKE_REGULAR_KEY
assert "authorization" not in updated_headers
def test_passthrough_missing_credentials_raises_authentication_error(self):
"""Passthrough endpoint should raise locally instead of forwarding an unauthenticated request."""
from unittest.mock import patch as mock_patch
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
config = AnthropicMessagesConfig()
with mock_patch.dict("os.environ", {}, clear=True):
with pytest.raises(litellm.AuthenticationError, match="Missing Anthropic API Key"):
config.validate_anthropic_messages_environment(
headers={},
model="claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Hello"}],
optional_params={},
litellm_params={},
api_key=None,
api_base=None,
)
@pytest.mark.parametrize("header_name", ["x-api-key", "X-Api-Key", "X-API-KEY"])
def test_passthrough_client_x_api_key_header_is_kept(self, header_name):
"""A client-forwarded x-api-key header, whatever its casing, should satisfy validation without env credentials."""
from unittest.mock import patch as mock_patch
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
config = AnthropicMessagesConfig()
with mock_patch.dict("os.environ", {}, clear=True):
updated_headers, _ = config.validate_anthropic_messages_environment(
headers={header_name: FAKE_REGULAR_KEY},
model="claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Hello"}],
optional_params={},
litellm_params={},
api_key=None,
api_base=None,
)
assert [name for name in updated_headers if name.lower() == "x-api-key"] == [header_name]
assert updated_headers[header_name] == FAKE_REGULAR_KEY
def test_passthrough_get_complete_url_honours_base_url_env(self):
"""get_complete_url should use ANTHROPIC_BASE_URL when api_base is None."""
from unittest.mock import patch as mock_patch
@ -1290,14 +1329,8 @@ class TestAnthropicThinkingSignatureSelfHeal:
)
assert is_anthropic_invalid_thinking_signature_error("") is False
assert (
is_anthropic_invalid_thinking_signature_error("rate limit exceeded")
is False
)
assert (
is_anthropic_invalid_thinking_signature_error("invalid_request_error: model not found")
is False
)
assert is_anthropic_invalid_thinking_signature_error("rate limit exceeded") is False
assert is_anthropic_invalid_thinking_signature_error("invalid_request_error: model not found") is False
assert is_anthropic_invalid_thinking_signature_error("thinking signature is malformed") is False
def test_strip_thinking_blocks_from_anthropic_messages(self):
@ -1688,10 +1721,7 @@ class TestAnthropicThinkingSignatureSelfHeal:
base = "call_abc123"
sig = "CiIBDDnWx+/a=="
assert (
normalize_anthropic_tool_use_id(f"{base}{THOUGHT_SIGNATURE_SEPARATOR}{sig}")
== base
)
assert normalize_anthropic_tool_use_id(f"{base}{THOUGHT_SIGNATURE_SEPARATOR}{sig}") == base
def test_anthropic_messages_config_http_retry_helpers(self):
import httpx
@ -1715,15 +1745,11 @@ class TestAnthropicThinkingSignatureSelfHeal:
resp_bad = httpx.Response(400, request=req, text="rate limit exceeded")
err_bad = httpx.HTTPStatusError("bad", request=req, response=resp_bad)
assert (
config.should_retry_anthropic_messages_on_http_error(err_bad, {}) is False
)
assert config.should_retry_anthropic_messages_on_http_error(err_bad, {}) is False
resp_500 = httpx.Response(500, request=req, text=err_text)
err_500 = httpx.HTTPStatusError("bad", request=req, response=resp_500)
assert (
config.should_retry_anthropic_messages_on_http_error(err_500, {}) is False
)
assert config.should_retry_anthropic_messages_on_http_error(err_500, {}) is False
data = {
"model": "claude-sonnet-4-20250514",
@ -1746,7 +1772,6 @@ class TestAnthropicThinkingSignatureSelfHeal:
assert data["messages"] == []
class TestClaudeOpus48AdaptiveThinking:
"""Opus 4.8 requires adaptive thinking (``thinking.type='adaptive'`` +
``output_config.effort``). Detection is driven by the
@ -1776,9 +1801,7 @@ class TestClaudeOpus48AdaptiveThinking:
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True
def test_resolver_reads_flag_through_bedrock_invoke_prefix(
self, local_model_cost_map
):
def test_resolver_reads_flag_through_bedrock_invoke_prefix(self, local_model_cost_map):
"""The resolver fix: ``bedrock/invoke/...`` resolves to the flagged
Bedrock entry. Pure ``_supports_factory`` without prefix-stripping
returns False here, which is why the data-only fix alone was not enough."""
@ -1828,9 +1851,7 @@ class TestClaudeOpus48AdaptiveThinking:
"claude-sonnet-4.6",
],
)
def test_adaptive_thinking_detected_for_opus_4_6_4_7_and_sonnet_4_6(
self, local_model_cost_map, model
):
def test_adaptive_thinking_detected_for_opus_4_6_4_7_and_sonnet_4_6(self, local_model_cost_map, model):
"""Opus 4.6/4.7 and Sonnet 4.6 carry the ``supports_adaptive_thinking`` flag,
so detection holds purely from the cost map with no version-rule
fallback. Each alias form the Bedrock/anthropic paths see resolves to a flagged
@ -1850,9 +1871,7 @@ class TestClaudeOpus48AdaptiveThinking:
"claude-fable-preview",
],
)
def test_unmapped_aliases_without_parseable_version_stay_non_adaptive(
self, local_model_cost_map, model
):
def test_unmapped_aliases_without_parseable_version_stay_non_adaptive(self, local_model_cost_map, model):
"""An alias absent from the map, not matched by any ``fallback_generalizations``
rule, and without any parseable family version stays non-adaptive. ``fable``
without a major version matches neither the core-family 4.6+ gate nor the
@ -1878,9 +1897,7 @@ class TestClaudeOpus48AdaptiveThinking:
"us.anthropic.claude-fable-5-preview",
],
)
def test_adaptive_thinking_version_fallback_for_unmapped_high_versions(
self, local_model_cost_map, model
):
def test_adaptive_thinking_version_fallback_for_unmapped_high_versions(self, local_model_cost_map, model):
"""Provider-prefixed or suffixed Claude names that resolve to no mapped entry
still resolve to adaptive when the id carries claude-<family>- at version 4.6
or higher, bare 5+ majors included. The version gate is the declarative
@ -1901,9 +1918,7 @@ class TestClaudeOpus48AdaptiveThinking:
"us.anthropic.claude-opus-4-20250514",
],
)
def test_adaptive_thinking_not_detected_for_unmapped_low_versions(
self, local_model_cost_map, model
):
def test_adaptive_thinking_not_detected_for_unmapped_low_versions(self, local_model_cost_map, model):
"""Unmapped Claude names below 4.6 stay non-adaptive through the declarative path.
The eight-digit dated Opus 4.0 id (``...-4-20250514``) is the date-safety case: the
version rule caps the minor at two digits, so the date is not misread as a >= 4.6
@ -1942,14 +1957,11 @@ class TestDefaultSuffixAdaptiveThinking:
"vertex_ai/claude-fable-5@default",
],
)
def test_default_suffix_models_are_adaptive_thinking(
self, local_model_cost_map, model: str
) -> None:
def test_default_suffix_models_are_adaptive_thinking(self, local_model_cost_map, model: str) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert AnthropicModelInfo._is_adaptive_thinking_model(model, "anthropic") is True, (
f"{model} not classified as adaptive thinking. "
"Check _model_map_lookup_candidates strips @default suffix."
f"{model} not classified as adaptive thinking. Check _model_map_lookup_candidates strips @default suffix."
)
@pytest.mark.parametrize(
@ -1959,15 +1971,11 @@ class TestDefaultSuffixAdaptiveThinking:
("vertex_ai/claude-sonnet-4-6@default", "claude-sonnet-4-6"),
],
)
def test_lookup_candidates_include_bare_name(
self, model: str, expected_bare: str
) -> None:
def test_lookup_candidates_include_bare_name(self, model: str, expected_bare: str) -> None:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
candidates = AnthropicModelInfo._model_map_lookup_candidates(model)
assert expected_bare in candidates, (
f"Expected '{expected_bare}' in candidates for '{model}', got: {candidates}"
)
assert expected_bare in candidates, f"Expected '{expected_bare}' in candidates for '{model}', got: {candidates}"
class TestCapabilityProbeUsesCallerProvider:
@ -1980,42 +1988,27 @@ class TestCapabilityProbeUsesCallerProvider:
BEDROCK_MODEL = "global.anthropic.claude-opus-4-8"
def test_exact_bedrock_entry_flag_is_authoritative_for_bedrock_caller(
self, local_model_cost_map, monkeypatch
):
def test_exact_bedrock_entry_flag_is_authoritative_for_bedrock_caller(self, local_model_cost_map, monkeypatch):
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
assert (
AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock")
is True
)
assert AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") is True
monkeypatch.setitem(
litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False
)
monkeypatch.setitem(litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False)
litellm.get_model_info.cache_clear()
assert (
AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock")
is False
)
assert AnthropicModelInfo._is_adaptive_thinking_model(self.BEDROCK_MODEL, "bedrock") is False
def test_native_anthropic_probe_still_reads_anthropic_entry(
self, local_model_cost_map, monkeypatch
):
def test_native_anthropic_probe_still_reads_anthropic_entry(self, local_model_cost_map, monkeypatch):
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
monkeypatch.setitem(
litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False
)
monkeypatch.setitem(litellm.model_cost[self.BEDROCK_MODEL], "supports_adaptive_thinking", False)
litellm.get_model_info.cache_clear()
assert (
AnthropicModelInfo._is_adaptive_thinking_model("claude-opus-4-8", "anthropic")
is True
)
assert AnthropicModelInfo._is_adaptive_thinking_model("claude-opus-4-8", "anthropic") is True
def test_create_anthropic_model_list_response_shape():
from litellm.llms.anthropic.common_utils import (
create_anthropic_model_list_response,
@ -2100,4 +2093,4 @@ def test_create_anthropic_model_list_response_empty():
assert response["data"] == []
assert response["has_more"] is False
assert response["first_id"] is None
assert response["last_id"] is None
assert response["last_id"] is None

216
tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py
View file

@ -559,3 +559,219 @@ async def test_async_realtime_default_maintains_backwards_compatibility():
mock_realtime_streaming.call_args.kwargs["backend_uses_beta_protocol"]
is True
)
class _DummyAsyncContextManager:
def __init__(self, value):
self.value = value
async def __aenter__(self):
return self.value
async def __aexit__(self, exc_type, exc, tb):
return None
@pytest.mark.asyncio
async def test_async_realtime_uses_bearer_token_when_no_api_key():
"""
Entra ID-only Azure realtime deployments have no static api-key, so the handshake must
authenticate with `Authorization: Bearer <azure_ad_token>` and must not send `api-key`.
Regression test for https://github.com/BerriAI/litellm/issues/34654
"""
from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime
handler = AzureOpenAIRealtime()
mock_backend_ws = AsyncMock()
with (
patch(
"websockets.connect",
return_value=_DummyAsyncContextManager(mock_backend_ws),
) as mock_ws_connect,
patch( # test-quality-ok: handler owns the streaming loop, only the handshake headers are under test
"litellm.llms.azure.realtime.handler.RealTimeStreaming"
) as mock_realtime_streaming,
):
mock_realtime_streaming.return_value.bidirectional_forward = AsyncMock()
await handler.async_realtime(
model="gpt-realtime-whisper",
websocket=AsyncMock(),
logging_obj=MagicMock(),
api_base="https://my-endpoint.openai.azure.com",
api_key=None,
api_version="2024-10-01-preview",
azure_ad_token="my-entra-token",
)
headers = mock_ws_connect.call_args.kwargs["additional_headers"]
assert headers == {"Authorization": "Bearer my-entra-token"}
def test_get_auth_headers_prefers_api_key_and_never_sends_both():
from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime
assert AzureOpenAIRealtime.get_auth_headers(api_key="test-key", azure_ad_token="my-entra-token") == {
"api-key": "test-key"
}
def test_get_auth_headers_without_credentials_raises():
from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime
with pytest.raises(ValueError, match="Missing Azure credentials"):
AzureOpenAIRealtime.get_auth_headers(api_key=None, azure_ad_token=None)
@pytest.mark.asyncio
async def test_arealtime_resolves_azure_ad_token_when_no_api_key(monkeypatch):
"""
`_arealtime` must resolve an Azure AD token (managed identity, service principal, etc.)
and forward it to the handler when the deployment has no api_key.
Regression test for https://github.com/BerriAI/litellm/issues/34654
"""
from litellm.realtime_api import main as realtime_main
mock_async_realtime = AsyncMock()
monkeypatch.setattr(realtime_main, "azure_realtime", MagicMock(async_realtime=mock_async_realtime))
monkeypatch.setattr(
realtime_main,
"get_llm_provider",
lambda model, api_base=None, api_key=None: (
"gpt-realtime-whisper",
"azure",
None,
"https://my-endpoint.openai.azure.com",
),
)
monkeypatch.delenv("AZURE_API_KEY", raising=False)
captured_params = {}
def fake_get_azure_ad_token(litellm_params):
captured_params["tenant_id"] = litellm_params.get("tenant_id")
return "my-entra-token"
monkeypatch.setattr(realtime_main, "get_azure_ad_token", fake_get_azure_ad_token)
await realtime_main._arealtime(
model="azure/gpt-realtime-whisper",
websocket=MagicMock(),
api_version="2024-10-01-preview",
litellm_logging_obj=MagicMock(),
tenant_id="my-tenant",
client_id="my-client",
client_secret="my-secret",
)
assert mock_async_realtime.call_args.kwargs["azure_ad_token"] == "my-entra-token"
assert captured_params["tenant_id"] == "my-tenant"
@pytest.mark.asyncio
async def test_arealtime_does_not_resolve_azure_ad_token_when_api_key_present(monkeypatch):
from litellm.realtime_api import main as realtime_main
mock_async_realtime = AsyncMock()
monkeypatch.setattr(realtime_main, "azure_realtime", MagicMock(async_realtime=mock_async_realtime))
monkeypatch.setattr(
realtime_main,
"get_llm_provider",
lambda model, api_base=None, api_key=None: (
"gpt-realtime-whisper",
"azure",
"test-key",
"https://my-endpoint.openai.azure.com",
),
)
def fail_get_azure_ad_token(litellm_params):
raise AssertionError("should not resolve an AD token when an api_key is configured")
monkeypatch.setattr(realtime_main, "get_azure_ad_token", fail_get_azure_ad_token)
await realtime_main._arealtime(
model="azure/gpt-realtime-whisper",
websocket=MagicMock(),
api_key="test-key",
api_version="2024-10-01-preview",
litellm_logging_obj=MagicMock(),
)
assert mock_async_realtime.call_args.kwargs["azure_ad_token"] is None
@pytest.mark.asyncio
async def test_realtime_health_check_uses_bearer_token_when_no_api_key(monkeypatch):
"""
An Entra ID-only realtime deployment must also pass its realtime health check.
Regression test for https://github.com/BerriAI/litellm/issues/34654
"""
from litellm.realtime_api import main as realtime_main
connect_calls = []
monkeypatch.setattr(
realtime_main,
"get_azure_ad_token",
lambda litellm_params: "my-entra-token",
)
def fake_connect(url, **kwargs):
connect_calls.append(kwargs)
return _DummyAsyncContextManager(MagicMock())
monkeypatch.setattr("websockets.connect", fake_connect)
assert (
await realtime_main._realtime_health_check(
model="gpt-realtime-whisper",
custom_llm_provider="azure",
api_key=None,
api_base="https://my-endpoint.openai.azure.com",
api_version="2024-10-01-preview",
model_params={"tenant_id": "my-tenant"},
)
is True
)
assert connect_calls[0]["additional_headers"] == {"Authorization": "Bearer my-entra-token"}
@pytest.mark.asyncio
async def test_arealtime_forwards_deployment_azure_ad_token(monkeypatch):
"""
The router binds a deployment's `azure_ad_token` to `_arealtime`'s named parameter rather than
**kwargs, so it must still reach the handler.
Regression test for https://github.com/BerriAI/litellm/issues/34654
"""
from litellm.realtime_api import main as realtime_main
mock_async_realtime = AsyncMock()
monkeypatch.setattr(realtime_main, "azure_realtime", MagicMock(async_realtime=mock_async_realtime))
monkeypatch.setattr(
realtime_main,
"get_llm_provider",
lambda model, api_base=None, api_key=None: (
"gpt-realtime-whisper",
"azure",
None,
"https://my-endpoint.openai.azure.com",
),
)
monkeypatch.delenv("AZURE_API_KEY", raising=False)
monkeypatch.setattr(realtime_main.litellm, "api_key", None)
await realtime_main._arealtime(
model="azure/gpt-realtime-whisper",
websocket=MagicMock(),
api_version="2024-10-01-preview",
azure_ad_token="deployment-entra-token",
litellm_logging_obj=MagicMock(),
)
assert mock_async_realtime.call_args.kwargs["azure_ad_token"] == "deployment-entra-token"

2
tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py
View file

@ -322,7 +322,7 @@ async def test_query_param_key_not_leaked_with_dummy_caller_key(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
fake_get,
):
with pytest.raises(litellm.APIConnectionError):
with pytest.raises(litellm.InternalServerError):
await litellm.asearch(
query="secrets",
search_provider=provider,

65
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -284,6 +284,71 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto():
assert optional_params["tool_choice"] == {"auto": {}}
@pytest.mark.parametrize(
"model",
[
"us.openai.gpt-5.6-sol",
"global.openai.gpt-5.6-terra",
"bedrock/converse/us.openai.gpt-5.6-luna",
],
)
def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(model, local_model_cost_map):
"""OpenAI GPT-5.x on Bedrock Converse routes reasoning_effort to
``additionalModelRequestFields.reasoning.effort`` rather than Anthropic ``thinking``."""
config = AmazonConverseConfig()
assert "reasoning_effort" in config.get_supported_openai_params(model=model)
optional_params = config.map_openai_params(
non_default_params={"reasoning_effort": "high"},
optional_params={},
model=model,
drop_params=False,
)
assert optional_params["reasoning"] == {"effort": "high"}
assert "thinking" not in optional_params
assert "reasoning_effort" not in optional_params
_, additional_request_params, _, _ = config._prepare_request_params(optional_params, model)
assert additional_request_params["reasoning"] == {"effort": "high"}
assert "thinking" not in additional_request_params
@pytest.mark.parametrize(
"model",
[
"us.openai.gpt-5.6-sol",
"bedrock/converse/global.openai.gpt-5.6-luna",
],
)
def test_openai_gpt5_converse_never_forwards_thinking(model, local_model_cost_map):
"""GPT-5.x on Converse must never send Anthropic ``thinking``/``output_config`` (Bedrock rejects them).
Regression: ``thinking`` is not advertised as supported, and even when supplied alongside
``reasoning_effort`` in either order it never survives into the request."""
config = AmazonConverseConfig()
supported = config.get_supported_openai_params(model=model)
assert "thinking" not in supported
assert "output_config" not in supported
thinking_block = {"type": "enabled", "budget_tokens": 2048}
for non_default_params in (
{"reasoning_effort": "high", "thinking": thinking_block},
{"thinking": thinking_block, "reasoning_effort": "high"},
):
optional_params = config.map_openai_params(
non_default_params=dict(non_default_params),
optional_params={},
model=model,
drop_params=False,
)
_, additional_request_params, _, _ = config._prepare_request_params(optional_params, model)
assert additional_request_params["reasoning"] == {"effort": "high"}
assert "thinking" not in additional_request_params
@pytest.mark.parametrize(
"model",
[

11
tests/test_litellm/llms/bedrock/test_cross_region_inference_profile_mapping.py
View file

@ -293,15 +293,16 @@ def test_bedrock_gpt_5_6_advertises_only_converse_supported_features(
@pytest.mark.parametrize("profile", GPT_5_6_PROFILES, ids=lambda p: p.model_id)
def test_bedrock_gpt_5_6_offers_tools_but_not_reasoning(profile, local_model_cost_map):
"""Converse rejects the Anthropic-shaped thinking block LiteLLM emits for
reasoning_effort, so neither reasoning param may be offered yet, while the tool
params these models do accept must be."""
def test_bedrock_gpt_5_6_offers_tools_and_reasoning_effort_but_not_thinking(profile, local_model_cost_map):
"""GPT-5.x on Converse maps reasoning_effort to reasoning.effort, so reasoning_effort
is offered while the Anthropic-only thinking/output_config are not, alongside the tool
params these models accept."""
supported = AmazonConverseConfig().get_supported_openai_params(
model=f"bedrock/{profile.model_id}"
)
assert "tools" in supported
assert "tool_choice" in supported
assert "reasoning_effort" not in supported
assert "reasoning_effort" in supported
assert "thinking" not in supported
assert "output_config" not in supported

30
tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
View file

@ -1673,7 +1673,7 @@ class TestBedrockMantleResponsesPricing:
assert info["input_cost_per_token"] == pytest.approx(5.5e-06)
assert info["output_cost_per_token"] == pytest.approx(3.3e-05)
assert info["cache_read_input_token_cost"] == pytest.approx(5.5e-07)
assert info["max_input_tokens"] == 272000
assert info["max_input_tokens"] == 1050000
def test_gpt_5_4_pricing_and_mode(self, local_cost_map):
info = litellm.get_model_info("bedrock_mantle/openai.gpt-5.4")
@ -1681,7 +1681,7 @@ class TestBedrockMantleResponsesPricing:
assert info["input_cost_per_token"] == pytest.approx(2.75e-06)
assert info["output_cost_per_token"] == pytest.approx(1.65e-05)
assert info["cache_read_input_token_cost"] == pytest.approx(2.75e-07)
assert info["max_input_tokens"] == 272000
assert info["max_input_tokens"] == 1050000
@pytest.mark.parametrize(
"model, input_cost, cache_creation_cost, cache_read_cost, output_cost",
@ -1753,13 +1753,14 @@ def _repo_cost_map(map_name: str) -> dict[str, dict[str, object]]:
return json.loads(paths[map_name].read_text())
class TestGpt56MantleRegistryEntries:
"""Locks the gpt-5.6 frontier entries to Bedrock Mantle's live behavior.
class TestMantleGptRegistryEntries:
"""Locks the OpenAI GPT entries to Bedrock Mantle's live behavior.
Mantle enforces a 1,050,000-token prompt maximum for gpt-5.6 sol/terra/luna
(oversize requests 400 with "prompt tokens (N) exceed model maximum
(1050000)", and a 1,030,590-token request completes), matching the OpenAI
Bedrock guide. mode must stay "responses": Mantle's native
and for gpt-5.5 and gpt-5.4 (oversize requests 400 with "prompt tokens (N)
exceed model maximum (1050000)", and a 1,030,590-token request completes
on every one of them), while the AWS model cards still quote 272K for
gpt-5.5 and gpt-5.4. mode must stay "responses": Mantle's native
/v1/chat/completions rejects function tools unless reasoning_effort is
"none", so chat traffic has to keep bridging to the Responses API
(see the responses_api_bridge tests above).
@ -1781,3 +1782,18 @@ class TestGpt56MantleRegistryEntries:
assert entry["mode"] == "responses"
assert entry["use_openai_responses_path"] is True
assert entry["supported_endpoints"] == ["/v1/chat/completions", "/v1/responses"]
@pytest.mark.parametrize("map_name", ("root", "bundled_backup"))
@pytest.mark.parametrize(
"key",
(
"bedrock_mantle/openai.gpt-5.5",
"bedrock_mantle/openai.gpt-5.4",
),
)
def test_gpt_55_and_54_entries_match_mantle_enforced_limits(self, map_name, key):
entry = _repo_cost_map(map_name)[key]
assert entry["max_input_tokens"] == 1050000
assert entry["max_output_tokens"] == 128000
assert entry["mode"] == "responses"
assert entry["use_openai_responses_path"] is True

2
tests/test_litellm/llms/compactifai/test_compactifai.py
View file

@ -172,7 +172,7 @@ def test_compactifai_authentication_error(respx_mock):
json=mock_error, status_code=401
)
with pytest.raises(litellm.APIConnectionError) as exc_info:
with pytest.raises(litellm.AuthenticationError) as exc_info:
litellm.completion(
model="compactifai/cai-llama-3-1-8b-slim",
messages=[{"role": "user", "content": "test"}],

2
tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py
View file

@ -233,7 +233,7 @@ def test_langflow_extra_body_cannot_inject_tweaks_into_run_payload():
return resp
with patch.object(HTTPHandler, "post", side_effect=fake_post):
with pytest.raises(litellm.APIConnectionError):
with pytest.raises(litellm.BadRequestError):
litellm.completion(
model="langflow/my-flow",
messages=[{"role": "user", "content": "hello"}],

199
tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
View file

@ -20,11 +20,24 @@ TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
UNMAPPED_MODEL = "example-org/brand-new-model"
NO_TOOLS_MODEL = "example-org/no-tools-model"
NO_SCHEMA_MODEL = "example-org/no-schema-model"
TOOL_PARAMS = ("tools", "tool_choice", "function_call")
WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
VOICE_NOTE_SCHEMA = {
"type": "object",
"properties": {"title": {"type": "string"}, "summary": {"type": "string"}},
"required": ["title", "summary"],
"additionalProperties": False,
}
JSON_SCHEMA_RESPONSE_FORMAT = {
"type": "json_schema",
"json_schema": {"name": "voice_note", "schema": VOICE_NOTE_SCHEMA, "strict": True},
}
REGEX_RESPONSE_FORMAT = {"type": "regex", "pattern": "(positive|neutral|negative)"}
@pytest.fixture(autouse=True)
def force_local_model_cost(monkeypatch):
@ -48,6 +61,15 @@ def registry_disables_function_calling(monkeypatch):
)
@pytest.fixture
def registry_disables_response_schema(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
f"together_ai/{NO_SCHEMA_MODEL}",
{"litellm_provider": "together_ai", "mode": "chat", "supports_response_schema": False},
)
@pytest.fixture
def together_warning_log(caplog):
from litellm._logging import verbose_logger
@ -70,7 +92,7 @@ def test_supported_params_unmapped_model_keeps_tool_params():
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
assert "response_format" in supported
assert "stream" in supported
assert "temperature" in supported
@ -80,7 +102,7 @@ def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_fun
for param in TOOL_PARAMS:
assert param in supported
assert "response_format" not in supported
assert "response_format" in supported
def test_map_openai_params_tool_calling_model_passes_tools():
@ -148,21 +170,17 @@ def test_map_openai_params_reasoning_model_passes_sampling_params():
assert mapped["max_tokens"] == 512
def test_map_openai_params_drops_text_response_format():
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": {"type": "text"}, "temperature": 0.5},
optional_params={},
model=REASONING_MODEL,
drop_params=False,
)
assert "response_format" not in mapped
assert mapped["temperature"] == 0.5
def test_map_openai_params_keeps_json_response_format():
response_format = {"type": "json_object"}
@pytest.mark.parametrize(
"response_format",
[
{"type": "text"},
{"type": "json_object"},
{"type": "json_object", "schema": VOICE_NOTE_SCHEMA},
JSON_SCHEMA_RESPONSE_FORMAT,
REGEX_RESPONSE_FORMAT,
],
)
def test_map_openai_params_schema_model_passes_response_format_through(response_format):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": response_format},
optional_params={},
@ -173,6 +191,46 @@ def test_map_openai_params_keeps_json_response_format():
assert mapped["response_format"] == response_format
@pytest.mark.parametrize("drop_params", [False, True])
def test_map_openai_params_unmapped_model_passes_response_format_through(drop_params, together_warning_log):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
optional_params={},
model=UNMAPPED_MODEL,
drop_params=drop_params,
)
assert mapped["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
assert UNMAPPED_MODEL in together_warning_log.text
assert "passing response_format through" in together_warning_log.text
def test_map_openai_params_no_schema_model_drops_response_format_with_warning(
registry_disables_response_schema, together_warning_log
):
mapped = TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT, "temperature": 0.5},
optional_params={},
model=NO_SCHEMA_MODEL,
drop_params=True,
)
assert "response_format" not in mapped
assert mapped["temperature"] == 0.5
assert NO_SCHEMA_MODEL in together_warning_log.text
assert "dropping response_format" in together_warning_log.text
def test_map_openai_params_no_schema_model_raises_without_drop_params(registry_disables_response_schema):
with pytest.raises(UnsupportedParamsError, match="response_format"):
TogetherAIChatConfig().map_openai_params(
non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
optional_params={},
model=NO_SCHEMA_MODEL,
drop_params=False,
)
def _transform_response(message: dict) -> ModelResponse:
raw_response_json = {
"id": "chatcmpl-test",
@ -206,26 +264,20 @@ def _transform_response(message: dict) -> ModelResponse:
def test_transform_response_maps_reasoning_to_reasoning_content():
result = _transform_response(
{"role": "assistant", "content": "4", "reasoning": "2+2 equals 4"}
)
result = _transform_response({"role": "assistant", "content": "4", "reasoning": "2+2 equals 4"})
assert result.choices[0].message.content == "4"
assert result.choices[0].message.reasoning_content == "2+2 equals 4"
def test_transform_response_preserves_reasoning_content_field():
result = _transform_response(
{"role": "assistant", "content": "4", "reasoning_content": "adding 2 and 2"}
)
result = _transform_response({"role": "assistant", "content": "4", "reasoning_content": "adding 2 and 2"})
assert result.choices[0].message.reasoning_content == "adding 2 and 2"
def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
iterator = TogetherAIChatConfig().get_model_response_iterator(
streaming_response=iter(()), sync_stream=True
)
iterator = TogetherAIChatConfig().get_model_response_iterator(streaming_response=iter(()), sync_stream=True)
assert isinstance(iterator, OpenAIChatCompletionStreamingHandler)
parsed = iterator.chunk_parser(
@ -241,9 +293,7 @@ def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
def test_streaming_chunk_preserves_tool_call_index_and_id():
iterator = TogetherAIChatConfig().get_model_response_iterator(
streaming_response=iter(()), sync_stream=True
)
iterator = TogetherAIChatConfig().get_model_response_iterator(streaming_response=iter(()), sync_stream=True)
def parse_tool_call_chunk(tool_call: dict):
parsed = iterator.chunk_parser(
@ -374,9 +424,7 @@ def test_together_ai_config_alias_points_at_chat_config():
def test_provider_config_manager_returns_together_chat_config():
from litellm.utils import ProviderConfigManager
config = ProviderConfigManager.get_provider_chat_config(
model=REASONING_MODEL, provider=LlmProviders.TOGETHER_AI
)
config = ProviderConfigManager.get_provider_chat_config(model=REASONING_MODEL, provider=LlmProviders.TOGETHER_AI)
assert isinstance(config, TogetherAIChatConfig)
@ -484,6 +532,72 @@ def test_completion_unmapped_model_sends_tools_to_together():
assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}
def _capture_completion_request(model: str, **completion_kwargs) -> dict:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
captured_requests = []
def respond(request: httpx.Request) -> httpx.Response:
captured_requests.append(request)
return httpx.Response(
200,
json={
"id": "chatcmpl-together-structured",
"object": "chat.completion",
"created": 1234567890,
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": '{"title": "t", "summary": "s"}'},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
)
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
litellm.completion(
model=f"together_ai/{model}",
messages=[{"role": "user", "content": "Summarize with a title and summary."}],
api_key="fake-key",
client=client,
**completion_kwargs,
)
return json.loads(captured_requests[0].content)
def test_completion_unmapped_model_sends_json_schema_to_together():
request_body = _capture_completion_request(
UNMAPPED_MODEL, response_format=JSON_SCHEMA_RESPONSE_FORMAT, drop_params=True
)
assert request_body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
def test_completion_pydantic_response_format_sends_json_schema_to_together():
from pydantic import BaseModel
class VoiceNote(BaseModel):
title: str
summary: str
request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=VoiceNote)
sent = request_body["response_format"]
assert sent["type"] == "json_schema"
assert sent["json_schema"]["name"] == "VoiceNote"
assert sent["json_schema"]["strict"] is True
assert sent["json_schema"]["schema"]["required"] == ["title", "summary"]
def test_completion_regex_response_format_sends_pattern_to_together():
request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=REGEX_RESPONSE_FORMAT)
assert request_body["response_format"] == REGEX_RESPONSE_FORMAT
TOGETHER_CHAT_URL = "https://api.together.ai/v1/chat/completions"
WEATHER_AND_TIME_TOOLS = [
@ -552,14 +666,23 @@ PARALLEL_TOOL_CALL_STREAM = (
_chunk(
{
"tool_calls": [
{"index": 0, "id": "call_weather", "type": "function", "function": {"name": "get_weather", "arguments": ""}}
{
"index": 0,
"id": "call_weather",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}
]
}
),
_chunk({"tool_calls": [{"index": 0, "function": {"arguments": '{"city": "San'}}]}),
_chunk({"tool_calls": [{"index": 0, "function": {"arguments": ' Francisco"}'}}]}),
_chunk(
{"tool_calls": [{"index": 1, "id": "call_time", "type": "function", "function": {"name": "get_time", "arguments": ""}}]}
{
"tool_calls": [
{"index": 1, "id": "call_time", "type": "function", "function": {"name": "get_time", "arguments": ""}}
]
}
),
_chunk({"tool_calls": [{"index": 1, "function": {"arguments": '{"tz": "PST"}'}}]}, finish_reason="tool_calls"),
)
@ -802,10 +925,14 @@ def test_anthropic_messages_streams_together_tool_call_as_input_json_delta():
if event["type"] == "content_block_start" and event["content_block"]["type"] == "tool_use"
}
input_json_deltas = [
event for event in events if event["type"] == "content_block_delta" and event["delta"]["type"] == "input_json_delta"
event
for event in events
if event["type"] == "content_block_delta" and event["delta"]["type"] == "input_json_delta"
]
tool_inputs = {
block["name"]: json.loads("".join(delta["delta"]["partial_json"] for delta in input_json_deltas if delta["index"] == index))
block["name"]: json.loads(
"".join(delta["delta"]["partial_json"] for delta in input_json_deltas if delta["index"] == index)
)
for index, block in tool_starts.items()
}
assert {block["id"] for block in tool_starts.values()} == {"call_weather", "call_time"}

6
tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py
View file

@ -238,7 +238,7 @@ class TestVertexGemmaCompletion:
Expected: Proper error handling when 'predictions' field is missing
"""
from litellm.exceptions import APIConnectionError
from litellm.exceptions import BadRequestError
# Invalid response without predictions field
invalid_response = {
@ -260,8 +260,8 @@ class TestVertexGemmaCompletion:
mock_client.post = AsyncMock(return_value=mock_response)
mock_get_client.return_value = mock_client
# Should raise exception (wrapped as APIConnectionError by LiteLLM)
with pytest.raises(APIConnectionError) as exc_info:
# Should raise exception (wrapped as BadRequestError by LiteLLM)
with pytest.raises(BadRequestError) as exc_info:
await litellm.acompletion(
model="vertex_ai/gemma/gemma-3-12b-it",
messages=[{"role": "user", "content": "Test"}],

279
tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py
View file

@ -7,8 +7,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
from starlette.datastructures import Headers
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
@ -5097,13 +5095,10 @@ class TestMCPDcrBridgeDelegateAdmission:
"""Admission-side arm for a DCR-bridge ``oauth_delegate`` client that authenticates with
a single envelope bearer (LIT-4338).
The arm fires only for a single ``is_dcr_bridge`` ``is_oauth_delegate`` target carrying an
envelope-shaped Authorization. It opens the litellm-signed envelope, reloads the live key
record the sealed ``key_hash`` references so the caller is admitted under the key's current
authorization context (team/org/object-permission) and revocation state, and injects the inner
upstream token under the server's per-server auth-header key so egress forwards it. A key that
is missing, blocked, or expired fails closed with a 401. Everything else must stay on its
existing admission path.
A credential-free request reaches the named MCP handler so it can issue the initial OAuth
challenge. Every bearer on that same route enters envelope resolution. A valid envelope opens
under its live authorization context, while invalid envelopes and non-envelope bearers receive
a named ``invalid_token`` challenge. Everything else stays on its existing admission path.
"""
_MASTER_KEY = "sk-bridge-master-key-for-envelope-derivation"
@ -5138,6 +5133,8 @@ class TestMCPDcrBridgeDelegateAdmission:
minted_at=None,
master_key=None,
):
from pydantic import SecretStr
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (
envelope_keys_from_master_key,
)
@ -5148,7 +5145,6 @@ class TestMCPDcrBridgeDelegateAdmission:
mint_envelope,
user_identity,
)
from pydantic import SecretStr
identity = (
user_identity(server_id=server_id, user_id=user_id)
@ -5272,6 +5268,92 @@ class TestMCPDcrBridgeDelegateAdmission:
request.body = mock_body
return request
async def test_bridge_target_requires_literal_boolean_opt_ins(self):
"""Truthy proxy values must not opt an unresolved server into bridge admission."""
for delegate_value, bridge_value in ((MagicMock(), True), (True, MagicMock())):
server = MagicMock()
server.is_oauth_delegate = delegate_value
server.is_dcr_bridge = bridge_value
server.server_name = "bridge_delegate_server"
server.alias = None
with patch( # test-quality-ok: isolate the MCP registry when testing target selection
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr:
mock_mgr.get_mcp_server_by_name.return_value = server
assert (
MCPRequestHandler._single_dcr_bridge_delegate_target(
path="/mcp/bridge_delegate_server",
mcp_servers=None,
client_ip=None,
)
is None
)
async def test_credential_free_named_bridge_request_reaches_mcp_handler(self):
scope = {
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [],
}
with (
patch( # test-quality-ok: observe the auth boundary while testing admission orchestration
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
new_callable=AsyncMock,
) as mock_auth,
patch( # test-quality-ok: isolate the MCP registry used by request admission
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr,
):
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server()
(
auth_result,
_mcp_auth_header,
_mcp_servers,
mcp_server_auth_headers,
_oauth2_headers,
_raw_headers,
) = await MCPRequestHandler.process_mcp_request(scope)
mock_auth.assert_not_called()
assert auth_result == UserAPIKeyAuth()
assert mcp_server_auth_headers == {}
@pytest.mark.parametrize(
"headers",
(
[(b"x-mcp-auth", b"Bearer upstream-token")],
[(b"x-mcp-bridge_delegate_server-authorization", b"Bearer upstream-token")],
),
ids=("deprecated-mcp-auth", "per-server-auth"),
)
async def test_client_mcp_credentials_do_not_receive_keyless_bridge_admission(self, headers):
scope = {
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": headers,
}
with (
patch( # test-quality-ok: force credential rejection through request admission
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
new_callable=AsyncMock,
side_effect=HTTPException(status_code=401, detail="Invalid key"),
) as mock_auth,
patch( # test-quality-ok: isolate the MCP registry used by request admission
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr,
):
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server()
with pytest.raises(HTTPException) as exc_info:
await MCPRequestHandler.process_mcp_request(scope)
assert exc_info.value.status_code == 401
mock_auth.assert_awaited_once()
async def test_valid_envelope_reloads_live_key_and_admits_its_authorization_context(self):
"""A valid envelope admits under the LIVE key record the sealed key_hash references, not a
blank identity: the reload is keyed by that exact hash, and the admitted auth carries the
@ -5809,6 +5891,7 @@ class TestMCPDcrBridgeDelegateAdmission:
):
_auth, new_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=self._bridge_delegate_server(server_name="bridge_name", alias="bridge_alias"),
requested_name="bridge_name",
authorization_value=f"Bearer {envelope}",
mcp_server_auth_headers=attacker_forwarded,
request=self._mcp_request(),
@ -5845,6 +5928,7 @@ class TestMCPDcrBridgeDelegateAdmission:
):
_auth, new_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=server,
requested_name="bridge_delegate_server",
authorization_value=f"Bearer {envelope}",
mcp_server_auth_headers=None,
request=self._mcp_request(),
@ -5887,8 +5971,7 @@ class TestMCPDcrBridgeDelegateAdmission:
mock_auth.assert_called_once()
async def test_expired_envelope_fails_closed_401(self):
"""An envelope whose exp is in the past must fail closed with a 401, never fall through to
anonymous admission."""
"""An expired envelope fails closed and tells the client where to reauthorize."""
expired = self._mint_bridge_envelope(
expires_in=60,
minted_at=datetime.now(timezone.utc) - timedelta(hours=2),
@ -5897,7 +5980,10 @@ class TestMCPDcrBridgeDelegateAdmission:
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", f"Bearer {expired}".encode("latin-1"))],
"headers": [
(b"host", b"testserver"),
(b"authorization", f"Bearer {expired}".encode("latin-1")),
],
}
with (
@ -5914,6 +6000,12 @@ class TestMCPDcrBridgeDelegateAdmission:
assert exc_info.value.status_code == 401
mock_auth.assert_not_called()
assert exc_info.value.headers == {
"www-authenticate": (
'Bearer error="invalid_token", '
'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/bridge_delegate_server"'
)
}
async def test_envelope_minted_for_a_different_server_fails_closed_401(self):
"""An envelope sealed for another server_id must be rejected when presented to this server,
@ -5924,7 +6016,10 @@ class TestMCPDcrBridgeDelegateAdmission:
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", f"Bearer {wrong_server}".encode("latin-1"))],
"headers": [
(b"host", b"testserver"),
(b"authorization", f"Bearer {wrong_server}".encode("latin-1")),
],
}
with (
@ -5941,6 +6036,12 @@ class TestMCPDcrBridgeDelegateAdmission:
assert exc_info.value.status_code == 401
mock_auth.assert_not_called()
assert exc_info.value.headers == {
"www-authenticate": (
'Bearer error="invalid_token", '
'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/bridge_delegate_server"'
)
}
async def test_envelope_under_wrong_master_key_fails_closed_401(self):
"""An envelope-shaped bearer whose signature does not verify under the proxy's derived keys
@ -5950,7 +6051,10 @@ class TestMCPDcrBridgeDelegateAdmission:
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", f"Bearer {foreign}".encode("latin-1"))],
"headers": [
(b"host", b"testserver"),
(b"authorization", f"Bearer {foreign}".encode("latin-1")),
],
}
with (
@ -5967,26 +6071,71 @@ class TestMCPDcrBridgeDelegateAdmission:
assert exc_info.value.status_code == 401
mock_auth.assert_not_called()
assert exc_info.value.headers == {
"www-authenticate": (
'Bearer error="invalid_token", '
'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/bridge_delegate_server"'
)
}
async def test_non_envelope_bearer_on_bridge_server_falls_through_to_oauth2_arm(self):
"""A plain (non-envelope) bearer on the same bridge server must NOT be admitted by the
envelope arm: it falls through to the oauth2 arm, which validates it as a LiteLLM key and
401s here. Proves the arm is gated on envelope shape, not merely on the target being a
bridge server."""
@pytest.mark.parametrize("requested_name", ["bridge_name", "bridge_alias"])
async def test_invalid_envelope_challenge_names_the_requested_spelling(self, requested_name):
"""A server reachable under both its server_name and a distinct alias must challenge with
metadata for the exact spelling the caller used, matching the per-server well-known
document, so the client rediscovers against the resource it actually asked for."""
foreign = self._mint_bridge_envelope(master_key="a-different-master-key-entirely")
scope = {
"type": "http",
"method": "POST",
"path": f"/mcp/{requested_name}",
"headers": [
(b"host", b"testserver"),
(b"authorization", f"Bearer {foreign}".encode("latin-1")),
],
}
with (
patch( # test-quality-ok: prove standard admission is never consulted for an envelope bearer
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
new_callable=AsyncMock,
) as mock_auth,
patch( # test-quality-ok: isolate the MCP registry, same seam as the sibling challenge tests
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr,
patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY), # test-quality-ok: envelope keys derive from the proxy master_key module global
):
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server(
server_name="bridge_name", alias="bridge_alias"
)
with pytest.raises(HTTPException) as exc_info:
await MCPRequestHandler.process_mcp_request(scope)
assert exc_info.value.status_code == 401
mock_auth.assert_not_called()
assert exc_info.value.headers == {
"www-authenticate": (
'Bearer error="invalid_token", '
f'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/{requested_name}"'
)
}
async def test_non_envelope_bearer_on_bridge_server_returns_named_challenge(self):
"""A raw provider bearer cannot authorize a bridge route and triggers reauthorization."""
scope = {
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", b"Bearer plain-upstream-bearer-not-an-envelope")],
"headers": [
(b"host", b"testserver"),
(b"authorization", b"Bearer plain-upstream-bearer-not-an-envelope"),
],
}
async def mock_user_api_key_auth_fails(api_key, request):
raise HTTPException(status_code=401, detail="Invalid API key")
with (
patch(
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
side_effect=mock_user_api_key_auth_fails,
new_callable=AsyncMock,
side_effect=HTTPException(status_code=401, detail="Invalid key"),
) as mock_auth,
patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr,
patch("litellm.proxy.proxy_server.master_key", self._MASTER_KEY),
@ -5996,8 +6145,77 @@ class TestMCPDcrBridgeDelegateAdmission:
await MCPRequestHandler.process_mcp_request(scope)
assert exc_info.value.status_code == 401
# The envelope arm was skipped, so the oauth2 arm ran and validated the bearer.
mock_auth.assert_called_once()
mock_auth.assert_awaited_once()
assert exc_info.value.headers == {
"www-authenticate": (
'Bearer error="invalid_token", '
'resource_metadata="http://testserver/.well-known/oauth-protected-resource/mcp/bridge_delegate_server"'
)
}
async def test_valid_litellm_authorization_key_uses_standard_admission(self):
scope = {
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", b"Bearer sk-valid-litellm-key")],
}
admitted = UserAPIKeyAuth(api_key="hashed-key", user_id="litellm-key-user")
with (
patch( # test-quality-ok: supply standard key admission through the auth boundary
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
new_callable=AsyncMock,
return_value=admitted,
) as mock_auth,
patch( # test-quality-ok: isolate the MCP registry used by request admission
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr,
patch( # test-quality-ok: configure key classification for the orchestration test
"litellm.proxy.proxy_server.master_key", self._MASTER_KEY
),
):
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server()
(
auth_result,
_mcp_auth,
_servers,
mcp_server_auth_headers,
_oauth,
_raw,
) = await MCPRequestHandler.process_mcp_request(scope)
assert auth_result is admitted
assert mcp_server_auth_headers == {}
assert mock_auth.await_args.kwargs["api_key"] == "Bearer sk-valid-litellm-key"
async def test_non_401_litellm_key_failure_is_not_converted_to_oauth_challenge(self):
scope = {
"type": "http",
"method": "POST",
"path": "/mcp/bridge_delegate_server",
"headers": [(b"authorization", b"Bearer sk-blocked-litellm-key")],
}
with (
patch( # test-quality-ok: force a non-401 auth result through request admission
"litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth",
new_callable=AsyncMock,
side_effect=HTTPException(status_code=403, detail="Key blocked"),
),
patch( # test-quality-ok: isolate the MCP registry used by request admission
"litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager"
) as mock_mgr,
patch( # test-quality-ok: configure key classification for the orchestration test
"litellm.proxy.proxy_server.master_key", self._MASTER_KEY
),
):
mock_mgr.get_mcp_server_by_name.return_value = self._bridge_delegate_server()
with pytest.raises(HTTPException) as exc_info:
await MCPRequestHandler.process_mcp_request(scope)
assert exc_info.value.status_code == 403
assert not exc_info.value.headers
async def test_explicit_litellm_key_wins_over_envelope_arm(self):
"""An explicit x-litellm-api-key is always a LiteLLM credential and its arm precedes the
@ -6126,6 +6344,7 @@ class TestMCPDcrBridgeDelegateAdmission:
):
auth_result, new_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=self._bridge_delegate_server(),
requested_name="bridge_delegate_server",
authorization_value=f"Bearer {envelope}",
mcp_server_auth_headers=existing,
request=self._mcp_request(),
@ -6150,6 +6369,7 @@ class TestMCPDcrBridgeDelegateAdmission:
with pytest.raises(HTTPException) as exc_info:
await MCPRequestHandler._admit_dcr_bridge_delegate(
server=self._bridge_delegate_server(),
requested_name="bridge_delegate_server",
authorization_value=f"Bearer {envelope}",
mcp_server_auth_headers=None,
request=self._mcp_request(),
@ -6168,6 +6388,7 @@ class TestMCPDcrBridgeDelegateAdmission:
with pytest.raises(HTTPException) as exc_info:
await MCPRequestHandler._admit_dcr_bridge_delegate(
server=self._bridge_delegate_server(),
requested_name="bridge_delegate_server",
authorization_value=f"Bearer {envelope}",
mcp_server_auth_headers=None,
request=self._mcp_request(),
@ -6480,8 +6701,8 @@ class TestGatewaySessionAdmission:
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
SessionPrincipal,
mint_session_token,
mint_session_refresh_token,
mint_session_token,
)
keys = session_keys_from_master_key(self._MASTER_KEY)
@ -6947,8 +7168,8 @@ class TestUserSubjectTeamUnion:
def _manager_with(self, server_ids, allow_all=()):
from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager
from litellm.types.mcp_server.mcp_server_manager import MCPServer
from litellm.types.mcp import MCPTransport
from litellm.types.mcp_server.mcp_server_manager import MCPServer
manager = MCPServerManager()
for sid in server_ids:

38
tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py
View file

@ -3,9 +3,9 @@
The envelope is the single client-held bearer carrying both a litellm identity and the
encrypted upstream grant, with zero server-side storage. These tests pin the security
contract: an envelope opens only under the exact keys that minted it, tampering with any
signed byte is detected, expiry is enforced against the injected clock (capped by the
module TTL ceiling), oversized envelopes are rejected rather than truncated, and no
error value, model repr, or raised exception ever contains the inner access token.
signed byte is detected, expiry is enforced against the injected clock and provider
lifetime, oversized envelopes are rejected rather than truncated, and no error value,
model repr, or raised exception ever contains the inner access token.
"""
import base64
@ -30,6 +30,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import
DecryptFailed,
EnvelopeIdentity,
EnvelopeKeys,
EnvelopeLifetimeUnrepresentable,
EnvelopeMintError,
EnvelopeTooLarge,
Expired,
MalformedPayload,
@ -159,6 +161,20 @@ def test_claim_layout_and_no_plaintext_token_in_envelope():
assert _REFRESH_TOKEN not in json.dumps(claims)
def test_unrepresentable_access_lifetime_is_a_typed_mint_error():
grant = UpstreamTokenGrant(
access_token=SecretStr(_ACCESS_TOKEN),
token_type="Bearer",
expires_in=10**30,
)
result = mint_envelope(_IDENTITY, grant, _KEYS, _NOW)
assert isinstance(result, EnvelopeLifetimeUnrepresentable)
assert result.tag == "envelope_lifetime_unrepresentable"
assert result.expires_in == 10**30
def _refresh_credential() -> RefreshCredential:
return RefreshCredential(refresh_token=SecretStr(_REFRESH_TOKEN), scope="read:tools", expires_in=None)
@ -243,11 +259,11 @@ def test_refresh_envelope_never_leaks_the_refresh_token_in_plaintext():
"expires_in, expected_ttl",
[
(600, 600),
(MAX_ENVELOPE_TTL_SECONDS + 82800, MAX_ENVELOPE_TTL_SECONDS),
(MAX_ENVELOPE_TTL_SECONDS + 82800, MAX_ENVELOPE_TTL_SECONDS + 82800),
(None, MAX_ENVELOPE_TTL_SECONDS),
],
)
def test_exp_is_min_of_upstream_expires_in_and_cap(expires_in, expected_ttl):
def test_exp_matches_upstream_lifetime_or_uses_missing_lifetime_fallback(expires_in: int | None, expected_ttl: int):
grant = UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN), token_type="Bearer", expires_in=expires_in)
sealed = mint_envelope(_IDENTITY, grant, _KEYS, _NOW)
assert isinstance(sealed, SealedEnvelope)
@ -261,13 +277,13 @@ def test_expiry_honored_against_injected_clock():
assert isinstance(open_envelope(token, _KEYS, _NOW + timedelta(seconds=601)), Expired)
def test_ttl_cap_enforced_on_open_even_when_upstream_token_lives_longer():
def test_upstream_token_lifetime_is_enforced_on_open():
grant = UpstreamTokenGrant(access_token=SecretStr(_ACCESS_TOKEN), token_type="Bearer", expires_in=86400)
token = _sealed_token(grant)
just_before_cap = _NOW + timedelta(seconds=MAX_ENVELOPE_TTL_SECONDS - 1)
at_cap = _NOW + timedelta(seconds=MAX_ENVELOPE_TTL_SECONDS)
assert isinstance(open_envelope(token, _KEYS, just_before_cap), OpenedEnvelope)
assert isinstance(open_envelope(token, _KEYS, at_cap), Expired)
just_before_expiry = _NOW + timedelta(seconds=86399)
at_expiry = _NOW + timedelta(seconds=86400)
assert isinstance(open_envelope(token, _KEYS, just_before_expiry), OpenedEnvelope)
assert isinstance(open_envelope(token, _KEYS, at_expiry), Expired)
def test_tampering_any_payload_or_signature_byte_is_bad_signature():
@ -420,7 +436,7 @@ def test_decryptable_blob_that_is_not_a_grant_is_malformed_payload():
assert isinstance(open_envelope(forged, _KEYS, _NOW), MalformedPayload)
def _mint_with_token_len(n: int) -> SealedEnvelope | EnvelopeTooLarge:
def _mint_with_token_len(n: int) -> SealedEnvelope | EnvelopeMintError:
grant = UpstreamTokenGrant(access_token=SecretStr("a" * n), token_type="Bearer")
return mint_envelope(_IDENTITY, grant, _KEYS, _NOW)

24
tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
View file

@ -5539,6 +5539,30 @@ async def test_bridge_envelope_too_large_upstream_token_is_502():
assert json.loads(response.body)["error"] == "server_error"
@pytest.mark.asyncio
async def test_bridge_envelope_unrepresentable_upstream_lifetime_is_502():
from litellm.types.mcp import MCPAuth
server = _bridge_server(auth_type=MCPAuth.oauth_delegate)
upstream = {
"access_token": "UPSTREAM-SECRET-TOKEN",
"token_type": "Bearer",
"expires_in": 10**30,
}
response = await _exchange_for_bridge_server(
server,
upstream,
key_hash="hashed-litellm-key-77",
)
assert response.status_code == 502
assert json.loads(response.body) == {
"error": "server_error",
"error_description": "the upstream token response reports an unrepresentable lifetime",
}
@pytest.mark.asyncio
async def test_bridge_access_envelope_never_carries_upstream_refresh_token():
"""The upstream refresh token is never sealed into the ACCESS envelope, the bearer forwarded upstream

4
tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
View file

@ -1948,14 +1948,14 @@ class FakePodLockManager:
if self.redis_cache is not None:
self.redis_cache.async_get_cache = AsyncMock(return_value="another-pod" if held_by_other else None)
self._acquired = acquired
self.acquire_calls: List[Dict[str, Any]] = []
self.acquire_calls: List[Dict[str, str | int | None]] = []
self.release_calls: List[str] = []
@staticmethod
def get_redis_lock_key(cronjob_id: str) -> str:
return f"cronjob_lock:{cronjob_id}"
async def acquire_lock(self, cronjob_id: str, ttl: Any = None) -> bool:
async def acquire_lock(self, cronjob_id: str, ttl: int | None = None) -> bool:
self.acquire_calls.append({"cronjob_id": cronjob_id, "ttl": ttl})
return self._acquired

14
tests/test_litellm/proxy/common_utils/test_sse_keepalive.py
View file

@ -10,6 +10,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
ANTHROPIC_PING_SSE_CHUNK,
SSE_COMMENT_PING_BYTES,
resolve_ttft_keepalive_interval,
split_complete_sse_frames,
wrap_passthrough_sse_bytes_with_keepalive_pings,
wrap_sse_stream_with_keepalive_pings,
)
@ -18,6 +19,19 @@ MESSAGE_START_CHUNK: Final = 'data: {"type": "message_start"}\n\n'
TEXT_DELTA_CHUNK: Final = 'data: {"type": "content_block_delta"}\n\n'
@pytest.mark.parametrize("delimiter", [b"\n\n", b"\r\n\r\n", b"\r\r"])
def test_split_complete_sse_frames_recognizes_every_sse_frame_delimiter(delimiter: bytes):
newline: Final = delimiter[: len(delimiter) // 2]
frame: Final = b"event: response.created" + newline + b"data: {}" + delimiter
tail: Final = b"data: partial"
assert split_complete_sse_frames(frame + tail) == (frame, tail)
def test_split_complete_sse_frames_holds_bytes_with_no_complete_frame():
assert split_complete_sse_frames(b"data: unterminated") == (b"", b"data: unterminated")
@pytest.mark.asyncio
async def test_pings_fill_mid_stream_silence_and_preserve_chunk_order():
async def gappy_stream() -> AsyncGenerator[str, None]:

78
tests/test_litellm/proxy/db/test_db_url_settings.py
View file

@ -34,6 +34,7 @@ def _apply() -> bool:
_MANAGED_DB_ENV_VARS = (
"IAM_TOKEN_DB_AUTH",
"AZURE_POSTGRESQL_AUTH",
"DATABASE_DISABLE_PREPARED_STATEMENTS",
"DATABASE_URL",
"DIRECT_URL",
"DATABASE_URL_READ_REPLICA",
@ -656,6 +657,83 @@ def test_reader_url_left_alone_when_writer_has_no_params(monkeypatch):
)
# ---------------------------------------------------------------------------
# DATABASE_DISABLE_PREPARED_STATEMENTS
# ---------------------------------------------------------------------------
def test_disable_prepared_statements_appends_pgbouncer_to_assembled_writer(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "true")
monkeypatch.setenv("DATABASE_HOST", "writer.example.com")
monkeypatch.setenv("DATABASE_USER", "litellm")
monkeypatch.setenv("DATABASE_NAME", "litellm_db")
monkeypatch.setenv("DATABASE_PASSWORD", "s3cr3t")
assert _apply() is True
assert os.environ["DATABASE_URL"] == (
"postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?pgbouncer=true"
)
assert "DIRECT_URL" not in os.environ
def test_disable_prepared_statements_appends_pgbouncer_to_pinned_writer(monkeypatch):
"""The componentized entrypoints (gateway / backend / migrations) receive a
pinned DATABASE_URL and call apply_to_env; without the pgbouncer param Prisma
keeps named prepared statements and 42P05 collisions surface behind a
transaction-pooling pgbouncer."""
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "true")
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
assert _apply() is False
assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=true"
def test_disable_prepared_statements_respects_a_pinned_pgbouncer_value(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "true")
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=false")
_apply()
assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=false"
def test_disable_prepared_statements_applies_to_direct_url(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "true")
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@direct.example.com:5432/litellm_db")
_apply()
assert os.environ["DIRECT_URL"] == "postgresql://u:p@direct.example.com:5432/litellm_db?pgbouncer=true"
def test_reader_inherits_pgbouncer_from_disable_prepared_statements(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "true")
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db")
monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "postgresql://u:p@reader.example.com:5432/db")
_apply()
query = urllib.parse.parse_qs(urllib.parse.urlsplit(os.environ["DATABASE_URL_READ_REPLICA"]).query)
assert query["pgbouncer"] == ["true"]
def test_disable_prepared_statements_off_leaves_urls_alone(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "false")
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
_apply()
assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db"
def test_disable_prepared_statements_rejects_an_unreadable_value(monkeypatch):
monkeypatch.setenv("DATABASE_DISABLE_PREPARED_STATEMENTS", "enabled")
with pytest.raises(ValidationError, match="DATABASE_DISABLE_PREPARED_STATEMENTS"):
DatabaseURLSettings.from_env()
def test_unsupported_db_scheme_message_names_var_and_scheme():
msg = unsupported_db_scheme_message("DIRECT_URL", "sqlite")
assert "DIRECT_URL" in msg

38
tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py
View file

@ -6,11 +6,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from fastapi import FastAPI
from fastapi.testclient import TestClient
from prisma.errors import ClientNotConnectedError, HTTPClientClosedError, PrismaError
import litellm
import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module
from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@ -2674,3 +2677,38 @@ class TestNoRedisWarning:
):
details = await _health_endpoints_module._get_health_readiness_details()
assert details["show_no_redis_warning"] is False
def test_test_model_connection_accepts_image_edit_mode(monkeypatch):
"""
Regression: /health/test_connection rejected mode=image_edit with a 422
before image_edit was added to its mode Literal, breaking the UI Test
Connection button for image edit deployments.
"""
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
app = FastAPI()
app.include_router(_health_endpoints_module.router)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN
)
client = TestClient(app)
with (
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: the endpoint reads the proxy-global DB client and 500s when it is None; it has no injection seam
respx.mock(assert_all_called=True) as respx_mock,
):
respx_mock.post(host="api.openai.com", path="/v1/images/edits").respond(
json={"created": 1700000000, "data": [{"b64_json": TEST_IMAGE_BASE64}]}
)
response = client.post(
"/health/test_connection",
json={
"mode": "image_edit",
"litellm_params": {"model": "openai/gpt-image-2", "api_key": "sk-test"},
},
)
assert response.status_code == 200, response.text
assert response.json()["status"] == "success"

129
tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py
View file

@ -780,3 +780,132 @@ class TestBlockRequestsForModelsWithoutPricing:
assert response.status_code == 500
assert "error" in response.json()["detail"]
AN_ALIAS = "onprem/alias"
AN_UNDERLYING_MODEL = "vendor/model"
A_MAPPED_MODEL = "openai/mapped-only-model"
INPUT_TOKENS = 1000
OUTPUT_TOKENS = 500
def _router_pricing(**pricing: float) -> MagicMock:
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
{
"model_name": AN_ALIAS,
"litellm_params": {
"model": AN_UNDERLYING_MODEL,
"custom_llm_provider": "openai",
**pricing,
},
"model_info": {},
}
]
return mock_router
async def _estimate(mock_router: MagicMock | None, model: str = AN_ALIAS, **overrides: int):
from litellm.proxy._types import CostEstimateRequest
from litellm.proxy.management_endpoints.cost_tracking_settings import estimate_cost
request = CostEstimateRequest(
model=model,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
**overrides,
)
with patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
"litellm.proxy.proxy_server.llm_router", mock_router
):
return await estimate_cost(request=request, user_api_key_dict=MagicMock())
class TestEstimateCostPartiallyPricedDeployments:
@pytest.mark.asyncio
async def test_a_deployment_that_prices_only_input_bills_output_at_zero(self):
response = await _estimate(_router_pricing(input_cost_per_token=0.000001))
assert response.input_cost_per_token == pytest.approx(0.000001)
assert response.output_cost_per_token == 0.0
assert response.cost_per_request == pytest.approx(0.001)
@pytest.mark.asyncio
async def test_a_deployment_that_prices_only_output_bills_input_at_zero(self):
response = await _estimate(_router_pricing(output_cost_per_token=0.000002))
assert response.input_cost_per_token == 0.0
assert response.output_cost_per_token == pytest.approx(0.000002)
assert response.cost_per_request == pytest.approx(0.001)
@pytest.mark.asyncio
async def test_a_model_priced_only_by_the_cost_map_reports_that_price_and_provider(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 0.000005,
"output_cost_per_token": 0.000006,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate(None, model=A_MAPPED_MODEL)
assert response.input_cost_per_token == pytest.approx(0.000005)
assert response.output_cost_per_token == pytest.approx(0.000006)
assert response.provider == "openai"
class TestEstimateCostPeriodTotals:
@pytest.mark.asyncio
async def test_zero_requests_a_day_reports_no_daily_cost_rather_than_zero(self):
response = await _estimate(
_router_pricing(input_cost_per_token=0.000001, output_cost_per_token=0.000002),
num_requests_per_day=0,
)
assert response.daily_cost is None
assert response.daily_input_cost is None
assert response.daily_output_cost is None
@pytest.mark.asyncio
async def test_daily_totals_scale_every_component_by_the_request_count(self):
response = await _estimate(
_router_pricing(input_cost_per_token=0.000001, output_cost_per_token=0.000002),
num_requests_per_day=100,
)
assert response.input_cost_per_request == pytest.approx(0.001)
assert response.output_cost_per_request == pytest.approx(0.001)
assert response.daily_input_cost == pytest.approx(0.1)
assert response.daily_output_cost == pytest.approx(0.1)
assert response.daily_cost == pytest.approx(0.2)
@pytest.mark.asyncio
async def test_a_month_and_a_day_are_totalled_from_their_own_request_counts(self):
response = await _estimate(
_router_pricing(input_cost_per_token=0.000001, output_cost_per_token=0.000002),
num_requests_per_day=100,
num_requests_per_month=3000,
)
assert response.daily_cost == pytest.approx(0.2)
assert response.monthly_cost == pytest.approx(6.0)
assert response.monthly_input_cost == pytest.approx(3.0)
assert response.monthly_output_cost == pytest.approx(3.0)
@pytest.mark.asyncio
async def test_a_configured_margin_is_totalled_per_period_like_the_other_components(self, monkeypatch):
monkeypatch.setattr(litellm, "cost_margin_config", {"openai": 0.10})
response = await _estimate(
_router_pricing(input_cost_per_token=0.000001, output_cost_per_token=0.000002),
num_requests_per_day=100,
)
assert response.margin_cost_per_request == pytest.approx(0.0002)
assert response.cost_per_request == pytest.approx(0.0022)
assert response.daily_margin_cost == pytest.approx(0.02)
assert response.daily_cost == pytest.approx(0.22)

32
tests/test_litellm/proxy/management_endpoints/test_ui_sso.py
View file

@ -33,6 +33,15 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
TeamMappings,
)
_SSO_PROVIDER_ENV_VARS = (
"DISABLE_ADMIN_UI",
"MICROSOFT_CLIENT_ID",
"GOOGLE_CLIENT_ID",
"GENERIC_CLIENT_ID",
"SAML_IDP_METADATA_URL",
"SAML_IDP_METADATA_XML",
)
def _wire_team_create_tx(prisma_client):
"""`/team/new` inserts the team and mirrors it onto the access groups in one transaction,
@ -2796,10 +2805,15 @@ class TestCLIKeyRegenerationFlow:
mock_request.base_url = "https://proxy.example.com/"
mock_cache = MagicMock(redis_cache=None)
mock_cache.get_cache.return_value = {"poll_secret_hash": "h"}
env_without_sso_providers = {
name: value
for name, value in os.environ.items()
if name not in _SSO_PROVIDER_ENV_VARS
}
async def drive(enabled: bool):
with (
patch.dict(os.environ, {}, clear=True),
patch.dict(os.environ, env_without_sso_providers, clear=True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch("litellm.proxy.proxy_server.user_api_key_cache", mock_cache),
@ -2825,15 +2839,13 @@ class TestCLIKeyRegenerationFlow:
return_value=None,
) as mock_get_cli_state,
):
try:
await google_login(
request=mock_request,
source="litellm-cli",
key="cli-validsessionkey123456",
user_code="WXYZ-2345",
)
except Exception:
pass
await google_login(
request=mock_request,
source="litellm-cli",
key="cli-validsessionkey123456",
user_code="WXYZ-2345",
)
assert mock_get_cli_state.called
return mock_get_cli_state.call_args.kwargs["user_code"]
assert await drive(enabled=True) == "WXYZ-2345"

Some files were not shown because too many files have changed in this diff Show more