scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_decrease_anys_opus5_0826

Browse source 
# Conflicts:
#	basedpyright-code-budget.json
#	ruff-strict-budget.json
#	type-discipline-budget.json
This commit is contained in:
mateo-berri 2026-08-27 10:39:56 +00:00
commit 2483a34dbc
410 changed files with 20928 additions and 3572 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/mutmut-coverage.rc vendored Normal file
View file

@ -0,0 +1,5 @@
# mutmut's gather_coverage() looks covered lines up by absolute path, so the
# repo's `relative_files = true` makes every lookup miss and mutmut generates
# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
[run]
relative_files = false

10
.github/workflows/mutation-test.yml vendored
View file

@ -87,11 +87,20 @@ jobs:
run: |
uv pip uninstall pytest-retry || true
# Ends before the job's own deadline so a run that outlasts the budget is
# still followed by the report and upload steps. mutmut saves after every
# mutant result, to mutants/<source path>.meta, so an interrupted run
# still scores the mutants it finished and export-cicd-stats can read
# them; a cancelled job skips those steps and publishes nothing at all.
- name: Run mutmut
timeout-minutes: 300
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
# Without this mutmut finds no covered lines and generates 0 mutants.
# See the file itself for why.
COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
run: |
set -o pipefail
mkdir -p mutants
@ -130,6 +139,7 @@ jobs:
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/**/*.meta
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

2
.gitignore vendored
View file

@ -3,6 +3,8 @@
tests/e2e/.fixtures/
.venv-typecheck
.venv_policy_test
.venv-mutmut
mutants/
.env
.claude
CLAUDE.local.md

2
CLAUDE.md
View file

@ -66,6 +66,8 @@ Commit and push your work when you're done without asking
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

36
basedpyright-code-budget.json
View file

@ -1,9 +1,9 @@
{
"reportAny": {
"limit": 17283
"limit": 18483
},
"reportArgumentType": {
"limit": 2551
"limit": 2564
},
"reportAssignmentType": {
"limit": 320
@ -12,25 +12,25 @@
"limit": 483
},
"reportCallIssue": {
"limit": 112
"limit": 113
},
"reportConstantRedefinition": {
"limit": 40
},
"reportDeprecated": {
"limit": 212
"limit": 213
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 5498
"limit": 5960
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 150
"limit": 154
},
"reportIncompatibleMethodOverride": {
"limit": 56
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5658
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15447
"limit": 15484
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1055
"limit": 1058
},
"reportOptionalOperand": {
"limit": 0
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1810
"limit": 1808
},
"reportRedeclaration": {
"limit": 8
@ -93,25 +93,25 @@
"limit": 213
},
"reportTypedDictNotRequiredAccess": {
"limit": 25
"limit": 26
},
"reportUndefinedVariable": {
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44530
"limit": 44528
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38770
"limit": 38804
},
"reportUnknownParameterType": {
"limit": 19798
"limit": 19829
},
"reportUnknownVariableType": {
"limit": 30330
"limit": 30355
},
"reportUnnecessaryCast": {
"limit": 117
@ -123,7 +123,7 @@
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 831
"limit": 833
},
"reportUntypedBaseClass": {
"limit": 0
@ -135,12 +135,12 @@
"limit": 21
},
"reportUnusedFunction": {
"limit": 139
"limit": 138
},
"reportUnusedImport": {
"limit": 544
},
"reportUnusedVariable": {
"limit": 140
"limit": 145
}
}

3
ci_cd/generate_model_prices_schema.py
View file

@ -157,6 +157,9 @@ COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",

10
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -1,6 +1,8 @@
"""
Polls LiteLLM_ManagedObjectTable to check if the response is complete.
Cost tracking is handled automatically by the get-responses call.
Cost tracking is handled by the get-responses call, which prices normally only because the
poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -113,7 +117,8 @@ class CheckResponsesCost:
Check if background responses are complete and track their cost.
- Get all status="queued" or "in_progress" and file_purpose="response" jobs
- Query the provider to check if response is complete
- Cost is automatically tracked by the get-responses call
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
"""
try:
@ -153,6 +158,7 @@ class CheckResponsesCost:
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available

130
litellm/_redis.py
View file

@ -12,8 +12,9 @@ import json
# s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
import os
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
from urllib.parse import urlsplit, urlunsplit
import redis
import redis.asyncio as async_redis
@ -50,6 +51,7 @@ def _get_redis_kwargs():
include_args: Final = {
"url",
"redis_connect_func",
"credential_provider",
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
@ -155,7 +157,8 @@ def _get_redis_cluster_kwargs(client=None):
def _get_redis_env_kwarg_mapping():
PREFIX: Final = "REDIS_"
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs()}
exclude_from_environment: Final = frozenset({"credential_provider"})
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment}
def _redis_kwargs_from_environment():
@ -353,6 +356,12 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _url_without_userinfo(url: str) -> str:
parts: Final = urlsplit(url)
netloc: Final = parts.netloc.rsplit("@", 1)[-1]
return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
def _get_redis_client_logic(**env_overrides):
"""
Common functionality across sync + async redis client implementations
@ -410,54 +419,58 @@ def _get_redis_client_logic(**env_overrides):
if _service_name is not None:
redis_kwargs["service_name"] = _service_name
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
if redis_kwargs.get("credential_provider") is None:
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str(
"REDIS_GCP_SERVICE_ACCOUNT"
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str("AZURE_CLIENT_SECRET")
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str(
"AZURE_CLIENT_SECRET"
)
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
@ -465,6 +478,13 @@ def _get_redis_client_logic(**env_overrides):
redis_kwargs.pop("azure_tenant_id", None)
redis_kwargs.pop("azure_client_secret", None)
if redis_kwargs.get("credential_provider") is not None:
redis_kwargs.pop("redis_connect_func", None)
redis_kwargs.pop("username", None)
redis_kwargs.pop("password", None)
if redis_kwargs.get("url") is not None:
redis_kwargs["url"] = _url_without_userinfo(redis_kwargs["url"])
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
# Only strip host/port/db/password when not routing to a cluster.
# When startup_nodes is also present the cluster path takes priority and
@ -532,8 +552,7 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
service_name: Final = redis_kwargs.get("service_name")
connection_kwargs: Final = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs: Final = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
sentinel_kwargs: Final = _sentinel_auth_kwargs(connection_kwargs, sentinel_password)
if not sentinel_nodes or not service_name:
raise ValueError("Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.")
@ -605,7 +624,12 @@ def _async_credential_provider(redis_connect_func: object | None) -> CredentialP
def _async_auth_kwargs(redis_kwargs: dict) -> dict:
"""Swaps a connect func an async path cannot run for the equivalent credential provider,
which supersedes any static username or password redis-py would otherwise reject it with."""
credential_provider: Final = _async_credential_provider(redis_kwargs.get("redis_connect_func"))
explicit_provider: Final = redis_kwargs.get("credential_provider")
credential_provider: Final = (
explicit_provider
if explicit_provider is not None
else _async_credential_provider(redis_kwargs.get("redis_connect_func"))
)
if credential_provider is None:
return redis_kwargs
@ -738,8 +762,20 @@ def get_redis_connection_pool(
return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)
def _redis_kwargs_for_logging(redis_kwargs: Mapping[str, object]) -> Mapping[str, object]:
return {
key: "<credential provider>"
if key == "credential_provider" and value is not None
else "<redis connect function>"
if key == "redis_connect_func" and value is not None
else value
for key, value in redis_kwargs.items()
}
def _pretty_print_redis_config(redis_kwargs: dict) -> None:
"""Pretty print the Redis configuration using rich with sensitive data masking"""
redis_kwargs_for_logging: Final = _redis_kwargs_for_logging(redis_kwargs)
try:
import logging
@ -757,7 +793,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
masker = SensitiveDataMasker()
# Mask sensitive data in redis_kwargs
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
# Create main panel title
title: Final = Text("Redis Configuration", style="bold blue")
@ -820,7 +856,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
except ImportError:
# Fallback to simple logging if rich is not available
masker = SensitiveDataMasker()
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
verbose_logger.info("Redis configuration: %s", masked_redis_kwargs)
except Exception as e:
verbose_logger.error("Error pretty printing Redis configuration: %s", e)

4
litellm/batches/batch_utils.py
View file

@ -551,7 +551,7 @@ def _get_batch_job_usage_from_response_body(
return usage
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict:
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
"""
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
@ -563,7 +563,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
def _get_response_from_batch_job_output_file(
batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
) -> Any:
) -> Mapping[str, Any]:
"""
Get the response from the batch job output file
"""

43
litellm/caching/caching_handler.py
View file

@ -18,7 +18,7 @@ import asyncio
import datetime
import inspect
import time
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import BaseModel
@ -27,6 +27,7 @@ import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache
from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
update_response_metadata,
)
@ -124,6 +125,29 @@ def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") ->
return details.model_dump(exclude_none=True) if hasattr(details, "model_dump") else {}
_PENDING_CACHE_WRITES: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs to pending write tasks
async def _complete_cache_write_despite_cancellation(write_factory: Callable[[], Awaitable[None]]) -> None:
try:
await write_factory()
except asyncio.CancelledError:
try:
await asyncio.wait_for(write_factory(), timeout=CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS)
except Exception as flush_error: # noqa: BLE001 # shutdown flush failures are logged, never raised
verbose_logger.warning(
"LiteLLM Cache: pending cache write failed during event loop shutdown: %s", flush_error
)
raise
def create_cache_write_task(write_factory: Callable[[], Awaitable[None]]) -> "asyncio.Task[None]":
task: Final = asyncio.create_task(_complete_cache_write_despite_cancellation(write_factory))
_PENDING_CACHE_WRITES.add(task)
task.add_done_callback(_PENDING_CACHE_WRITES.discard)
return task
def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None:
"""Read the caller-supplied ``cache_key`` off the request kwargs."""
return request_kwargs.get("cache_key", None)
@ -983,6 +1007,7 @@ class LLMCachingHandler:
if litellm.cache is None:
return
cache: Final = litellm.cache
new_kwargs: Final = kwargs.copy()
new_kwargs.update(
@ -1004,24 +1029,24 @@ class LLMCachingHandler:
):
if (
isinstance(result, EmbeddingResponse)
and litellm.cache is not None
and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude.
and not isinstance(cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude.
):
asyncio.create_task(
litellm.cache.async_add_cache_pipeline(
create_cache_write_task(
lambda: cache.async_add_cache_pipeline(
result, dynamic_cache_object=self.dual_cache, **new_kwargs
)
)
else:
asyncio.create_task(
litellm.cache.async_add_cache(
result.model_dump_json(),
result_json: Final = result.model_dump_json()
create_cache_write_task(
lambda: cache.async_add_cache(
result_json,
dynamic_cache_object=self.dual_cache,
**new_kwargs,
)
)
else:
asyncio.create_task(litellm.cache.async_add_cache(result, **new_kwargs))
create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
def sync_set_cache(
self,

13
litellm/caching/redis_cache.py
View file

@ -175,6 +175,10 @@ _RedisCallResult = TypeVar("_RedisCallResult")
_swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0)
def _opaque_kwarg_key(value: object) -> str:
return f"{type(value).__name__}-{id(value)}"
@functools.lru_cache(maxsize=1)
def _redis_health_error_types() -> tuple[type, ...]:
"""Exception types that mean the Redis backend itself is unhealthy.
@ -399,10 +403,9 @@ class RedisCache(BaseCache):
Generate a cache key for the async Redis client based on connection parameters.
This ensures different Redis configurations use different cached clients.
"""
# Create a stable representation of redis_kwargs for hashing
# Sort keys to ensure consistent hash regardless of parameter order
sorted_kwargs: Final = sorted(self.redis_kwargs.items())
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True)
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True, default=_opaque_kwarg_key)
kwargs_hash: Final = hashlib.sha256(kwargs_str.encode()).hexdigest()[:16]
return f"async-redis-client-{kwargs_hash}"
@ -432,7 +435,7 @@ class RedisCache(BaseCache):
"""
if key is None:
return key
if self.namespace is not None and not key.startswith(self.namespace):
if self.namespace and not key.startswith(self.namespace + ":"):
key = self.namespace + ":" + key
return key
@ -1384,10 +1387,10 @@ class RedisCache(BaseCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from .._redis import get_redis_async_client
# Create a fresh Redis client with current settings
redis_client: Final = redis_async.Redis(**self.redis_kwargs)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

17
litellm/caching/redis_cluster_cache.py
View file

@ -64,22 +64,9 @@ class RedisClusterCache(RedisCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from redis.cluster import ClusterNode
from .._redis import get_redis_async_client
# Create ClusterNode objects from startup_nodes
cluster_kwargs: Final = self.redis_kwargs.copy()
startup_nodes: Final = cluster_kwargs.pop("startup_nodes", [])
new_startup_nodes: Final[list[ClusterNode]] = []
for item in startup_nodes:
new_startup_nodes.append(ClusterNode(**item))
# Create a fresh Redis Cluster client with current settings
redis_client: Final = redis_async.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

40
litellm/constants.py
View file

@ -381,6 +381,7 @@ AZURE_OPERATION_POLLING_TIMEOUT: Final = int(os.getenv("AZURE_OPERATION_POLLING_
AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: Final = str(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2024-11-30"))
AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = int(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96))
REDIS_SOCKET_TIMEOUT: Final = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1))
CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS: Final[float] = 5.0
REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5))
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
@ -1363,8 +1364,6 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@ -1813,6 +1812,43 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
# A retrieved response replays the usage of the call that created it, so pricing these
# read/management routes like inference bills the same tokens twice.
NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(
{
"get_responses",
"aget_responses",
"delete_responses",
"adelete_responses",
"cancel_responses",
"acancel_responses",
"list_input_items",
"alist_input_items",
"vector_store_create",
"avector_store_create",
"vector_store_retrieve",
"avector_store_retrieve",
"vector_store_list",
"avector_store_list",
"vector_store_update",
"avector_store_update",
"vector_store_delete",
"avector_store_delete",
"vector_store_file_create",
"avector_store_file_create",
"vector_store_file_list",
"avector_store_file_list",
"vector_store_file_retrieve",
"avector_store_file_retrieve",
"vector_store_file_content",
"avector_store_file_content",
"vector_store_file_update",
"avector_store_file_update",
"vector_store_file_delete",
"avector_store_file_delete",
}
)
# PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this
# sentinel api_key so PTU flat cost stays distinguishable from real per-request
# spend under the table's composite unique constraint.

115
litellm/cost_calculator.py
View file

@ -2,6 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -591,6 +592,7 @@ def cost_per_token(
prompt_characters=prompt_characters,
completion_characters=completion_characters,
usage=usage_block,
service_tier=service_tier,
vertex_location=vertex_location,
)
elif cost_router == "cost_per_token":
@ -794,14 +796,27 @@ def _select_model_name_for_cost_calc(
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
if region_name is not None:
return_model = f"{custom_llm_provider}/{region_name}/{return_model}"
else:
return_model = f"{custom_llm_provider}/{return_model}"
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
return return_model
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069)."""
segments: Final = model.split("/")
if "/".join(segments[1:]) in litellm.model_cost:
return model
head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1
head: Final = "/".join(segments[:head_len])
tail: Final = segments[head_len:]
strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail))
candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1))
return next((candidate for candidate in candidates if candidate in litellm.model_cost), model)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _model_contains_known_llm_provider(model: str) -> bool:
"""
@ -832,9 +847,11 @@ def _get_response_model(completion_response: object) -> str | None:
_GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
# ON_DEMAND_PRIORITY maps to "priority" — selects input_cost_per_token_priority, etc.
"ON_DEMAND_PRIORITY": "priority",
# FLEX / BATCH maps to "flex" — selects input_cost_per_token_flex, etc.
# FLEX / BATCH / ON_DEMAND_FLEX maps to "flex" — selects input_cost_per_token_flex, etc.
# Vertex AI reports flex/shared-capacity traffic as ON_DEMAND_FLEX, not FLEX.
"FLEX": "flex",
"BATCH": "flex",
"ON_DEMAND_FLEX": "flex",
# ON_DEMAND is standard pricing — no service_tier suffix applied
"ON_DEMAND": None,
}
@ -849,9 +866,9 @@ def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
trafficType values seen in practice
------------------------------------
ON_DEMAND -> standard pricing (service_tier = None)
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
FLEX / BATCH -> batch/flex pricing (service_tier = "flex")
ON_DEMAND -> standard pricing (service_tier = None)
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
FLEX / BATCH / ON_DEMAND_FLEX -> batch/flex pricing (service_tier = "flex")
"""
if traffic_type is None:
return None
@ -2357,6 +2374,64 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
def _candidate_realtime_token_costs(
model_name: str,
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float] | None:
try:
return generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
return None
def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool:
entries: Final = (
litellm.model_cost.get(model_name),
litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"),
)
return any(
entry is not None and any("cost_per" in field and value is not None for field, value in entry.items())
for entry in entries
)
def _first_priced_realtime_token_costs(
potential_model_names: Sequence[str | None],
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float]:
candidate_costs: Final = (
(model_name, costs)
for model_name in potential_model_names
if model_name is not None
and (
costs := _candidate_realtime_token_costs(
model_name=model_name,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
)
is not None
)
return next(
(
costs
for model_name, costs in candidate_costs
if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider)
),
(0.0, 0.0),
)
def handle_realtime_stream_cost_calculation(
results: OpenAIRealtimeStreamList,
combined_usage_object: Usage,
@ -2381,24 +2456,12 @@ def handle_realtime_stream_cost_calculation(
potential_model_names.append(received_model)
potential_model_names.append(litellm_model_name)
input_cost_per_token = 0.0
output_cost_per_token = 0.0
for model_name in potential_model_names:
try:
if model_name is None:
continue
_input_cost_per_token, _output_cost_per_token = generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
continue
input_cost_per_token += _input_cost_per_token
output_cost_per_token += _output_cost_per_token
break # exit if we find a valid model
input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs(
potential_model_names=potential_model_names,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
transcription_cost: Final = (
handle_realtime_transcription_cost_calculation(
results=results,

12
litellm/endpoints/speech/speech_to_completion_bridge/transformation.py
View file

@ -8,6 +8,14 @@ if TYPE_CHECKING:
from litellm.types.utils import ModelResponse
def _completion_response_cost(model_response: "ModelResponse") -> float | None:
hidden_params: Final = getattr(model_response, "_hidden_params", None)
if not isinstance(hidden_params, dict):
return None
response_cost: Final = hidden_params.get("response_cost")
return response_cost if isinstance(response_cost, float) else None
class SpeechToCompletionBridgeTransformationHandler:
def transform_request(
self,
@ -123,4 +131,6 @@ class SpeechToCompletionBridgeTransformationHandler:
# Create an httpx.Response object
response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
return HttpxBinaryResponseContent(response)
binary_response: Final = HttpxBinaryResponseContent(response)
binary_response.set_response_cost(_completion_response_cost(model_response))
return binary_response

15
litellm/experimental_mcp_client/client.py
View file

@ -7,6 +7,7 @@ import base64
import os
from collections.abc import Awaitable, Callable, Generator
from datetime import timedelta
from importlib import metadata
from typing import Any, Final, TypeVar
import httpx
@ -21,6 +22,18 @@ try:
streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
except ImportError:
pass
MCP_STREAMABLE_HTTP_REQUIREMENT: Final = "mcp>=1.28.1"
def missing_streamable_http_client_error() -> ImportError:
return ImportError(
f"MCP streamable HTTP transport requires {MCP_STREAMABLE_HTTP_REQUIREMENT}, but the installed "
f"mcp {metadata.version('mcp')} does not provide streamable_http_client. "
"Fix with: pip install 'litellm[mcp]' (or upgrade mcp directly: pip install -U mcp)"
)
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import (
@ -323,7 +336,7 @@ class MCPClient:
)
# HTTP transport (default)
if streamable_http_client is None:
raise ImportError("streamable_http_client is not available. Please install mcp with HTTP support.")
raise missing_streamable_http_client_error()
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)

12
litellm/google_genai/streaming_iterator.py
View file

@ -2,6 +2,7 @@ import asyncio
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
@ -65,6 +66,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
litellm_logging_obj: LiteLLMLoggingObj,
request_body: dict,
model: str,
custom_llm_provider: str,
hidden_params: dict[str, Any] | None = None,
):
self.litellm_logging_obj = litellm_logging_obj
@ -72,6 +74,10 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
self.start_time = datetime.now()
self.collected_chunks: list[bytes] = []
self.model = model
self.custom_llm_provider = custom_llm_provider
self.endpoint_type: Final = (
EndpointType.GEMINI if custom_llm_provider == litellm.LlmProviders.GEMINI.value else EndpointType.VERTEX_AI
)
self._hidden_params: dict[str, Any] = hidden_params or {}
async def _handle_async_streaming_logging(
@ -89,7 +95,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/generateContent",
request_body=self.request_body or {},
endpoint_type=EndpointType.VERTEX_AI,
endpoint_type=self.endpoint_type,
start_time=self.start_time,
raw_bytes=self.collected_chunks,
end_time=end_time,
@ -118,13 +124,13 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
custom_llm_provider=custom_llm_provider,
hidden_params=hidden_params,
)
self.response = response
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
self.custom_llm_provider = custom_llm_provider
# Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.iter_lines()
@ -169,13 +175,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
custom_llm_provider=custom_llm_provider,
hidden_params=hidden_params,
)
self.response = response
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
self.custom_llm_provider = custom_llm_provider
# Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.aiter_lines()

36
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -104,6 +104,13 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
# rather than spending them on a list that is still missing some of their targets.
CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points"
class AnthropicCacheControlHook(CustomPromptManagement):
def get_chat_completion_prompt(
self,
@ -128,6 +135,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
- non_default_params: dict - params with any global cache controls
"""
# Extract cache control injection points
carry_unmatched: Final = bool(non_default_params.pop(CARRY_UNMATCHED_MESSAGE_POINTS, False))
injection_points: Final[list[CacheControlInjectionPoint]] = non_default_params.pop(
"cache_control_injection_points", []
)
@ -161,12 +169,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
non_default_params.get("prompt_cache_options"),
)
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
# budget. An ordinal names a different message once a later layer builds its own
# list, so it is placed here or not at all.
carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
tuple(point for point in message_points if point.get("index") is None) if carry_unmatched else ()
)
applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
tuple(point for point in message_points if point.get("index") is not None)
if carry_unmatched
else tuple(message_points)
)
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=message_points,
points=applied_message_points,
messages=processed_messages,
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
openai_dialect=openai_dialect,
@ -177,10 +198,15 @@ class AnthropicCacheControlHook(CustomPromptManagement):
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
# Pass through non-message injection points for provider-specific handling
if remaining_points:
# Points this pass did not place: non-message ones for the provider transform, and
# the deferred role-targeted ones. Deferring is what reaches the Responses API's
# `instructions`, which is only a system message once the bridge builds one. The
# judged stamp is what makes it safe: the next pass must not re-judge points
# against messages this pass already marked (see `_should_stand_down`).
carried_points: Final[Sequence[CacheControlInjectionPoint]] = (*remaining_points, *carried_message_points)
if carried_points:
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
remaining_points
carried_points
)
return model, processed_messages, non_default_params
@ -218,7 +244,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
@staticmethod
def _apply_message_injections(
points: list[CacheControlMessageInjectionPoint],
points: Sequence[CacheControlMessageInjectionPoint],
messages: list[AllMessageValues],
max_blocks: int,
openai_dialect: bool = False,

12
litellm/integrations/callback_configs.json
View file

@ -220,6 +220,12 @@
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
},
"langfuse_environment": {
"type": "text",
"ui_name": "Tracing Environment",
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
"required": false
}
},
"description": "Langfuse v2 Logging Integration"
@ -247,6 +253,12 @@
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
},
"langfuse_environment": {
"type": "text",
"ui_name": "Tracing Environment",
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
"required": false
}
},
"description": "Langfuse v3 OTEL Logging Integration"

2
litellm/integrations/custom_batch_logger.py
View file

@ -45,7 +45,7 @@ class CustomBatchLogger(CustomLogger):
super().__init__(**kwargs)
async def periodic_flush(self):
async def periodic_flush(self) -> None:
while True:
await asyncio.sleep(self.flush_interval)
verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval)

2
litellm/integrations/custom_logger.py
View file

@ -295,7 +295,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_post_call_failure_deployment_hook(
self,
request_data: Mapping[str, Any],
request_data: Mapping[str, object],
exception: Exception,
call_type: CallTypes | None,
fallback_depth: int | None = None,

6
litellm/integrations/dotprompt/__init__.py
View file

@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
from .prompt_manager import strip_version_suffix
registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id
try:
dot_prompt_manager: Final = DotpromptManager(
prompt_directory=prompt_directory,
prompt_data=prompt_data,
prompt_file=prompt_file,
prompt_id=prompt_id,
prompt_id=registration_prompt_id,
)
return dot_prompt_manager

4
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement):
if prompt_id is None:
return False
try:
return prompt_id in self.prompt_manager.list_prompts()
return self.prompt_manager.get_prompt(prompt_id) is not None
except Exception:
# If there's any error accessing prompts, don't run prompt management
return False
@ -209,6 +209,8 @@ class DotpromptManager(CustomPromptManagement):
prompt_spec=prompt_spec,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
async def async_get_chat_completion_prompt(

31
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
return base
return None
class PromptTemplate:
"""Represents a single prompt template with metadata and content."""
@ -124,11 +131,13 @@ class PromptManager:
"content": "template content",
"metadata": {"model": "gpt-4", "temperature": 0.7, ...}
} + prompt_id
"""
if prompt_id:
prompt_data = {prompt_id: prompt_data}
for prompt_id, prompt_info in prompt_data.items():
A dict carrying a "content" key is a single flat template registered under
prompt_id; anything else is treated as already keyed by template ID.
"""
keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data
for template_id, prompt_info in keyed_prompts.items():
try:
content = prompt_info.get("content", "")
metadata = prompt_info.get("metadata", {})
@ -136,11 +145,11 @@ class PromptManager:
template = PromptTemplate(
content=content,
metadata=metadata,
template_id=prompt_id,
template_id=template_id,
)
self.prompts[prompt_id] = template
self.prompts[template_id] = template
except Exception:
# Optional: print(f"Error loading prompt from JSON: {prompt_id}")
# Optional: print(f"Error loading prompt from JSON: {template_id}")
pass
def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:
@ -272,8 +281,12 @@ class PromptManager:
if versioned_id in self.prompts:
return self.prompts[versioned_id]
# Fall back to base prompt_id
return self.prompts.get(prompt_id)
direct_match: Final = self.prompts.get(prompt_id)
if direct_match is not None:
return direct_match
base_prompt_id: Final = strip_version_suffix(prompt_id)
return self.prompts.get(base_prompt_id) if base_prompt_id else None
def list_prompts(self) -> list[str]:
"""Get a list of all available prompt IDs."""

26
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -416,17 +416,8 @@ class GenericPromptManager(CustomPromptManagement):
tools=tools,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=(
ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
if prompt_spec
else False
),
ignore_prompt_manager_optional_params=(
ignore_prompt_manager_optional_params
or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
if prompt_spec
else False
),
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
def get_chat_completion_prompt(
@ -457,17 +448,8 @@ class GenericPromptManager(CustomPromptManagement):
prompt_spec=prompt_spec,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=(
ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
if prompt_spec
else False
),
ignore_prompt_manager_optional_params=(
ignore_prompt_manager_optional_params
or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
if prompt_spec
else False
),
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
def clear_cache(self) -> None:

11
litellm/integrations/langfuse/langfuse.py
View file

@ -1,5 +1,6 @@
#### What this does ####
# On success, logs events to Langfuse
import inspect
import os
import traceback
from collections.abc import Callable, Iterable, Mapping
@ -21,6 +22,9 @@ from litellm.litellm_core_utils.core_helpers import (
reconstruct_model_name,
safe_deep_copy,
)
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_langfuse_environment_value,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@ -140,6 +144,7 @@ class LangFuseLogger:
langfuse_public_key=None,
langfuse_secret=None,
langfuse_host=None,
langfuse_environment: str | None = None,
flush_interval=1,
allow_env_credentials: bool = True,
):
@ -159,6 +164,10 @@ class LangFuseLogger:
if not (self.langfuse_host.startswith("http://") or self.langfuse_host.startswith("https://")):
# add http:// if unset, assume communicating over private network - e.g. render
self.langfuse_host = "http://" + self.langfuse_host
_env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
self.langfuse_environment = _env_override or os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
if self.langfuse_environment:
validate_langfuse_environment_value(self.langfuse_environment)
self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
@ -182,6 +191,8 @@ class LangFuseLogger:
}
self.langfuse_sdk_version: str = langfuse.version.__version__
if "environment" in inspect.signature(Langfuse.__init__).parameters:
parameters["environment"] = self.langfuse_environment
if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
parameters["sdk_integration"] = "litellm"
self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)

25
litellm/integrations/langfuse/langfuse_handler.py
View file

@ -1,3 +1,5 @@
import os
"""
This file contains the LangFuseHandler class
@ -108,6 +110,7 @@ class LangFuseHandler:
langfuse_public_key=credentials.get("langfuse_public_key"),
langfuse_secret=credentials.get("langfuse_secret") or credentials.get("langfuse_secret_key"),
langfuse_host=credentials.get("langfuse_host"),
langfuse_environment=credentials.get("langfuse_environment"),
allow_env_credentials=credentials.get("langfuse_host") is None,
)
in_memory_dynamic_logger_cache.set_cache(
@ -135,8 +138,29 @@ class LangFuseHandler:
or standard_callback_dynamic_params.get("langfuse_secret_key"),
langfuse_public_key=standard_callback_dynamic_params.get("langfuse_public_key"),
langfuse_host=standard_callback_dynamic_params.get("langfuse_host"),
langfuse_environment=LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params),
)
@staticmethod
def _meaningful_dynamic_environment(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> str | None:
"""Return the per-request environment only when it changes behavior.
Empty/whitespace values and values equal to the deployment-wide
LANGFUSE_TRACING_ENVIRONMENT fallback are treated as absent so an
environment-only override that matches the default does not mint a
duplicate SDK client (each client costs threads and counts against
MAX_LANGFUSE_INITIALIZED_CLIENTS).
"""
raw = standard_callback_dynamic_params.get("langfuse_environment")
if raw is None:
return None
value = str(raw).strip()
if not value or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT"):
return None
return value
@staticmethod
def _dynamic_langfuse_credentials_are_passed(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
@ -153,6 +177,7 @@ class LangFuseHandler:
or standard_callback_dynamic_params.get("langfuse_public_key") is not None
or standard_callback_dynamic_params.get("langfuse_secret") is not None
or standard_callback_dynamic_params.get("langfuse_secret_key") is not None
or LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params) is not None
):
return True
return False

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -231,7 +231,10 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
langfuse_environment: Final = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
if langfuse_environment:
safe_set_attribute(
span,

395
litellm/integrations/newrelic/newrelic_metrics.py Normal file
View file

@ -0,0 +1,395 @@
"""
New Relic Metric API Integration - sends per-team cost/usage metrics to /metric/v1
NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/
`async_log_success_event` / `async_log_failure_event` queue one record per request;
at flush the queue is aggregated by (team, model group, model, provider, status)
into count/summary metrics. `interval.ms` is the real window between flushes,
computed at flush time.
Team-scoped by construction: the ingest key is injected explicitly and there is
deliberately no environment-variable fallback, so a team's metrics are never sent
with the proxy operator's credentials (mirrors ``allow_env_credentials=False`` on
the Datadog team logger).
Error policy on flush: 4xx drops the batch (a retry would fail identically; 403
is a permanent credential failure), 5xx/network re-queues capped at
``max_queue_size`` records with the oldest dropped.
For batching specific details see CustomBatchLogger class
"""
import asyncio
import gzip
import time
import traceback
from collections.abc import Mapping
from math import ceil
from types import MappingProxyType
from typing import Final
from httpx import HTTPStatusError, Response
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.newrelic import (
NEWRELIC_DEFAULT_REGION,
NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN,
NEWRELIC_METRIC_COMPLETION_TOKENS,
NEWRELIC_METRIC_COST_USD,
NEWRELIC_METRIC_ENDPOINT_BY_REGION,
NEWRELIC_METRIC_PROMPT_TOKENS,
NEWRELIC_METRIC_REQUEST_DURATION_MS,
NEWRELIC_METRIC_REQUESTS,
NEWRELIC_METRIC_TOTAL_TOKENS,
NEWRELIC_METRICS_MAX_BATCH_SIZE,
NEWRELIC_METRICS_MAX_DRAIN_PASSES,
NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
NewRelicCountMetric,
NewRelicMetric,
NewRelicMetricCommon,
NewRelicMetricEnvelope,
NewRelicMetricRecord,
NewRelicSummaryMetric,
NewRelicSummaryValue,
)
from litellm.types.utils import StandardLoggingPayload
# 408 (request timeout) and 429 (rate limit) are transient client errors the
# Metric API expects a retry on, unlike 400/403 which a retry would only repeat.
_RETRYABLE_CLIENT_STATUSES: Final = frozenset({408, 429})
def resolve_newrelic_metric_endpoint(newrelic_region: str | None) -> str:
if not newrelic_region:
return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
endpoint: Final = NEWRELIC_METRIC_ENDPOINT_BY_REGION.get(newrelic_region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
newrelic_region,
", ".join(sorted(NEWRELIC_METRIC_ENDPOINT_BY_REGION)),
)
return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
return endpoint
def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) -> NewRelicMetricRecord:
metadata: Final = standard_logging_object.get("metadata")
team_id: Final = ((metadata.get("user_api_key_team_id") or metadata.get("team_id")) if metadata else None) or ""
team_alias: Final = (
(metadata.get("user_api_key_team_alias") or metadata.get("team_alias")) if metadata else None
) or ""
return NewRelicMetricRecord(
team_id=team_id,
team_alias=team_alias,
model_group=standard_logging_object.get("model_group") or "",
model=standard_logging_object.get("model") or "",
custom_llm_provider=standard_logging_object.get("custom_llm_provider") or "",
status=str(standard_logging_object.get("status") or "success"),
response_cost=float(standard_logging_object.get("response_cost") or 0.0),
prompt_tokens=int(standard_logging_object.get("prompt_tokens") or 0),
completion_tokens=int(standard_logging_object.get("completion_tokens") or 0),
total_tokens=int(standard_logging_object.get("total_tokens") or 0),
duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0,
)
def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]:
first: Final = bucket_records[0]
attributes: Final[Mapping[str, str]] = { # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType
key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN]
for key, value in (
("team_id", first.team_id),
("team_alias", first.team_alias),
("model_group", first.model_group),
("model", first.model),
("custom_llm_provider", first.custom_llm_provider),
("status", first.status),
)
if value
}
durations: Final = tuple(record.duration_ms for record in bucket_records)
counts: Final[tuple[tuple[str, float], ...]] = (
(NEWRELIC_METRIC_REQUESTS, float(len(bucket_records))),
(NEWRELIC_METRIC_COST_USD, sum(record.response_cost for record in bucket_records)),
(NEWRELIC_METRIC_PROMPT_TOKENS, float(sum(record.prompt_tokens for record in bucket_records))),
(NEWRELIC_METRIC_COMPLETION_TOKENS, float(sum(record.completion_tokens for record in bucket_records))),
(NEWRELIC_METRIC_TOTAL_TOKENS, float(sum(record.total_tokens for record in bucket_records))),
)
count_metrics: Final[tuple[NewRelicMetric, ...]] = tuple(
NewRelicCountMetric(name=name, type="count", value=value, attributes=attributes) for name, value in counts
)
summary_metric: Final = NewRelicSummaryMetric(
name=NEWRELIC_METRIC_REQUEST_DURATION_MS,
type="summary",
value=NewRelicSummaryValue(
count=len(durations),
sum=sum(durations),
min=min(durations),
max=max(durations),
),
attributes=attributes,
)
return (*count_metrics, summary_metric)
def build_metric_payload(
records: tuple[NewRelicMetricRecord, ...],
*,
window_start: float,
now: float,
) -> tuple[NewRelicMetricEnvelope, ...]:
"""Aggregates records into one Metric API envelope for the flush window."""
interval_ms: Final = max(1, int((now - window_start) * 1000))
bucket_keys: Final = tuple(dict.fromkeys(record.bucket_key for record in records))
metrics: Final = tuple(
metric
for key in bucket_keys
for metric in _bucket_metrics(tuple(record for record in records if record.bucket_key == key))
)
common: Final[NewRelicMetricCommon] = {
"timestamp": int(window_start * 1000),
"interval.ms": interval_ms,
}
return (NewRelicMetricEnvelope(common=common, metrics=metrics),)
class NewRelicMetricsLogger(CustomBatchLogger):
def __init__(
self,
newrelic_api_key: str,
newrelic_region: str | None = None,
) -> None:
if not newrelic_api_key:
raise ValueError(
"newrelic_api_key is required for NewRelicMetricsLogger; "
"team-scoped metrics never fall back to environment credentials"
)
self.newrelic_api_key: Final = newrelic_api_key
self.metric_api_url: Final = resolve_newrelic_metric_endpoint(newrelic_region)
self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
self._stopped: bool = False
self._drain_lock = asyncio.Lock()
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
super().__init__(
flush_lock=self.flush_lock,
batch_size=NEWRELIC_METRICS_MAX_BATCH_SIZE,
max_queue_size=NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
)
def stop(self) -> None:
"""Ends the periodic flush loop; called on DynamicLoggingCache eviction.
Schedules one final drain of anything still queued, so eviction never
silently discards records. Guarded so it can never raise into the
cache's eviction path.
"""
self._stopped = True
try:
asyncio.get_running_loop().create_task(self._final_drain())
except Exception: # noqa: BLE001 # no running loop / shutdown; the periodic loop's final drain still runs
verbose_logger.debug("New Relic Metrics: could not schedule final drain on stop()", exc_info=True)
async def _drain_with_retry(self) -> None:
"""Deliver everything queued on a stopped logger, or drop it with a log.
A stopped logger has no periodic loop left, so every post-stop path
funnels through here. ``_drain_lock`` serializes drains: a callback that
appends and starts its own drain queues behind the running one instead
of racing it. Each pass attempts the whole current queue in
``batch_size`` chunks, unlike the periodic path it does not stop at the
first failing chunk, so a persistently failing head never starves the
tail. Only after ``_MAX_DRAIN_PASSES`` against a permanently failing
destination is the remainder dropped, and then only the records that were
queued when this drain began, so every dropped record got the full retry
budget: a record a callback appended mid-drain is not in that snapshot,
so it is left for its own serialized drain rather than dropped after
fewer attempts, and is never stranded.
"""
async with self._drain_lock:
attempted: Final = tuple(self.log_queue)
for _pass in range(NEWRELIC_METRICS_MAX_DRAIN_PASSES):
await self._drain_flush_once()
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)
dropped: Final = len(self.log_queue) - len(survivors)
if dropped:
verbose_logger.warning(
"New Relic Metrics: dropping %s records after %s drain passes",
dropped,
NEWRELIC_METRICS_MAX_DRAIN_PASSES,
)
self.log_queue[:] = list(survivors) # mutable-ok: leave late arrivals for the next serialized drain
async def _drain_flush_once(self) -> None:
"""Attempt every queued record once, in ``batch_size`` chunks, without
stopping at the first failing chunk so a persistently failing head does
not starve the tail (the periodic ``flush_queue`` deliberately stops
instead). Takes the queue under ``flush_lock`` and re-queues only the
chunks a 5xx/network error left undelivered, so records a concurrent
request appends during the sends survive for the next pass."""
async with self.flush_lock:
pending: Final = tuple(self.log_queue)
window_start: Final = self.last_flush_time
self.last_flush_time = time.time()
del self.log_queue[:]
if not pending:
return
chunks: Final = tuple(
pending[start : start + self.batch_size] for start in range(0, len(pending), self.batch_size)
)
delivered: Final = tuple([await self._classify_and_send(chunk, window_start) for chunk in chunks])
failed: Final = tuple(record for chunk, ok in zip(chunks, delivered) for record in (() if ok else chunk))
if failed:
self._requeue(failed)
async def _final_drain(self) -> None:
await self._drain_with_retry()
async def periodic_flush(self) -> None:
while not self._stopped:
await asyncio.sleep(self.flush_interval)
if self._stopped:
break
await self.flush_queue()
await self._final_drain()
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
try:
await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
except Exception as e: # noqa: BLE001 # logging must never break the request path
verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
try:
await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
except Exception as e: # noqa: BLE001 # logging must never break the request path
verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
async def _log_async_event(self, standard_logging_object: StandardLoggingPayload | None) -> None:
if standard_logging_object is None:
raise ValueError("standard_logging_object not found in kwargs")
self.log_queue.append(_metric_record_from_payload(standard_logging_object))
if self._stopped:
# A stopped logger has no periodic loop left; an in-flight callback
# that appends after the eviction drain delivers its own record.
await self._drain_with_retry()
return
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
async def flush_queue(self) -> None:
async with self.flush_lock:
window_start: Final = self.last_flush_time
self.last_flush_time = time.time()
queued: Final = len(self.log_queue)
if not queued:
return
verbose_logger.debug("New Relic Metrics: Flushing %s queued records", queued)
# Bounded by what is queued now: records appended mid-flush belong to
# the next window, and looping until empty would never end under load.
for _chunk in range(ceil(queued / self.batch_size)):
if not await self.async_send_batch(window_start=window_start):
return
async def async_send_batch(self, window_start: float | None = None) -> bool:
"""Sends the oldest ``batch_size`` records only, so a queue grown past that
by re-queues cannot breach the Metric API data point cap in one request.
Returns False once a chunk fails and is re-queued, so the caller stops."""
if not self.log_queue:
return False
batch_to_send: Final[tuple[NewRelicMetricRecord, ...]] = tuple(self.log_queue[: self.batch_size])
del self.log_queue[: len(batch_to_send)]
delivered: Final = await self._classify_and_send(
batch_to_send, window_start if window_start is not None else self.last_flush_time
)
if not delivered:
self._requeue(batch_to_send)
return delivered
async def _classify_and_send(self, batch: tuple[NewRelicMetricRecord, ...], window_start: float) -> bool:
"""Send one chunk and classify the outcome, never touching the queue.
Returns True when the batch is done with (delivered on any 2xx, or a 4xx
a retry would only repeat, 403 being a permanent bad-key rejection), and
False when a 5xx or network error means the caller should re-queue it.
``AsyncHTTPHandler.post`` raises ``HTTPStatusError`` on any non-2xx, so a
4xx never returns a response here; the status is read off the raised
error to keep the client-error path (drop) distinct from 5xx (retry)."""
payload: Final = build_metric_payload(records=batch, window_start=window_start, now=time.time())
try:
status = (
await self.async_send_compressed_data(payload)
).status_code # rebind-ok: reassigned from the raised HTTPStatusError below
except HTTPStatusError as e:
status = e.response.status_code
except Exception as e: # noqa: BLE001 # transport/network failure re-queues the batch
verbose_logger.warning(
"New Relic Metrics: network error sending %s records, will retry - %s",
len(batch),
e,
)
return False
if 200 <= status < 300:
return True
if 400 <= status < 500 and status not in _RETRYABLE_CLIENT_STATUSES:
verbose_logger.warning(
"New Relic Metrics: %s from Metric API%s, dropping %s records.",
status,
" (permanent credential failure: invalid or revoked team ingest key)" if status == 403 else "",
len(batch),
)
return True
verbose_logger.warning(
"New Relic Metrics: %s from Metric API, will retry %s records",
status,
len(batch),
)
return False
def _requeue(self, batch: tuple[NewRelicMetricRecord, ...]) -> None:
"""Prepends ``batch`` in place (never by assignment: records appended by
concurrent requests during the flush await must survive), keeping
chronological order so the cap drops the oldest records first."""
self.log_queue[:0] = batch
overflow: Final = len(self.log_queue) - self.max_queue_size
if overflow > 0:
del self.log_queue[:overflow]
verbose_logger.warning(
"New Relic Metrics: retry queue exceeded max_queue_size=%s; dropped %s oldest records.",
self.max_queue_size,
overflow,
)
async def async_send_compressed_data(self, payload: tuple[NewRelicMetricEnvelope, ...]) -> Response:
compressed_data: Final = gzip.compress(safe_dumps(payload).encode("utf-8"))
headers: Final[Mapping[str, str]] = MappingProxyType(
{
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Api-Key": self.newrelic_api_key,
}
)
return await self.async_client.post(
url=self.metric_api_url,
data=compressed_data,
headers=headers,
)

90
litellm/integrations/newrelic/newrelic_team_handler.py Normal file
View file

@ -0,0 +1,90 @@
"""
New Relic Team Handler
Used to get the NewRelicMetricsLogger for a given request.
Handles Key/Team Based New Relic metrics, following the same pattern as DataDogHandler.
"""
from typing import TYPE_CHECKING, Final
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
from .newrelic_metrics import NewRelicMetricsLogger
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
class NewRelicLoggingConfig(TypedDict):
newrelic_api_key: ReadOnly[str | None]
newrelic_region: ReadOnly[str | None]
class NewRelicHandler:
@staticmethod
def get_newrelic_logger_for_request(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
in_memory_dynamic_logger_cache: "DynamicLoggingCache",
) -> NewRelicMetricsLogger:
"""
Get a team-scoped NewRelicMetricsLogger for a given request.
Resolves and caches per-team NewRelicMetricsLogger instances using
DynamicLoggingCache, keyed by the team's New Relic credentials. Each unique
set of credentials gets its own logger instance with its own batch/flush loop.
Note: This handler is only called when a team-scoped newrelic_api_key is
present. The trace logger for the ``newrelic`` callback (OTel v2 / legacy
agent) is managed separately by _init_custom_logger_compatible_class via
_in_memory_loggers.
"""
_credentials: Final = NewRelicHandler.get_dynamic_newrelic_logging_config(
standard_callback_dynamic_params=standard_callback_dynamic_params,
)
temp_newrelic_logger = in_memory_dynamic_logger_cache.get_cache(
credentials=_credentials, service_name="newrelic"
)
if temp_newrelic_logger is None:
temp_newrelic_logger = NewRelicHandler._create_newrelic_logger_from_credentials(
credentials=_credentials,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
return temp_newrelic_logger
@staticmethod
def _create_newrelic_logger_from_credentials(
credentials: NewRelicLoggingConfig,
in_memory_dynamic_logger_cache: "DynamicLoggingCache",
) -> NewRelicMetricsLogger:
newrelic_logger: Final = NewRelicMetricsLogger(
newrelic_api_key=credentials.get("newrelic_api_key") or "",
newrelic_region=credentials.get("newrelic_region"),
)
in_memory_dynamic_logger_cache.set_cache(
credentials=credentials,
service_name="newrelic",
logging_obj=newrelic_logger,
)
verbose_logger.debug("New Relic: Created and cached new NewRelicMetricsLogger for team-scoped credentials")
return newrelic_logger
@staticmethod
def get_dynamic_newrelic_logging_config(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> NewRelicLoggingConfig:
return NewRelicLoggingConfig(
newrelic_api_key=standard_callback_dynamic_params.get("newrelic_api_key"),
newrelic_region=standard_callback_dynamic_params.get("newrelic_region"),
)
@staticmethod
def _dynamic_newrelic_credentials_are_passed(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> bool:
return standard_callback_dynamic_params.get("newrelic_api_key") is not None

42
litellm/integrations/opentelemetry.py
View file

@ -22,6 +22,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
)
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.service_tier_utils import (
@ -1643,7 +1644,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if self._operation_duration_histogram:
self._operation_duration_histogram.record(duration_s, attributes=common_attrs)
if response_obj and (usage := response_obj.get("usage")) and self._token_usage_histogram:
if (
self._token_usage_histogram
and response_obj
and not is_unbilled_non_inference_call_from_params(kwargs.get("call_type"), params, response_obj)
and (usage := response_obj.get("usage"))
):
in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
out_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
self._token_usage_histogram.record(usage.get("prompt_tokens", 0), attributes=in_attrs)
@ -1719,6 +1725,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if not self._time_per_output_token_histogram:
return
if is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
):
return
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
@ -2049,6 +2060,26 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# serialise to JSON once so set_attribute never coerces.
guardrail_span.set_attribute("guardrail_violation_categories", safe_dumps(violation_categories))
# Billable usage counters and USD cost stamped by the provider hook
# (e.g. Azure Prompt Shield text records, Bedrock policy units).
guardrail_usage = guardrail_information.get("guardrail_usage")
if guardrail_usage is not None:
guardrail_span.set_attribute("guardrail_usage", safe_dumps(guardrail_usage))
guardrail_cost = guardrail_information.get("guardrail_cost")
if guardrail_cost is not None:
self.safe_set_attribute(
span=guardrail_span,
key="guardrail_cost",
value=guardrail_cost,
)
guardrail_cost_in_spend = guardrail_information.get("guardrail_cost_in_spend")
if isinstance(guardrail_cost_in_spend, bool):
self.safe_set_attribute(
span=guardrail_span,
key="guardrail_cost_in_spend",
value=guardrail_cost_in_spend,
)
self._set_team_attributes_from_kwargs(guardrail_span, kwargs)
guardrail_span.end(end_time=self._to_ns(end_time_datetime))
@ -2468,7 +2499,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
usage: Final = response_obj and response_obj.get("usage")
usage: Final = (
response_obj.get("usage")
if response_obj
and not is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), litellm_params, response_obj
)
else None
)
if usage:
self.safe_set_attribute(
span=span,

3
litellm/integrations/otel/mappers/genai.py
View file

@ -136,6 +136,9 @@ class GenAIMapper:
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
LiteLLM.GUARDRAIL_USAGE: lambda d: d.usage_json,
LiteLLM.GUARDRAIL_COST: lambda d: d.cost,
LiteLLM.GUARDRAIL_COST_IN_SPEND: lambda d: d.cost_in_spend,
}
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {

14
litellm/integrations/otel/model/payloads.py
View file

@ -190,6 +190,15 @@ class GuardrailSpanData:
guardrail_id: str | None = None
policy_template: str | None = None
detection_method: str | None = None
# Provider-reported billable usage counters (JSON-serialized) and the USD cost
# priced from them by the provider hook (``guardrail_usage`` /
# ``guardrail_cost`` on ``StandardLoggingGuardrailInformation``).
usage_json: str | None = None
cost: float | None = None
# Whether ``cost`` participates in the request's billed spend (absent means
# billed, the default; False means report-only). Mirrors
# ``guardrail_cost_in_spend`` so trace consumers can avoid double-counting.
cost_in_spend: bool | None = None
# Set when the guardrail intervened/blocked or failed, so the emitter marks
# the span ERROR — a blocking guardrail is an error outcome for that span.
error: SpanError | None = None
@ -209,6 +218,8 @@ class GuardrailSpanData:
get: Final = cast(Mapping[str, object], entry).get
status: Final = as_str(get("guardrail_status"))
response: Final = get("guardrail_response")
usage: Final = get("guardrail_usage")
in_spend: Final = get("guardrail_cost_in_spend")
error: Final = (
SpanError(error_type=status, message=as_str(get("guardrail_action")))
if status in cls._ERROR_STATUSES
@ -231,6 +242,9 @@ class GuardrailSpanData:
guardrail_id=as_str(get("guardrail_id")),
policy_template=as_str(get("policy_template")),
detection_method=as_str(get("detection_method")),
usage_json=_json_or_none(usage) if usage is not None else None,
cost=as_float(get("guardrail_cost")),
cost_in_spend=in_spend if isinstance(in_spend, bool) else None,
error=error,
)

18
litellm/integrations/otel/model/semconv.py
View file

@ -32,6 +32,7 @@ class GenAIOperation(str, Enum):
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
LITELLM_MODERATION = "litellm.moderation"
@ -307,6 +308,15 @@ class LiteLLM:
GUARDRAIL_ID: Final = "litellm.guardrail.id"
GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"
GUARDRAIL_DETECTION_METHOD: Final = "litellm.guardrail.detection_method"
# Provider-reported billable usage counters, JSON-serialized into one value.
GUARDRAIL_USAGE: Final = "litellm.guardrail.usage"
# Numeric USD cost of the guardrail invocation; lives under the litellm.cost.*
# namespace (COST_PREFIX) beside the LLM call's litellm.cost.total.
GUARDRAIL_COST: Final = "litellm.cost.guardrail"
# Whether litellm.cost.guardrail is already inside litellm.cost.total (True,
# the billed default) or reported alongside it (False) — without this a trace
# consumer cannot tell whether adding the two double-counts.
GUARDRAIL_COST_IN_SPEND: Final = "litellm.guardrail.cost_in_spend"
SERVICE_NAME: Final = "litellm.service.name"
SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
@ -374,6 +384,14 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"get_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"aget_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"delete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"adelete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"cancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"acancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"list_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"alist_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"image_generation": GenAIOperation.GENERATE_CONTENT,
"aimage_generation": GenAIOperation.GENERATE_CONTENT,
"moderation": GenAIOperation.LITELLM_MODERATION,

10
litellm/integrations/otel/plumbing/metrics.py
View file

@ -32,6 +32,7 @@ from litellm.integrations.otel.model.semconv import (
resolve_provider,
)
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -198,16 +199,21 @@ class GenAIMetricRecorder:
) -> None:
common_attrs: Final = self._filter_attributes(self._bounded_attributes(kwargs))
duration_s: Final = (end_time - start_time).total_seconds()
usage_is_replayed: Final = is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
)
self._metrics.operation_duration.record(duration_s, attributes=common_attrs)
self._record_token_usage(response_obj, common_attrs)
if not usage_is_replayed:
self._record_token_usage(response_obj, common_attrs)
cost: Final = kwargs.get("response_cost")
if cost:
self._metrics.token_cost.record(cost, attributes=common_attrs)
self._record_time_to_first_token(kwargs, common_attrs)
self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
if not usage_is_replayed:
self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
self._record_response_duration(kwargs, end_time, common_attrs)
def record_failure(

5
litellm/integrations/otel/plumbing/routing.py
View file

@ -15,7 +15,7 @@ from collections import OrderedDict
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, TypeAlias
from typing import Final, TypeAlias
from urllib.parse import quote
from opentelemetry.sdk.trace import TracerProvider
@ -32,6 +32,7 @@ from litellm.integrations.otel.presets import (
dynamic_otlp_headers,
project_routing_headers,
)
from litellm.types.utils import StandardCallbackDynamicParams
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
_NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
@ -166,7 +167,7 @@ class TenantTracerCache:
def route_for(
self,
default: Tracer,
dynamic_params: Any,
dynamic_params: StandardCallbackDynamicParams | None,
auth_metadata: Mapping[str, str] | None = None,
) -> TenantRoute:
"""Return the tracer (and trace-detachment flag) for this request.

16
litellm/integrations/prometheus.py
View file

@ -2495,12 +2495,12 @@ class PrometheusLogger(CustomLogger):
return None
def _get_user_email() -> str | None:
val = _metadata.get("user_api_key_user_email")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_user_email")
if val is not None:
return val
from_metadata: Final = _metadata.get("user_api_key_user_email")
if from_metadata is not None:
return from_metadata
from_params: Final = _litellm_params_metadata.get("user_api_key_user_email")
if from_params is not None:
return from_params
if user_api_key_auth is not None:
return self._safe_get(user_api_key_auth, "user_email")
return None
@ -3576,7 +3576,9 @@ class PrometheusLogger(CustomLogger):
except Exception as e:
verbose_logger.exception("Error initializing user/team count metrics: %s", e)
async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]):
async def _set_key_list_budget_metrics(
self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]
) -> None:
"""Helper function to set budget metrics for a list of keys"""
for key in keys:
if isinstance(key, UserAPIKeyAuth):

31
litellm/integrations/prompt_management_base.py
View file

@ -19,6 +19,19 @@ class PromptManagementClient(TypedDict):
completed_messages: list[AllMessageValues] | None
def resolve_prompt_manager_ignore_flags(
prompt_spec: PromptSpec | None,
ignore_prompt_manager_model: bool | None,
ignore_prompt_manager_optional_params: bool | None,
) -> tuple[bool, bool]:
spec_params: Final = prompt_spec.litellm_params if prompt_spec is not None else None
return (
bool(ignore_prompt_manager_model) or bool(spec_params is not None and spec_params.ignore_prompt_manager_model),
bool(ignore_prompt_manager_optional_params)
or bool(spec_params is not None and spec_params.ignore_prompt_manager_optional_params),
)
class PromptManagementBase(ABC):
@property
@abstractmethod
@ -182,13 +195,18 @@ class PromptManagementBase(ABC):
prompt_version=prompt_version,
)
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,
messages=messages,
non_default_params=non_default_params,
model=model,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_model=resolved_ignore_model,
ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
)
async def async_get_chat_completion_prompt(
@ -224,11 +242,16 @@ class PromptManagementBase(ABC):
prompt_version=prompt_version,
)
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,
messages=messages,
non_default_params=non_default_params,
model=model,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_model=resolved_ignore_model,
ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
)

138
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -550,6 +550,13 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"AnthropicException - {error_str}",
llm_provider="anthropic",
model=model,
response=original_exception.response,
)
elif original_exception.status_code == 400 or original_exception.status_code == 413:
raise BadRequestError(
message=f"AnthropicException - {error_str}",
@ -755,12 +762,19 @@ def _map_openai_like_exception(
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 401 or original_exception.status_code == 403:
elif original_exception.status_code == 401:
raise AuthenticationError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=403),
)
elif original_exception.status_code == 400:
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
@ -2187,6 +2201,120 @@ def _map_openrouter_exception(
)
def _response_or_stub(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
response: Final = original_exception.response if hasattr(original_exception, "response") else None
if response is not None:
return response
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs")
)
def _map_exception_by_status(
*,
model: str,
original_exception: _ProviderHTTPException,
custom_llm_provider: str,
error_str: str,
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:
case 401:
raise AuthenticationError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 403:
raise PermissionDeniedError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=status_code),
litellm_debug_info=extra_information,
)
case 404:
raise NotFoundError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 408:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
case 429:
raise RateLimitError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 500:
raise InternalServerError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 502:
raise BadGatewayError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 503:
raise ServiceUnavailableError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 504:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=status_code,
)
case _ if status_code < 500:
raise BadRequestError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case _:
raise APIError(
status_code=status_code,
message=message,
llm_provider=custom_llm_provider,
model=model,
request=original_exception.request if hasattr(original_exception, "request") else None,
litellm_debug_info=extra_information,
)
def exception_type(
model,
original_exception,
@ -2501,6 +2629,14 @@ def exception_type(
For unmapped exceptions - raise the exception with traceback - https://github.com/BerriAI/litellm/issues/4201
"""
exception_mapping_worked = True
_map_exception_by_status(
model=model,
original_exception=mappable_exception,
custom_llm_provider=custom_llm_provider,
error_str=error_str,
exception_provider=exception_provider,
extra_information=extra_information,
)
if hasattr(original_exception, "request"):
raise APIConnectionError(
message=f"{exception_provider} - {error_str}",

36
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -2,17 +2,32 @@
Helper functions for health check calls.
"""
from collections.abc import Callable
import base64
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, Literal
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import ImageResponse
# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
# Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG
TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC"
IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
"Add a small yellow star in the top right corner of this simple drawing of a blue circle on a white background"
)
def get_image_file_for_health_check() -> bytes:
"""Return the image used for health checks."""
return base64.b64decode(TEST_IMAGE_BASE64)
class HealthCheckHelpers:
@staticmethod
@ -112,6 +127,17 @@ class HealthCheckHelpers:
else:
return await litellm.acompletion(**model_params)
@staticmethod
async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse":
import litellm
try:
return await edit_request()
except litellm.BadRequestError as e:
if isinstance(e, litellm.ContentPolicyViolationError) or "moderation_blocked" in str(e):
return litellm.ImageResponse()
raise
@staticmethod
def get_mode_handlers(
model: str,
@ -127,6 +153,7 @@ class HealthCheckHelpers:
"audio_speech",
"audio_transcription",
"image_generation",
"image_edit",
"video_generation",
"rerank",
"realtime",
@ -185,6 +212,13 @@ class HealthCheckHelpers:
**_filter_model_params(model_params=model_params),
prompt=prompt,
),
"image_edit": lambda: HealthCheckHelpers._image_edit_health_check(
edit_request=lambda: litellm.aimage_edit(
**_filter_model_params(model_params=model_params),
image=get_image_file_for_health_check(),
prompt=IMAGE_EDIT_HEALTH_CHECK_PROMPT,
),
),
"video_generation": lambda: litellm.avideo_generation(
**_filter_model_params(model_params=model_params),
prompt=prompt or "test video generation",

18
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,3 +1,4 @@
import re
from collections.abc import Iterator, Mapping
from typing import Any, Final
@ -45,12 +46,29 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
_raise_env_reference_error(param, source=source)
# Langfuse rejects events whose environment does not match this pattern
# (lowercase alphanumerics, hyphens, underscores; no "langfuse" prefix).
# Validating here fails fast at config/init time instead of silently
# dropping every trace server-side.
LANGFUSE_ENVIRONMENT_PATTERN: Final = r"^(?!langfuse)[a-z0-9-_]+$"
def validate_langfuse_environment_value(value: str) -> None:
if not re.match(LANGFUSE_ENVIRONMENT_PATTERN, value):
raise ValueError(
f"Invalid langfuse_environment {value!r}: must be lowercase "
"alphanumerics/hyphens/underscores and must not start with "
f"'langfuse' (pattern {LANGFUSE_ENVIRONMENT_PATTERN})"
)
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
"langfuse_host",
"langfuse_environment",
"langfuse_prompt_version",
"langsmith_api_key",
"langsmith_project",

58
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -20,8 +20,8 @@ from __future__ import annotations
from collections.abc import Mapping
from typing import Final
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.types.utils import InternalCallOrigin
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, NON_INFERENCE_CALL_TYPES
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, InternalCallOrigin
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
@ -45,6 +45,60 @@ budget-checked like the request that spawned it. Everything else on the parent's
be a lie on a sub-call that runs after it returned."""
def is_background_response(response: object) -> bool:
"""Whether a retrieved object is a response created with ``background=true``.
Such a create returns ``status="queued"`` and no usage at all, so nothing has billed the
job by the time anyone reads it back. Accepts the response as a mapping or a model,
because the callers hold it in both shapes.
"""
if isinstance(response, Mapping):
return response.get("background") is True
return getattr(response, "background", None) is True
def is_unbilled_non_inference_call(
call_type: str | None,
metadata: Mapping[str, object] | None,
response: object,
) -> bool:
"""A read/management route priced at zero, because the usage it reports belongs to the
call that created the object it just read.
Retrieving a background response is the exception, and the enterprise cost poller's read
is the same exception seen from the other side: that job's create billed nothing, so its
retrieval is the only place the spend is ever visible. Pricing those at zero would lose
the spend rather than deduplicate it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
if is_background_response(response):
return False
if metadata is None:
return True
return metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
def is_unbilled_non_inference_call_from_params(
call_type: str | None,
litellm_params: Mapping[str, object] | None,
response: object,
) -> bool:
""":func:`is_unbilled_non_inference_call` for callers holding raw ``litellm_params``.
The call-type membership test runs first so that inference traffic, which is every
request in a normal workload, never pays for the metadata merge behind it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
metadata: Final = (
StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None
)
return is_unbilled_non_inference_call(call_type, metadata, response)
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

86
litellm/litellm_core_utils/litellm_logging.py
View file

@ -64,6 +64,7 @@ from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
cost_breakdown_with_guardrail,
guardrail_information_cost,
@ -615,37 +616,60 @@ class Logging(LiteLLMLoggingBaseClass):
processed_list: Final[list[str | Callable | CustomLogger]] = []
for callback in callback_list:
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
# For callbacks that support team-scoped credentials (e.g. datadog),
# pass only the relevant dynamic params as custom_logger_init_args.
_custom_logger_init_args: dict | None = None
if callback == "datadog":
# dd_* params are blocked from standard_callback_dynamic_params
# (request-level security); only the proxy-stamped team/key
# callback vars are admin-configured and trusted.
_custom_logger_init_args = {k: v for k, v in self._trusted_callback_vars if k.startswith("dd_")}
callback_class = _init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
custom_logger_init_args=_custom_logger_init_args,
)
if callback_class is not None:
processed_list.append(callback_class)
for callback_instance in self._resolve_dynamic_callback_string(callback):
processed_list.append(callback_instance)
# If processing dynamic_success_callbacks, add to dynamic_async_success_callbacks
if dynamic_callbacks_type == "success":
if self.dynamic_async_success_callbacks is None:
self.dynamic_async_success_callbacks = []
self.dynamic_async_success_callbacks.append(callback_class)
self.dynamic_async_success_callbacks.append(callback_instance)
elif dynamic_callbacks_type == "failure":
if self.dynamic_async_failure_callbacks is None:
self.dynamic_async_failure_callbacks = []
self.dynamic_async_failure_callbacks.append(callback_class)
self.dynamic_async_failure_callbacks.append(callback_instance)
else:
processed_list.append(callback)
return processed_list
def _resolve_dynamic_callback_string(self, callback: str) -> "tuple[CustomLogger, ...]":
"""
Resolve a known callback name to the logger instance(s) it dispatches to.
For callbacks that support team-scoped credentials (datadog, newrelic),
only the proxy-stamped team/key callback vars are passed as
custom_logger_init_args: dd_*/newrelic_* params are blocked from
standard_callback_dynamic_params (request-level security), so the
trusted-vars channel is the only way credentials reach a per-team logger.
"""
_trusted_var_prefix: Final = "dd_" if callback == "datadog" else "newrelic_" if callback == "newrelic" else None
_custom_logger_init_args: Final[dict | None] = (
{k: v for k, v in self._trusted_callback_vars if k.startswith(_trusted_var_prefix)}
if _trusted_var_prefix is not None
else None
)
callback_class: Final = _init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
custom_logger_init_args=_custom_logger_init_args,
)
if callback_class is None:
return ()
# With team creds, "newrelic" resolves to the per-team METRICS logger;
# resolve the name again without creds so the trace logger (OTel v2 /
# legacy agent) keeps receiving this request.
_newrelic_trace_class: Final = (
_init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
if callback == "newrelic" and _custom_logger_init_args and _custom_logger_init_args.get("newrelic_api_key")
else None
)
if _newrelic_trace_class is not None and _newrelic_trace_class is not callback_class:
return (callback_class, _newrelic_trace_class)
return (callback_class,)
def initialize_standard_callback_dynamic_params(self, kwargs: dict | None = None) -> StandardCallbackDynamicParams:
"""
Initialize the standard callback dynamic params from the kwargs
@ -1589,11 +1613,16 @@ class Logging(LiteLLMLoggingBaseClass):
if cache_hit is True:
return 0.0
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):
return 0.0
transformed_result: Final = self._generate_content_result_as_model_response(result)
if transformed_result is not None:
result = transformed_result
if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"):
if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"):
hidden_params: Final = getattr(result, "_hidden_params", {})
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
@ -4639,6 +4668,19 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
if custom_logger_init_args.get("newrelic_api_key"):
# Team-scoped credentials: per-team METRICS logger, isolated per
# credential set via DynamicLoggingCache. The trace logger for
# this name stays on the global path below.
from litellm.integrations.newrelic.newrelic_team_handler import (
NewRelicHandler,
)
return NewRelicHandler.get_newrelic_logger_for_request(
standard_callback_dynamic_params=custom_logger_init_args,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
@ -5060,7 +5102,7 @@ class StandardLoggingPayloadSetup:
return messages
@staticmethod
def merge_litellm_metadata(litellm_params: dict) -> dict:
def merge_litellm_metadata(litellm_params: Mapping[str, object]) -> dict:
"""
Merge both litellm_metadata and metadata from litellm_params.
@ -5822,7 +5864,7 @@ def get_standard_logging_object_payload(
cache_hit: Final = kwargs.get("cache_hit", False)
# Extract usage as a plain dict, avoiding Pydantic round-trip
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=response_obj,
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
)
usage_dict: Final = (

54
litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py
View file

@ -21,11 +21,13 @@ class GuardrailCostEntry(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
guardrail_cost: float | None = None
# ``bool | None`` because the TypedDict sanctions None; None means "not set"
# and keeps the default billed behavior, so a None-carrying entry must not
# fail union validation and silently zero a sibling entry's real cost.
guardrail_cost_in_spend: bool | None = True
GuardrailInformationShape = tuple[GuardrailCostEntry, ...] | GuardrailCostEntry | None
_GUARDRAIL_INFORMATION_ADAPTER: Final[TypeAdapter[GuardrailInformationShape]] = TypeAdapter(GuardrailInformationShape)
_GUARDRAIL_COST_ENTRY_ADAPTER: Final[TypeAdapter[GuardrailCostEntry]] = TypeAdapter(GuardrailCostEntry)
def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing | None:
@ -47,23 +49,55 @@ def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str
return sum(units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items())
AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records"
def azure_prompt_shield_guardrail_cost(
usage_units: Mapping[str, int],
cost_tier: str | None,
price_per_1000_text_records: float | None,
) -> float | None:
"""USD cost of an Azure Prompt Shield invocation from its text-record count.
Returns 0.0 on the free tier, ``text_records * price / 1000`` when a price is
configured, and None when pricing is not configured (usage-only tracking).
"""
if cost_tier == "free":
return 0.0
if price_per_1000_text_records is None:
return None
return usage_units.get(AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT, 0) * price_per_1000_text_records / 1000.0
def _billable_entry_cost(entry: GuardrailCostEntry) -> float:
if entry.guardrail_cost_in_spend is False:
return 0.0
cost: Final = entry.guardrail_cost
if cost is None or not math.isfinite(cost) or cost <= 0.0:
return 0.0
return cost
def guardrail_information_cost(guardrail_information: object) -> float:
def _validated_entry_cost(raw: object) -> float:
"""Billable cost of one raw ``guardrail_information`` entry.
Validated per entry so one malformed entry (e.g. a custom hook stamping a
non-boolean ``guardrail_cost_in_spend``) prices to 0.0 by itself instead of
failing a whole-payload validation and silently zeroing a sibling entry's
real billable cost."""
try:
parsed: Final = _GUARDRAIL_INFORMATION_ADAPTER.validate_python(guardrail_information)
except ValidationError:
return _billable_entry_cost(_GUARDRAIL_COST_ENTRY_ADAPTER.validate_python(raw))
except ValidationError as e:
verbose_logger.warning("Ignoring malformed guardrail_information entry for guardrail cost: %s", e)
return 0.0
if parsed is None:
def guardrail_information_cost(guardrail_information: object) -> float:
if guardrail_information is None:
return 0.0
if isinstance(parsed, GuardrailCostEntry):
return _billable_entry_cost(parsed)
return sum(_billable_entry_cost(entry) for entry in parsed)
if isinstance(guardrail_information, (list, tuple)):
return sum(_validated_entry_cost(entry) for entry in guardrail_information)
return _validated_entry_cost(guardrail_information)
def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None:

79
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -7,7 +7,7 @@ from typing import Any, Final, Literal
import litellm
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
from litellm.types.llms.openai import (
FileSearchTool,
ResponsesAPIResponse,
@ -64,11 +64,17 @@ class StandardBuiltInToolCostTracking:
"""
standard_built_in_tools_params = standard_built_in_tools_params or {}
google_maps_grounding_cost: Final = StandardBuiltInToolCostTracking._handle_google_maps_grounding_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
)
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
):
return StandardBuiltInToolCostTracking._handle_web_search_cost(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
@ -78,19 +84,56 @@ class StandardBuiltInToolCostTracking:
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object):
return StandardBuiltInToolCostTracking._handle_file_search_cost(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
@staticmethod
def _resolve_model_info(model: str, custom_llm_provider: str | None) -> tuple[ModelInfo | None, str | None]:
direct: Final = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if direct is not None:
return direct, custom_llm_provider or direct["litellm_provider"]
if "/" not in model:
return None, custom_llm_provider
by_prefix: Final = StandardBuiltInToolCostTracking._safe_get_model_info(model=model)
if by_prefix is None:
return None, custom_llm_provider
return by_prefix, by_prefix["litellm_provider"]
@staticmethod
def _handle_google_maps_grounding_cost(
model: str,
custom_llm_provider: str | None,
usage: Usage | None,
) -> float:
from litellm.llms import get_cost_for_google_maps_grounding_request
from litellm.llms.gemini.cost_calculator import google_maps_grounding_requests
if usage is None or google_maps_grounding_requests(usage) is None:
return 0.0
model_info, resolved_provider = StandardBuiltInToolCostTracking._resolve_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if model_info is None or resolved_provider is None:
return 0.0
return (
get_cost_for_google_maps_grounding_request(
custom_llm_provider=resolved_provider, usage=usage, model_info=model_info
)
or 0.0
)
@staticmethod
def _handle_web_search_cost(
model: str,
@ -102,29 +145,21 @@ class StandardBuiltInToolCostTracking:
"""Handle web search cost calculation."""
from litellm.llms import get_cost_for_web_search_request
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
# A provider-prefixed model (e.g. gemini/gemini-3.1-flash-lite) may not map under the
# request's custom_llm_provider. _resolve_model_info re-resolves from the prefix and adopts
# that provider so the cost is routed and priced with the model_info that was actually
# resolved, instead of feeding a re-resolved model into the original provider's calculator.
model_info, resolved_provider = StandardBuiltInToolCostTracking._resolve_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
# A provider-prefixed model (e.g. gemini/gemini-3.1-flash-lite) may not map under the
# request's custom_llm_provider. Re-resolve from the prefix and adopt that provider so the
# cost is routed and priced with the model_info that was actually resolved, instead of
# feeding a re-resolved model into the original provider's calculator.
if model_info is None and "/" in model:
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(model=model)
if model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
if custom_llm_provider is None and model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
resolved_usage: Final = StandardBuiltInToolCostTracking._usage_with_anthropic_web_search(
usage=usage, response_object=response_object
)
if model_info is not None and resolved_usage is not None and custom_llm_provider is not None:
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
)
@ -333,7 +368,7 @@ class StandardBuiltInToolCostTracking:
get_anthropic_web_search_requests_from_response,
)
if usage is not None and (_get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None):
if usage is not None and (get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None):
return usage
web_search_requests: Final = get_anthropic_web_search_requests_from_response(response_object)
if web_search_requests is None:
@ -381,7 +416,7 @@ class StandardBuiltInToolCostTracking:
# Anthropic Claude (direct API and Vertex AI) uses server_tool_use.web_search_requests.
# Without this check, Claude ModelResponse always falls through to return False
# and _handle_web_search_cost() is never called.
if hasattr(usage, "server_tool_use") and _get_web_search_requests(usage.server_tool_use) is not None:
if hasattr(usage, "server_tool_use") and get_web_search_requests(usage.server_tool_use) is not None:
return True
# xAI reports usage.server_side_tool_usage_details.web_search_calls; a searched
# answer with no url_citation annotations has no other chat-path signal
@ -396,7 +431,7 @@ class StandardBuiltInToolCostTracking:
elif usage is not None:
if (
hasattr(usage, "server_tool_use")
and _get_web_search_requests(usage.server_tool_use) is not None
and get_web_search_requests(usage.server_tool_use) is not None
or (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None

37
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,6 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from typing import Any, Final
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
@ -39,7 +39,7 @@ class TranscriptionUsageObjectTransformation:
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
_INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
@ -59,7 +59,7 @@ def _token_count(value: object) -> int:
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
@ -69,10 +69,13 @@ def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, i
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
entries: Final = usage_object.get("grounding_tool_count")
if not isinstance(entries, Sequence):
return 0
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
for entry in entries
if isinstance(entry, Mapping) and entry.get("type") == "google_search"
)
@ -112,30 +115,30 @@ class InteractionsUsageObjectTransformation:
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
input_sums: Final = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
reasoning_tokens: Final = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
prompt_tokens: Final = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
completion_tokens: Final = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens: Final = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
web_search_requests: Final = _google_search_query_count(usage_object)
prompt_tokens_details: Final = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
@ -144,7 +147,7 @@ class InteractionsUsageObjectTransformation:
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
completion_tokens_details: Final = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,

33
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -72,7 +72,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
def _get_web_search_requests(server_tool_use: Any) -> int | None:
def get_web_search_requests(server_tool_use: Any) -> int | None:
"""
Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance,
@ -889,11 +889,22 @@ def generic_cost_per_token(
total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting:
text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens
if has_double_counting:
# cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a
# modality can only bill what the cache did not already cover or the overlap is billed twice
uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
billable_audio: Final = min(audio_tokens, uncached_budget)
billable_image: Final = min(image_tokens, uncached_budget - billable_audio)
billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
prompt_tokens_details["audio_tokens"] = billable_audio
prompt_tokens_details["image_tokens"] = billable_image
prompt_tokens_details["video_tokens"] = billable_video
prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video
elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0:
# Clamp to zero: inconsistent streaming usage
text_tokens = max(text_tokens, 0)
prompt_tokens_details["text_tokens"] = text_tokens
prompt_tokens_details["text_tokens"] = max(
usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
)
(
prompt_base_cost,
@ -1063,15 +1074,17 @@ def get_token_type_cost_breakdown(
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
# Reasoning is billed at the selected tier's reasoning rate for tiered models,
# else at the explicit per-reasoning-token rate when the model defines one,
# otherwise at the standard output-token rate - this mirrors how the total
# completion cost is computed, so the breakdown can never diverge from it.
# else at the service-tier-aware per-reasoning-token rate - this mirrors how the
# total completion cost is computed, so the breakdown can never diverge from it.
tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
flat_reasoning_rate: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
reasoning_rate: Final = (
tiered_reasoning_rate
if tiered_reasoning_rate is not None
else (flat_reasoning_rate if flat_reasoning_rate is not None else completion_base_cost)
else _resolve_reasoning_token_cost(
model_info=model_info,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
)
)
reasoning_cost = float(reasoning_tokens) * reasoning_rate

2
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -178,7 +178,7 @@ def update_response_metadata(
- response._hidden_params["litellm_overhead_time_ms"]
- response.response_time_ms
"""
if result is None:
if result is None or not hasattr(result, "_hidden_params"):
return
metadata: Final = ResponseMetadata(result)

69
litellm/litellm_core_utils/logging_worker.py
View file

@ -4,6 +4,7 @@
import asyncio
import atexit
import contextvars
import inspect
import logging
from collections.abc import Coroutine, Iterator
from typing import Final
@ -53,6 +54,7 @@ class LoggingWorker:
self._queue: asyncio.Queue[LoggingTask] | None = None
self._worker_task: asyncio.Task | None = None
self._running_tasks: set[asyncio.Task] = set()
self._dequeued_tasks: dict[int, LoggingTask] = {} # mutable-ok: refs so flush can rescue never-started tasks
self._sem: asyncio.Semaphore | None = None
self._bound_loop: asyncio.AbstractEventLoop | None = None
self._last_aggressive_clear_time: float = 0.0
@ -61,6 +63,38 @@ class LoggingWorker:
# Register cleanup handler to flush remaining events on exit
atexit.register(self._flush_on_exit)
def _track_dequeued(self, task: LoggingTask) -> None:
self._dequeued_tasks[id(task)] = task
def _untrack_dequeued(self, task: LoggingTask) -> None:
self._dequeued_tasks.pop(id(task), None)
def _unstarted_dequeued_tasks(self) -> tuple[LoggingTask, ...]:
return tuple(
task
for task in self._dequeued_tasks.values()
if inspect.getcoroutinestate(task["coroutine"]) == inspect.CORO_CREATED
)
def _requeue_unstarted_dequeued(self, new_queue: "asyncio.Queue[LoggingTask]") -> int:
revived: Final = self._unstarted_dequeued_tasks()
self._dequeued_tasks.clear()
for index, revived_task in enumerate(revived):
try:
new_queue.put_nowait(revived_task)
except asyncio.QueueFull:
for leftover in revived[index:]:
self._track_dequeued(leftover)
return index
return len(revived)
def _run_coroutine_silently(self, loop: asyncio.AbstractEventLoop, coroutine: Coroutine) -> bool:
try:
loop.run_until_complete(asyncio.wait_for(coroutine, timeout=self.timeout))
except (Exception, asyncio.CancelledError): # noqa: BLE001 # atexit flush must never break the user's program
return False
return True
@staticmethod
def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
"""Pop every task still queued, without awaiting them, so they can be moved to another queue."""
@ -90,10 +124,12 @@ class LoggingWorker:
new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
for carried_task in carried_over:
new_queue.put_nowait(carried_task)
if carried_over:
revived_count: Final = self._requeue_unstarted_dequeued(new_queue)
if carried_over or revived_count:
verbose_logger.warning(
"LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
"LoggingWorker: event loop changed; carried %d pending and revived %d dequeued logging task(s) onto the new loop",
len(carried_over),
revived_count,
)
else:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
@ -129,6 +165,7 @@ class LoggingWorker:
except Exception as e:
verbose_logger.exception("LoggingWorker error: %s", e)
finally:
self._untrack_dequeued(task)
self._queue.task_done()
finally:
# Always release semaphore, even if queue is None
@ -146,6 +183,7 @@ class LoggingWorker:
await self._sem.acquire()
try:
task = await self._queue.get()
self._track_dequeued(task)
# Track each spawned coroutine so we can cancel on shutdown.
processing_task = asyncio.create_task(self._process_log_task(task, self._sem))
self._running_tasks.add(processing_task)
@ -298,9 +336,10 @@ class LoggingWorker:
extracted_tasks: Final = []
for _ in range(items_to_extract):
try:
extracted_tasks.append(self._queue.get_nowait())
extracted_tasks.append(extracted := self._queue.get_nowait())
except asyncio.QueueEmpty:
break
self._track_dequeued(extracted)
return extracted_tasks
@ -318,6 +357,7 @@ class LoggingWorker:
# Add new task to extracted tasks to process directly
if new_task is not None:
self._track_dequeued(new_task)
extracted_tasks.append(new_task)
# Process extracted tasks directly
@ -343,6 +383,7 @@ class LoggingWorker:
# Suppress errors during processing to ensure we keep going
pass
finally:
self._untrack_dequeued(task)
self._queue.task_done()
async def _process_extracted_tasks(self, tasks: list[LoggingTask]) -> None:
@ -486,11 +527,12 @@ class LoggingWorker:
self._safe_log("debug", "[LoggingWorker] atexit: No queue initialized")
return
if self._queue.empty():
unstarted_dequeued: Final = self._unstarted_dequeued_tasks()
if self._queue.empty() and not unstarted_dequeued:
self._safe_log("debug", "[LoggingWorker] atexit: Queue is empty")
return
queue_size: Final = self._queue.qsize()
queue_size: Final = self._queue.qsize() + len(unstarted_dequeued)
self._safe_log("info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events...")
# Create a new event loop since the original is closed
@ -509,6 +551,16 @@ class LoggingWorker:
previous_raise_exceptions: Final = logging.raiseExceptions
logging.raiseExceptions = False
try:
for pending in unstarted_dequeued:
if (
processed >= MAX_ITERATIONS_TO_CLEAR_QUEUE
or loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE
):
break
if self._run_coroutine_silently(loop, pending["coroutine"]):
processed += 1
self._untrack_dequeued(pending)
while not self._queue.empty() and processed < MAX_ITERATIONS_TO_CLEAR_QUEUE:
if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
self._safe_log(
@ -526,11 +578,8 @@ class LoggingWorker:
# Note: We run the coroutine directly, not via create_task,
# since we're in a new event loop context
try:
loop.run_until_complete(task["coroutine"])
processed += 1
except Exception:
# Silent failure to not break user's program
pass
if self._run_coroutine_silently(loop, task["coroutine"]):
processed += 1
finally:
# Clear reference to prevent memory leaks
task = None

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -511,9 +511,6 @@ def update_messages_with_model_file_ids(
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
@ -588,9 +585,6 @@ def update_responses_input_with_model_file_ids(
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)

1
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -28,6 +28,7 @@ PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in MirroredPricingParams.model_
"cache_creation_input_token_cost_above_1hr",
"cache_creation_input_token_cost_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
"google_maps_grounding_cost_per_query",
)
# tiered_pricing is emptied rather than zeroed: its tiers outrank the zeros written beside
# them, so a zero here would leave the cost map's tiers billing the traffic the reserved

3
litellm/litellm_core_utils/redact_messages.py
View file

@ -10,6 +10,7 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import litellm
@ -191,7 +192,7 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_tool_calls_dict(message: dict) -> None:
def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
"""Redact tool call / function_call arguments in a dict-form message or delta."""
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list):

10
litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py
View file

@ -13,6 +13,7 @@ import json
from typing import Any, Final
import litellm
from litellm._logging import verbose_logger
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
from ...caching import InMemoryCache
@ -46,6 +47,15 @@ class LangfuseInMemoryCache(InMemoryCache):
_created_langfuse_logger.Langfuse.flush()
_created_langfuse_logger.Langfuse.shutdown()
# Loggers with a periodic flush task (e.g. NewRelicMetricsLogger) expose
# stop() so eviction actually ends the task instead of leaking it.
_evicted_stop: Final = getattr(self.cache_dict[key], "stop", None)
if callable(_evicted_stop):
try:
_evicted_stop()
except Exception: # noqa: BLE001 # a failing stop() must not block eviction
verbose_logger.debug("DynamicLoggingCache: stop() raised during eviction", exc_info=True)
#########################################################
# Call parent class to remove key from cache
#########################################################

43
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -173,6 +173,27 @@ def attach_cache_creation_token_details(
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
def apply_grounding_request_counts(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
web_search_requests: int | None,
google_maps_grounding_requests: int | None,
) -> PromptTokensDetailsWrapper | None:
updates: Final = MappingProxyType(
{
field: value
for field, value in (
("web_search_requests", web_search_requests),
("google_maps_grounding_requests", google_maps_grounding_requests),
)
if value is not None
}
)
if not updates:
return prompt_tokens_details
counted: Final = prompt_tokens_details if prompt_tokens_details is not None else PromptTokensDetailsWrapper()
return counted.model_copy(update=updates)
class ChunkProcessor:
def __init__(self, chunks: list, messages: list | None = None):
self.chunks = self._sort_chunks(chunks)
@ -778,6 +799,7 @@ class ChunkProcessor:
server_tool_use: ServerToolUse | None = None
web_search_requests: int | None = None
google_maps_grounding_requests: int | None = None
completion_tokens_details: CompletionTokensDetails | None = None
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
# Anthropic emits the cache-creation TTL breakdown (5m/1h split) only on
@ -827,6 +849,13 @@ class ChunkProcessor:
)
if chunk_web_search_requests is not None:
web_search_requests = chunk_web_search_requests
chunk_google_maps_grounding_requests: int | None = getattr(
usage_chunk_dict["prompt_tokens_details"],
"google_maps_grounding_requests",
None,
)
if chunk_google_maps_grounding_requests is not None:
google_maps_grounding_requests = chunk_google_maps_grounding_requests
prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"] or prompt_tokens_details
@ -852,6 +881,7 @@ class ChunkProcessor:
cache_read_input_tokens=cache_read_input_tokens,
server_tool_use=server_tool_use,
web_search_requests=web_search_requests,
google_maps_grounding_requests=google_maps_grounding_requests,
completion_tokens_details=completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
cost=cost,
@ -939,6 +969,7 @@ class ChunkProcessor:
server_tool_use: Final[ServerToolUse | None] = calculated_usage_per_chunk["server_tool_use"]
web_search_requests: Final[int | None] = calculated_usage_per_chunk["web_search_requests"]
google_maps_grounding_requests: Final[int | None] = calculated_usage_per_chunk["google_maps_grounding_requests"]
completion_tokens_details: Final[CompletionTokensDetails | None] = calculated_usage_per_chunk[
"completion_tokens_details"
]
@ -998,13 +1029,11 @@ class ChunkProcessor:
if server_tool_use is not None:
returned_usage.server_tool_use = server_tool_use
if web_search_requests is not None:
if returned_usage.prompt_tokens_details is None:
returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper(
web_search_requests=web_search_requests
)
else:
returned_usage.prompt_tokens_details.web_search_requests = web_search_requests
returned_usage.prompt_tokens_details = apply_grounding_request_counts(
returned_usage.prompt_tokens_details,
web_search_requests,
google_maps_grounding_requests,
)
if cost is not None:
setattr(returned_usage, "cost", cost)

15
litellm/llms/__init__.py
View file

@ -14,6 +14,21 @@ if TYPE_CHECKING:
from litellm.types.utils import ModelInfo, Usage
def get_cost_for_google_maps_grounding_request(
custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo"
) -> float | None:
"""
Get the cost of Grounding with Google Maps for a given model. Only Gemini models on the
Gemini API and Vertex AI can populate the Maps grounding counter, so every other provider
returns None.
"""
if custom_llm_provider != "gemini" and not custom_llm_provider.startswith("vertex_ai"):
return None
from .gemini.cost_calculator import cost_per_google_maps_grounding_request
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
"""
Get the cost for a web search request for a given model.

5
litellm/llms/anthropic/chat/handler.py
View file

@ -712,11 +712,14 @@ class ModelResponseIterator:
def _handle_usage(self, anthropic_usage_chunk: dict | UsageDelta) -> Usage:
reasoning_content: Final = "".join(self.reasoning_content_chunks) if self.reasoning_content_chunks else None
return AnthropicConfig().calculate_usage(
usage: Final = AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=reasoning_content,
speed=self.speed,
)
if usage.speed is not None:
self.speed = usage.speed
return usage
def _content_block_delta_helper(
self, chunk: dict

4
litellm/llms/anthropic/chat/transformation.py
View file

@ -2281,6 +2281,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
str | None,
_usage.get("service_tier"),
)
raw_speed: Final = _usage.get("speed")
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
iterations: Final[list[Any] | None] = _usage.get("iterations")
if iterations:
@ -2355,7 +2357,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else None
),
inference_geo=inference_geo,
speed=speed,
speed=resolved_speed,
service_tier=service_tier,
)
return usage

23
litellm/llms/anthropic/common_utils.py
View file

@ -4,7 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import re
from collections.abc import Mapping, Sequence
from collections.abc import Mapping, MutableMapping, Sequence
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Any, Final, Literal
@ -93,8 +93,8 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
"""
Handle Anthropic OAuth token detection and header setup.
If an OAuth token is detected in the Authorization header, extracts it
and sets the required OAuth headers.
If an OAuth token is detected in the Authorization header (any casing),
extracts it and sets the required OAuth headers.
Args:
headers: Request headers dict
@ -104,16 +104,21 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
Tuple of (updated headers, api_key)
"""
# Check Authorization header (passthrough / forwarded requests)
auth_header: Final = headers.get("authorization", "")
if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.replace("Bearer ", "")
headers.pop("x-api-key", None)
auth_header: Final = next((value for name, value in headers.items() if name.lower() == "authorization"), "")
if auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.removeprefix("Bearer ")
for name in tuple(
header_name for header_name in headers if header_name.lower() in ("x-api-key", "authorization")
):
headers.pop(name)
headers["authorization"] = auth_header
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
# Check api_key directly (standard chat/completion flow)
if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX):
headers.pop("x-api-key", None)
for name in tuple(header_name for header_name in headers if header_name.lower() == "x-api-key"):
headers.pop(name)
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
@ -468,7 +473,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
@staticmethod
def maybe_drop_disabled_thinking(
model: str,
optional_params: dict, # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param, as in _maybe_drop_speed_param
custom_llm_provider: str,
) -> None:
"""Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models

47
litellm/llms/anthropic/cost_calculation.py
View file

@ -8,12 +8,9 @@ from typing import TYPE_CHECKING, Final, Optional
from pydantic import BaseModel, ValidationError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_token_base_cost,
_get_web_search_requests,
calculate_cache_writing_cost,
generic_cost_per_token,
get_provider_specific_geo_multiplier,
parse_prompt_tokens_details,
get_web_search_requests,
)
if TYPE_CHECKING:
@ -21,43 +18,6 @@ if TYPE_CHECKING:
import litellm
def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None) -> float:
"""
Return only the cache-related portion of the prompt cost (cache read + cache write).
These costs must NOT be scaled by the ``fast`` speed multiplier because the old
explicit ``fast/`` model entries carried unchanged cache rates while
multiplying only the regular input/output token costs. Regional pricing, by
contrast, uplifts every token type, so the geo multiplier does scale them.
"""
if usage.prompt_tokens_details is None:
return 0.0
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
(
_,
_,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(model_info=model_info, usage=usage, service_tier=service_tier)
cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
if (
prompt_tokens_details["cache_creation_tokens"]
or prompt_tokens_details["cache_creation_token_details"] is not None
):
cache_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
)
return cache_cost
def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -89,8 +49,7 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
)
if speed_multiplier != 1.0:
cache_cost: Final = _compute_cache_only_cost(model_info=model_info, usage=usage, service_tier=service_tier)
prompt_cost = (prompt_cost - cache_cost) * speed_multiplier + cache_cost
prompt_cost *= speed_multiplier
completion_cost *= speed_multiplier
if geo_multiplier != 1.0:
@ -145,7 +104,7 @@ def get_cost_for_anthropic_web_search(
if usage is None:
return 0.0
web_search_requests: Final = _get_web_search_requests(getattr(usage, "server_tool_use", None))
web_search_requests: Final = get_web_search_requests(getattr(usage, "server_tool_use", None))
if web_search_requests is None:
return 0.0

20
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -99,6 +99,7 @@ from litellm.types.llms.anthropic import (
ContextManagementResponse,
MessageBlockDelta,
MessageDelta,
ServerToolUsage,
StreamingContentBlockDeltaType,
UsageDelta,
UsageIteration,
@ -1354,10 +1355,24 @@ class LiteLLMAnthropicMessagesAdapter:
return explicit_value
return cls._first_positive_prompt_tokens_detail_value(usage, ("cache_creation_tokens", "cache_write_tokens"))
@classmethod
def _get_web_search_request_count(cls, usage: Usage) -> int:
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests,
)
from_server_tool_use: Final = cls._positive_int(
get_web_search_requests(getattr(usage, "server_tool_use", None))
)
if from_server_tool_use > 0:
return from_server_tool_use
return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",))
@classmethod
def _translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta:
cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage)
cache_creation_input_tokens: Final = cls._get_cache_creation_input_tokens(usage)
web_search_requests: Final = cls._get_web_search_request_count(usage)
input_tokens: Final = max(
(usage.prompt_tokens or 0) - cache_read_input_tokens - cache_creation_input_tokens,
0,
@ -1371,6 +1386,11 @@ class LiteLLMAnthropicMessagesAdapter:
usage_delta["cache_creation_input_tokens"] = cache_creation_input_tokens
if cache_read_input_tokens > 0:
usage_delta["cache_read_input_tokens"] = cache_read_input_tokens
if web_search_requests > 0:
return UsageDelta(
**usage_delta,
server_tool_use=ServerToolUsage(web_search_requests=web_search_requests),
)
return usage_delta
@classmethod

4
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -352,8 +352,8 @@ async def _check_summary_model_budget(
)
return False
user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
user_id: Final = getattr(user_api_key_auth, "user_id", None)
user_model_max_budget: Final = user_api_key_auth.user_model_max_budget
user_id: Final = user_api_key_auth.user_id
if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
try:
await model_max_budget_limiter.is_user_within_model_budget(

21
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,7 @@ from functools import partial
from typing import Any, Final, cast
import litellm
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
flatten_unencrypted_web_search_results_in_anthropic_messages,
@ -21,6 +22,7 @@ from litellm.llms.anthropic.common_utils import (
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -382,13 +384,18 @@ async def anthropic_messages(
)
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
try:
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
return await init_response
return init_response
except BaseLLMException as e:
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
extra_kwargs=kwargs,
)
def validate_anthropic_api_metadata(metadata: dict | None = None) -> dict | None:

17
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -307,10 +308,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# Check for Anthropic OAuth token in Authorization header
headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
if "x-api-key" not in headers and "authorization" not in headers:
header_names: Final = frozenset(name.lower() for name in headers)
if "x-api-key" not in header_names and "authorization" not in header_names:
auth_header: Final = AnthropicModelInfo.get_auth_header(api_key)
if auth_header is not None:
headers.update(auth_header)
if auth_header is None:
raise AuthenticationError(
message=(
"Missing Anthropic API Key - A call is being made to anthropic but no key is set "
"either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` "
"or `ANTHROPIC_AUTH_TOKEN` in your environment vars"
),
llm_provider=self._resolved_provider,
model=model,
)
headers.update(auth_header)
if "anthropic-version" not in headers:
headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
if "content-type" not in headers:

2
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -582,7 +582,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": True,
"strict": output_format.get("strict", False),
}
}

23
litellm/llms/azure/realtime/handler.py
View file

@ -4,6 +4,8 @@ This file contains the calling Azure OpenAI's `/openai/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, cast
from litellm._logging import _redact_string, verbose_proxy_logger
@ -30,6 +32,21 @@ async def forward_messages(client_ws: Any, backend_ws: Any):
class AzureOpenAIRealtime(AzureChatCompletion):
@staticmethod
def get_auth_headers(api_key: str | None, azure_ad_token: str | None) -> Mapping[str, str]:
"""
Build the websocket handshake auth headers, preferring a static api-key and falling back to
an Azure AD (Entra ID) bearer token. Never sends both.
"""
if api_key:
return MappingProxyType({"api-key": api_key})
if azure_ad_token:
return MappingProxyType({"Authorization": f"Bearer {azure_ad_token}"})
raise ValueError(
"Missing Azure credentials for the realtime endpoint. Set an api_key, or configure Azure AD auth "
"(azure_ad_token, tenant_id/client_id/client_secret, or a managed identity)"
)
def _construct_url(
self,
api_base: str,
@ -117,13 +134,13 @@ class AzureOpenAIRealtime(AzureChatCompletion):
query_params=query_params,
)
auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
try:
ssl_context: Final = get_shared_realtime_ssl_context()
async with websockets.connect(
url,
additional_headers={
"api-key": api_key,
},
additional_headers=auth_headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
) as backend_ws:

30
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -65,6 +65,7 @@ from litellm.types.llms.openai import (
OpenAIMessageContentListBlock,
)
from litellm.types.utils import (
CacheCreationTokenDetails,
ChatCompletionMessageToolCall,
CompletionTokensDetailsWrapper,
Function,
@ -421,12 +422,16 @@ class AmazonConverseConfig(BaseConfig):
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS models: passed through unchanged via additionalModelRequestFields.
- OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model:
optional_params["reasoning_effort"] = reasoning_effort
elif "openai.gpt-5" in model:
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
optional_params["reasoning"] = reasoning
elif self._is_nova_2_model(model):
reasoning_config: Final = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
optional_params.update(reasoning_config)
@ -558,7 +563,7 @@ class AmazonConverseConfig(BaseConfig):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice")
if "gpt-oss" in model:
if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model:
supported_params.append("reasoning_effort")
elif self._is_nova_2_model(model):
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
@ -906,7 +911,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["_parallel_tool_use_config"] = {
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking":
if param == "thinking" and "openai.gpt-5" not in model:
if (
isinstance(value, dict)
and value.get("type") == "adaptive"
@ -1806,6 +1811,26 @@ class AmazonConverseConfig(BaseConfig):
thinking_blocks_list.append(_redacted_block)
return thinking_blocks_list
@staticmethod
def _parse_cache_details(usage: ConverseTokenUsageBlock) -> "CacheCreationTokenDetails | None":
"""Split ``cacheDetails`` into 5m/1h buckets, or ``None`` unless the split fully
accounts for ``cacheWriteInputTokens``, since a partial or unrecognized-ttl
breakdown would understate the cache-write cost.
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_CacheDetail.html
"""
cache_details: Final = usage.get("cacheDetails")
if not cache_details:
return None
tokens_5m: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "5m")
tokens_1h: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "1h")
if tokens_5m + tokens_1h != usage.get("cacheWriteInputTokens", 0):
return None
return CacheCreationTokenDetails(
ephemeral_5m_input_tokens=tokens_5m,
ephemeral_1h_input_tokens=tokens_1h,
)
@staticmethod
def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None:
"""Converse omits thinking tokens from its usage block; they only arrive under
@ -1877,6 +1902,7 @@ class AmazonConverseConfig(BaseConfig):
prompt_tokens_details: Final = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_creation_input_tokens,
cache_creation_token_details=self._parse_cache_details(usage),
text_tokens=raw_input_tokens,
)
estimated_reasoning_tokens: Final = (

8
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -243,7 +243,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
@staticmethod
def _agent_message_text(item: "Mapping[str, Any]") -> str:
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
@ -254,7 +254,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
)
@classmethod
def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = cls._agent_message_text(item)
if not text:
return None
@ -266,7 +266,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
@ -274,7 +274,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None

33
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -1,3 +1,4 @@
import ssl
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Final, cast
@ -18,6 +19,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_ssl_configuration,
)
from litellm.types.llms.openai import FileTypes
from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
@ -58,7 +60,11 @@ class BaseLLMAIOHTTPHandler:
# Create a transport using AsyncHTTPHandler's logic
try:
self.transport = AsyncHTTPHandler._create_aiohttp_transport()
ssl_config: Final = get_ssl_configuration()
self.transport = AsyncHTTPHandler._create_aiohttp_transport(
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
)
self._owns_transport = True
return self.transport
except Exception:
@ -81,20 +87,19 @@ class BaseLLMAIOHTTPHandler:
def _create_client_session_with_transport(self) -> ClientSession:
"""Create a new client session using transport or connector configuration."""
connector: Final = self._get_connector()
if self.transport is None:
connector: Final = self._get_connector()
if connector:
return aiohttp.ClientSession(connector=connector)
if self.transport and hasattr(self.transport, "_get_valid_client_session"):
# Use transport's session creation if available
session = self.transport._get_valid_client_session()
return session
elif connector:
# Use provided connector
session = aiohttp.ClientSession(connector=connector)
return session
else:
# Default session creation
session = aiohttp.ClientSession()
return session
transport: Final = self.transport or self._get_or_create_transport()
if transport is not None and hasattr(transport, "_get_valid_client_session"):
try:
return transport._get_valid_client_session()
except RuntimeError:
pass
return aiohttp.ClientSession()
def _get_async_client_session(self, dynamic_client_session: ClientSession | None = None) -> ClientSession:
if dynamic_client_session:

6
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5621,10 +5621,9 @@ class BaseLLMHTTPHandler:
kwargs=hook_kwargs,
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_agentic_loop [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)
@ -5646,10 +5645,9 @@ class BaseLLMHTTPHandler:
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)

100
litellm/llms/deepseek/chat/transformation.py
View file

@ -2,16 +2,17 @@
Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
"""
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping, Sequence
from typing import Any, Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
convert_content_list_to_str,
extract_search_results_text,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_reasoning
from litellm.utils import supports_reasoning, supports_vision
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -117,13 +118,98 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]:
"""
DeepSeek does not support content in list format.
DeepSeek vision models accept image_url content blocks in user
messages (https://api-docs.deepseek.com/guides/vision), so those
content lists are forwarded as-is, with any search_results text
appended as a trailing text block. Every other message keeps the
historical string collapse (which also folds search_results text
into string content); a list with no extractable text stays
unchanged, matching what DeepSeek historically received.
"""
messages = handle_messages_with_content_list_to_str_conversion(messages)
forward_images: Final = any(
isinstance(message.get("content"), list) for message in messages
) and supports_vision(model=model, custom_llm_provider="deepseek")
transformed: Final = [ # mutable-ok: provider messages must stay JSON-array lists the base transform mutates
self._forward_or_collapse_content(message=message, forward_images=forward_images) for message in messages
]
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed, model=model, is_async=False)
def _forward_or_collapse_content(self, message: AllMessageValues, forward_images: bool) -> AllMessageValues:
"""
Returns the vision-forwardable message with any search_results text
appended as a text block; every other message keeps the historical
string collapse, which extracts the text from a content list and
folds search_results text into string content.
"""
content: Final = message.get("content")
if (
forward_images
and isinstance(content, list)
and self._is_vision_forwardable_content(message=message, content=content)
):
return self._with_search_results_text_block(message=message, content=content)
collapsed: Final = convert_content_list_to_str(message=message)
if not collapsed or collapsed == content:
return message
collapsed_message: Final = {**message, "content": collapsed} # mutable-ok: wire messages are plain JSON dicts
return cast(AllMessageValues, collapsed_message) # cast-ok: TypedDict spread narrows to dict
def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool:
"""
True only for a user message whose content list holds well-formed
text and image_url blocks with at least one image; a block missing
its payload falls back to the string collapse instead of crashing
or reaching the wire malformed. The model capability gate lives in
the caller.
"""
if message.get("role") != "user":
return False
if not all(self._is_forwardable_block(block) for block in content):
return False
return any(isinstance(block, dict) and block.get("type") == "image_url" for block in content)
@staticmethod
def _is_forwardable_block(block: object) -> bool:
"""A dict block typed text or image_url that carries its payload."""
if not isinstance(block, dict):
return False
block_type: Final = block.get("type")
if block_type == "image_url":
return DeepSeekChatConfig._is_image_url_payload(block.get("image_url"))
if block_type == "text":
return isinstance(block.get("text"), str)
return False
@staticmethod
def _is_image_url_payload(payload: object) -> bool:
"""A url string or an object carrying one, per the OpenAI image_url shape."""
if isinstance(payload, str):
return bool(payload)
if not isinstance(payload, Mapping):
return False
url: Final = payload.get("url")
return isinstance(url, str) and bool(url)
def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues:
"""
Appends the message's search_results text as a trailing text block,
keeping the context that the string collapse used to fold in, and
drops the non-OpenAI search_results key from the wire message.
"""
message_fields: Final = cast(Mapping[str, object], message) # cast-ok: search_results is not on the TypedDicts
search_text: Final = extract_search_results_text(message_fields.get("search_results"))
if not search_text:
return message
forwarded_content: Final = [*content, {"type": "text", "text": search_text}] # mutable-ok: JSON-array content
forwarded: Final = { # mutable-ok: wire messages are plain JSON dicts
**{key: value for key, value in message_fields.items() if key != "search_results"},
"content": forwarded_content,
}
return cast(AllMessageValues, forwarded) # cast-ok: TypedDict spread narrows to dict
def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
"""

10
litellm/llms/fireworks_ai/common_utils.py
View file

@ -13,6 +13,13 @@ class FireworksAIException(BaseLLMException):
def get_fireworks_session_id(litellm_params: dict) -> str | None:
"""
Session id to send as `x-session-affinity`, or None when the caller gave none.
Deliberately does not fall back to `litellm_trace_id`: that is generated per
request (`str(uuid.uuid4())` when absent), so using it pins every request to a
different Fireworks node and prompt caching never hits.
"""
params: Final = litellm_params
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
@ -23,9 +30,6 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None:
value = metadata.get("session_id")
if value:
return str(value)
value = params.get("litellm_trace_id")
if value:
return str(value)
return None

70
litellm/llms/gemini/cost_calculator.py
View file

@ -39,25 +39,69 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
``model_info`` when available, falling back to $0.035 for models not
yet updated in the pricing JSON.
"""
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests
from litellm.types.utils import PromptTokensDetailsWrapper
_DEFAULT_COST: Final = 35e-3
search_costs: Final = model_info.get("search_context_cost_per_query") or {}
_cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST)
number_of_web_search_requests = 0
if (
usage is not None
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
):
number_of_web_search_requests = usage.prompt_tokens_details.web_search_requests
requests_from_prompt_details: Final = (
usage.prompt_tokens_details.web_search_requests
if (
usage is not None
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
)
else None
)
requests_from_server_tool_use: Final = get_web_search_requests(getattr(usage, "server_tool_use", None))
number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0
# per_prompt billing: clamp to 1 (flat fee per grounded API call)
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
if number_of_web_search_requests > 0 and billing_mode == "per_prompt":
number_of_web_search_requests = 1
billable_requests: Final = (
1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests
)
return _cost * number_of_web_search_requests
return _cost * billable_requests
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT: Final = 25e-3
def google_maps_grounding_requests(usage: "Usage | None") -> int | None:
from litellm.types.utils import PromptTokensDetailsWrapper
details: Final = usage.prompt_tokens_details if usage is not None else None
if not isinstance(details, PromptTokensDetailsWrapper) or not hasattr(details, "google_maps_grounding_requests"):
return None
return details.google_maps_grounding_requests
def cost_per_google_maps_grounding_request(usage: "Usage", model_info: "ModelInfo") -> float:
"""
Calculates the cost of Grounding with Google Maps.
Billing follows ``web_search_billing_unit`` in model_info the same way Google Search grounding
does: ``"per_query"`` (Gemini 3.x) multiplies the executed Maps queries, ``"per_prompt"``
(default, Gemini 2.x) charges one flat fee per grounded prompt.
The rate comes from ``google_maps_grounding_cost_per_query`` in ``model_info``, falling back
to Google's list price for that billing unit when the pricing JSON has no entry yet.
"""
requests: Final = google_maps_grounding_requests(usage)
if not requests or requests <= 0:
return 0.0
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
default_cost: Final = (
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY
if billing_mode == "per_query"
else GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT
)
configured_cost: Final = model_info.get("google_maps_grounding_cost_per_query")
cost: Final = default_cost if configured_cost is None else configured_cost
billed_requests: Final = requests if billing_mode == "per_query" else 1
return cost * billed_requests

37
litellm/llms/gemini/realtime/transformation.py
View file

@ -4,6 +4,7 @@ This file contains the transformation logic for the Gemini realtime API.
import json
from collections import OrderedDict
from collections.abc import Mapping
from typing import Any, Final, cast
import litellm
@ -72,6 +73,28 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig()._map_audio_params({"voice": voice})
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
@ -282,12 +305,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "voice":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
vertex_gemini_config = VertexGeminiConfig()
speech_config = vertex_gemini_config._map_audio_params({"voice": value})
speech_config = _gemini_live_speech_config(value)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
@ -365,10 +383,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
entry: Final = GeminiRealtimeConfig._model_cost_entry(model)
return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
@staticmethod
def _is_native_audio_model(model: str) -> bool:
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
"""Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
@ -384,7 +398,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
"""Drop fields Gemini Live native-audio rejects on ``setup``."""
generation_config: Final = setup.get("generationConfig")
if isinstance(generation_config, dict):
modalities: Final = generation_config.get("responseModalities")
@ -392,8 +405,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities(
model, modalities
)
if GeminiRealtimeConfig._is_native_audio_model(model):
generation_config.pop("speechConfig", None)
return setup
def _handle_session_update(

22
litellm/llms/minimax/messages/transformation.py
View file

@ -2,7 +2,7 @@
MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API
"""
from typing import Final
from typing import Any, Final # noqa: TID251 # override below must mirror the legacy base signature
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
@ -49,6 +49,26 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig):
"""
return api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/anthropic/v1/messages"
def validate_anthropic_messages_environment(
self,
headers: dict, # mutable-ok: mirrors the legacy base override signature
model: str,
messages: list[Any], # mutable-ok: mirrors the legacy base override signature
optional_params: dict, # mutable-ok: mirrors the legacy base override signature
litellm_params: dict, # mutable-ok: mirrors the legacy base override signature
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict, str | None]: # mutable-ok: mirrors the legacy base override signature
return super().validate_anthropic_messages_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=self.get_api_key(api_key=api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: str | None,

81
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,8 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Container, Coroutine
from types import MappingProxyType
from collections.abc import Callable, Container, Coroutine
from typing import (
Final,
Literal,
@ -17,26 +16,42 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling
from litellm.utils import supports_function_calling, supports_response_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
LITELLM_INTERNAL_ASSISTANT_FIELDS: Final = frozenset({"thinking_blocks", "provider_specific_fields"})
PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs"
def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None:
try:
if check(model):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get(flag) is False:
return False
return None
def _function_calling_verdict(model: str) -> bool | None:
try:
if supports_function_calling(model, custom_llm_provider="together_ai"):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
return False
return None
return _registry_verdict(
model,
"supports_function_calling",
lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"),
)
def _response_schema_verdict(model: str) -> bool | None:
return _registry_verdict(
model,
"supports_response_schema",
lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"),
)
def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
@ -68,6 +83,32 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
)
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
if "response_format" not in passed_params:
return False
verdict: Final = _response_schema_verdict(model)
if verdict is True:
return False
if verdict is None:
verbose_logger.warning(
"together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return False
if drop_params or litellm.drop_params:
verbose_logger.warning(
"together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return True
raise UnsupportedParamsError(
status_code=500,
message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
)
def _without_litellm_internal_fields(message: AllMessageValues) -> AllMessageValues:
if message["role"] != "assistant" or LITELLM_INTERNAL_ASSISTANT_FIELDS.isdisjoint(message):
return message
@ -112,18 +153,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
return super()._transform_messages(stripped, model, is_async=True)
return super()._transform_messages(stripped, model, is_async=False)
def get_supported_openai_params(self, model: str) -> list:
supports_fc: Final = _function_calling_verdict(model)
supported_params: Final = super().get_supported_openai_params(model)
if supports_fc is True:
return supported_params
verbose_logger.debug(
"Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
)
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
param for param in supported_params if param != "response_format"
]
def map_openai_params(
self,
non_default_params: dict,
@ -134,6 +163,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
mapped_openai_params.pop(param)
if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
if _drop_response_format(mapped_openai_params, model, drop_params):
mapped_openai_params.pop("response_format")
return mapped_openai_params

7
litellm/llms/vertex_ai/cost_calculator.py
View file

@ -64,6 +64,7 @@ def cost_per_character(
usage: Usage,
prompt_characters: float | None = None,
completion_characters: float | None = None,
service_tier: str | None = None,
vertex_location: str | None = None,
) -> tuple[float, float]:
"""
@ -74,6 +75,8 @@ def cost_per_character(
- custom_llm_provider: str, "vertex_ai-*"
- prompt_characters: float, the number of input characters
- completion_characters: float, the number of output characters
- service_tier: optional tier derived from Gemini trafficType
("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch).
- vertex_location: the Vertex AI location serving the request; non-global
locations apply the model's regional-endpoint uplift multiplier
@ -92,6 +95,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
else:
try:
@ -123,6 +127,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
## CALCULATE OUTPUT COST
@ -131,6 +136,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
else:
completion_tokens: Final = usage.completion_tokens
@ -162,6 +168,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)

56
litellm/llms/vertex_ai/gemini/grounding_requests.py Normal file
View file

@ -0,0 +1,56 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final
@dataclass(frozen=True, slots=True)
class GroundingRequests:
web_search_requests: int | None
google_maps_grounding_requests: int | None
def has_billable_grounding(self) -> bool:
return bool(self.web_search_requests or self.google_maps_grounding_requests)
def _chunk_kinds(item: Mapping[str, object]) -> frozenset[str]:
chunks: Final = item.get("groundingChunks")
if not isinstance(chunks, list):
return frozenset()
return frozenset(kind for chunk in chunks if isinstance(chunk, Mapping) for kind in chunk)
def _queries(item: Mapping[str, object]) -> frozenset[str]:
queries: Final = item.get("webSearchQueries")
if not isinstance(queries, list):
return frozenset()
return frozenset(query for query in queries if isinstance(query, str) and query)
def _is_maps_item(item: Mapping[str, object]) -> bool:
return "maps" in _chunk_kinds(item) or bool(item.get("googleMapsWidgetContextToken"))
def _attributes_queries_to_maps(item: Mapping[str, object]) -> bool:
return _is_maps_item(item) and "web" not in _chunk_kinds(item)
def calculate_grounding_requests(grounding_metadata: Sequence[Mapping[str, object]]) -> GroundingRequests:
"""Billable grounding requests across candidates, counting each distinct query once.
Duplicate queries within and across grounding metadata items collapse to the
distinct-query count (#36377), and empty strings are ignored. Maps grounding is
floored at one request whenever a candidate carries maps chunks or a widget token,
since per-prompt billing charges the prompt even when no query is reported.
"""
items: Final = tuple(item for item in grounding_metadata if isinstance(item, Mapping))
web_queries: Final = frozenset(
query for item in items if not _attributes_queries_to_maps(item) for query in _queries(item)
)
maps_queries: Final = frozenset(
query for item in items if _attributes_queries_to_maps(item) for query in _queries(item)
)
has_maps: Final = any(_is_maps_item(item) for item in items)
return GroundingRequests(
web_search_requests=len(web_queries) or None,
google_maps_grounding_requests=max(len(maps_queries), 1) if has_maps else None,
)

44
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -89,6 +89,7 @@ from ..common_utils import (
supports_response_json_schema,
)
from ..vertex_llm_base import VertexBase
from .grounding_requests import calculate_grounding_requests
from .transformation import (
_gemini_convert_messages_with_history,
async_transform_request_body,
@ -1717,14 +1718,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_response: GenerateContentResponseBody | BidiGenerateContentServerMessage,
) -> bool:
"""
Whether the response used Grounding with Google Search, detected via
groundingMetadata.webSearchQueries (an actual web search was performed).
Whether the response used Grounding with Google Search or Grounding with Google Maps,
detected via groundingMetadata.webSearchQueries (an actual web search was performed) or
groundingMetadata.groundingChunks[].maps (a Maps lookup was performed).
Google bills grounding-with-Google-Search retrieved tokens separately (a per-request /
per-query search fee) and excludes them from input token billing, unlike URL context /
File Search / code execution whose tool-use tokens are charged at the input token rate.
URL context also emits groundingMetadata (with groundingChunks but no webSearchQueries),
so presence of groundingMetadata alone is not a sufficient signal.
Google bills both groundings separately (a per-request / per-query fee) and excludes their
retrieved tokens from input token billing, unlike URL context / File Search / code execution
whose tool-use tokens are charged at the input token rate. URL context also emits
groundingMetadata (with web groundingChunks but no webSearchQueries), so presence of
groundingMetadata alone is not a sufficient signal.
See https://ai.google.dev/gemini-api/docs/pricing and
https://github.com/BerriAI/litellm/discussions/33198
"""
@ -1732,7 +1734,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return False
for candidate in completion_response["candidates"] or []:
grounding_metadata, _, _, _ = VertexGeminiConfig._extract_candidate_metadata(candidate)
if VertexGeminiConfig._calculate_web_search_requests(grounding_metadata):
if calculate_grounding_requests(grounding_metadata).has_billable_grounding():
return True
return False
@ -1979,16 +1981,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
@staticmethod
def _calculate_web_search_requests(grounding_metadata: list[dict]) -> int | None:
web_search_requests: int | None = None
return calculate_grounding_requests(grounding_metadata).web_search_requests
if grounding_metadata and isinstance(grounding_metadata, list) and len(grounding_metadata) > 0:
for grounding_metadata_item in grounding_metadata:
web_search_queries = grounding_metadata_item.get("webSearchQueries")
if web_search_queries and web_search_requests:
web_search_requests += len([q for q in web_search_queries if q])
elif web_search_queries:
web_search_requests = len([q for q in web_search_queries if q])
return web_search_requests
@staticmethod
def _set_grounding_usage_counters(usage: Usage, grounding_metadata: Sequence[Mapping[str, object]]) -> None:
grounding_requests: Final = calculate_grounding_requests(grounding_metadata)
details: Final = cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details)
if grounding_requests.web_search_requests is not None:
details.web_search_requests = grounding_requests.web_search_requests
if grounding_requests.google_maps_grounding_requests is not None:
details.google_maps_grounding_requests = grounding_requests.google_maps_grounding_requests
@staticmethod
def _create_streaming_choice(
@ -2454,9 +2456,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
usage: Final = VertexGeminiConfig._calculate_usage(completion_response=completion_response)
web_search_requests: Final = VertexGeminiConfig._calculate_web_search_requests(grounding_metadata)
if web_search_requests is not None:
cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details).web_search_requests = web_search_requests
VertexGeminiConfig._set_grounding_usage_counters(usage, grounding_metadata)
setattr(model_response, "usage", usage)
@ -3221,9 +3221,7 @@ class ModelResponseIterator:
completion_response=processed_chunk,
)
web_search_requests: Final = VertexGeminiConfig._calculate_web_search_requests(grounding_metadata)
if web_search_requests is not None:
cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details).web_search_requests = web_search_requests
VertexGeminiConfig._set_grounding_usage_counters(usage, grounding_metadata)
traffic_type: Final = processed_chunk.get("usageMetadata", {}).get("trafficType")
if traffic_type:

2
litellm/main.py
View file

@ -8013,7 +8013,7 @@ def speech(
if max_retries is None:
max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES
litellm_params_dict: Final = get_litellm_params(**kwargs)
litellm_params_dict: Final = get_litellm_params(metadata=metadata, api_key=api_key or dynamic_api_key, **kwargs)
# Get provider-specific text-to-speech config and map parameters
text_to_speech_provider_config = ProviderConfigManager.get_provider_text_to_speech_config(

609
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

123
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,6 +1,8 @@
import re
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, cast
from fastapi import HTTPException
@ -13,6 +15,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.oauth_utils import (
get_passthrough_resource_metadata_url,
get_passthrough_www_authenticate,
get_request_base_url,
well_known_root_suffix,
)
@ -43,6 +46,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import (
_get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth
_run_centralized_common_checks,
user_api_key_auth,
)
@ -298,6 +302,16 @@ def _admission_failure_fallback(
raise exc
@dataclass(frozen=True, slots=True)
class DcrBridgeTarget:
"""The single DCR-bridge server a request targets, paired with the exact name the caller
used to reach it (alias or server_name, whichever they typed), which is the spelling an
``invalid_token`` challenge must echo back."""
requested_name: str
server: MCPServer
class MCPRequestHandler:
"""
Class to handle MCP request processing, including:
@ -416,7 +430,10 @@ class MCPRequestHandler:
# An explicit x-litellm-api-key is always a LiteLLM credential, even
# for a delegated server, so validate it: identity / spend / rate
# limits resolve and any stored upstream token can be forwarded.
validated_user_api_key_auth = await user_api_key_auth(api_key=litellm_api_key, request=request)
validated_user_api_key_auth = await user_api_key_auth(
api_key=f"Bearer {_get_bearer_token_or_received_api_key(litellm_api_key)}",
request=request,
)
elif MCPRequestHandler._target_servers_delegate_auth_to_upstream(
path=request_route,
mcp_servers=mcp_servers,
@ -437,27 +454,33 @@ class MCPRequestHandler:
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
) or (
MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
is not None
and not oauth2_headers
and not mcp_server_auth_headers
and not mcp_auth_header
):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
(
bridge_delegate_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
bridge_delegate_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
is not None
and oauth2_headers
and is_bridge_envelope_shaped(oauth2_headers["Authorization"])
):
# A single DCR-bridge oauth_delegate target carrying an envelope-shaped
# Authorization: open the envelope, admit under its recovered identity, and
# inject the inner upstream token for egress. A non-envelope bearer on the same
# server is NOT admitted here — it falls through to the oauth2 arm, which 401s.
validated_user_api_key_auth, mcp_server_auth_headers = await MCPRequestHandler._admit_dcr_bridge_delegate(
server=bridge_delegate_target,
) is not None and oauth2_headers:
(
validated_user_api_key_auth,
mcp_server_auth_headers,
) = await MCPRequestHandler._admit_dcr_bridge_authorization(
server=bridge_delegate_target.server,
requested_name=bridge_delegate_target.requested_name,
authorization_value=oauth2_headers["Authorization"],
litellm_api_key=litellm_api_key,
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=request_route,
@ -723,10 +746,10 @@ class MCPRequestHandler:
@staticmethod
def _single_dcr_bridge_delegate_target(
path: str, mcp_servers: list[str] | None, client_ip: str | None
) -> MCPServer | None:
) -> DcrBridgeTarget | None:
"""The one DCR-bridge ``oauth_delegate`` server this request targets, or ``None``.
Returns the server only when EXACTLY ONE target resolves and it is both
Returns the target only when EXACTLY ONE name resolves and its server is both
``is_oauth_delegate`` and ``is_dcr_bridge``. Fails closed (``None``) on a
multi-target request, an unresolved target, or a non-matching server, so the
envelope admission arm never fires for an aggregate scope or a server that did not
@ -740,17 +763,21 @@ class MCPRequestHandler:
if len(target_names) != 1:
return None
server: Final = global_mcp_server_manager.get_mcp_server_by_name(target_names[0], client_ip=client_ip)
if server is None or not server.is_oauth_delegate or not server.is_dcr_bridge:
# Both flags are security-sensitive opt-ins. Require literal booleans so
# partially populated objects and truthy proxy values cannot enable bridge
# admission accidentally.
if server is None or server.is_oauth_delegate is not True or server.is_dcr_bridge is not True:
return None
# Egress resolves the injected per-server token only by alias / server_name; a server with
# neither cannot receive the forwarded token, so fail closed rather than admit-and-drop.
if not (server.server_name or server.alias):
return None
return server
return DcrBridgeTarget(requested_name=target_names[0], server=server)
@staticmethod
async def _admit_dcr_bridge_delegate(
server: MCPServer,
requested_name: str,
authorization_value: str,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
request: Request,
@ -798,10 +825,62 @@ class MCPRequestHandler:
new_headers: Final = {**(mcp_server_auth_headers or {}), **injected}
return admitted, new_headers
case BridgeEnvelopeInvalid() | NotBridgeEnvelope():
raise HTTPException(status_code=401, detail="Invalid or expired credential")
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
requested_name=requested_name, request=request
)
case _:
assert_never(result)
@staticmethod
async def _admit_dcr_bridge_authorization(
server: MCPServer,
requested_name: str,
authorization_value: str,
litellm_api_key: str,
mcp_server_auth_headers: dict[str, dict[str, str]] | None, # mutable-ok: existing MCP sink shape
request: Request,
route: str,
) -> tuple[UserAPIKeyAuth, dict[str, dict[str, str]] | None]: # mutable-ok: existing MCP sink shape
if is_bridge_envelope_shaped(authorization_value):
return await MCPRequestHandler._admit_dcr_bridge_delegate(
server=server,
requested_name=requested_name,
authorization_value=authorization_value,
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=route,
)
try:
admitted: Final = await user_api_key_auth(api_key=litellm_api_key, request=request)
except (HTTPException, ProxyException) as exc:
if not _is_litellm_auth_admission_error(exc):
raise
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
requested_name=requested_name, request=request
) from exc
return admitted, mcp_server_auth_headers
@staticmethod
def _dcr_bridge_invalid_token_challenge(requested_name: str, request: Request) -> HTTPException:
"""The RFC 6750 ``invalid_token`` challenge for a failed bridge admission.
Named by the exact spelling the caller requested, matching the per-server well-known
document and the other challenge emitters, so ``resource_metadata`` always points at the
resource the client actually asked for even when alias and server_name differ."""
return HTTPException(
status_code=401,
detail="Invalid or expired credential",
headers=MappingProxyType(
{
"www-authenticate": get_passthrough_www_authenticate(
scope=request.scope,
server_name=requested_name,
invalid_token=True,
)
}
),
)
@staticmethod
async def _admit_gateway_session(
authorization_value: str,

24
litellm/proxy/_experimental/mcp_server/bridge_token_flow.py
View file

@ -306,15 +306,15 @@ _UpstreamGrantRejection = Literal["no_access_token", "expired_lifetime"]
- ``expired_lifetime``: the response reports a parseable, non-positive ``expires_in``, i.e. an upstream
token that is already dead, so sealing it would forward a bearer the edge cannot use
An absent or unparseable ``expires_in`` is NOT a rejection; the lifetime is merely unknown and the
envelope caps it, the by-design behaviour for an upstream that omits the field."""
envelope uses its fallback lifetime, the by-design behaviour for an upstream that omits the field."""
def _classify_upstream_lifetime(raw_expires_in: object) -> "int | Literal['unspecified', 'expired']":
"""Classify an upstream ``expires_in`` into a positive number of seconds, ``"unspecified"`` (absent
or unparseable, so the envelope caps it), or ``"expired"`` (a non-positive value the upstream reports
or unparseable, so the envelope uses its fallback), or ``"expired"`` (a non-positive value the upstream reports
as already elapsed). Telling "we do not know the lifetime" apart from "the upstream says it is
already dead" is what stops an explicitly-expired token from silently receiving the envelope's 1h
cap. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
already dead" is what stops an explicitly-expired token from silently receiving the envelope's
one-hour fallback. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
positive sub-second lifetime in ``(0, 1)`` is not truncated to ``0`` and misread as elapsed; the
envelope works in whole seconds, so such a lifetime clamps up to its 1s floor. ``bool`` is excluded
(an ``int`` subclass but never a real lifetime), and the conversions can raise on ``NaN`` /
@ -335,7 +335,7 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
"""Validate an upstream OAuth token response into a typed grant, or say why it cannot back an
envelope. Each field is isinstance-checked so nothing untyped from ``response.json()`` reaches the
grant. ``expires_in`` is read three ways (see :func:`_classify_upstream_lifetime`): an unknown
lifetime leaves the grant ``expires_in`` ``None`` for the envelope to cap, a positive value is
lifetime leaves the grant ``expires_in`` ``None`` for the envelope fallback, a positive value is
honoured, and an explicit already-elapsed value is a rejection rather than a silent fall-through to
the cap."""
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import
@ -357,8 +357,8 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
token_type=token_type if isinstance(token_type, str) and token_type else "Bearer",
# The upstream refresh_token is deliberately NOT sealed: the edge never consumes it (it forwards
# only token_type + access_token), so it would be dead weight embedding a long-lived upstream
# credential in the client-held bearer, and it enlarges the envelope. Refresh support is a
# follow-up (a dedicated refresh-envelope); the client re-runs authorization_code at the cap.
# credential in the client-held bearer, and it enlarges the envelope. The dedicated refresh
# envelope carries that credential separately.
refresh_token=None,
scope=scope if isinstance(scope, str) and scope else None,
expires_in=lifetime if isinstance(lifetime, int) else None,
@ -387,6 +387,7 @@ _BridgeMintError = Literal[
"not_configured",
"no_upstream_token",
"upstream_token_expired",
"upstream_lifetime_unrepresentable",
"too_large",
]
@ -456,6 +457,12 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
"server_error",
"the upstream token response reports an already-expired lifetime",
)
case "upstream_lifetime_unrepresentable":
status, code, desc = (
502,
"server_error",
"the upstream token response reports an unrepresentable lifetime",
)
case "too_large":
status, code, desc = (
502,
@ -619,6 +626,7 @@ def _finish_bridge_mint(
build_bridge_token_response,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # inline import avoids a module-load circular import
EnvelopeLifetimeUnrepresentable,
SealedEnvelope,
UpstreamTokenGrant,
)
@ -627,6 +635,8 @@ def _finish_bridge_mint(
if not isinstance(grant, UpstreamTokenGrant):
return _upstream_rejection_to_mint_error(grant)
sealed: Final = build_bridge_token_response(ready.identity, grant, ready.keys, now)
if isinstance(sealed, EnvelopeLifetimeUnrepresentable):
return "upstream_lifetime_unrepresentable"
if not isinstance(sealed, SealedEnvelope):
return "too_large"
# Report expires_in from the JWT's own second-truncated exp, rounding the elapsed portion up, so the

4
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -1600,7 +1600,7 @@ async def refresh_user_oauth_token(
) -> OAuthCredentialPayload | None:
"""Attempt to refresh a per-user OAuth2 token using its stored refresh_token.
POSTs to ``server.token_url`` with ``grant_type=refresh_token``.
POSTs to ``server.effective_token_url`` with ``grant_type=refresh_token``.
On success: persists the new credential via ``store_user_oauth_credential``
and returns the updated payload dict.
@ -1609,7 +1609,7 @@ async def refresh_user_oauth_token(
stale credential and triggering re-authentication.
"""
refresh_token: Final[str | None] = cred.get("refresh_token")
token_url: Final[str | None] = getattr(server, "token_url", None)
token_url: Final[str | None] = getattr(server, "effective_token_url", None) or getattr(server, "token_url", None)
server_id: Final[str] = getattr(server, "server_id", "")
client_id: Final[str | None] = getattr(server, "client_id", None)
client_secret: Final[str | None] = getattr(server, "client_secret", None)

183
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -3,7 +3,7 @@ import html as _html
import json
import secrets
import time
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Final, Literal, Optional
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
@ -663,6 +663,26 @@ def _endpoint_not_configured_detail(
)
async def _server_with_oauth_endpoints(
mcp_server: MCPServer,
needed_endpoint: Callable[[MCPServer], str | None],
) -> MCPServer:
"""Join deferred OAuth discovery only when the endpoint this caller needs is still missing.
Admin-entered endpoints live on ``configured_*`` after an anchored issuer empties the
resolved fields. A caller whose needed endpoint already resolves never awaits discovery
and cannot 503 over a leftover pin. A server still missing it joins the deferred task;
no slot is a no-op and the caller 400s.
"""
if needed_endpoint(mcp_server) is not None:
return mcp_server
from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load
global_mcp_server_manager,
)
return await global_mcp_server_manager.ensure_oauth_metadata_discovered(mcp_server)
def _raise_unless_oauth2_discovery_server(
mcp_server: MCPServer | None,
mcp_server_name: str | None,
@ -697,7 +717,7 @@ def _dcr_bridge_relays_client_registration(mcp_server: MCPServer) -> bool:
returns directly to the client's redirect URI without transiting the gateway. Gateway-side
redirect trust and the ``/callback`` state relay therefore only apply to the short-circuit
arm, where the upstream only knows the gateway's own callback."""
return mcp_server.is_dcr_bridge and bool(mcp_server.registration_url) and not mcp_server.client_id
return mcp_server.is_dcr_bridge and bool(mcp_server.effective_registration_url) and not mcp_server.client_id
def _require_s256_pkce(
@ -745,7 +765,7 @@ def _redirect_to_upstream_authorize(
**({"scope": scope_value} if scope_value else {}),
**({"resource": upstream_resource} if upstream_resource else {}),
}
parsed_auth_url: Final = urlparse(mcp_server.authorization_url or "")
parsed_auth_url: Final = urlparse(mcp_server.effective_authorization_url or "")
merged_params: Final = {**dict(parse_qsl(parsed_auth_url.query)), **passthrough_params}
return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params))))
@ -812,18 +832,19 @@ async def authorize_with_server(
ephemeral_dcr_client: "EphemeralDcrClient | None" = None,
):
_raise_if_not_oauth2(mcp_server)
if mcp_server.authorization_url is None:
resolved_server: Final = await _server_with_oauth_endpoints(mcp_server, _register_flow_needed_endpoint)
if resolved_server.effective_authorization_url is None:
raise HTTPException(
status_code=400,
detail=_endpoint_not_configured_detail(
mcp_server,
resolved_server,
"authorization url",
"set Authorization URL and Token URL manually",
"set Issuer to discover them from the identity provider (RFC 8414)",
),
)
if mcp_server.is_dcr_bridge:
if resolved_server.is_dcr_bridge:
# Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated,
# now-non-optional pair to the upstream authorize; the short-circuit arm keeps
# calling this for its enforcement side effect, then falls through to the gateway
@ -832,9 +853,9 @@ async def authorize_with_server(
# A gateway-minted ephemeral client is registered against {base}/callback, so its
# flow must run the short-circuit arm; the relay arm is only for clients that
# registered themselves through the front door and hold their own redirect binding.
if _dcr_bridge_relays_client_registration(mcp_server) and ephemeral_dcr_client is None:
if _dcr_bridge_relays_client_registration(resolved_server) and ephemeral_dcr_client is None:
return _redirect_to_upstream_authorize(
mcp_server=mcp_server,
mcp_server=resolved_server,
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
@ -860,7 +881,7 @@ async def authorize_with_server(
# litellm key, so the browser session is the only identity source; without one there is nothing to
# bind, so send the user through login first. Every other oauth2 server keeps the identity-less state.
litellm_user_id: str | None = None
if mcp_server.is_dcr_bridge and mcp_server.is_oauth_delegate:
if resolved_server.is_dcr_bridge and resolved_server.is_oauth_delegate:
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # inline import avoids a module-load circular import
_user_id_from_session_cookie,
)
@ -870,7 +891,7 @@ async def authorize_with_server(
return _redirect_to_litellm_login(request)
denial: Final = await _bridge_authorize_access_denial(
litellm_user_id=litellm_user_id,
mcp_server=mcp_server,
mcp_server=resolved_server,
redirect_uri=redirect_uri,
state=state,
)
@ -884,7 +905,7 @@ async def authorize_with_server(
code_challenge_method=code_challenge_method,
client_redirect_uri=redirect_uri,
litellm_user_id=litellm_user_id,
mcp_server_id=mcp_server.server_id if (litellm_user_id or ephemeral_dcr_client) else None,
mcp_server_id=resolved_server.server_id if (litellm_user_id or ephemeral_dcr_client) else None,
dcr_client_id=ephemeral_dcr_client.client_id if ephemeral_dcr_client else None,
dcr_client_secret=ephemeral_dcr_client.client_secret if ephemeral_dcr_client else None,
dcr_token_endpoint_auth_method=ephemeral_dcr_client.token_endpoint_auth_method
@ -894,26 +915,26 @@ async def authorize_with_server(
relay_state: Final = secrets.token_urlsafe(_OAUTH_STATE_HANDLE_BYTES)
params: Final = {
"client_id": mcp_server.client_id if mcp_server.client_id else client_id,
"client_id": resolved_server.client_id if resolved_server.client_id else client_id,
"redirect_uri": f"{request_base_url}/callback",
"state": relay_state,
"response_type": response_type or "code",
}
if scope:
params["scope"] = scope
elif mcp_server.scopes:
params["scope"] = " ".join(mcp_server.scopes)
elif resolved_server.scopes:
params["scope"] = " ".join(resolved_server.scopes)
if code_challenge:
params["code_challenge"] = code_challenge
if code_challenge_method:
params["code_challenge_method"] = code_challenge_method
upstream_resource: Final = resolve_upstream_resource(mcp_server)
upstream_resource: Final = resolve_upstream_resource(resolved_server)
if upstream_resource:
params["resource"] = upstream_resource
parsed_auth_url: Final = urlparse(mcp_server.authorization_url)
parsed_auth_url: Final = urlparse(resolved_server.effective_authorization_url)
existing_params: Final = dict(parse_qsl(parsed_auth_url.query))
existing_params.update(params)
final_url: Final = urlunparse(parsed_auth_url._replace(query=urlencode(existing_params)))
@ -946,11 +967,13 @@ async def exchange_token_with_server(
if grant_type not in ("authorization_code", "refresh_token"):
raise HTTPException(status_code=400, detail="Unsupported grant_type")
if mcp_server.token_url is None:
resolved_server: Final = await _server_with_oauth_endpoints(mcp_server, _token_flow_needed_endpoint)
token_url: Final = resolved_server.effective_token_url
if token_url is None:
raise HTTPException(
status_code=400,
detail=_endpoint_not_configured_detail(
mcp_server,
resolved_server,
"token url",
"set Token URL manually",
"set Issuer to discover it from the identity provider (RFC 8414)",
@ -965,16 +988,16 @@ async def exchange_token_with_server(
# recovered from a sealed code) must authenticate the way its own registration was granted,
# not the way the server row is configured; callers that carry no method keep the row's method
# as before.
resolved_client_id: Final = mcp_server.client_id if mcp_server.client_id else client_id
resolved_client_secret: Final = mcp_server.client_secret if mcp_server.client_id else client_secret
resolved_client_id: Final = resolved_server.client_id if resolved_server.client_id else client_id
resolved_client_secret: Final = resolved_server.client_secret if resolved_server.client_id else client_secret
resolved_auth_method: Final = (
mcp_server.token_endpoint_auth_method
if mcp_server.client_id
else (client_token_endpoint_auth_method or mcp_server.token_endpoint_auth_method)
resolved_server.token_endpoint_auth_method
if resolved_server.client_id
else (client_token_endpoint_auth_method or resolved_server.token_endpoint_auth_method)
)
try:
token_request: Final = build_upstream_oauth2_token_request(
mcp_server,
resolved_server,
auth_method=resolved_auth_method,
client_id=resolved_client_id,
client_secret=resolved_client_secret,
@ -987,14 +1010,14 @@ async def exchange_token_with_server(
bridge_upstream_refresh: SecretStr | None = None
bridge_upstream_scope: str | None = None
refresh_request_scope: str | None = None
is_bridge: Final = mcp_server.is_oauth_delegate and mcp_server.is_dcr_bridge
is_bridge: Final = resolved_server.is_oauth_delegate and resolved_server.is_dcr_bridge
if grant_type == "refresh_token":
# Phase 1 for a bridge refresh: open the client's refresh envelope, re-validate the sealed
# identity, and unwrap the real upstream refresh token BEFORE building token_data, so the exchange
# sends the upstream token and never the envelope. A failure returns without touching the upstream.
if is_bridge:
prepared_refresh: Final = await _prepare_bridge_refresh(mcp_server, refresh_token)
prepared_refresh: Final = await _prepare_bridge_refresh(resolved_server, refresh_token)
if not isinstance(prepared_refresh, _BridgeRefreshReady):
return _bridge_mint_error_response(prepared_refresh)
bridge_mint_ready = prepared_refresh.ready
@ -1031,13 +1054,13 @@ async def exchange_token_with_server(
# A raw upstream code (scripted path) opens to None and the code is used as-is.
bridge_identity = open_bridge_authorization_code(code)
if bridge_identity is not None:
if bridge_identity.mcp_server_id != mcp_server.server_id:
if bridge_identity.mcp_server_id != resolved_server.server_id:
raise HTTPException(
status_code=400,
detail="Authorization code was issued for a different MCP server",
)
code = bridge_identity.upstream_code
bridge_token_relay: Final = _dcr_bridge_relays_client_registration(mcp_server)
bridge_token_relay: Final = _dcr_bridge_relays_client_registration(resolved_server)
if bridge_token_relay and not redirect_uri:
raise HTTPException(
status_code=400,
@ -1059,7 +1082,7 @@ async def exchange_token_with_server(
# Phase 1 for a bridge authorization_code mint: resolve identity (the SSO user recovered above, or
# the presented litellm key) and the envelope keys BEFORE the exchange consumes the single-use code.
if is_bridge:
prepared: Final = await _prepare_bridge_mint(request, mcp_server, bridge_identity)
prepared: Final = await _prepare_bridge_mint(request, resolved_server, bridge_identity)
if not isinstance(prepared, _BridgeMintReady):
return _bridge_mint_error_response(prepared)
bridge_mint_ready = prepared
@ -1067,7 +1090,7 @@ async def exchange_token_with_server(
async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
try:
response: Final = await async_client.post(
mcp_server.token_url,
token_url,
headers={"Accept": "application/json", **token_request.headers},
data=token_data,
)
@ -1076,8 +1099,8 @@ async def exchange_token_with_server(
except httpx.HTTPStatusError as exc:
fault: Final = classify_upstream_token_rejection(
exc.response,
credential_source=_token_credential_source(mcp_server),
log_context=mcp_server.server_id,
credential_source=_token_credential_source(resolved_server),
log_context=resolved_server.server_id,
)
upstream_rejected_bridge_refresh: Final = (
is_bridge
@ -1090,7 +1113,7 @@ async def exchange_token_with_server(
"bridge refresh: the upstream rejected the sealed refresh token for server=%s with "
"invalid_grant (revoked or expired at the IdP); returning invalid_grant so the client "
"re-runs authorization_code rather than an opaque upstream error",
mcp_server.server_id,
resolved_server.server_id,
)
return _bridge_mint_error_response("invalid_refresh")
return render_token_fault(fault)
@ -1103,22 +1126,22 @@ async def exchange_token_with_server(
# Validate token response against server-configured rules before any storage.
# This rejects tokens from wrong Slack workspaces, Atlassian orgs, etc.
if mcp_server.token_validation and isinstance(mcp_server.token_validation, dict):
if resolved_server.token_validation and isinstance(resolved_server.token_validation, dict):
_validate_token_response(
token_response=token_response,
validation_rules=mcp_server.token_validation,
server_id=mcp_server.server_id,
validation_rules=resolved_server.token_validation,
server_id=resolved_server.server_id,
)
# Store server-side when the server is configured for per-user OAuth and
# the calling client has provided a valid LiteLLM identity.
# Errors are non-fatal: the token is still returned to the client.
if mcp_server.needs_user_oauth_token:
if resolved_server.needs_user_oauth_token:
user_id: Final = await _extract_user_id_from_request(request)
if user_id:
try:
await _store_per_user_token_server_side(
server=mcp_server,
server=resolved_server,
user_id=user_id,
token_response=token_response,
)
@ -1126,7 +1149,7 @@ async def exchange_token_with_server(
verbose_logger.warning(
"exchange_token_with_server: server-side storage failed for user=%s server=%s: %s",
user_id,
mcp_server.server_id,
resolved_server.server_id,
exc,
)
else:
@ -1136,7 +1159,7 @@ async def exchange_token_with_server(
"requires the stored token, so the client will be challenged with 401 on reconnect. "
"Ensure the request carries a valid LiteLLM key (x-litellm-api-key or Authorization), "
"or store it via POST /mcp/server/{id}/oauth-user-credential.",
mcp_server.server_id,
resolved_server.server_id,
)
# A DCR-bridge oauth_delegate server hands the client a gateway-bound envelope (identity plus the
@ -1147,7 +1170,9 @@ async def exchange_token_with_server(
token_response = {**token_response, "scope": refresh_request_scope}
# Phase 3: seal the upstream grant into the client-held envelope; failures map through the same
# OAuth-shaped response as the phase-1 preconditions.
minted: Final = _finish_bridge_mint(bridge_mint_ready, mcp_server, token_response, datetime.now(timezone.utc))
minted: Final = _finish_bridge_mint(
bridge_mint_ready, resolved_server, token_response, datetime.now(timezone.utc)
)
return minted if isinstance(minted, JSONResponse) else _bridge_mint_error_response(minted)
raw_access_token: Final = token_response.get("access_token") if isinstance(token_response, dict) else None
@ -1551,7 +1576,8 @@ async def mint_ephemeral_dcr_client(request: Request, mcp_server: MCPServer) ->
bounded by the server count even when the request origin varies) so parallel authorize requests
cannot each register an upstream client; the cache stamps nothing onto the server record and
correctness never depends on it because the sealed state carries the client through the flow."""
if mcp_server.registration_url is None:
registration_url: Final = mcp_server.effective_registration_url
if registration_url is None:
return None
request_base_url: Final = get_request_base_url(request)
cache_key: Final = f"mcp_ephemeral_dcr_client:{mcp_server.server_id}:{request_base_url}"
@ -1571,7 +1597,7 @@ async def mint_ephemeral_dcr_client(request: Request, mcp_server: MCPServer) ->
"token_endpoint_auth_method": "none",
}
response: Final = await _post_dcr_registration(
registration_url=mcp_server.registration_url,
registration_url=registration_url,
register_data=register_data,
server_id=mcp_server.server_id,
)
@ -1617,7 +1643,7 @@ async def resolve_ephemeral_dcr_client(
usable to generate orphan IdP clients)."""
if not (mcp_server.is_true_passthrough or (mcp_server.is_oauth_delegate and not mcp_server.is_dcr_bridge)):
return None
if mcp_server.authorization_url is None:
if mcp_server.effective_authorization_url is None:
raise HTTPException(
status_code=400,
detail="MCP server authorization url is not set",
@ -1627,6 +1653,29 @@ async def resolve_ephemeral_dcr_client(
return await mint_ephemeral_dcr_client(request, mcp_server)
def _register_flow_needed_endpoint(mcp_server: MCPServer) -> str | None:
"""The register flow's deferred-discovery join gate. A DCR bridge with no admin-configured
client can only register callers through the upstream's registration endpoint
(``_oauth_endpoints_unresolved`` keeps its discovery slot armed for exactly this shape), so
the flow must keep joining discovery while registration is still missing instead of silently
degrading to the dummy short-circuit. Every other shape only needs the authorization url."""
if mcp_server.is_dcr_bridge and not mcp_server.client_id and mcp_server.effective_registration_url is None:
return None
return mcp_server.effective_authorization_url
def _token_flow_needed_endpoint(mcp_server: MCPServer) -> str | None:
"""The token exchange's deferred-discovery join gate. The exchange's relay-vs-callback arm
(:func:`_dcr_bridge_relays_client_registration`) reads the registration url, so a clientless
DCR bridge rebuilt without its discovered registration endpoint must keep joining discovery
even when the token url already resolves; skipping it would select the gateway-callback arm
and the upstream would reject the code over a redirect_uri mismatch. Every other shape only
needs the token url."""
if mcp_server.is_dcr_bridge and not mcp_server.client_id and mcp_server.effective_registration_url is None:
return None
return mcp_server.effective_token_url
async def register_client_with_server(
request: Request,
mcp_server: MCPServer,
@ -1661,21 +1710,23 @@ async def register_client_with_server(
):
return dummy_return
if mcp_server.authorization_url is None:
resolved_server: Final = await _server_with_oauth_endpoints(mcp_server, _register_flow_needed_endpoint)
if resolved_server.effective_authorization_url is None:
raise HTTPException(
status_code=400,
detail=_endpoint_not_configured_detail(
mcp_server,
resolved_server,
"authorization url",
"set Authorization URL and Token URL manually",
"set Issuer to discover them from the identity provider (RFC 8414)",
),
)
if mcp_server.registration_url is None:
registration_url: Final = resolved_server.effective_registration_url
if registration_url is None:
return dummy_return
bridge_relay: Final = _dcr_bridge_relays_client_registration(mcp_server)
bridge_relay: Final = _dcr_bridge_relays_client_registration(resolved_server)
if bridge_relay and not client_redirect_uris:
raise HTTPException(
status_code=400,
@ -1690,15 +1741,17 @@ async def register_client_with_server(
"token_endpoint_auth_method": token_endpoint_auth_method or ("none" if bridge_relay else ""),
}
response: Final = await _post_dcr_registration(
registration_url=mcp_server.registration_url,
registration_url=registration_url,
register_data=register_data,
server_id=mcp_server.server_id,
server_id=resolved_server.server_id,
)
token_response = response.json()
if persist_credentials and not bridge_relay:
persistence_result = await _persist_dcr_client_registration(mcp_server, token_response, current_redirect_uri)
persistence_result = await _persist_dcr_client_registration(
resolved_server, token_response, current_redirect_uri
)
if persistence_result == "reused":
return dummy_return
@ -1755,17 +1808,10 @@ async def authorize(
lookup_name: Final[str | None] = mcp_server_name or client_id
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = (
await global_mcp_server_manager.get_resolved_mcp_server_by_name(lookup_name, client_ip=client_ip)
if lookup_name
else None
global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip) if lookup_name else None
)
if mcp_server is None and mcp_server_name is None:
unresolved_server: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
mcp_server = (
await global_mcp_server_manager.ensure_oauth_metadata_discovered(unresolved_server)
if unresolved_server is not None
else None
)
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
_raise_if_not_oauth2(mcp_server)
@ -1846,14 +1892,9 @@ async def token_endpoint(
lookup_name: Final = mcp_server_name or client_id
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
mcp_server = await global_mcp_server_manager.get_resolved_mcp_server_by_name(lookup_name, client_ip=client_ip)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(lookup_name, client_ip=client_ip)
if mcp_server is None and mcp_server_name is None:
unresolved_server: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
mcp_server = (
await global_mcp_server_manager.ensure_oauth_metadata_discovered(unresolved_server)
if unresolved_server is not None
else None
)
mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
return await exchange_token_with_server(
@ -2684,10 +2725,9 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
return await register_aggregate_client(request=request, request_body=data)
resolved: Final = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
if resolved:
resolved_server: Final = await global_mcp_server_manager.ensure_oauth_metadata_discovered(resolved)
return await register_client_with_server(
request=request,
mcp_server=resolved_server,
mcp_server=resolved,
client_name=data.get("client_name", ""),
grant_types=data.get("grant_types", []),
response_types=data.get("response_types", []),
@ -2697,10 +2737,7 @@ async def register_client(request: Request, mcp_server_name: str | None = None):
)
return dummy_return
mcp_server: Final = await global_mcp_server_manager.get_resolved_mcp_server_by_name(
mcp_server_name,
client_ip=client_ip,
)
mcp_server: Final = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name, client_ip=client_ip)
if mcp_server is None:
return dummy_return
return await register_client_with_server(

14
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -523,7 +523,7 @@ def _oauth_endpoints_unresolved(server: MCPServer) -> bool:
# can come from resource discovery, so a server that resolved its endpoints but no scopes is
# still unresolved for its flow.
return True
if server.is_dcr_bridge and not server.client_id and server.registration_url is None:
if server.is_dcr_bridge and not server.client_id and server.effective_registration_url is None:
# A DCR bridge with no admin-configured client can only register callers through the
# upstream's registration endpoint, so a build that resolved the authorize and token
# endpoints but not registration_endpoint (partial metadata) is still unresolved for its
@ -535,8 +535,8 @@ def _oauth_endpoints_unresolved(server: MCPServer) -> bool:
return _flow_endpoints_missing(
server.auth_type,
MCPServerManager.effective_oauth2_flow(server),
server.authorization_url,
server.token_url,
server.effective_authorization_url,
server.effective_token_url,
server.token_exchange_endpoint,
)
@ -6205,14 +6205,6 @@ class MCPServerManager:
return server
return None
async def get_resolved_mcp_server_by_name(
self,
server_name: str,
client_ip: str | None = None,
) -> MCPServer | None:
server: Final = self.get_mcp_server_by_name(server_name, client_ip=client_ip)
return await self.ensure_oauth_metadata_discovered(server) if server is not None else None
def get_filtered_registry(self, client_ip: str | None = None) -> dict[str, MCPServer]:
"""
Get registry filtered by client IP access control.

13
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
View file

@ -67,7 +67,7 @@ class MCPOAuth2TokenCache(InMemoryCache):
rest of the identity rather than stored in a key."""
material: Final = "\x00".join(
(
server.token_url or "",
server.effective_token_url or "",
server.client_id or "",
server.client_secret or "",
" ".join(server.scopes or ()),
@ -82,7 +82,7 @@ class MCPOAuth2TokenCache(InMemoryCache):
@staticmethod
def _has_client_credentials_config(server: "MCPServer") -> bool:
return bool(server.client_id and server.client_secret and server.token_url)
return bool(server.client_id and server.client_secret and server.effective_token_url)
async def async_get_token(self, server: "MCPServer") -> str | None:
"""Return a valid access token, fetching or refreshing as needed.
@ -112,19 +112,20 @@ class MCPOAuth2TokenCache(InMemoryCache):
return token
async def _fetch_token(self, server: "MCPServer") -> tuple[str, int]:
"""POST to ``token_url`` with ``grant_type=client_credentials``.
"""POST to ``effective_token_url`` with ``grant_type=client_credentials``.
Returns ``(access_token, ttl_seconds)`` where ttl accounts for the
expiry buffer so the cache entry expires before the real token does.
"""
client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
if not server.client_id or not server.client_secret or not server.token_url:
token_url: Final = server.effective_token_url
if not server.client_id or not server.client_secret or not token_url:
raise ValueError(
f"MCP server '{server.server_id}' missing required OAuth2 fields: "
f"client_id={bool(server.client_id)}, "
f"client_secret={bool(server.client_secret)}, "
f"token_url={bool(server.token_url)}"
f"token_url={bool(token_url)}"
)
token_request: Final = build_upstream_oauth2_token_request(
@ -146,7 +147,7 @@ class MCPOAuth2TokenCache(InMemoryCache):
)
try:
response: Final = await client.post(server.token_url, data=data, headers=token_request.headers or None)
response: Final = await client.post(token_url, data=data, headers=token_request.headers or None)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise ValueError(

4
litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py
View file

@ -142,7 +142,7 @@ def _client_credentials_spec(server: MCPServer, resource: str) -> ServerSpec:
config=ClientCredentialsConfig(
client_id=server.client_id,
client_secret=SecretStr(server.client_secret) if server.client_secret else None,
token_url=server.token_url,
token_url=server.effective_token_url,
scopes=tuple(server.scopes or ()),
audience=server.audience,
upstream_resource=resolve_upstream_resource(server),
@ -163,7 +163,7 @@ def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec | None:
normalizes to ``rfc8693`` so a bad config value cannot crash spec-building. ``audience`` is
forwarded only when the operator set it; a missing one is omitted, not derived.
"""
endpoint: Final = server.token_exchange_endpoint or server.token_url
endpoint: Final = server.token_exchange_endpoint or server.effective_token_url
if not server.client_id or not server.client_secret:
return None
profile: Final[Literal["rfc8693", "entra_obo"]] = (

7
litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py
View file

@ -88,7 +88,10 @@ class AuthorizationCodeRefresher:
if token.refresh_token is None:
return None
server: Final = self._server_lookup(server_id)
if server is None or not server.token_url:
if server is None:
return None
token_url: Final = server.effective_token_url
if not token_url:
return None
try:
@ -106,7 +109,7 @@ class AuthorizationCodeRefresher:
"refresh_token": token.refresh_token,
**token_request.body,
}
body: Final = await self._token_endpoint(server.token_url, form, token_request.headers)
body: Final = await self._token_endpoint(token_url, form, token_request.headers)
if body is None:
return None
access_token: Final = body.get("access_token")

2
litellm/proxy/_experimental/mcp_server/outbound_credentials/bridge_credentials.py
View file

@ -92,7 +92,7 @@ def build_bridge_token_response(
The producer mirror of :func:`resolve_bridge_envelope`: a thin, pure wrapper over
:func:`mint_envelope` that returns the sealed envelope, or the mint error as a value
(an oversized grant) for the caller to map onto an OAuth error response.
for the caller to map onto an OAuth error response.
"""
return mint_envelope(identity, grant, keys, now)

44
litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py
View file

@ -19,17 +19,16 @@ in plaintext anywhere in the envelope.
Failures are values: :func:`open_envelope` returns one of the frozen
``EnvelopeOpenError`` variants (discriminated on ``tag``) for invalid, expired,
tampered, or undecryptable input, and :func:`mint_envelope` returns
``EnvelopeTooLarge`` for oversized grants. Error values carry tags and sizes only,
never token material.
tampered, or undecryptable input, and :func:`mint_envelope` returns a typed error
for oversized grants or an unrepresentable provider lifetime. Error values carry
tags and metadata only, never token material.
The pydantic input models reject programmer errors at construction (e.g. a
non-positive ``expires_in`` or an empty required field). :func:`open_envelope` is
additionally total over hostile, attacker-controlled input: it never raises, only
returns an ``EnvelopeOpenError``. :func:`mint_envelope` operates on a
gateway-supplied grant (an upstream IdP's UTF-8 JSON token response), so it does not
defend against non-UTF-8 field content that cannot survive JSON parsing; its only
value-typed failure is ``EnvelopeTooLarge``.
defend against non-UTF-8 field content that cannot survive JSON parsing.
"""
from __future__ import annotations
@ -57,10 +56,11 @@ ENVELOPE_ISSUER: Final = "litellm-mcp-bridge"
"""``iss`` claim stamped into every envelope and required back on open."""
MAX_ENVELOPE_TTL_SECONDS: Final = 3600
"""Hard ceiling on ACCESS envelope lifetime. ``exp`` is ``min(upstream expires_in, this cap)``
(the cap alone when the upstream omits ``expires_in``), matching the 1h lifetime of the
BYOK session bearer this module's signing approach is borrowed from: a client-held
credential should never outlive a bounded window even when the upstream token does."""
"""Fallback ACCESS envelope lifetime when the upstream omits ``expires_in``.
The historical exported name is retained for import compatibility. When the upstream
reports a positive lifetime, the envelope matches it so a renewal does not consume a
still-valid provider refresh grant."""
MAX_REFRESH_ENVELOPE_TTL_SECONDS: Final = 1209600
"""Hard ceiling on REFRESH envelope lifetime (14 days). A refresh envelope only renews the short-lived
@ -202,7 +202,15 @@ class EnvelopeTooLarge(BaseModel):
max_bytes: int
EnvelopeMintError: TypeAlias = EnvelopeTooLarge
class EnvelopeLifetimeUnrepresentable(BaseModel):
"""A positive provider lifetime cannot be represented as a Python datetime."""
model_config = ConfigDict(frozen=True)
tag: Literal["envelope_lifetime_unrepresentable"] = "envelope_lifetime_unrepresentable"
expires_in: int
EnvelopeMintError: TypeAlias = EnvelopeTooLarge | EnvelopeLifetimeUnrepresentable
class NotAnEnvelope(BaseModel):
@ -307,11 +315,17 @@ def mint_envelope(
) -> SealedEnvelope | EnvelopeMintError:
"""Seal ``grant`` for ``identity`` into a client-held envelope.
``exp`` is ``min(grant.expires_in, MAX_ENVELOPE_TTL_SECONDS)`` seconds from ``now``
(the cap alone when ``expires_in`` is absent). Returns ``EnvelopeTooLarge`` when the
serialized envelope exceeds ``MAX_ENVELOPE_BYTES``.
``exp`` is ``grant.expires_in`` seconds from ``now`` when the upstream reports a
lifetime, or ``MAX_ENVELOPE_TTL_SECONDS`` when it does not. Returns
``EnvelopeLifetimeUnrepresentable`` when that positive lifetime cannot be represented
as a Python datetime, or ``EnvelopeTooLarge`` when the serialized envelope exceeds
``MAX_ENVELOPE_BYTES``.
"""
expires_at: Final = now + timedelta(seconds=_envelope_ttl_seconds(grant.expires_in))
ttl_seconds: Final = _envelope_ttl_seconds(grant.expires_in)
try:
expires_at: Final = now + timedelta(seconds=ttl_seconds)
except OverflowError:
return EnvelopeLifetimeUnrepresentable(expires_in=ttl_seconds)
return _seal(
kind="access",
prefix=ENVELOPE_PREFIX,
@ -457,7 +471,7 @@ def _open_claims(
def _envelope_ttl_seconds(upstream_expires_in: int | None) -> int:
if upstream_expires_in is None:
return MAX_ENVELOPE_TTL_SECONDS
return min(upstream_expires_in, MAX_ENVELOPE_TTL_SECONDS)
return upstream_expires_in
def _refresh_ttl_seconds(upstream_refresh_expires_in: int | None) -> int:

6
litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py
View file

@ -54,9 +54,9 @@ the envelope issuer so a token of one family can never validate in the other eve
hypothetical shared signing key."""
SESSION_TTL_SECONDS: Final = 3600
"""Session ACCESS token lifetime (1h), matching the access-envelope and BYOK session bearer
windows: a client-held credential never outlives a bounded window, and each refresh
re-validates the live user before re-minting."""
"""Session ACCESS token lifetime (1h), matching the BYOK session bearer window: a
client-held credential never outlives a bounded window, and each refresh re-validates
the live user before re-minting."""
SESSION_REFRESH_TTL_SECONDS: Final = 1209600
"""Session REFRESH token lifetime (14 days), matching the refresh-envelope bound. Each

32
litellm/proxy/_lazy_features.py
View file

@ -8,8 +8,8 @@ omits each feature's routes until the feature is warmed.
import asyncio
import importlib
import sys
from collections.abc import Callable
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final
@ -397,11 +397,27 @@ def _make_warmup_router(app: "FastAPI") -> "APIRouter":
return router
def inject_lazy_stubs(schema: dict) -> dict:
"""Inject openapi entries for unloaded features. Uses the snapshot file
when available (full route info), otherwise falls back to a single
placeholder per feature. Any failure logs and returns the schema unchanged
so /openapi.json never 500s on a cosmetic injection bug."""
def loaded_lazy_modules(app: "FastAPI") -> frozenset[str]:
"""The set of lazy feature modules whose routers are actually registered
on this app (tracked by _force_load), empty before the middleware ever ran.
sys.modules is the wrong signal: boot code imports several feature modules
(mcp_management, cloudzero, vantage, config_overrides) without mounting
their routers, and their stubs must still be injected."""
loaded: Final = getattr(app.state, "lazy_loaded", None)
if not isinstance(loaded, set):
return frozenset()
return frozenset(m for m in loaded if isinstance(m, str))
def inject_lazy_stubs(
schema: dict,
loaded_modules: AbstractSet[str],
features: tuple[LazyFeature, ...] = LAZY_FEATURES,
) -> dict:
"""Inject openapi entries for features not in loaded_modules. Uses the
snapshot file when available (full route info), otherwise falls back to a
single placeholder per feature. Any failure logs and returns the schema
unchanged so /openapi.json never 500s on a cosmetic injection bug."""
try:
from litellm.proxy._lazy_openapi_snapshot import load_snapshot
@ -409,8 +425,8 @@ def inject_lazy_stubs(schema: dict) -> dict:
paths: Final = schema.setdefault("paths", {})
schemas: Final = schema.setdefault("components", {}).setdefault("schemas", {})
for feat in LAZY_FEATURES:
if feat.module_path in sys.modules and not feat.persistent_swagger_stub:
for feat in features:
if feat.module_path in loaded_modules and not feat.persistent_swagger_stub:
continue
fragment = (snapshot or {}).get(feat.name)

2
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -23538,7 +23538,7 @@
"paths": {
"/prompts": {
"post": {
"description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer <your_api_key>\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"json_prompt\",\n \"prompt_integration\": \"dotprompt\",\n ### EITHER prompt_directory OR prompt_data MUST BE PROVIDED\n \"prompt_directory\": \"/path/to/dotprompt/folder\",\n \"prompt_data\": {\"json_prompt\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```",
"description": "Create a new prompt\n\n\ud83d\udc49 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management)\n\nExample Request:\n```bash\ncurl -X POST \"http://localhost:4000/prompts\" \\\n -H \"Authorization: Bearer <your_api_key>\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"prompt_id\": \"my_prompt\",\n \"litellm_params\": {\n \"prompt_id\": \"my_prompt\",\n \"prompt_integration\": \"dotprompt\",\n \"prompt_data\": {\"content\": \"This is a prompt\", \"metadata\": {\"model\": \"gpt-4\"}}\n },\n \"prompt_info\": {\n \"prompt_type\": \"config\"\n }\n }'\n```",
"operationId": "create_prompt_prompts_post",
"requestBody": {
"content": {

20
litellm/proxy/_types.py
View file

@ -20,6 +20,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
from litellm._uuid import uuid
from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_langfuse_environment_value,
validate_no_callback_env_reference,
)
from litellm.types.integrations.compression_interception import (
@ -2027,6 +2028,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}")
callback_vars[key] = str(value)
validate_no_callback_env_reference(key, callback_vars[key], source="key/team callback metadata")
if key == "langfuse_environment":
validate_langfuse_environment_value(callback_vars[key])
return values
@ -2507,6 +2510,17 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"are skipped for on-demand GET /health as well as the background health loop."
),
)
model_list_healthy_only: bool | None = Field(
None,
description=(
"When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing "
"deployments are all unhealthy, for every caller, without needing `healthy_only=true` "
"per request. Requires `background_health_checks: true`, and keeps deployment health "
"state cached without turning on `enable_health_check_routing`, so routing is "
"unaffected. With no health state nothing is hidden. Hiding is presentation-only, a "
"hidden model can still be called."
),
)
alerting: list | None = Field(
None,
description="List of alerting integrations. Today, just slack - `alerting: ['slack']`",
@ -2819,7 +2833,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
# Values stay `object` rather than BudgetConfig: this is the raw JSON column,
# and validating it here would make one malformed row fail auth outright.
# resolve_model_budget validates the single entry a request actually needs.
user_model_max_budget: dict[str, object] | None = None
user_model_max_budget: Mapping[str, object] | None = None
request_route: str | None = None
is_session_token: bool = False
# Server-only marker set exclusively by the MCP gateway admission path
@ -2997,8 +3011,8 @@ class UserInfoV2Response(LiteLLMPydanticObjectBase):
sso_user_id: str | None = None
teams: list[str] = [] # Just team IDs, not full team objects
object_permission: LiteLLM_ObjectPermissionTable | None = None
model_max_budget: dict | None = None
model_max_budget_usage: dict | None = None
model_max_budget: Mapping[str, object] | None = None
model_max_budget_usage: Mapping[str, Mapping[str, object]] | None = None
from litellm.models.config import LiteLLM_Config as LiteLLM_Config # noqa: E402

24
litellm/proxy/auth/auth_checks.py
View file

@ -2588,13 +2588,29 @@ async def _delete_cache_key_object(
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging | None,
):
"""
Evict one key object, best-effort, matching `delete_cache_team_object` and
`delete_cache_key_objects`.
Every caller runs this after its own write has already committed, and the in-memory entry is
dropped before the Redis round trip. Letting a cache-backend error raise here therefore reports
failure for work that succeeded without making the cache any less stale; the leftover Redis
entry expires at its TTL either way.
"""
key: Final = hashed_token
user_api_key_cache.delete_cache(key=key)
try:
user_api_key_cache.delete_cache(key=key)
## UPDATE REDIS CACHE ##
if proxy_logging_obj is not None:
await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)
## UPDATE REDIS CACHE ##
if proxy_logging_obj is not None:
await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)
except Exception as e: # noqa: BLE001 # best-effort: a cache error must not fail a committed write
verbose_proxy_logger.warning(
"Failed to invalidate cached key entry %s; a stale key object may be served until its TTL expires: %s",
key,
e,
)
async def delete_cache_key_objects(

11
litellm/proxy/auth/user_api_key_auth.py
View file

@ -212,9 +212,9 @@ async def _read_user_model_max_budget(
user_id: str | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
parent_otel_span: object,
parent_otel_span: Span | None,
proxy_logging_obj: ProxyLogging,
) -> dict | None:
) -> Mapping[str, object] | None:
"""The user row's `model_max_budget`, or None when the row cannot be read.
A user whose row is missing must not be refused: this is a budget lookup,
@ -228,13 +228,13 @@ async def _read_user_model_max_budget(
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
parent_otel_span=parent_otel_span, # pyright: ignore[reportArgumentType] # Span is a runtime union, not usable in an annotation here
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except Exception as e: # noqa: BLE001 # mirrors the main path's tolerance
verbose_logger.debug("Unable to read user for the per-model budget check: %s", e)
return None
return getattr(user_obj, "model_max_budget", None)
return user_obj.model_max_budget if user_obj is not None else None
async def _check_user_model_budget(
@ -3267,8 +3267,7 @@ async def _run_post_custom_auth_checks(
# loaded the user row yet. The attach is unconditional because the post-call
# spend hook reads this field off the token: gating it on the same condition
# as enforcement would leave the user's counter uncharged whenever this
# request was not itself enforceable, which is the untracked-spend bug this
# PR exists to fix.
# request was not itself enforceable, so its spend would go untracked.
user_budget: Final = await _read_user_model_max_budget(
user_id=valid_token.user_id,
prisma_client=prisma_client,

114
litellm/proxy/common_request_processing.py
View file

@ -4,7 +4,7 @@ import json
import logging
import math
import traceback
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
@ -21,14 +21,13 @@ import litellm
from litellm._logging import _redact_string, verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
AUTO_ROUTED_REQUEST_METADATA_KEY,
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
DEFAULT_MAX_RECURSE_DEPTH,
LITELLM_DETAILED_TIMING,
LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
NON_INFERENCE_CALL_TYPES,
RETURN_RAW_MODEL_NAME_METADATA_KEY,
ROUTER_MODEL_NAME_RESPONSE_FIELD,
STREAM_SSE_DATA_PREFIX,
STREAM_SSE_KEEPALIVE_PING_BYTES,
UNSAFE_PROXY_RESPONSE_HEADERS,
@ -39,6 +38,7 @@ from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
@ -284,7 +284,7 @@ def _deferred_stream_logging_is_armed(request_data: dict) -> bool:
)
def _assembled_model_came_from_a_later_chunk(chunks: list, assembled_model: object) -> bool:
def _assembled_model_came_from_a_later_chunk(chunks: Sequence[object], assembled_model: object) -> bool:
"""Report whether stream_chunk_builder picked a model the first chunk did not carry.
Azure Model Router puts the routed model on the chunks after the first one, and the
@ -306,7 +306,10 @@ def _assembled_model_came_from_a_later_chunk(chunks: list, assembled_model: obje
)
def _assembled_model_is_the_name_the_client_asked_for(request_data: dict, assembled_model: object) -> bool:
def _assembled_model_is_the_name_the_client_asked_for(
request_data: Mapping[str, object],
assembled_model: object,
) -> bool:
"""Report whether the assembled model is the public name the proxy stamps onto chunks.
That stamp is what leaves an unpriced alias on the partial response, so the deployment's
@ -1299,15 +1302,51 @@ def _uncached_input_cost(
return input_cost - (cache_read_cost or 0.0) - (cache_creation_cost or 0.0)
_ZERO_COST_BREAKDOWN: Final = CostBreakdownHeaderValues(
original_cost=0.0,
discount_amount=0.0,
margin_total_amount=0.0,
margin_percent=0.0,
input_cost=0.0,
output_cost=0.0,
tool_usage_cost=0.0,
)
"""The component split a call priced at zero advertises, so a client reading the cost headers off a
read or management route still finds the whole family rather than a partially populated one."""
def _totals_to_zero(response_cost: float | str | None) -> bool:
"""Whether the total these headers carry is zero, counting a total no route ever priced as one.
A component split is only reported as zero alongside a total that agrees with it, so a read
that did price normally never advertises a real total beside an all-zero split.
"""
if response_cost is None or response_cost == "":
return True
try:
return float(response_cost) == 0.0
except (TypeError, ValueError):
return False
def _get_cost_breakdown_from_logging_obj(
litellm_logging_obj: LiteLLMLoggingObj | None,
response_cost: float | str | None = None,
) -> CostBreakdownHeaderValues:
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown."""
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown.
A non-inference call that priced at zero never records a breakdown, so its components are
reported as zero here. Any such call that did price normally (retrieving a background response,
and the cost poller's read of one) reports the breakdown it stored, or nothing at all when the
breakdown has not landed yet.
"""
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
return CostBreakdownHeaderValues()
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
if not cost_breakdown:
if litellm_logging_obj.call_type in NON_INFERENCE_CALL_TYPES and _totals_to_zero(response_cost):
return _ZERO_COST_BREAKDOWN
return CostBreakdownHeaderValues()
return CostBreakdownHeaderValues(
@ -1456,7 +1495,9 @@ class ProxyBaseLLMRequestProcessing:
exclude_values: Final = {"", None, "None"}
hidden_params = hidden_params or {}
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj)
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=litellm_logging_obj, response_cost=response_cost
)
# Calculate updated spend for header (include current response_cost)
current_spend: Final = user_api_key_dict.spend or 0.0
@ -2033,54 +2074,6 @@ class ProxyBaseLLMRequestProcessing:
return deployment
return None
@staticmethod
def get_router_selected_model_name(
litellm_logging_obj: LiteLLMLoggingObj | None,
) -> str | None:
"""Model group an auto-routing strategy selected, or None if none fired.
The marker and ``deployment_model_name`` are written by different bucket
resolvers (``get_or_create_metadata_bucket`` vs
``_get_router_metadata_variable_name``), so they can land in different
buckets on the same request. Resolve each across both.
"""
litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None)
if not isinstance(litellm_params, dict):
return None
buckets: Final = tuple(
bucket for key in ("litellm_metadata", "metadata") if isinstance(bucket := litellm_params.get(key), dict)
)
if not any(bucket.get(AUTO_ROUTED_REQUEST_METADATA_KEY) is True for bucket in buckets):
return None
return next(
(
model_group
for bucket in buckets
if isinstance(model_group := bucket.get("deployment_model_name"), str) and model_group
),
None,
)
@staticmethod
def set_router_selected_model_field(
*,
response_obj: object,
router_model_name: str | None,
) -> None:
if not router_model_name:
return
if isinstance(response_obj, dict):
response_obj[ROUTER_MODEL_NAME_RESPONSE_FIELD] = router_model_name
return
try:
setattr(response_obj, ROUTER_MODEL_NAME_RESPONSE_FIELD, router_model_name)
except (AttributeError, TypeError, ValueError):
verbose_proxy_logger.debug(
"Could not set %s on response object of type %s",
ROUTER_MODEL_NAME_RESPONSE_FIELD,
type(response_obj),
)
@staticmethod
def _response_cost_from_logging_obj(
*,
@ -2579,20 +2572,21 @@ class ProxyBaseLLMRequestProcessing:
log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
return_raw_model_name=_should_return_raw_model_name(self.data),
)
self.set_router_selected_model_field(
response_obj=response,
router_model_name=self.get_router_selected_model_name(logging_obj),
)
hidden_params = get_hidden_params_dict(response) # get any updated response headers
additional_headers = hidden_params.get("additional_headers", {}) or {}
recover_response_cost: Final = not response_cost and hidden_params.get("response_cost") is None
llm_cost_for_headers: Final = (
computed_cost_for_headers: Final = (
self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj) or ""
if recover_response_cost
else response_cost
)
llm_cost_for_headers: Final = (
0.0
if is_unbilled_non_inference_call_from_params(logging_obj.call_type, logging_obj.litellm_params, response)
else computed_cost_for_headers
)
_, request_metadata_bucket = get_or_create_metadata_bucket(self.data)
guardrail_cost_for_headers: Final = guardrail_information_cost(
request_metadata_bucket.get("standard_logging_guardrail_information")

27
litellm/proxy/common_utils/callback_config_validation.py
View file

@ -14,11 +14,36 @@ _NEWRELIC_VAR_PREFIX: Final = "newrelic_"
def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None:
if callback_name != _NEWRELIC_CALLBACK or not callback_vars:
if not callback_vars:
return None
env_error: Final = _langfuse_environment_error(callback_vars)
if env_error is not None:
return env_error
if callback_name != _NEWRELIC_CALLBACK:
return None
return _newrelic_config_error(callback_vars)
def _langfuse_environment_error(callback_vars: Mapping[str, str]) -> str | None:
"""Reject langfuse_environment values Langfuse ingestion would drop.
Accepting an invalid value here would 200 the config write and then
silently lose every trace for that key/team at request time.
"""
value: Final = callback_vars.get("langfuse_environment")
if value is None:
return None
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_langfuse_environment_value,
)
try:
validate_langfuse_environment_value(value)
except ValueError as e:
return str(e)
return None
def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None:
"""Validate every ``logging`` entry of a team/key metadata payload."""
if not metadata:

79
litellm/proxy/common_utils/healthy_model_filter.py Normal file
View file

@ -0,0 +1,79 @@
"""Opt-in health filtering shared by the model listing endpoints.
`/v1/models`, `GET /v1/models/{id}` and `/v1/model/info` hide models whose
backing deployments are all marked unhealthy by background health checks, either
per request via `healthy_only=true` or proxy-wide via
`general_settings.model_list_healthy_only: true`. Both are opt-in: with neither
set the listings are returned unfiltered and no health lookup runs at all.
The proxy-wide setting is what an operator turns on so every client (UI, SDK,
raw API) sees only reachable models without having to pass the query parameter.
It also makes the background health check loop keep the deployment health cache
populated, so `background_health_checks: true` is the only other setting needed.
The per-request parameter reads that same cache, so on its own it needs the
cache to be filled by either this setting or `enable_health_check_routing`.
Filtering is presentation-only and always fails open: it answers "should this
model be advertised?", never "should a request for it be attempted?". A hidden
model stays callable, and an absent, stale or empty health state hides nothing.
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from litellm.router import Router
MODEL_LIST_HEALTHY_ONLY_SETTING: Final = "model_list_healthy_only"
def is_healthy_only_listing_default(general_settings: Mapping[str, object]) -> bool:
"""Whether `model_list_healthy_only` filters every listing on this proxy.
Only a real `true` counts, so a quoted YAML value never silently starts
hiding models. This also tells the background health check loop to keep the
deployment health cache populated, which is the state the filter reads.
"""
return general_settings.get(MODEL_LIST_HEALTHY_ONLY_SETTING, False) is True
def is_healthy_only_enabled(
healthy_only: bool | None,
general_settings: Mapping[str, object],
) -> bool:
"""Whether the health filter applies to this request.
The per-request `healthy_only=true` and the proxy-wide
`model_list_healthy_only` setting are independent opt-ins: either one turns
the filter on, and a request cannot turn the proxy-wide setting back off
(`healthy_only=false` is the unset default, indistinguishable from absent).
"""
if healthy_only:
return True
return is_healthy_only_listing_default(general_settings)
async def get_hidden_unhealthy_model_names(
healthy_only: bool | None,
general_settings: Mapping[str, object],
llm_router: Router | None,
) -> set[str]:
"""Model names to hide from a listing, empty when the filter is off.
Empty is also the fail-open answer whenever the router cannot report health
(no router, no background health checks, stale state, `allowed_fails_policy`
configured), so callers apply it unconditionally and simply hide nothing.
"""
if llm_router is None or not is_healthy_only_enabled(healthy_only, general_settings):
return set()
unhealthy_names: Final = await llm_router.async_get_fully_unhealthy_model_names()
if not unhealthy_names:
verbose_proxy_logger.debug(
"healthy-only model listing is enabled but no unhealthy deployment state is "
"available (requires background_health_checks); returning unfiltered model list"
)
return unhealthy_names

2
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -1182,7 +1182,7 @@ class ResetBudgetJob:
if not raw:
continue
row_id: str = row[source.id_column]
windows: list = raw if isinstance(raw, list) else json.loads(raw)
windows: list[dict[str, object]] = raw if isinstance(raw, list) else json.loads(raw)
changed = False
for window in windows:
counter_key = f"{source.counter_prefix}:{row_id}:window:{window['budget_duration']}"

Some files were not shown because too many files have changed in this diff Show more