scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_together_cache_pricing

Browse source 
# Conflicts:
#	litellm/model_prices_and_context_window_backup.json
#	model_prices_and_context_window.json
#	tests/test_litellm/test_cost_calculator.py
This commit is contained in:
mateo-berri 2026-08-27 14:29:46 -07:00
commit dbadee7210
591 changed files with 47915 additions and 7343 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/mutmut-coverage.rc vendored Normal file
View file

@ -0,0 +1,5 @@
# mutmut's gather_coverage() looks covered lines up by absolute path, so the
# repo's `relative_files = true` makes every lookup miss and mutmut generates
# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
[run]
relative_files = false

18
.github/workflows/check-ui-api-types.yml vendored
View file

@ -83,6 +83,24 @@ jobs:
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Regenerate the lazy OpenAPI snapshot
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
- name: Fail if the lazy OpenAPI snapshot is stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
echo ""
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
echo "To fix, run from the repo root:"
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
exit 1
fi
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
- name: Set up Node.js
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

10
.github/workflows/mutation-test.yml vendored
View file

@ -87,11 +87,20 @@ jobs:
run: |
uv pip uninstall pytest-retry || true
# Ends before the job's own deadline so a run that outlasts the budget is
# still followed by the report and upload steps. mutmut saves after every
# mutant result, to mutants/<source path>.meta, so an interrupted run
# still scores the mutants it finished and export-cicd-stats can read
# them; a cancelled job skips those steps and publishes nothing at all.
- name: Run mutmut
timeout-minutes: 300
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
# Without this mutmut finds no covered lines and generates 0 mutants.
# See the file itself for why.
COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
run: |
set -o pipefail
mkdir -p mutants
@ -130,6 +139,7 @@ jobs:
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/**/*.meta
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

2
.gitignore vendored
View file

@ -3,6 +3,8 @@
tests/e2e/.fixtures/
.venv-typecheck
.venv_policy_test
.venv-mutmut
mutants/
.env
.claude
CLAUDE.local.md

2
CLAUDE.md
View file

@ -66,6 +66,8 @@ Commit and push your work when you're done without asking
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

38
basedpyright-code-budget.json
View file

@ -1,18 +1,18 @@
{
"reportAny": {
"limit": 19949
"limit": 18483
},
"reportArgumentType": {
"limit": 2566
"limit": 2564
},
"reportAssignmentType": {
"limit": 320
},
"reportAttributeAccessIssue": {
"limit": 488
"limit": 483
},
"reportCallIssue": {
"limit": 114
"limit": 113
},
"reportConstantRedefinition": {
"limit": 40
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 6049
"limit": 5960
},
"reportFunctionMemberAccess": {
"limit": 7
@ -45,7 +45,7 @@
"limit": 35
},
"reportInvalidTypeForm": {
"limit": 35
"limit": 34
},
"reportInvalidTypeVarUse": {
"limit": 2
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5661
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15555
"limit": 15484
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1061
"limit": 1058
},
"reportOptionalOperand": {
"limit": 0
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1810
"limit": 1808
},
"reportRedeclaration": {
"limit": 8
@ -99,31 +99,31 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44655
"limit": 44528
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 39009
"limit": 38804
},
"reportUnknownParameterType": {
"limit": 19883
"limit": 19829
},
"reportUnknownVariableType": {
"limit": 30569
"limit": 30355
},
"reportUnnecessaryCast": {
"limit": 117
},
"reportUnnecessaryComparison": {
"limit": 699
"limit": 697
},
"reportUnnecessaryContains": {
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 836
"limit": 833
},
"reportUntypedBaseClass": {
"limit": 0
@ -135,12 +135,12 @@
"limit": 21
},
"reportUnusedFunction": {
"limit": 139
"limit": 138
},
"reportUnusedImport": {
"limit": 545
"limit": 544
},
"reportUnusedVariable": {
"limit": 146
"limit": 145
}
}

3
ci_cd/generate_model_prices_schema.py
View file

@ -157,6 +157,9 @@ COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",

5
db_scripts/partition_spend_logs.sql
View file

@ -10,6 +10,11 @@
-- partitioned, so existing installs are unaffected until you run this.
--
-- IMPORTANT
-- * After partitioning, `prisma db push` (including the proxy's
-- --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
-- the primary key back to ("request_id"), which Postgres rejects on a
-- partitioned table. The proxy detects this and exits with guidance.
-- Use the default startup path (`prisma migrate deploy`) instead.
-- * Test on a staging copy first and take a backup.
-- * Postgres cannot convert a populated table to partitioned in place, so this
-- renames the old table aside and creates a fresh partitioned table.

4
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -14,6 +14,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from prisma import models as prisma_models
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -351,7 +353,7 @@ class CheckBatchCost:
return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
async def _finalize_unbilled_terminal_job(
self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
) -> None:
"""Persist a terminal batch that has nothing billable, converting any raw
provider file ids to managed ids, and take it out of the poll page."""

10
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -1,6 +1,8 @@
"""
Polls LiteLLM_ManagedObjectTable to check if the response is complete.
Cost tracking is handled automatically by the get-responses call.
Cost tracking is handled by the get-responses call, which prices normally only because the
poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -113,7 +117,8 @@ class CheckResponsesCost:
Check if background responses are complete and track their cost.
- Get all status="queued" or "in_progress" and file_purpose="response" jobs
- Query the provider to check if response is complete
- Cost is automatically tracked by the get-responses call
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
"""
try:
@ -153,6 +158,7 @@ class CheckResponsesCost:
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.59"
version = "0.1.61"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.59"
version = "0.1.61"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

141
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -40,6 +40,65 @@ def _get_prisma_env() -> dict:
_MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
)
_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
re.IGNORECASE,
)
PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
"LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
"so its primary key must include the partition key (\"startTime\"). `prisma db push` "
"reconciles the database against schema.prisma, which declares the unpartitioned "
"primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
"constraint on partitioned table must include all partitioning columns. Start the "
"proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
"applies shipped migrations and leaves the partitioned primary key alone."
)
def _without_sql_comments(statement: str) -> str:
return "\n".join(
line
for line in statement.splitlines()
if line.strip() and not line.strip().startswith("--")
).strip()
def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
if not prefix_match:
return statement
kept = tuple(
clause.strip()
for clause in statement[prefix_match.end():].split(",\n")
if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
)
if not kept:
return None
return statement[: prefix_match.end()] + ",\n".join(kept)
def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
"""Drop statements from a `prisma migrate diff` script that fight the
SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
partitioned table, and drops of runbook artifacts such as
"LiteLLM_SpendLogs_legacy"."""
kept = tuple(
filtered
for statement in diff_sql.split(";")
for bare in (_without_sql_comments(statement),)
if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
for filtered in (_without_spend_logs_pk_clauses(bare),)
if filtered is not None
)
return "".join(f"{statement};\n\n" for statement in kept)
def _migration_timestamp(name: str) -> int:
"""Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@ -355,7 +414,24 @@ class ProxyExtrasDBManager:
return
logger.info(f"Migration diff created at {diff_sql_path}")
if ProxyExtrasDBManager.spend_logs_is_partitioned():
filtered_sql = filter_partitioned_spend_logs_diff(
diff_sql_path.read_text()
)
diff_sql_path.write_text(filtered_sql)
logger.info(
"LiteLLM_SpendLogs is partitioned; removed its primary-key "
"rewrite and partitioning artifacts from the drift script"
)
if not filtered_sql.strip():
logger.info("Drift script is empty after filtering; nothing to apply")
if not mark_all_applied:
return
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
return
# 2. Run prisma db execute to apply the migration
applied_ok = False
try:
logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run(
@ -376,6 +452,7 @@ class ProxyExtrasDBManager:
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
applied_ok = True
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to apply migration diff: {e.stderr}")
except subprocess.TimeoutExpired:
@ -384,6 +461,16 @@ class ProxyExtrasDBManager:
# 3. Mark all migrations as applied
if not mark_all_applied:
return
if not applied_ok:
logger.warning(
"Drift script failed to apply; NOT marking migrations as "
"applied so a later migration run can retry them"
)
return
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
@staticmethod
def _mark_migrations_applied(migrations_dir: str):
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
logger.info(f"Resolving {len(migration_names)} migrations")
for migration_name in migration_names:
@ -410,6 +497,55 @@ class ProxyExtrasDBManager:
f"Failed to resolve migration {migration_name}: {e.stderr}"
)
@staticmethod
def spend_logs_is_partitioned() -> bool:
"""True when the connected database's LiteLLM_SpendLogs is a
partitioned table in Prisma's target schema (the `schema` URL param,
falling back to Prisma's default target, public), i.e. the operator
ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
unavailable or the database cannot be reached, preserving the
pre-existing behavior in those cases."""
database_url = os.getenv("DATABASE_URL")
if not database_url:
return False
try:
import psycopg
except ImportError:
return False
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
try:
with psycopg.connect(
cleaned_url, connect_timeout=10, autocommit=True
) as conn:
row = conn.execute(
"SELECT 1 "
"FROM pg_partitioned_table pt "
"JOIN pg_class c ON c.oid = pt.partrelid "
"JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE c.relname = 'LiteLLM_SpendLogs' "
" AND n.nspname = %s",
(
ProxyExtrasDBManager._prisma_schema_param(database_url)
or "public",
),
).fetchone()
except (psycopg.OperationalError, psycopg.DatabaseError):
return False
return row is not None
@staticmethod
def _prisma_schema_param(url: str) -> Optional[str]:
"""The `schema` query param Prisma uses to pick its target schema,
or None when the URL does not set one."""
from urllib.parse import urlparse, parse_qsl
return next(
(v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
None,
)
@staticmethod
def _strip_prisma_query_params(url: str) -> str:
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
@ -528,7 +664,8 @@ class ProxyExtrasDBManager:
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
if not use_migrate:
# Preserve `prisma db push` path unchanged.
if ProxyExtrasDBManager.spend_logs_is_partitioned():
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
original_dir = os.getcwd()
os.chdir(migrations_dir)
try:
@ -972,6 +1109,8 @@ class ProxyExtrasDBManager:
)
raise
else:
if ProxyExtrasDBManager.spend_logs_is_partitioned():
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
# Use prisma db push with increased timeout
subprocess.run(
[_get_prisma_command(), "db", "push", "--accept-data-loss"],

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.89"
version = "0.4.90"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.89"
version = "0.4.90"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

6
litellm/__init__.py
View file

@ -445,6 +445,7 @@ max_ui_session_budget: Optional[float] = (
1.0 # USD budget for each dashboard login session (playground, test connection)
)
internal_user_budget_duration: Optional[str] = None
budget_rollover: bool = False # carry spend beyond max_budget into the next window instead of zeroing it
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
max_end_user_budget: Optional[float] = None
max_end_user_budget_id: Optional[str] = None
@ -464,6 +465,11 @@ prometheus_metrics_config: Optional[List] = None
prometheus_exclude_metrics: Optional[List[str]] = None
prometheus_exclude_labels: Optional[List[str]] = None
prometheus_emit_stream_label: bool = False
prometheus_deployment_and_latency_caller_identity: Literal[
"api_key_alias",
"user_email",
"both",
] = "api_key_alias"
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
# pre-unification label set so existing dashboards / recording rules keyed on

130
litellm/_redis.py
View file

@ -12,8 +12,9 @@ import json
# s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
import os
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
from urllib.parse import urlsplit, urlunsplit
import redis
import redis.asyncio as async_redis
@ -50,6 +51,7 @@ def _get_redis_kwargs():
include_args: Final = {
"url",
"redis_connect_func",
"credential_provider",
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
@ -155,7 +157,8 @@ def _get_redis_cluster_kwargs(client=None):
def _get_redis_env_kwarg_mapping():
PREFIX: Final = "REDIS_"
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs()}
exclude_from_environment: Final = frozenset({"credential_provider"})
return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment}
def _redis_kwargs_from_environment():
@ -353,6 +356,12 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _url_without_userinfo(url: str) -> str:
parts: Final = urlsplit(url)
netloc: Final = parts.netloc.rsplit("@", 1)[-1]
return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
def _get_redis_client_logic(**env_overrides):
"""
Common functionality across sync + async redis client implementations
@ -410,54 +419,58 @@ def _get_redis_client_logic(**env_overrides):
if _service_name is not None:
redis_kwargs["service_name"] = _service_name
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
if redis_kwargs.get("credential_provider") is None:
# Handle GCP IAM authentication
_gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str(
"REDIS_GCP_SERVICE_ACCOUNT"
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
_gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
if _gcp_service_account is not None:
verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Only enable SSL if explicitly requested AND SSL CA certs are provided
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
_azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str("AZURE_CLIENT_SECRET")
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
_azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
_azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str(
"AZURE_CLIENT_SECRET"
)
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
redis_kwargs.pop("gcp_service_account", None)
redis_kwargs.pop("gcp_ssl_ca_certs", None)
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
@ -465,6 +478,13 @@ def _get_redis_client_logic(**env_overrides):
redis_kwargs.pop("azure_tenant_id", None)
redis_kwargs.pop("azure_client_secret", None)
if redis_kwargs.get("credential_provider") is not None:
redis_kwargs.pop("redis_connect_func", None)
redis_kwargs.pop("username", None)
redis_kwargs.pop("password", None)
if redis_kwargs.get("url") is not None:
redis_kwargs["url"] = _url_without_userinfo(redis_kwargs["url"])
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
# Only strip host/port/db/password when not routing to a cluster.
# When startup_nodes is also present the cluster path takes priority and
@ -532,8 +552,7 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
service_name: Final = redis_kwargs.get("service_name")
connection_kwargs: Final = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs: Final = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
sentinel_kwargs: Final = _sentinel_auth_kwargs(connection_kwargs, sentinel_password)
if not sentinel_nodes or not service_name:
raise ValueError("Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.")
@ -605,7 +624,12 @@ def _async_credential_provider(redis_connect_func: object | None) -> CredentialP
def _async_auth_kwargs(redis_kwargs: dict) -> dict:
"""Swaps a connect func an async path cannot run for the equivalent credential provider,
which supersedes any static username or password redis-py would otherwise reject it with."""
credential_provider: Final = _async_credential_provider(redis_kwargs.get("redis_connect_func"))
explicit_provider: Final = redis_kwargs.get("credential_provider")
credential_provider: Final = (
explicit_provider
if explicit_provider is not None
else _async_credential_provider(redis_kwargs.get("redis_connect_func"))
)
if credential_provider is None:
return redis_kwargs
@ -738,8 +762,20 @@ def get_redis_connection_pool(
return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)
def _redis_kwargs_for_logging(redis_kwargs: Mapping[str, object]) -> Mapping[str, object]:
return {
key: "<credential provider>"
if key == "credential_provider" and value is not None
else "<redis connect function>"
if key == "redis_connect_func" and value is not None
else value
for key, value in redis_kwargs.items()
}
def _pretty_print_redis_config(redis_kwargs: dict) -> None:
"""Pretty print the Redis configuration using rich with sensitive data masking"""
redis_kwargs_for_logging: Final = _redis_kwargs_for_logging(redis_kwargs)
try:
import logging
@ -757,7 +793,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
masker = SensitiveDataMasker()
# Mask sensitive data in redis_kwargs
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
# Create main panel title
title: Final = Text("Redis Configuration", style="bold blue")
@ -820,7 +856,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
except ImportError:
# Fallback to simple logging if rich is not available
masker = SensitiveDataMasker()
masked_redis_kwargs = masker.mask_dict(redis_kwargs)
masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
verbose_logger.info("Redis configuration: %s", masked_redis_kwargs)
except Exception as e:
verbose_logger.error("Error pretty printing Redis configuration: %s", e)

4
litellm/batches/batch_utils.py
View file

@ -551,7 +551,7 @@ def _get_batch_job_usage_from_response_body(
return usage
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict:
def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
"""
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
@ -563,7 +563,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
def _get_response_from_batch_job_output_file(
batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
) -> Any:
) -> Mapping[str, Any]:
"""
Get the response from the batch job output file
"""

43
litellm/caching/caching_handler.py
View file

@ -18,7 +18,7 @@ import asyncio
import datetime
import inspect
import time
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import BaseModel
@ -27,6 +27,7 @@ import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache
from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
update_response_metadata,
)
@ -124,6 +125,29 @@ def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") ->
return details.model_dump(exclude_none=True) if hasattr(details, "model_dump") else {}
_PENDING_CACHE_WRITES: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs to pending write tasks
async def _complete_cache_write_despite_cancellation(write_factory: Callable[[], Awaitable[None]]) -> None:
try:
await write_factory()
except asyncio.CancelledError:
try:
await asyncio.wait_for(write_factory(), timeout=CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS)
except Exception as flush_error: # noqa: BLE001 # shutdown flush failures are logged, never raised
verbose_logger.warning(
"LiteLLM Cache: pending cache write failed during event loop shutdown: %s", flush_error
)
raise
def create_cache_write_task(write_factory: Callable[[], Awaitable[None]]) -> "asyncio.Task[None]":
task: Final = asyncio.create_task(_complete_cache_write_despite_cancellation(write_factory))
_PENDING_CACHE_WRITES.add(task)
task.add_done_callback(_PENDING_CACHE_WRITES.discard)
return task
def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None:
"""Read the caller-supplied ``cache_key`` off the request kwargs."""
return request_kwargs.get("cache_key", None)
@ -983,6 +1007,7 @@ class LLMCachingHandler:
if litellm.cache is None:
return
cache: Final = litellm.cache
new_kwargs: Final = kwargs.copy()
new_kwargs.update(
@ -1004,24 +1029,24 @@ class LLMCachingHandler:
):
if (
isinstance(result, EmbeddingResponse)
and litellm.cache is not None
and not isinstance(litellm.cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude.
and not isinstance(cache.cache, S3Cache) # s3 doesn't support bulk writing. Exclude.
):
asyncio.create_task(
litellm.cache.async_add_cache_pipeline(
create_cache_write_task(
lambda: cache.async_add_cache_pipeline(
result, dynamic_cache_object=self.dual_cache, **new_kwargs
)
)
else:
asyncio.create_task(
litellm.cache.async_add_cache(
result.model_dump_json(),
result_json: Final = result.model_dump_json()
create_cache_write_task(
lambda: cache.async_add_cache(
result_json,
dynamic_cache_object=self.dual_cache,
**new_kwargs,
)
)
else:
asyncio.create_task(litellm.cache.async_add_cache(result, **new_kwargs))
create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
def sync_set_cache(
self,

13
litellm/caching/redis_cache.py
View file

@ -175,6 +175,10 @@ _RedisCallResult = TypeVar("_RedisCallResult")
_swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0)
def _opaque_kwarg_key(value: object) -> str:
return f"{type(value).__name__}-{id(value)}"
@functools.lru_cache(maxsize=1)
def _redis_health_error_types() -> tuple[type, ...]:
"""Exception types that mean the Redis backend itself is unhealthy.
@ -399,10 +403,9 @@ class RedisCache(BaseCache):
Generate a cache key for the async Redis client based on connection parameters.
This ensures different Redis configurations use different cached clients.
"""
# Create a stable representation of redis_kwargs for hashing
# Sort keys to ensure consistent hash regardless of parameter order
sorted_kwargs: Final = sorted(self.redis_kwargs.items())
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True)
kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True, default=_opaque_kwarg_key)
kwargs_hash: Final = hashlib.sha256(kwargs_str.encode()).hexdigest()[:16]
return f"async-redis-client-{kwargs_hash}"
@ -432,7 +435,7 @@ class RedisCache(BaseCache):
"""
if key is None:
return key
if self.namespace is not None and not key.startswith(self.namespace):
if self.namespace and not key.startswith(self.namespace + ":"):
key = self.namespace + ":" + key
return key
@ -1384,10 +1387,10 @@ class RedisCache(BaseCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from .._redis import get_redis_async_client
# Create a fresh Redis client with current settings
redis_client: Final = redis_async.Redis(**self.redis_kwargs)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

17
litellm/caching/redis_cluster_cache.py
View file

@ -64,22 +64,9 @@ class RedisClusterCache(RedisCache):
dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
"""
try:
import redis.asyncio as redis_async
from redis.cluster import ClusterNode
from .._redis import get_redis_async_client
# Create ClusterNode objects from startup_nodes
cluster_kwargs: Final = self.redis_kwargs.copy()
startup_nodes: Final = cluster_kwargs.pop("startup_nodes", [])
new_startup_nodes: Final[list[ClusterNode]] = []
for item in startup_nodes:
new_startup_nodes.append(ClusterNode(**item))
# Create a fresh Redis Cluster client with current settings
redis_client: Final = redis_async.RedisCluster(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)
redis_client: Final = get_redis_async_client(**self.redis_kwargs)
# Test the connection
ping_result: Final = await redis_client.ping()

51
litellm/constants.py
View file

@ -49,6 +49,8 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
@ -379,6 +381,7 @@ AZURE_OPERATION_POLLING_TIMEOUT: Final = int(os.getenv("AZURE_OPERATION_POLLING_
AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: Final = str(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2024-11-30"))
AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = int(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96))
REDIS_SOCKET_TIMEOUT: Final = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1))
CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS: Final[float] = 5.0
REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5))
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
@ -462,6 +465,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
]
STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@ -1359,8 +1364,6 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@ -1470,6 +1473,12 @@ LITELLM_PROXY_MASTER_KEY_ALIAS: Final = "litellm_proxy_master_key"
# ``ProxyLogging._handle_logging_proxy_only_error``.
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: Final = "litellm_no_upstream_llm_call"
# Key/team metadata fields naming the OTel Resource ``service.name``, highest
# precedence first. Shared between the OTel v2 tenant router (which reads them
# out of ``user_api_key_auth_metadata``) and proxy request setup (which re-applies
# the key's values after the team metadata merge so a key outranks its team).
OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_service_name")
# Key Rotation Constants
LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int(
@ -1643,6 +1652,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
"max_ui_session_budget",
"budget_rollover",
]
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
@ -1809,6 +1819,43 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
# A retrieved response replays the usage of the call that created it, so pricing these
# read/management routes like inference bills the same tokens twice.
NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(
{
"get_responses",
"aget_responses",
"delete_responses",
"adelete_responses",
"cancel_responses",
"acancel_responses",
"list_input_items",
"alist_input_items",
"vector_store_create",
"avector_store_create",
"vector_store_retrieve",
"avector_store_retrieve",
"vector_store_list",
"avector_store_list",
"vector_store_update",
"avector_store_update",
"vector_store_delete",
"avector_store_delete",
"vector_store_file_create",
"avector_store_file_create",
"vector_store_file_list",
"avector_store_file_list",
"vector_store_file_retrieve",
"avector_store_file_retrieve",
"vector_store_file_content",
"avector_store_file_content",
"vector_store_file_update",
"avector_store_file_update",
"vector_store_file_delete",
"avector_store_file_delete",
}
)
# PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this
# sentinel api_key so PTU flat cost stays distinguishable from real per-request
# spend under the table's composite unique constraint.

118
litellm/cost_calculator.py
View file

@ -2,6 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -559,9 +560,10 @@ def cost_per_token(
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return openai_cost_per_token(
return generic_cost_per_token(
model=model_without_prefix,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
data_residency=data_residency,
)
@ -594,6 +596,7 @@ def cost_per_token(
prompt_characters=prompt_characters,
completion_characters=completion_characters,
usage=usage_block,
service_tier=service_tier,
vertex_location=vertex_location,
)
elif cost_router == "cost_per_token":
@ -797,14 +800,27 @@ def _select_model_name_for_cost_calc(
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
if region_name is not None:
return_model = f"{custom_llm_provider}/{region_name}/{return_model}"
else:
return_model = f"{custom_llm_provider}/{return_model}"
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
return return_model
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069)."""
segments: Final = model.split("/")
if "/".join(segments[1:]) in litellm.model_cost:
return model
head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1
head: Final = "/".join(segments[:head_len])
tail: Final = segments[head_len:]
strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail))
candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1))
return next((candidate for candidate in candidates if candidate in litellm.model_cost), model)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _model_contains_known_llm_provider(model: str) -> bool:
"""
@ -835,9 +851,11 @@ def _get_response_model(completion_response: object) -> str | None:
_GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
# ON_DEMAND_PRIORITY maps to "priority" — selects input_cost_per_token_priority, etc.
"ON_DEMAND_PRIORITY": "priority",
# FLEX / BATCH maps to "flex" — selects input_cost_per_token_flex, etc.
# FLEX / BATCH / ON_DEMAND_FLEX maps to "flex" — selects input_cost_per_token_flex, etc.
# Vertex AI reports flex/shared-capacity traffic as ON_DEMAND_FLEX, not FLEX.
"FLEX": "flex",
"BATCH": "flex",
"ON_DEMAND_FLEX": "flex",
# ON_DEMAND is standard pricing — no service_tier suffix applied
"ON_DEMAND": None,
}
@ -852,9 +870,9 @@ def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
trafficType values seen in practice
------------------------------------
ON_DEMAND -> standard pricing (service_tier = None)
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
FLEX / BATCH -> batch/flex pricing (service_tier = "flex")
ON_DEMAND -> standard pricing (service_tier = None)
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
FLEX / BATCH / ON_DEMAND_FLEX -> batch/flex pricing (service_tier = "flex")
"""
if traffic_type is None:
return None
@ -2359,6 +2377,64 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
def _candidate_realtime_token_costs(
model_name: str,
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float] | None:
try:
return generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
return None
def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool:
entries: Final = (
litellm.model_cost.get(model_name),
litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"),
)
return any(
entry is not None and any("cost_per" in field and value is not None for field, value in entry.items())
for entry in entries
)
def _first_priced_realtime_token_costs(
potential_model_names: Sequence[str | None],
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float]:
candidate_costs: Final = (
(model_name, costs)
for model_name in potential_model_names
if model_name is not None
and (
costs := _candidate_realtime_token_costs(
model_name=model_name,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
)
is not None
)
return next(
(
costs
for model_name, costs in candidate_costs
if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider)
),
(0.0, 0.0),
)
def handle_realtime_stream_cost_calculation(
results: OpenAIRealtimeStreamList,
combined_usage_object: Usage,
@ -2383,24 +2459,12 @@ def handle_realtime_stream_cost_calculation(
potential_model_names.append(received_model)
potential_model_names.append(litellm_model_name)
input_cost_per_token = 0.0
output_cost_per_token = 0.0
for model_name in potential_model_names:
try:
if model_name is None:
continue
_input_cost_per_token, _output_cost_per_token = generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
continue
input_cost_per_token += _input_cost_per_token
output_cost_per_token += _output_cost_per_token
break # exit if we find a valid model
input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs(
potential_model_names=potential_model_names,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
transcription_cost: Final = (
handle_realtime_transcription_cost_calculation(
results=results,

12
litellm/endpoints/speech/speech_to_completion_bridge/transformation.py
View file

@ -8,6 +8,14 @@ if TYPE_CHECKING:
from litellm.types.utils import ModelResponse
def _completion_response_cost(model_response: "ModelResponse") -> float | None:
hidden_params: Final = getattr(model_response, "_hidden_params", None)
if not isinstance(hidden_params, dict):
return None
response_cost: Final = hidden_params.get("response_cost")
return response_cost if isinstance(response_cost, float) else None
class SpeechToCompletionBridgeTransformationHandler:
def transform_request(
self,
@ -123,4 +131,6 @@ class SpeechToCompletionBridgeTransformationHandler:
# Create an httpx.Response object
response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
return HttpxBinaryResponseContent(response)
binary_response: Final = HttpxBinaryResponseContent(response)
binary_response.set_response_cost(_completion_response_cost(model_response))
return binary_response

15
litellm/experimental_mcp_client/client.py
View file

@ -7,6 +7,7 @@ import base64
import os
from collections.abc import Awaitable, Callable, Generator
from datetime import timedelta
from importlib import metadata
from typing import Any, Final, TypeVar
import httpx
@ -21,6 +22,18 @@ try:
streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
except ImportError:
pass
MCP_STREAMABLE_HTTP_REQUIREMENT: Final = "mcp>=1.28.1"
def missing_streamable_http_client_error() -> ImportError:
return ImportError(
f"MCP streamable HTTP transport requires {MCP_STREAMABLE_HTTP_REQUIREMENT}, but the installed "
f"mcp {metadata.version('mcp')} does not provide streamable_http_client. "
"Fix with: pip install 'litellm[mcp]' (or upgrade mcp directly: pip install -U mcp)"
)
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import (
@ -323,7 +336,7 @@ class MCPClient:
)
# HTTP transport (default)
if streamable_http_client is None:
raise ImportError("streamable_http_client is not available. Please install mcp with HTTP support.")
raise missing_streamable_http_client_error()
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)

12
litellm/google_genai/streaming_iterator.py
View file

@ -2,6 +2,7 @@ import asyncio
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
@ -65,6 +66,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
litellm_logging_obj: LiteLLMLoggingObj,
request_body: dict,
model: str,
custom_llm_provider: str,
hidden_params: dict[str, Any] | None = None,
):
self.litellm_logging_obj = litellm_logging_obj
@ -72,6 +74,10 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
self.start_time = datetime.now()
self.collected_chunks: list[bytes] = []
self.model = model
self.custom_llm_provider = custom_llm_provider
self.endpoint_type: Final = (
EndpointType.GEMINI if custom_llm_provider == litellm.LlmProviders.GEMINI.value else EndpointType.VERTEX_AI
)
self._hidden_params: dict[str, Any] = hidden_params or {}
async def _handle_async_streaming_logging(
@ -89,7 +95,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
url_route="/v1/generateContent",
request_body=self.request_body or {},
endpoint_type=EndpointType.VERTEX_AI,
endpoint_type=self.endpoint_type,
start_time=self.start_time,
raw_bytes=self.collected_chunks,
end_time=end_time,
@ -118,13 +124,13 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
custom_llm_provider=custom_llm_provider,
hidden_params=hidden_params,
)
self.response = response
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
self.custom_llm_provider = custom_llm_provider
# Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.iter_lines()
@ -169,13 +175,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
custom_llm_provider=custom_llm_provider,
hidden_params=hidden_params,
)
self.response = response
self.model = model
self.generate_content_provider_config = generate_content_provider_config
self.litellm_metadata = litellm_metadata
self.custom_llm_provider = custom_llm_provider
# Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
# large inlineData payloads (e.g. image/jpeg) intact within one event.
self.stream_iterator = response.aiter_lines()

36
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -104,6 +104,13 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
# rather than spending them on a list that is still missing some of their targets.
CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points"
class AnthropicCacheControlHook(CustomPromptManagement):
def get_chat_completion_prompt(
self,
@ -128,6 +135,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
- non_default_params: dict - params with any global cache controls
"""
# Extract cache control injection points
carry_unmatched: Final = bool(non_default_params.pop(CARRY_UNMATCHED_MESSAGE_POINTS, False))
injection_points: Final[list[CacheControlInjectionPoint]] = non_default_params.pop(
"cache_control_injection_points", []
)
@ -161,12 +169,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
non_default_params.get("prompt_cache_options"),
)
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
# budget. An ordinal names a different message once a later layer builds its own
# list, so it is placed here or not at all.
carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
tuple(point for point in message_points if point.get("index") is None) if carry_unmatched else ()
)
applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
tuple(point for point in message_points if point.get("index") is not None)
if carry_unmatched
else tuple(message_points)
)
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=message_points,
points=applied_message_points,
messages=processed_messages,
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
openai_dialect=openai_dialect,
@ -177,10 +198,15 @@ class AnthropicCacheControlHook(CustomPromptManagement):
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
# Pass through non-message injection points for provider-specific handling
if remaining_points:
# Points this pass did not place: non-message ones for the provider transform, and
# the deferred role-targeted ones. Deferring is what reaches the Responses API's
# `instructions`, which is only a system message once the bridge builds one. The
# judged stamp is what makes it safe: the next pass must not re-judge points
# against messages this pass already marked (see `_should_stand_down`).
carried_points: Final[Sequence[CacheControlInjectionPoint]] = (*remaining_points, *carried_message_points)
if carried_points:
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
remaining_points
carried_points
)
return model, processed_messages, non_default_params
@ -218,7 +244,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
@staticmethod
def _apply_message_injections(
points: list[CacheControlMessageInjectionPoint],
points: Sequence[CacheControlMessageInjectionPoint],
messages: list[AllMessageValues],
max_blocks: int,
openai_dialect: bool = False,

12
litellm/integrations/callback_configs.json
View file

@ -220,6 +220,12 @@
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
},
"langfuse_environment": {
"type": "text",
"ui_name": "Tracing Environment",
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
"required": false
}
},
"description": "Langfuse v2 Logging Integration"
@ -247,6 +253,12 @@
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
},
"langfuse_environment": {
"type": "text",
"ui_name": "Tracing Environment",
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
"required": false
}
},
"description": "Langfuse v3 OTEL Logging Integration"

4
litellm/integrations/compression_interception/handler.py
View file

@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
import time
import uuid
from typing import Any, Final, cast
from typing import Any, ClassVar, Final, cast
from litellm._logging import verbose_logger
from litellm.compression import compress
@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
4. Build typed rerun plan with tool_result blocks from the compressed cache.
"""
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
def __init__(
self,
enabled: bool = True,

2
litellm/integrations/custom_batch_logger.py
View file

@ -45,7 +45,7 @@ class CustomBatchLogger(CustomLogger):
super().__init__(**kwargs)
async def periodic_flush(self):
async def periodic_flush(self) -> None:
while True:
await asyncio.sleep(self.flush_interval)
verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval)

53
litellm/integrations/custom_logger.py
View file

@ -2,8 +2,8 @@
# On success, logs events to Promptlayer
import re
import traceback
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any, Final, Optional
from collections.abc import AsyncGenerator, Mapping
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from pydantic import BaseModel
@ -60,6 +60,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
# Class variables or attributes
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
enforces_request_content: bool = False
"""
@ -292,6 +293,54 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
Allow modifying / reviewing the response just after it's received from the deployment.
"""
async def async_post_call_failure_deployment_hook(
self,
request_data: Mapping[str, object],
exception: Exception,
call_type: CallTypes | None,
fallback_depth: int | None = None,
) -> None:
"""
Called once per failed deployment attempt - attempt 1, every retry, and
every fallback chain step - because the router re-invokes the wrapped
function on each attempt, re-entering this hook's call site fresh
every time.
This is a DEPLOYMENT-LEVEL signal, distinct from the REQUEST-LEVEL
``async_log_failure_event``, which fires once per logical client
request behind a dedup gate. ``request_data`` is mostly this
attempt's own kwargs, with one exception: it omits
``attempted_targets``, the router's own bookkeeping of which fallback
targets this request has already tried, since that one object *is*
shared by reference across every hop of the live fallback walk.
Pairs with ``async_pre_call_deployment_hook`` and
``async_post_call_success_deployment_hook`` to complete the
pre-call/success/failure lifecycle for a single deployment attempt.
``fallback_depth`` is best-effort: ``None`` on the first attempt and on
any call made without a ``Router`` (a bare SDK call has no fallback
chain to be at a depth in), ``1`` on the first fallback hop, ``2`` on
the second, and so on. It reflects ``Router``'s own internal fallback
bookkeeping (``kwargs["fallback_depth"]``), not a value this hook
computes or guarantees the shape of across versions. It tracks
fallback hops only, not retries within the same model group - a
retry-only failure (no fallback yet) also reports ``None``. If an
override predates this field it's simply never passed, rather than
raising - safe to leave off an override written before it existed.
``exception`` is a same-class snapshot, not the exact object about to
be re-raised to the real caller: read it freely, but setting an
attribute on it (e.g. ``status_code``) has no effect on what the
caller actually receives.
Default: no-op. Opt in by overriding. Keep overrides fast - this
runs on the request's exception path, so a slow implementation
delays error propagation to the caller. The reported failure
duration is captured before this hook runs, so a slow override
doesn't inflate that metric, but the caller still waits for it.
"""
async def async_post_call_streaming_deployment_hook(
self,
request_data: dict,

6
litellm/integrations/dotprompt/__init__.py
View file

@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
from .prompt_manager import strip_version_suffix
registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id
try:
dot_prompt_manager: Final = DotpromptManager(
prompt_directory=prompt_directory,
prompt_data=prompt_data,
prompt_file=prompt_file,
prompt_id=prompt_id,
prompt_id=registration_prompt_id,
)
return dot_prompt_manager

4
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement):
if prompt_id is None:
return False
try:
return prompt_id in self.prompt_manager.list_prompts()
return self.prompt_manager.get_prompt(prompt_id) is not None
except Exception:
# If there's any error accessing prompts, don't run prompt management
return False
@ -209,6 +209,8 @@ class DotpromptManager(CustomPromptManagement):
prompt_spec=prompt_spec,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
async def async_get_chat_completion_prompt(

30
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
return base
return None
class PromptTemplate:
"""Represents a single prompt template with metadata and content."""
@ -124,11 +131,13 @@ class PromptManager:
"content": "template content",
"metadata": {"model": "gpt-4", "temperature": 0.7, ...}
} + prompt_id
"""
if prompt_id:
prompt_data = {prompt_id: prompt_data}
for prompt_id, prompt_info in prompt_data.items():
A dict carrying a "content" key is a single flat template registered under
prompt_id; anything else is treated as already keyed by template ID.
"""
keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data
for template_id, prompt_info in keyed_prompts.items():
try:
content = prompt_info.get("content", "")
metadata = prompt_info.get("metadata", {})
@ -136,11 +145,10 @@ class PromptManager:
template = PromptTemplate(
content=content,
metadata=metadata,
template_id=prompt_id,
template_id=template_id,
)
self.prompts[prompt_id] = template
self.prompts[template_id] = template
except Exception:
# Optional: print(f"Error loading prompt from JSON: {prompt_id}")
pass
def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:
@ -272,8 +280,12 @@ class PromptManager:
if versioned_id in self.prompts:
return self.prompts[versioned_id]
# Fall back to base prompt_id
return self.prompts.get(prompt_id)
direct_match: Final = self.prompts.get(prompt_id)
if direct_match is not None:
return direct_match
base_prompt_id: Final = strip_version_suffix(prompt_id)
return self.prompts.get(base_prompt_id) if base_prompt_id else None
def list_prompts(self) -> list[str]:
"""Get a list of all available prompt IDs."""

26
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -416,17 +416,8 @@ class GenericPromptManager(CustomPromptManagement):
tools=tools,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=(
ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
if prompt_spec
else False
),
ignore_prompt_manager_optional_params=(
ignore_prompt_manager_optional_params
or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
if prompt_spec
else False
),
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
def get_chat_completion_prompt(
@ -457,17 +448,8 @@ class GenericPromptManager(CustomPromptManagement):
prompt_spec=prompt_spec,
prompt_label=prompt_label,
prompt_version=prompt_version,
ignore_prompt_manager_model=(
ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
if prompt_spec
else False
),
ignore_prompt_manager_optional_params=(
ignore_prompt_manager_optional_params
or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
if prompt_spec
else False
),
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
def clear_cache(self) -> None:

11
litellm/integrations/langfuse/langfuse.py
View file

@ -1,5 +1,6 @@
#### What this does ####
# On success, logs events to Langfuse
import inspect
import os
import traceback
from collections.abc import Callable, Iterable, Mapping
@ -21,6 +22,9 @@ from litellm.litellm_core_utils.core_helpers import (
reconstruct_model_name,
safe_deep_copy,
)
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_langfuse_environment_value,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@ -140,6 +144,7 @@ class LangFuseLogger:
langfuse_public_key=None,
langfuse_secret=None,
langfuse_host=None,
langfuse_environment: str | None = None,
flush_interval=1,
allow_env_credentials: bool = True,
):
@ -159,6 +164,10 @@ class LangFuseLogger:
if not (self.langfuse_host.startswith("http://") or self.langfuse_host.startswith("https://")):
# add http:// if unset, assume communicating over private network - e.g. render
self.langfuse_host = "http://" + self.langfuse_host
_env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
self.langfuse_environment = _env_override or os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
if self.langfuse_environment:
validate_langfuse_environment_value(self.langfuse_environment)
self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
@ -182,6 +191,8 @@ class LangFuseLogger:
}
self.langfuse_sdk_version: str = langfuse.version.__version__
if "environment" in inspect.signature(Langfuse.__init__).parameters:
parameters["environment"] = self.langfuse_environment
if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
parameters["sdk_integration"] = "litellm"
self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)

25
litellm/integrations/langfuse/langfuse_handler.py
View file

@ -1,3 +1,5 @@
import os
"""
This file contains the LangFuseHandler class
@ -108,6 +110,7 @@ class LangFuseHandler:
langfuse_public_key=credentials.get("langfuse_public_key"),
langfuse_secret=credentials.get("langfuse_secret") or credentials.get("langfuse_secret_key"),
langfuse_host=credentials.get("langfuse_host"),
langfuse_environment=credentials.get("langfuse_environment"),
allow_env_credentials=credentials.get("langfuse_host") is None,
)
in_memory_dynamic_logger_cache.set_cache(
@ -135,8 +138,29 @@ class LangFuseHandler:
or standard_callback_dynamic_params.get("langfuse_secret_key"),
langfuse_public_key=standard_callback_dynamic_params.get("langfuse_public_key"),
langfuse_host=standard_callback_dynamic_params.get("langfuse_host"),
langfuse_environment=LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params),
)
@staticmethod
def _meaningful_dynamic_environment(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> str | None:
"""Return the per-request environment only when it changes behavior.
Empty/whitespace values and values equal to the deployment-wide
LANGFUSE_TRACING_ENVIRONMENT fallback are treated as absent so an
environment-only override that matches the default does not mint a
duplicate SDK client (each client costs threads and counts against
MAX_LANGFUSE_INITIALIZED_CLIENTS).
"""
raw = standard_callback_dynamic_params.get("langfuse_environment")
if raw is None:
return None
value = str(raw).strip()
if not value or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT"):
return None
return value
@staticmethod
def _dynamic_langfuse_credentials_are_passed(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
@ -153,6 +177,7 @@ class LangFuseHandler:
or standard_callback_dynamic_params.get("langfuse_public_key") is not None
or standard_callback_dynamic_params.get("langfuse_secret") is not None
or standard_callback_dynamic_params.get("langfuse_secret_key") is not None
or LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params) is not None
):
return True
return False

12
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
LangfuseLLMObsOTELAttributes,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
)
@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
)
elif item_type == "function_call":
arguments_str = getattr(item, "arguments", "{}")
arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
arguments_obj = (
safe_json_loads(arguments_str, default={})
if isinstance(arguments_str, str)
else arguments_str
)
langfuse_tool_call = {
"id": getattr(item, "id", ""),
"name": getattr(item, "name", ""),
@ -226,7 +231,10 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
langfuse_environment: Final = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
if langfuse_environment:
safe_set_attribute(
span,

395
litellm/integrations/newrelic/newrelic_metrics.py Normal file
View file

@ -0,0 +1,395 @@
"""
New Relic Metric API Integration - sends per-team cost/usage metrics to /metric/v1
NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/
`async_log_success_event` / `async_log_failure_event` queue one record per request;
at flush the queue is aggregated by (team, model group, model, provider, status)
into count/summary metrics. `interval.ms` is the real window between flushes,
computed at flush time.
Team-scoped by construction: the ingest key is injected explicitly and there is
deliberately no environment-variable fallback, so a team's metrics are never sent
with the proxy operator's credentials (mirrors ``allow_env_credentials=False`` on
the Datadog team logger).
Error policy on flush: 4xx drops the batch (a retry would fail identically; 403
is a permanent credential failure), 5xx/network re-queues capped at
``max_queue_size`` records with the oldest dropped.
For batching specific details see CustomBatchLogger class
"""
import asyncio
import gzip
import time
import traceback
from collections.abc import Mapping
from math import ceil
from types import MappingProxyType
from typing import Final
from httpx import HTTPStatusError, Response
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.integrations.newrelic import (
NEWRELIC_DEFAULT_REGION,
NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN,
NEWRELIC_METRIC_COMPLETION_TOKENS,
NEWRELIC_METRIC_COST_USD,
NEWRELIC_METRIC_ENDPOINT_BY_REGION,
NEWRELIC_METRIC_PROMPT_TOKENS,
NEWRELIC_METRIC_REQUEST_DURATION_MS,
NEWRELIC_METRIC_REQUESTS,
NEWRELIC_METRIC_TOTAL_TOKENS,
NEWRELIC_METRICS_MAX_BATCH_SIZE,
NEWRELIC_METRICS_MAX_DRAIN_PASSES,
NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
NewRelicCountMetric,
NewRelicMetric,
NewRelicMetricCommon,
NewRelicMetricEnvelope,
NewRelicMetricRecord,
NewRelicSummaryMetric,
NewRelicSummaryValue,
)
from litellm.types.utils import StandardLoggingPayload
# 408 (request timeout) and 429 (rate limit) are transient client errors the
# Metric API expects a retry on, unlike 400/403 which a retry would only repeat.
_RETRYABLE_CLIENT_STATUSES: Final = frozenset({408, 429})
def resolve_newrelic_metric_endpoint(newrelic_region: str | None) -> str:
if not newrelic_region:
return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
endpoint: Final = NEWRELIC_METRIC_ENDPOINT_BY_REGION.get(newrelic_region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
newrelic_region,
", ".join(sorted(NEWRELIC_METRIC_ENDPOINT_BY_REGION)),
)
return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
return endpoint
def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) -> NewRelicMetricRecord:
metadata: Final = standard_logging_object.get("metadata")
team_id: Final = ((metadata.get("user_api_key_team_id") or metadata.get("team_id")) if metadata else None) or ""
team_alias: Final = (
(metadata.get("user_api_key_team_alias") or metadata.get("team_alias")) if metadata else None
) or ""
return NewRelicMetricRecord(
team_id=team_id,
team_alias=team_alias,
model_group=standard_logging_object.get("model_group") or "",
model=standard_logging_object.get("model") or "",
custom_llm_provider=standard_logging_object.get("custom_llm_provider") or "",
status=str(standard_logging_object.get("status") or "success"),
response_cost=float(standard_logging_object.get("response_cost") or 0.0),
prompt_tokens=int(standard_logging_object.get("prompt_tokens") or 0),
completion_tokens=int(standard_logging_object.get("completion_tokens") or 0),
total_tokens=int(standard_logging_object.get("total_tokens") or 0),
duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0,
)
def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]:
first: Final = bucket_records[0]
attributes: Final[Mapping[str, str]] = { # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType
key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN]
for key, value in (
("team_id", first.team_id),
("team_alias", first.team_alias),
("model_group", first.model_group),
("model", first.model),
("custom_llm_provider", first.custom_llm_provider),
("status", first.status),
)
if value
}
durations: Final = tuple(record.duration_ms for record in bucket_records)
counts: Final[tuple[tuple[str, float], ...]] = (
(NEWRELIC_METRIC_REQUESTS, float(len(bucket_records))),
(NEWRELIC_METRIC_COST_USD, sum(record.response_cost for record in bucket_records)),
(NEWRELIC_METRIC_PROMPT_TOKENS, float(sum(record.prompt_tokens for record in bucket_records))),
(NEWRELIC_METRIC_COMPLETION_TOKENS, float(sum(record.completion_tokens for record in bucket_records))),
(NEWRELIC_METRIC_TOTAL_TOKENS, float(sum(record.total_tokens for record in bucket_records))),
)
count_metrics: Final[tuple[NewRelicMetric, ...]] = tuple(
NewRelicCountMetric(name=name, type="count", value=value, attributes=attributes) for name, value in counts
)
summary_metric: Final = NewRelicSummaryMetric(
name=NEWRELIC_METRIC_REQUEST_DURATION_MS,
type="summary",
value=NewRelicSummaryValue(
count=len(durations),
sum=sum(durations),
min=min(durations),
max=max(durations),
),
attributes=attributes,
)
return (*count_metrics, summary_metric)
def build_metric_payload(
records: tuple[NewRelicMetricRecord, ...],
*,
window_start: float,
now: float,
) -> tuple[NewRelicMetricEnvelope, ...]:
"""Aggregates records into one Metric API envelope for the flush window."""
interval_ms: Final = max(1, int((now - window_start) * 1000))
bucket_keys: Final = tuple(dict.fromkeys(record.bucket_key for record in records))
metrics: Final = tuple(
metric
for key in bucket_keys
for metric in _bucket_metrics(tuple(record for record in records if record.bucket_key == key))
)
common: Final[NewRelicMetricCommon] = {
"timestamp": int(window_start * 1000),
"interval.ms": interval_ms,
}
return (NewRelicMetricEnvelope(common=common, metrics=metrics),)
class NewRelicMetricsLogger(CustomBatchLogger):
def __init__(
self,
newrelic_api_key: str,
newrelic_region: str | None = None,
) -> None:
if not newrelic_api_key:
raise ValueError(
"newrelic_api_key is required for NewRelicMetricsLogger; "
"team-scoped metrics never fall back to environment credentials"
)
self.newrelic_api_key: Final = newrelic_api_key
self.metric_api_url: Final = resolve_newrelic_metric_endpoint(newrelic_region)
self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
self._stopped: bool = False
self._drain_lock = asyncio.Lock()
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
super().__init__(
flush_lock=self.flush_lock,
batch_size=NEWRELIC_METRICS_MAX_BATCH_SIZE,
max_queue_size=NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
)
def stop(self) -> None:
"""Ends the periodic flush loop; called on DynamicLoggingCache eviction.
Schedules one final drain of anything still queued, so eviction never
silently discards records. Guarded so it can never raise into the
cache's eviction path.
"""
self._stopped = True
try:
asyncio.get_running_loop().create_task(self._final_drain())
except Exception: # noqa: BLE001 # no running loop / shutdown; the periodic loop's final drain still runs
verbose_logger.debug("New Relic Metrics: could not schedule final drain on stop()", exc_info=True)
async def _drain_with_retry(self) -> None:
"""Deliver everything queued on a stopped logger, or drop it with a log.
A stopped logger has no periodic loop left, so every post-stop path
funnels through here. ``_drain_lock`` serializes drains: a callback that
appends and starts its own drain queues behind the running one instead
of racing it. Each pass attempts the whole current queue in
``batch_size`` chunks, unlike the periodic path it does not stop at the
first failing chunk, so a persistently failing head never starves the
tail. Only after ``_MAX_DRAIN_PASSES`` against a permanently failing
destination is the remainder dropped, and then only the records that were
queued when this drain began, so every dropped record got the full retry
budget: a record a callback appended mid-drain is not in that snapshot,
so it is left for its own serialized drain rather than dropped after
fewer attempts, and is never stranded.
"""
async with self._drain_lock:
attempted: Final = tuple(self.log_queue)
for _pass in range(NEWRELIC_METRICS_MAX_DRAIN_PASSES):
await self._drain_flush_once()
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)
dropped: Final = len(self.log_queue) - len(survivors)
if dropped:
verbose_logger.warning(
"New Relic Metrics: dropping %s records after %s drain passes",
dropped,
NEWRELIC_METRICS_MAX_DRAIN_PASSES,
)
self.log_queue[:] = list(survivors) # mutable-ok: leave late arrivals for the next serialized drain
async def _drain_flush_once(self) -> None:
"""Attempt every queued record once, in ``batch_size`` chunks, without
stopping at the first failing chunk so a persistently failing head does
not starve the tail (the periodic ``flush_queue`` deliberately stops
instead). Takes the queue under ``flush_lock`` and re-queues only the
chunks a 5xx/network error left undelivered, so records a concurrent
request appends during the sends survive for the next pass."""
async with self.flush_lock:
pending: Final = tuple(self.log_queue)
window_start: Final = self.last_flush_time
self.last_flush_time = time.time()
del self.log_queue[:]
if not pending:
return
chunks: Final = tuple(
pending[start : start + self.batch_size] for start in range(0, len(pending), self.batch_size)
)
delivered: Final = tuple([await self._classify_and_send(chunk, window_start) for chunk in chunks])
failed: Final = tuple(record for chunk, ok in zip(chunks, delivered) for record in (() if ok else chunk))
if failed:
self._requeue(failed)
async def _final_drain(self) -> None:
await self._drain_with_retry()
async def periodic_flush(self) -> None:
while not self._stopped:
await asyncio.sleep(self.flush_interval)
if self._stopped:
break
await self.flush_queue()
await self._final_drain()
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
try:
await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
except Exception as e: # noqa: BLE001 # logging must never break the request path
verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
try:
await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
except Exception as e: # noqa: BLE001 # logging must never break the request path
verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
async def _log_async_event(self, standard_logging_object: StandardLoggingPayload | None) -> None:
if standard_logging_object is None:
raise ValueError("standard_logging_object not found in kwargs")
self.log_queue.append(_metric_record_from_payload(standard_logging_object))
if self._stopped:
# A stopped logger has no periodic loop left; an in-flight callback
# that appends after the eviction drain delivers its own record.
await self._drain_with_retry()
return
if len(self.log_queue) >= self.batch_size:
await self.flush_queue()
async def flush_queue(self) -> None:
async with self.flush_lock:
window_start: Final = self.last_flush_time
self.last_flush_time = time.time()
queued: Final = len(self.log_queue)
if not queued:
return
verbose_logger.debug("New Relic Metrics: Flushing %s queued records", queued)
# Bounded by what is queued now: records appended mid-flush belong to
# the next window, and looping until empty would never end under load.
for _chunk in range(ceil(queued / self.batch_size)):
if not await self.async_send_batch(window_start=window_start):
return
async def async_send_batch(self, window_start: float | None = None) -> bool:
"""Sends the oldest ``batch_size`` records only, so a queue grown past that
by re-queues cannot breach the Metric API data point cap in one request.
Returns False once a chunk fails and is re-queued, so the caller stops."""
if not self.log_queue:
return False
batch_to_send: Final[tuple[NewRelicMetricRecord, ...]] = tuple(self.log_queue[: self.batch_size])
del self.log_queue[: len(batch_to_send)]
delivered: Final = await self._classify_and_send(
batch_to_send, window_start if window_start is not None else self.last_flush_time
)
if not delivered:
self._requeue(batch_to_send)
return delivered
async def _classify_and_send(self, batch: tuple[NewRelicMetricRecord, ...], window_start: float) -> bool:
"""Send one chunk and classify the outcome, never touching the queue.
Returns True when the batch is done with (delivered on any 2xx, or a 4xx
a retry would only repeat, 403 being a permanent bad-key rejection), and
False when a 5xx or network error means the caller should re-queue it.
``AsyncHTTPHandler.post`` raises ``HTTPStatusError`` on any non-2xx, so a
4xx never returns a response here; the status is read off the raised
error to keep the client-error path (drop) distinct from 5xx (retry)."""
payload: Final = build_metric_payload(records=batch, window_start=window_start, now=time.time())
try:
status = (
await self.async_send_compressed_data(payload)
).status_code # rebind-ok: reassigned from the raised HTTPStatusError below
except HTTPStatusError as e:
status = e.response.status_code
except Exception as e: # noqa: BLE001 # transport/network failure re-queues the batch
verbose_logger.warning(
"New Relic Metrics: network error sending %s records, will retry - %s",
len(batch),
e,
)
return False
if 200 <= status < 300:
return True
if 400 <= status < 500 and status not in _RETRYABLE_CLIENT_STATUSES:
verbose_logger.warning(
"New Relic Metrics: %s from Metric API%s, dropping %s records.",
status,
" (permanent credential failure: invalid or revoked team ingest key)" if status == 403 else "",
len(batch),
)
return True
verbose_logger.warning(
"New Relic Metrics: %s from Metric API, will retry %s records",
status,
len(batch),
)
return False
def _requeue(self, batch: tuple[NewRelicMetricRecord, ...]) -> None:
"""Prepends ``batch`` in place (never by assignment: records appended by
concurrent requests during the flush await must survive), keeping
chronological order so the cap drops the oldest records first."""
self.log_queue[:0] = batch
overflow: Final = len(self.log_queue) - self.max_queue_size
if overflow > 0:
del self.log_queue[:overflow]
verbose_logger.warning(
"New Relic Metrics: retry queue exceeded max_queue_size=%s; dropped %s oldest records.",
self.max_queue_size,
overflow,
)
async def async_send_compressed_data(self, payload: tuple[NewRelicMetricEnvelope, ...]) -> Response:
compressed_data: Final = gzip.compress(safe_dumps(payload).encode("utf-8"))
headers: Final[Mapping[str, str]] = MappingProxyType(
{
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Api-Key": self.newrelic_api_key,
}
)
return await self.async_client.post(
url=self.metric_api_url,
data=compressed_data,
headers=headers,
)

90
litellm/integrations/newrelic/newrelic_team_handler.py Normal file
View file

@ -0,0 +1,90 @@
"""
New Relic Team Handler
Used to get the NewRelicMetricsLogger for a given request.
Handles Key/Team Based New Relic metrics, following the same pattern as DataDogHandler.
"""
from typing import TYPE_CHECKING, Final
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
from .newrelic_metrics import NewRelicMetricsLogger
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
class NewRelicLoggingConfig(TypedDict):
newrelic_api_key: ReadOnly[str | None]
newrelic_region: ReadOnly[str | None]
class NewRelicHandler:
@staticmethod
def get_newrelic_logger_for_request(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
in_memory_dynamic_logger_cache: "DynamicLoggingCache",
) -> NewRelicMetricsLogger:
"""
Get a team-scoped NewRelicMetricsLogger for a given request.
Resolves and caches per-team NewRelicMetricsLogger instances using
DynamicLoggingCache, keyed by the team's New Relic credentials. Each unique
set of credentials gets its own logger instance with its own batch/flush loop.
Note: This handler is only called when a team-scoped newrelic_api_key is
present. The trace logger for the ``newrelic`` callback (OTel v2 / legacy
agent) is managed separately by _init_custom_logger_compatible_class via
_in_memory_loggers.
"""
_credentials: Final = NewRelicHandler.get_dynamic_newrelic_logging_config(
standard_callback_dynamic_params=standard_callback_dynamic_params,
)
temp_newrelic_logger = in_memory_dynamic_logger_cache.get_cache(
credentials=_credentials, service_name="newrelic"
)
if temp_newrelic_logger is None:
temp_newrelic_logger = NewRelicHandler._create_newrelic_logger_from_credentials(
credentials=_credentials,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
return temp_newrelic_logger
@staticmethod
def _create_newrelic_logger_from_credentials(
credentials: NewRelicLoggingConfig,
in_memory_dynamic_logger_cache: "DynamicLoggingCache",
) -> NewRelicMetricsLogger:
newrelic_logger: Final = NewRelicMetricsLogger(
newrelic_api_key=credentials.get("newrelic_api_key") or "",
newrelic_region=credentials.get("newrelic_region"),
)
in_memory_dynamic_logger_cache.set_cache(
credentials=credentials,
service_name="newrelic",
logging_obj=newrelic_logger,
)
verbose_logger.debug("New Relic: Created and cached new NewRelicMetricsLogger for team-scoped credentials")
return newrelic_logger
@staticmethod
def get_dynamic_newrelic_logging_config(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> NewRelicLoggingConfig:
return NewRelicLoggingConfig(
newrelic_api_key=standard_callback_dynamic_params.get("newrelic_api_key"),
newrelic_region=standard_callback_dynamic_params.get("newrelic_region"),
)
@staticmethod
def _dynamic_newrelic_credentials_are_passed(
standard_callback_dynamic_params: StandardCallbackDynamicParams,
) -> bool:
return standard_callback_dynamic_params.get("newrelic_api_key") is not None

42
litellm/integrations/opentelemetry.py
View file

@ -22,6 +22,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
)
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.service_tier_utils import (
@ -1643,7 +1644,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if self._operation_duration_histogram:
self._operation_duration_histogram.record(duration_s, attributes=common_attrs)
if response_obj and (usage := response_obj.get("usage")) and self._token_usage_histogram:
if (
self._token_usage_histogram
and response_obj
and not is_unbilled_non_inference_call_from_params(kwargs.get("call_type"), params, response_obj)
and (usage := response_obj.get("usage"))
):
in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
out_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
self._token_usage_histogram.record(usage.get("prompt_tokens", 0), attributes=in_attrs)
@ -1719,6 +1725,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if not self._time_per_output_token_histogram:
return
if is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
):
return
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
@ -2049,6 +2060,26 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# serialise to JSON once so set_attribute never coerces.
guardrail_span.set_attribute("guardrail_violation_categories", safe_dumps(violation_categories))
# Billable usage counters and USD cost stamped by the provider hook
# (e.g. Azure Prompt Shield text records, Bedrock policy units).
guardrail_usage = guardrail_information.get("guardrail_usage")
if guardrail_usage is not None:
guardrail_span.set_attribute("guardrail_usage", safe_dumps(guardrail_usage))
guardrail_cost = guardrail_information.get("guardrail_cost")
if guardrail_cost is not None:
self.safe_set_attribute(
span=guardrail_span,
key="guardrail_cost",
value=guardrail_cost,
)
guardrail_cost_in_spend = guardrail_information.get("guardrail_cost_in_spend")
if isinstance(guardrail_cost_in_spend, bool):
self.safe_set_attribute(
span=guardrail_span,
key="guardrail_cost_in_spend",
value=guardrail_cost_in_spend,
)
self._set_team_attributes_from_kwargs(guardrail_span, kwargs)
guardrail_span.end(end_time=self._to_ns(end_time_datetime))
@ -2468,7 +2499,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
usage: Final = response_obj and response_obj.get("usage")
usage: Final = (
response_obj.get("usage")
if response_obj
and not is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), litellm_params, response_obj
)
else None
)
if usage:
self.safe_set_attribute(
span=span,

6
litellm/integrations/otel/emitter.py
View file

@ -146,7 +146,7 @@ class SpanEmitter:
For callers that own and manage their own span lifecycle. ``tracer``
overrides the bound tracer for this span only, used for per-request
multi-tenant credential routing. ``links`` records related-but-not-parent
spans (e.g. the transport span of an MCP message, per MCP semconv).
spans (e.g. the trace context an MCP client propagated in ``params._meta``).
"""
return (tracer or self._tracer).start_span(
name,
@ -196,8 +196,8 @@ class SpanEmitter:
Return the span, or ``None`` if it was deduplicated away. ``tracer``
overrides the bound tracer for this span, used for per-request routing.
``links`` records related-but-not-parent spans (the transport span of an
MCP message).
``links`` records related-but-not-parent spans (e.g. the trace context an
MCP client propagated in ``params._meta``).
"""
# LLM-call and MCP tool-call spans carry a dedup key (their request's
# call id), so a sync+async double-firing coalesces. ``isinstance`` narrows

14
litellm/integrations/otel/logger.py
View file

@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger):
MCP tool calls reach the success/failure callbacks like any other request
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
semconv it parents to the trace context the client propagated in
``params._meta`` (or starts a new root) and links the transport span, rather
than nesting under the HTTP/session span. Returns whether it handled the
no ``pre_call`` carrier — so they get their own CLIENT span here. It nests
under the transport span of the request carrying this message, and trace
context the client propagated in ``params._meta`` is recorded as a span
link (see ``resolve_mcp_span_context``). Returns whether it handled the
event, so the caller skips the LLM-call path. The whole span is emitted at
once (there is no boundary to open it at), deduped on the call id.
"""
@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger):
Like a tool call, listing reaches the success/failure callbacks (here with
``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
context (or starts a new root) and links the transport span, rather than
nesting under the HTTP/session span. Returns whether it handled the event so
own CLIENT span, nested under the transport span of the request carrying
this message with any ``params._meta`` trace context recorded as a span
link (see ``resolve_mcp_span_context``). Returns whether it handled the event so
the caller skips the LLM-call path.
"""
raw_payload: Final = kwargs.get("standard_logging_object")

3
litellm/integrations/otel/mappers/genai.py
View file

@ -136,6 +136,9 @@ class GenAIMapper:
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
LiteLLM.GUARDRAIL_USAGE: lambda d: d.usage_json,
LiteLLM.GUARDRAIL_COST: lambda d: d.cost,
LiteLLM.GUARDRAIL_COST_IN_SPEND: lambda d: d.cost_in_spend,
}
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {

14
litellm/integrations/otel/model/payloads.py
View file

@ -190,6 +190,15 @@ class GuardrailSpanData:
guardrail_id: str | None = None
policy_template: str | None = None
detection_method: str | None = None
# Provider-reported billable usage counters (JSON-serialized) and the USD cost
# priced from them by the provider hook (``guardrail_usage`` /
# ``guardrail_cost`` on ``StandardLoggingGuardrailInformation``).
usage_json: str | None = None
cost: float | None = None
# Whether ``cost`` participates in the request's billed spend (absent means
# billed, the default; False means report-only). Mirrors
# ``guardrail_cost_in_spend`` so trace consumers can avoid double-counting.
cost_in_spend: bool | None = None
# Set when the guardrail intervened/blocked or failed, so the emitter marks
# the span ERROR — a blocking guardrail is an error outcome for that span.
error: SpanError | None = None
@ -209,6 +218,8 @@ class GuardrailSpanData:
get: Final = cast(Mapping[str, object], entry).get
status: Final = as_str(get("guardrail_status"))
response: Final = get("guardrail_response")
usage: Final = get("guardrail_usage")
in_spend: Final = get("guardrail_cost_in_spend")
error: Final = (
SpanError(error_type=status, message=as_str(get("guardrail_action")))
if status in cls._ERROR_STATUSES
@ -231,6 +242,9 @@ class GuardrailSpanData:
guardrail_id=as_str(get("guardrail_id")),
policy_template=as_str(get("policy_template")),
detection_method=as_str(get("detection_method")),
usage_json=_json_or_none(usage) if usage is not None else None,
cost=as_float(get("guardrail_cost")),
cost_in_spend=in_spend if isinstance(in_spend, bool) else None,
error=error,
)

18
litellm/integrations/otel/model/semconv.py
View file

@ -32,6 +32,7 @@ class GenAIOperation(str, Enum):
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
LITELLM_MODERATION = "litellm.moderation"
@ -307,6 +308,15 @@ class LiteLLM:
GUARDRAIL_ID: Final = "litellm.guardrail.id"
GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"
GUARDRAIL_DETECTION_METHOD: Final = "litellm.guardrail.detection_method"
# Provider-reported billable usage counters, JSON-serialized into one value.
GUARDRAIL_USAGE: Final = "litellm.guardrail.usage"
# Numeric USD cost of the guardrail invocation; lives under the litellm.cost.*
# namespace (COST_PREFIX) beside the LLM call's litellm.cost.total.
GUARDRAIL_COST: Final = "litellm.cost.guardrail"
# Whether litellm.cost.guardrail is already inside litellm.cost.total (True,
# the billed default) or reported alongside it (False) — without this a trace
# consumer cannot tell whether adding the two double-counts.
GUARDRAIL_COST_IN_SPEND: Final = "litellm.guardrail.cost_in_spend"
SERVICE_NAME: Final = "litellm.service.name"
SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
@ -374,6 +384,14 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"get_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"aget_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"delete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"adelete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"cancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"acancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"list_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"alist_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
"image_generation": GenAIOperation.GENERATE_CONTENT,
"aimage_generation": GenAIOperation.GENERATE_CONTENT,
"moderation": GenAIOperation.LITELLM_MODERATION,

44
litellm/integrations/otel/model/spans.py
View file

@ -10,6 +10,8 @@ Canonical hierarchy::
│ └── DB_CALL (CLIENT) # its key/user/team lookups nest here
├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL
├── LLM_CALL (CLIENT)
├── MCP_TOOL_CALL (CLIENT) # nests under the POST carrying the message
├── MCP_LIST_TOOLS (CLIENT) # (client-propagated context is a span link)
└── DB_CALL (CLIENT) # e.g. the spend-log write
Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
not a child of it. The emitter parents every span to the ambient OTel context
(the active server span), which matches this.
MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit
time by :func:`resolve_mcp_span_context`. When the client propagates trace context
in ``params._meta`` MCP and the HTTP transport are independent contexts per the
OTel GenAI MCP semconv, so the span parents to that propagated context and records
the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape
this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is
propagated (the common case) the span nests under the transport span of the request
carrying that message, so the tool call stays in one trace.
MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by
:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport
span of the request carrying that message, so the tool call stays in one trace.
Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as
a span *link*, never the parent — a remote parent would root the span in a trace
whose root never reaches the gateway's tracing backend. Links always target that
remote client context, never a registry role, so ``SpanSpec`` declares no link
field; the concrete transport parent is resolved per message at emit time.
Not every service call becomes a span — :func:`span_role_for_service` decides:
@ -85,25 +87,19 @@ class SpanSpec:
role: SpanRole
kind: LiteLLMSpanKind
parent: SpanRole | None
links: SpanRole | None = None
SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
# The proxy is an MCP client to the upstream server, so MCP spans are CLIENT
# spans. With trace context propagated in ``params._meta``, MCP and the HTTP
# transport are independent contexts (OTel GenAI MCP semconv): the span parents
# to the propagated context and records the PROXY_REQUEST transport span as a
# span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST``
# encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span
# under that message's transport span instead, keeping the call in one trace.
SpanRole.MCP_TOOL_CALL: SpanSpec(
SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
),
SpanRole.MCP_LIST_TOOLS: SpanSpec(
SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
),
# spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST
# transport span of the request carrying that message (resolved per message at
# emit time), keeping the call in one trace. Trace context the client
# propagated in ``params._meta`` becomes a span *link* to that remote context,
# which is not a registry role, so ``SpanSpec`` has no link field.
SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
def root_roles() -> list[SpanRole]:
"""Roles with no in-process parent. They start a new trace unless they adopt a
remote parent (e.g. an MCP span joining the client's propagated context)."""
"""Roles with no in-process parent, i.e. they start a new trace (only the
instrumentor-owned ``PROXY_REQUEST`` server span today)."""
return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
@ -227,8 +223,6 @@ def validate_registry(
raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
if spec.parent is not None and spec.parent not in reg:
raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
if spec.links is not None and spec.links not in reg:
raise ValueError(f"span role {role} declares unknown link target {spec.links}")
missing: Final = [role for role in SpanRole if role not in reg]
if missing:
raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")

63
litellm/integrations/otel/plumbing/context.py
View file

@ -57,8 +57,8 @@ def request_root_span() -> "Span | None":
# The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
# MCP client propagated in the current request's ``params._meta``. The MCP gateway
# sets it per message so the MCP span can parent to the client's span rather than
# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
# sets it per message so the MCP span can record the client's span as a span
# link. A ``ContextVar`` because, like the root-span anchor, it must
# ride the request task and be readable by the inline success-logging callback.
_mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar(
"litellm_otel_mcp_message_trace_carrier", default=None
@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None":
Prefers the transport the gateway published for this specific message; falls
back to the ambient request anchor for paths that emit an MCP span on the
request task itself (the REST MCP endpoints, the SDK). Parenting and linking
only need the immutable context, and unlike ``mcp_message_transport_span`` they
stay correct against a transport that has already finished, so this does not
require the span to still be recording.
request task itself (the REST MCP endpoints). Parenting needs only the
immutable context, and unlike ``mcp_message_transport_span`` it stays correct
against a transport that has already finished, so this does not require the
span to still be recording.
"""
published: Final = _mcp_message_transport_span.get()
if published is not None:
@ -222,25 +222,31 @@ def resolve_mcp_span_context(
) -> "tuple[Context, tuple[Link, ...]]":
"""Parent context + links for an MCP message span.
The span always nests under the transport span of the request carrying this
message, so a tool call and the ``POST`` that carried it stay in one trace.
The transport comes from :func:`_mcp_transport_span_context`, which is the
*current message's* POST rather than whatever request happened to open the
session, so a long-lived session does not glue every message under its first
request.
When the client propagates W3C trace context in the request's ``params._meta``
(SEP-414), MCP and the underlying transport are independent lifecycles — one
streamable-HTTP session multiplexes many messages, and the client's own span is
the truthful parent. So, per the OTel GenAI MCP semconv:
(SEP-414), that remote context is recorded as a span *link*, never the parent.
The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link),
but the gateway's tracing backend only ever receives the gateway's half of such
a trace: parenting into the client's trace id roots the span in a trace whose
root span never reaches the backend, so the span is unreachable from the trace
view and the transport transaction shows a dangling link (observed with
clients that propagate synthetic trace ids). Anchoring to the gateway's own
request and linking the client's context keeps every trace renderable while
preserving the client-side correlation.
* parent to the trace context the client propagated (a *remote* parent), and
* record the transport span as a *link*, never the parent.
Almost no client implements SEP-414 yet, so in practice nothing is propagated.
Rooting the span there splits a single tool call into two disconnected traces
joined only by a link, which is how it surfaces in APM: the ``POST`` transaction
and the ``tools/call`` span share no trace. With no remote parent to honor,
parent to the transport span of the request carrying this message instead, so
the call stays in one trace; no link is added since the transport is now the
real parent. The transport comes from :func:`_mcp_transport_span_context`, which
is the *current message's* POST rather than whatever request happened to open
the session, so a long-lived session does not glue every message under its
first request. With neither a remote parent nor a transport the returned context
carries no span and the span legitimately starts its own root trace.
With no transport at all the span starts its own root trace, still carrying
the link — the client context is only ever a link, so this event keeps one
shape everywhere. Both returned contexts are built on an explicitly empty
base, so ambient (stale session) state can never leak in, and the span
inherits the transport's sampling decision exactly like every other
request-level span — a client's sampled flag neither forces nor suppresses
recording.
Only trace context (``traceparent``/``tracestate``) is extracted, never the
client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
@ -251,13 +257,12 @@ def resolve_mcp_span_context(
never fall through to the ambient (stale session) span.
"""
source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get()
parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context())
propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context()))
links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else ()
transport: Final = _mcp_transport_span_context()
if is_recordable_span(get_current_span(parent)):
return parent, (Link(transport),) if transport is not None else ()
if transport is not None:
return context_from_span(NonRecordingSpan(transport)), ()
return parent, ()
if transport is None:
return Context(), links
return context_from_span(NonRecordingSpan(transport), context=Context()), links
def is_recordable_span(obj: object) -> bool:

10
litellm/integrations/otel/plumbing/metrics.py
View file

@ -32,6 +32,7 @@ from litellm.integrations.otel.model.semconv import (
resolve_provider,
)
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -198,16 +199,21 @@ class GenAIMetricRecorder:
) -> None:
common_attrs: Final = self._filter_attributes(self._bounded_attributes(kwargs))
duration_s: Final = (end_time - start_time).total_seconds()
usage_is_replayed: Final = is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
)
self._metrics.operation_duration.record(duration_s, attributes=common_attrs)
self._record_token_usage(response_obj, common_attrs)
if not usage_is_replayed:
self._record_token_usage(response_obj, common_attrs)
cost: Final = kwargs.get("response_cost")
if cost:
self._metrics.token_cost.record(cost, attributes=common_attrs)
self._record_time_to_first_token(kwargs, common_attrs)
self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
if not usage_is_replayed:
self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
self._record_response_duration(kwargs, end_time, common_attrs)
def record_failure(

75
litellm/integrations/otel/plumbing/routing.py
View file

@ -2,12 +2,13 @@
When a request carries team/key vendor credentials in
``standard_callback_dynamic_params``, or the key/team config resolved at auth
names a destination project, its spans must export through a
``TracerProvider`` whose OTLP headers carry those credentials / that project.
``TenantTracerCache`` builds and caches one provider per distinct
(credentials, project) pair, and otherwise hands back the logger's default
tracer. This lets a single logger fan requests out to many tenants without
needing a logger per tenant.
names a destination project or a service name, its spans must export through a
``TracerProvider`` whose OTLP headers carry those credentials / that project,
or whose Resource carries that ``service.name``. ``TenantTracerCache`` builds
and caches one provider per distinct (credentials, project, service name)
tuple, and otherwise hands back the logger's default tracer. This lets a
single logger fan requests out to many tenants without needing a logger per
tenant.
"""
import threading
@ -15,13 +16,14 @@ from collections import OrderedDict
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Final, TypeAlias
from typing import Final, TypeAlias
from urllib.parse import quote
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Tracer
from litellm._logging import verbose_logger
from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
@ -32,6 +34,7 @@ from litellm.integrations.otel.presets import (
dynamic_otlp_headers,
project_routing_headers,
)
from litellm.types.utils import StandardCallbackDynamicParams
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
_NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
@ -64,8 +67,30 @@ _MAX_RETIRED_PROVIDERS: Final = 64
_HeaderItems: TypeAlias = tuple[tuple[str, str], ...]
_RouteKey: TypeAlias = tuple[_HeaderItems, _HeaderItems, str | None, str | None]
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
#: Key/team config fields naming the Resource ``service.name``, highest
#: precedence first. Read only from ``user_api_key_auth_metadata`` (the config
#: the proxy resolved at auth), never from client-supplied request metadata:
#: the service name picks the dataset/service traces land in (Honeycomb routes
#: datasets by it), so a caller must not be able to choose one.
_SERVICE_NAME_KEYS: Final = OTEL_SERVICE_NAME_METADATA_KEYS
def tenant_service_name(auth_metadata: Mapping[str, str] | None) -> str | None:
"""The per-request ``service.name`` override for this key/team, if any.
``None`` keeps the env-configured default (``OTEL_SERVICE_NAME``).
"""
if not auth_metadata:
return None
return next(
(stripped for key in _SERVICE_NAME_KEYS if (stripped := (auth_metadata.get(key) or "").strip())),
None,
)
def _shutdown_provider(provider: TracerProvider) -> None:
"""Flush + stop an evicted provider's processors (reclaims their threads).
@ -115,7 +140,7 @@ class TenantRoute:
class TenantTracerCache:
"""Credential/project-scoped ``TracerProvider`` cache keyed by the routing headers."""
"""Tenant-scoped ``TracerProvider`` cache keyed by routing headers and service name."""
def __init__(
self,
@ -130,7 +155,7 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
self._providers: OrderedDict[_RouteKey, TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
@ -166,15 +191,16 @@ class TenantTracerCache:
def route_for(
self,
default: Tracer,
dynamic_params: Any,
dynamic_params: StandardCallbackDynamicParams | None,
auth_metadata: Mapping[str, str] | None = None,
) -> TenantRoute:
"""Return the tracer (and trace-detachment flag) for this request.
Use ``default`` unless the request's dynamic credentials or its key/team
project require a scoped tracer, in which case build (or reuse) one. The
cache is a bounded LRU: the least-recently-used provider is flushed and
shut down on overflow so its exporter threads don't accumulate.
Use ``default`` unless the request's dynamic credentials, its key/team
project, or its key/team service name require a scoped tracer, in
which case build (or reuse) one. The cache is a bounded LRU: the
least-recently-used provider is flushed and shut down on overflow so
its exporter threads don't accumulate.
A routed provider is returned already held — its open-span count is
incremented in the same critical section as the cache update — so a
@ -183,7 +209,8 @@ class TenantTracerCache:
"""
credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
service_name: Final = tenant_service_name(auth_metadata)
if not credential_headers and not project_headers and service_name is None:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
@ -192,9 +219,12 @@ class TenantTracerCache:
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
service_name,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
provider: Final = self._cached_provider_locked(
cache_key, credential_headers, project_headers, endpoint, service_name
)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -207,16 +237,19 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
cache_key: _RouteKey,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
service_name: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
built: Final = build_tracer_provider(
self._routed_config(credential_headers, project_headers, endpoint, service_name)
)
self._providers[cache_key] = built
return built
@ -266,6 +299,7 @@ class TenantTracerCache:
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
service_name: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -284,7 +318,10 @@ class TenantTracerCache:
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
update: Final = (
{"exporters": exporters} if service_name is None else {"exporters": exporters, "service_name": service_name}
)
return self._config.model_copy(update=update)
def _routed_exporter(
self,

30
litellm/integrations/prometheus.py
View file

@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import (
_sanitize_prometheus_label_name,
_sanitize_prometheus_label_value,
validate_prometheus_deployment_and_latency_caller_identity,
)
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
@ -96,7 +97,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
"""View a repository's prisma table through the pagination surface budget metrics need."""
return repository.table
return cast(
_PaginatedPrismaTable[_TableRowT],
repository.table, # cast-ok: prisma rows carry the budget columns the domain model declares
)
class _OrgBudgetRow(Protocol):
@ -172,6 +176,11 @@ class PrometheusLogger(CustomLogger):
try:
from prometheus_client import Counter, Gauge, Histogram
# Validate the caller-identity mode before any collector registers so an
# invalid value cannot leave partially-registered metrics behind in the
# process-global registry.
validate_prometheus_deployment_and_latency_caller_identity()
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
@ -2462,6 +2471,7 @@ class PrometheusLogger(CustomLogger):
else:
_metadata = {
"user_api_key_alias": getattr(_metadata_raw, "user_api_key_alias", None),
"user_api_key_user_email": getattr(_metadata_raw, "user_api_key_user_email", None),
"user_api_key_team_id": getattr(_metadata_raw, "user_api_key_team_id", None),
"user_api_key_team_alias": getattr(_metadata_raw, "user_api_key_team_alias", None),
"user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
@ -2484,6 +2494,17 @@ class PrometheusLogger(CustomLogger):
return getattr(user_api_key_auth, "key_alias", None)
return None
def _get_user_email() -> str | None:
from_metadata: Final = _metadata.get("user_api_key_user_email")
if from_metadata is not None:
return from_metadata
from_params: Final = _litellm_params_metadata.get("user_api_key_user_email")
if from_params is not None:
return from_params
if user_api_key_auth is not None:
return self._safe_get(user_api_key_auth, "user_email")
return None
def _get_team_id() -> str | None:
val = _metadata.get("user_api_key_team_id")
if val is not None:
@ -2519,6 +2540,7 @@ class PrometheusLogger(CustomLogger):
return {
"api_key_alias": _get_api_key_alias(),
"user_email": _get_user_email(),
"team": _get_team_id(),
"team_alias": _get_team_alias(),
"hashed_api_key": _get_hashed_api_key(),
@ -2576,6 +2598,7 @@ class PrometheusLogger(CustomLogger):
_metadata: Final = standard_logging_payload.get("metadata", {}) or {}
hashed_api_key: Final = fallback_values.get("hashed_api_key") or _metadata.get("user_api_key_hash")
api_key_alias: Final = fallback_values.get("api_key_alias") or _metadata.get("user_api_key_alias")
user_email: Final = fallback_values.get("user_email")
team: Final = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
team_alias: Final = fallback_values.get("team_alias") or _metadata.get("user_api_key_team_alias")
client_ip: Final = fallback_values.get("client_ip") or _metadata.get("requester_ip_address")
@ -2616,6 +2639,7 @@ class PrometheusLogger(CustomLogger):
requested_model=label_requested_model,
hashed_api_key=hashed_api_key,
api_key_alias=api_key_alias,
user_email=user_email,
team=team,
team_alias=team_alias,
tags=standard_logging_payload.get("request_tags", []),
@ -3552,7 +3576,9 @@ class PrometheusLogger(CustomLogger):
except Exception as e:
verbose_logger.exception("Error initializing user/team count metrics: %s", e)
async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]):
async def _set_key_list_budget_metrics(
self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]
) -> None:
"""Helper function to set budget metrics for a list of keys"""
for key in keys:
if isinstance(key, UserAPIKeyAuth):

31
litellm/integrations/prompt_management_base.py
View file

@ -19,6 +19,19 @@ class PromptManagementClient(TypedDict):
completed_messages: list[AllMessageValues] | None
def resolve_prompt_manager_ignore_flags(
prompt_spec: PromptSpec | None,
ignore_prompt_manager_model: bool | None,
ignore_prompt_manager_optional_params: bool | None,
) -> tuple[bool, bool]:
spec_params: Final = prompt_spec.litellm_params if prompt_spec is not None else None
return (
bool(ignore_prompt_manager_model) or bool(spec_params is not None and spec_params.ignore_prompt_manager_model),
bool(ignore_prompt_manager_optional_params)
or bool(spec_params is not None and spec_params.ignore_prompt_manager_optional_params),
)
class PromptManagementBase(ABC):
@property
@abstractmethod
@ -182,13 +195,18 @@ class PromptManagementBase(ABC):
prompt_version=prompt_version,
)
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,
messages=messages,
non_default_params=non_default_params,
model=model,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_model=resolved_ignore_model,
ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
)
async def async_get_chat_completion_prompt(
@ -224,11 +242,16 @@ class PromptManagementBase(ABC):
prompt_version=prompt_version,
)
resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
prompt_spec=prompt_spec,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
)
return self.post_compile_prompt_processing(
prompt_template=prompt_template,
messages=messages,
non_default_params=non_default_params,
model=model,
ignore_prompt_manager_model=ignore_prompt_manager_model,
ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
ignore_prompt_manager_model=resolved_ignore_model,
ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
)

43
litellm/litellm_core_utils/core_helpers.py
View file

@ -58,15 +58,56 @@ def safe_divide(
return numerator / denominator
def _is_litellm_limit_rejection(exception: BaseException) -> bool:
from litellm.exceptions import RateLimitErrorCategory
litellm_limit_categories: Final = frozenset(
(RateLimitErrorCategory.LITELLM_RATE_LIMIT.value, RateLimitErrorCategory.LITELLM_BATCH_RATE_LIMIT.value)
)
return getattr(exception, "category", None) in litellm_limit_categories
def _is_proxy_rejection(exception: BaseException) -> bool:
if _is_litellm_limit_rejection(exception):
return True
try:
from starlette.exceptions import HTTPException
except ImportError:
return False
return isinstance(exception, HTTPException)
def _is_provider_originated(exception: BaseException) -> bool:
if _is_proxy_rejection(exception):
return False
if getattr(exception, "llm_provider", None):
return True
from litellm.llms.base_llm.chat.transformation import BaseLLMException
return isinstance(exception, BaseLLMException)
def is_expected_client_error(exception: BaseException | None) -> bool:
"""
True when the exception maps to an HTTP 4xx status.
True when the proxy itself rejected the request with an HTTP 4xx before any
provider call (bad key, budget, unknown model, guardrail). A 4xx returned by
a provider is an upstream or deployment problem, so it is never an expected
client error and keeps its traceback: a mapped litellm exception carries
``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
raise before mapping (the /v1/messages route surfaces it as-is) is one too.
The proxy's own limiters raise ``HTTPException`` subclasses that also carry
an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection, and
so does any exception whose unified rate-limit ``category`` names litellm's
own limiter (``BudgetExceededError`` is a plain ``Exception`` that the auth
handler decorates with the requested model's provider).
ProxyException stores the status on .code (as a str), HTTPException and
litellm exceptions on .status_code.
"""
if exception is None:
return False
if _is_provider_originated(exception):
return False
code: Final[object] = getattr(exception, "code", None)
status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
if status_code is None or isinstance(status_code, bool):

140
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -550,6 +550,13 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"AnthropicException - {error_str}",
llm_provider="anthropic",
model=model,
response=original_exception.response,
)
elif original_exception.status_code == 400 or original_exception.status_code == 413:
raise BadRequestError(
message=f"AnthropicException - {error_str}",
@ -755,12 +762,19 @@ def _map_openai_like_exception(
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 401 or original_exception.status_code == 403:
elif original_exception.status_code == 401:
raise AuthenticationError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=403),
)
elif original_exception.status_code == 400:
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
@ -2187,6 +2201,120 @@ def _map_openrouter_exception(
)
def _response_or_stub(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
response: Final = original_exception.response if hasattr(original_exception, "response") else None
if response is not None:
return response
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs")
)
def _map_exception_by_status(
*,
model: str,
original_exception: _ProviderHTTPException,
custom_llm_provider: str,
error_str: str,
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:
case 401:
raise AuthenticationError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 403:
raise PermissionDeniedError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=status_code),
litellm_debug_info=extra_information,
)
case 404:
raise NotFoundError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 408:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
case 429:
raise RateLimitError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 500:
raise InternalServerError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 502:
raise BadGatewayError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 503:
raise ServiceUnavailableError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 504:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=status_code,
)
case _ if status_code < 500:
raise BadRequestError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case _:
raise APIError(
status_code=status_code,
message=message,
llm_provider=custom_llm_provider,
model=model,
request=original_exception.request if hasattr(original_exception, "request") else None,
litellm_debug_info=extra_information,
)
def exception_type(
model,
original_exception,
@ -2213,6 +2341,7 @@ def exception_type(
litellm_response_headers: Final = _get_response_headers(original_exception=original_exception)
try:
error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
extra_information = ""
if model or custom_llm_provider:
if hasattr(original_exception, "message"):
error_str = (
@ -2229,7 +2358,6 @@ def exception_type(
# Common Extra information needed for all providers
# We pass num retries, api_base, vertex_deployment etc to the exception here
################################################################################
extra_information = ""
try:
_api_base: Final = litellm.get_api_base(model=model, optional_params=extra_kwargs)
messages: Final = litellm.get_first_chars_messages(kwargs=completion_kwargs)
@ -2501,6 +2629,14 @@ def exception_type(
For unmapped exceptions - raise the exception with traceback - https://github.com/BerriAI/litellm/issues/4201
"""
exception_mapping_worked = True
_map_exception_by_status(
model=model,
original_exception=mappable_exception,
custom_llm_provider=custom_llm_provider,
error_str=error_str,
exception_provider=exception_provider,
extra_information=extra_information,
)
if hasattr(original_exception, "request"):
raise APIConnectionError(
message=f"{exception_provider} - {error_str}",

36
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -2,17 +2,32 @@
Helper functions for health check calls.
"""
from collections.abc import Callable
import base64
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, Literal
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import ImageResponse
# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
# Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG
TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC"
IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = (
"Add a small yellow star in the top right corner of this simple drawing of a blue circle on a white background"
)
def get_image_file_for_health_check() -> bytes:
"""Return the image used for health checks."""
return base64.b64decode(TEST_IMAGE_BASE64)
class HealthCheckHelpers:
@staticmethod
@ -112,6 +127,17 @@ class HealthCheckHelpers:
else:
return await litellm.acompletion(**model_params)
@staticmethod
async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse":
import litellm
try:
return await edit_request()
except litellm.BadRequestError as e:
if isinstance(e, litellm.ContentPolicyViolationError) or "moderation_blocked" in str(e):
return litellm.ImageResponse()
raise
@staticmethod
def get_mode_handlers(
model: str,
@ -127,6 +153,7 @@ class HealthCheckHelpers:
"audio_speech",
"audio_transcription",
"image_generation",
"image_edit",
"video_generation",
"rerank",
"realtime",
@ -185,6 +212,13 @@ class HealthCheckHelpers:
**_filter_model_params(model_params=model_params),
prompt=prompt,
),
"image_edit": lambda: HealthCheckHelpers._image_edit_health_check(
edit_request=lambda: litellm.aimage_edit(
**_filter_model_params(model_params=model_params),
image=get_image_file_for_health_check(),
prompt=IMAGE_EDIT_HEALTH_CHECK_PROMPT,
),
),
"video_generation": lambda: litellm.avideo_generation(
**_filter_model_params(model_params=model_params),
prompt=prompt or "test video generation",

18
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,3 +1,4 @@
import re
from collections.abc import Iterator, Mapping
from typing import Any, Final
@ -45,12 +46,29 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
_raise_env_reference_error(param, source=source)
# Langfuse rejects events whose environment does not match this pattern
# (lowercase alphanumerics, hyphens, underscores; no "langfuse" prefix).
# Validating here fails fast at config/init time instead of silently
# dropping every trace server-side.
LANGFUSE_ENVIRONMENT_PATTERN: Final = r"^(?!langfuse)[a-z0-9-_]+$"
def validate_langfuse_environment_value(value: str) -> None:
if not re.match(LANGFUSE_ENVIRONMENT_PATTERN, value):
raise ValueError(
f"Invalid langfuse_environment {value!r}: must be lowercase "
"alphanumerics/hyphens/underscores and must not start with "
f"'langfuse' (pattern {LANGFUSE_ENVIRONMENT_PATTERN})"
)
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
"langfuse_host",
"langfuse_environment",
"langfuse_prompt_version",
"langsmith_api_key",
"langsmith_project",

58
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -20,8 +20,8 @@ from __future__ import annotations
from collections.abc import Mapping
from typing import Final
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.types.utils import InternalCallOrigin
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, NON_INFERENCE_CALL_TYPES
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, InternalCallOrigin
BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"})
@ -45,6 +45,60 @@ budget-checked like the request that spawned it. Everything else on the parent's
be a lie on a sub-call that runs after it returned."""
def is_background_response(response: object) -> bool:
"""Whether a retrieved object is a response created with ``background=true``.
Such a create returns ``status="queued"`` and no usage at all, so nothing has billed the
job by the time anyone reads it back. Accepts the response as a mapping or a model,
because the callers hold it in both shapes.
"""
if isinstance(response, Mapping):
return response.get("background") is True
return getattr(response, "background", None) is True
def is_unbilled_non_inference_call(
call_type: str | None,
metadata: Mapping[str, object] | None,
response: object,
) -> bool:
"""A read/management route priced at zero, because the usage it reports belongs to the
call that created the object it just read.
Retrieving a background response is the exception, and the enterprise cost poller's read
is the same exception seen from the other side: that job's create billed nothing, so its
retrieval is the only place the spend is ever visible. Pricing those at zero would lose
the spend rather than deduplicate it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
if is_background_response(response):
return False
if metadata is None:
return True
return metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
def is_unbilled_non_inference_call_from_params(
call_type: str | None,
litellm_params: Mapping[str, object] | None,
response: object,
) -> bool:
""":func:`is_unbilled_non_inference_call` for callers holding raw ``litellm_params``.
The call-type membership test runs first so that inference traffic, which is every
request in a normal workload, never pays for the metadata merge behind it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
metadata: Final = (
StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None
)
return is_unbilled_non_inference_call(call_type, metadata, response)
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

86
litellm/litellm_core_utils/litellm_logging.py
View file

@ -64,6 +64,7 @@ from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
cost_breakdown_with_guardrail,
guardrail_information_cost,
@ -612,37 +613,60 @@ class Logging(LiteLLMLoggingBaseClass):
processed_list: Final[list[str | Callable | CustomLogger]] = []
for callback in callback_list:
if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks:
# For callbacks that support team-scoped credentials (e.g. datadog),
# pass only the relevant dynamic params as custom_logger_init_args.
_custom_logger_init_args: dict | None = None
if callback == "datadog":
# dd_* params are blocked from standard_callback_dynamic_params
# (request-level security); only the proxy-stamped team/key
# callback vars are admin-configured and trusted.
_custom_logger_init_args = {k: v for k, v in self._trusted_callback_vars if k.startswith("dd_")}
callback_class = _init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
custom_logger_init_args=_custom_logger_init_args,
)
if callback_class is not None:
processed_list.append(callback_class)
for callback_instance in self._resolve_dynamic_callback_string(callback):
processed_list.append(callback_instance)
# If processing dynamic_success_callbacks, add to dynamic_async_success_callbacks
if dynamic_callbacks_type == "success":
if self.dynamic_async_success_callbacks is None:
self.dynamic_async_success_callbacks = []
self.dynamic_async_success_callbacks.append(callback_class)
self.dynamic_async_success_callbacks.append(callback_instance)
elif dynamic_callbacks_type == "failure":
if self.dynamic_async_failure_callbacks is None:
self.dynamic_async_failure_callbacks = []
self.dynamic_async_failure_callbacks.append(callback_class)
self.dynamic_async_failure_callbacks.append(callback_instance)
else:
processed_list.append(callback)
return processed_list
def _resolve_dynamic_callback_string(self, callback: str) -> "tuple[CustomLogger, ...]":
"""
Resolve a known callback name to the logger instance(s) it dispatches to.
For callbacks that support team-scoped credentials (datadog, newrelic),
only the proxy-stamped team/key callback vars are passed as
custom_logger_init_args: dd_*/newrelic_* params are blocked from
standard_callback_dynamic_params (request-level security), so the
trusted-vars channel is the only way credentials reach a per-team logger.
"""
_trusted_var_prefix: Final = "dd_" if callback == "datadog" else "newrelic_" if callback == "newrelic" else None
_custom_logger_init_args: Final[dict | None] = (
{k: v for k, v in self._trusted_callback_vars if k.startswith(_trusted_var_prefix)}
if _trusted_var_prefix is not None
else None
)
callback_class: Final = _init_custom_logger_compatible_class(
callback,
internal_usage_cache=None,
llm_router=None,
custom_logger_init_args=_custom_logger_init_args,
)
if callback_class is None:
return ()
# With team creds, "newrelic" resolves to the per-team METRICS logger;
# resolve the name again without creds so the trace logger (OTel v2 /
# legacy agent) keeps receiving this request.
_newrelic_trace_class: Final = (
_init_custom_logger_compatible_class(callback, internal_usage_cache=None, llm_router=None)
if callback == "newrelic" and _custom_logger_init_args and _custom_logger_init_args.get("newrelic_api_key")
else None
)
if _newrelic_trace_class is not None and _newrelic_trace_class is not callback_class:
return (callback_class, _newrelic_trace_class)
return (callback_class,)
def initialize_standard_callback_dynamic_params(self, kwargs: dict | None = None) -> StandardCallbackDynamicParams:
"""
Initialize the standard callback dynamic params from the kwargs
@ -1586,11 +1610,16 @@ class Logging(LiteLLMLoggingBaseClass):
if cache_hit is True:
return 0.0
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):
return 0.0
transformed_result: Final = self._generate_content_result_as_model_response(result)
if transformed_result is not None:
result = transformed_result
if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"):
if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"):
hidden_params: Final = getattr(result, "_hidden_params", {})
if (
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
@ -4636,6 +4665,19 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
if custom_logger_init_args.get("newrelic_api_key"):
# Team-scoped credentials: per-team METRICS logger, isolated per
# credential set via DynamicLoggingCache. The trace logger for
# this name stays on the global path below.
from litellm.integrations.newrelic.newrelic_team_handler import (
NewRelicHandler,
)
return NewRelicHandler.get_newrelic_logger_for_request(
standard_callback_dynamic_params=custom_logger_init_args,
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
)
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
@ -5057,7 +5099,7 @@ class StandardLoggingPayloadSetup:
return messages
@staticmethod
def merge_litellm_metadata(litellm_params: dict) -> dict:
def merge_litellm_metadata(litellm_params: Mapping[str, object]) -> dict:
"""
Merge both litellm_metadata and metadata from litellm_params.
@ -5819,7 +5861,7 @@ def get_standard_logging_object_payload(
cache_hit: Final = kwargs.get("cache_hit", False)
# Extract usage as a plain dict, avoiding Pydantic round-trip
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=response_obj,
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
)
usage_dict: Final = (

54
litellm/litellm_core_utils/llm_cost_calc/guardrail_cost.py
View file

@ -21,11 +21,13 @@ class GuardrailCostEntry(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
guardrail_cost: float | None = None
# ``bool | None`` because the TypedDict sanctions None; None means "not set"
# and keeps the default billed behavior, so a None-carrying entry must not
# fail union validation and silently zero a sibling entry's real cost.
guardrail_cost_in_spend: bool | None = True
GuardrailInformationShape = tuple[GuardrailCostEntry, ...] | GuardrailCostEntry | None
_GUARDRAIL_INFORMATION_ADAPTER: Final[TypeAdapter[GuardrailInformationShape]] = TypeAdapter(GuardrailInformationShape)
_GUARDRAIL_COST_ENTRY_ADAPTER: Final[TypeAdapter[GuardrailCostEntry]] = TypeAdapter(GuardrailCostEntry)
def _bedrock_guardrail_pricing(aws_region_name: str | None) -> GuardrailPricing | None:
@ -47,23 +49,55 @@ def bedrock_guardrail_cost(usage_units: Mapping[str, int], aws_region_name: str
return sum(units * pricing.guardrail_cost_per_unit.get(counter, 0.0) for counter, units in usage_units.items())
AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT: Final = "text_records"
def azure_prompt_shield_guardrail_cost(
usage_units: Mapping[str, int],
cost_tier: str | None,
price_per_1000_text_records: float | None,
) -> float | None:
"""USD cost of an Azure Prompt Shield invocation from its text-record count.
Returns 0.0 on the free tier, ``text_records * price / 1000`` when a price is
configured, and None when pricing is not configured (usage-only tracking).
"""
if cost_tier == "free":
return 0.0
if price_per_1000_text_records is None:
return None
return usage_units.get(AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT, 0) * price_per_1000_text_records / 1000.0
def _billable_entry_cost(entry: GuardrailCostEntry) -> float:
if entry.guardrail_cost_in_spend is False:
return 0.0
cost: Final = entry.guardrail_cost
if cost is None or not math.isfinite(cost) or cost <= 0.0:
return 0.0
return cost
def guardrail_information_cost(guardrail_information: object) -> float:
def _validated_entry_cost(raw: object) -> float:
"""Billable cost of one raw ``guardrail_information`` entry.
Validated per entry so one malformed entry (e.g. a custom hook stamping a
non-boolean ``guardrail_cost_in_spend``) prices to 0.0 by itself instead of
failing a whole-payload validation and silently zeroing a sibling entry's
real billable cost."""
try:
parsed: Final = _GUARDRAIL_INFORMATION_ADAPTER.validate_python(guardrail_information)
except ValidationError:
return _billable_entry_cost(_GUARDRAIL_COST_ENTRY_ADAPTER.validate_python(raw))
except ValidationError as e:
verbose_logger.warning("Ignoring malformed guardrail_information entry for guardrail cost: %s", e)
return 0.0
if parsed is None:
def guardrail_information_cost(guardrail_information: object) -> float:
if guardrail_information is None:
return 0.0
if isinstance(parsed, GuardrailCostEntry):
return _billable_entry_cost(parsed)
return sum(_billable_entry_cost(entry) for entry in parsed)
if isinstance(guardrail_information, (list, tuple)):
return sum(_validated_entry_cost(entry) for entry in guardrail_information)
return _validated_entry_cost(guardrail_information)
def cost_breakdown_with_guardrail(cost_breakdown: CostBreakdown | None, guardrail_cost: float) -> CostBreakdown | None:

95
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -7,7 +7,9 @@ from typing import Any, Final, Literal
import litellm
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
)
from litellm.types.llms.openai import (
FileSearchTool,
ResponsesAPIResponse,
@ -64,11 +66,17 @@ class StandardBuiltInToolCostTracking:
"""
standard_built_in_tools_params = standard_built_in_tools_params or {}
google_maps_grounding_cost: Final = StandardBuiltInToolCostTracking._handle_google_maps_grounding_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
)
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
):
return StandardBuiltInToolCostTracking._handle_web_search_cost(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
@ -78,19 +86,56 @@ class StandardBuiltInToolCostTracking:
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object):
return StandardBuiltInToolCostTracking._handle_file_search_cost(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
@staticmethod
def _resolve_model_info(model: str, custom_llm_provider: str | None) -> tuple[ModelInfo | None, str | None]:
direct: Final = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if direct is not None:
return direct, custom_llm_provider or direct["litellm_provider"]
if "/" not in model:
return None, custom_llm_provider
by_prefix: Final = StandardBuiltInToolCostTracking._safe_get_model_info(model=model)
if by_prefix is None:
return None, custom_llm_provider
return by_prefix, by_prefix["litellm_provider"]
@staticmethod
def _handle_google_maps_grounding_cost(
model: str,
custom_llm_provider: str | None,
usage: Usage | None,
) -> float:
from litellm.llms import get_cost_for_google_maps_grounding_request
from litellm.llms.gemini.cost_calculator import google_maps_grounding_requests
if usage is None or google_maps_grounding_requests(usage) is None:
return 0.0
model_info, resolved_provider = StandardBuiltInToolCostTracking._resolve_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if model_info is None or resolved_provider is None:
return 0.0
return (
get_cost_for_google_maps_grounding_request(
custom_llm_provider=resolved_provider, usage=usage, model_info=model_info
)
or 0.0
)
@staticmethod
def _handle_web_search_cost(
model: str,
@ -102,29 +147,21 @@ class StandardBuiltInToolCostTracking:
"""Handle web search cost calculation."""
from litellm.llms import get_cost_for_web_search_request
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
# A provider-prefixed model (e.g. gemini/gemini-3.1-flash-lite) may not map under the
# request's custom_llm_provider. _resolve_model_info re-resolves from the prefix and adopts
# that provider so the cost is routed and priced with the model_info that was actually
# resolved, instead of feeding a re-resolved model into the original provider's calculator.
model_info, resolved_provider = StandardBuiltInToolCostTracking._resolve_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
# A provider-prefixed model (e.g. gemini/gemini-3.1-flash-lite) may not map under the
# request's custom_llm_provider. Re-resolve from the prefix and adopt that provider so the
# cost is routed and priced with the model_info that was actually resolved, instead of
# feeding a re-resolved model into the original provider's calculator.
if model_info is None and "/" in model:
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(model=model)
if model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
if custom_llm_provider is None and model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
resolved_usage: Final = StandardBuiltInToolCostTracking._usage_with_anthropic_web_search(
usage=usage, response_object=response_object
)
if model_info is not None and resolved_usage is not None and custom_llm_provider is not None:
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
)
@ -333,7 +370,7 @@ class StandardBuiltInToolCostTracking:
get_anthropic_web_search_requests_from_response,
)
if usage is not None and (_get_web_search_requests(getattr(usage, "server_tool_use", None)) is not None):
if usage is not None and (get_web_search_requests_from_usage(usage) is not None):
return usage
web_search_requests: Final = get_anthropic_web_search_requests_from_response(response_object)
if web_search_requests is None:
@ -381,7 +418,7 @@ class StandardBuiltInToolCostTracking:
# Anthropic Claude (direct API and Vertex AI) uses server_tool_use.web_search_requests.
# Without this check, Claude ModelResponse always falls through to return False
# and _handle_web_search_cost() is never called.
if hasattr(usage, "server_tool_use") and _get_web_search_requests(usage.server_tool_use) is not None:
if get_web_search_requests_from_usage(usage) is not None:
return True
# xAI reports usage.server_side_tool_usage_details.web_search_calls; a searched
# answer with no url_citation annotations has no other chat-path signal
@ -394,16 +431,12 @@ class StandardBuiltInToolCostTracking:
response_object=response_object, output_type="web_search_call"
)
elif usage is not None:
if (
hasattr(usage, "server_tool_use")
and _get_web_search_requests(usage.server_tool_use) is not None
or (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
)
if get_web_search_requests_from_usage(usage) is not None or (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
):
return True
if _usage_reports_server_side_web_search_calls(usage):

37
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,6 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from typing import Any, Final
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
@ -39,7 +39,7 @@ class TranscriptionUsageObjectTransformation:
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
_INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
@ -59,7 +59,7 @@ def _token_count(value: object) -> int:
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
@ -69,10 +69,13 @@ def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, i
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
entries: Final = usage_object.get("grounding_tool_count")
if not isinstance(entries, Sequence):
return 0
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
for entry in entries
if isinstance(entry, Mapping) and entry.get("type") == "google_search"
)
@ -112,30 +115,30 @@ class InteractionsUsageObjectTransformation:
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
input_sums: Final = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
reasoning_tokens: Final = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
prompt_tokens: Final = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
completion_tokens: Final = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens: Final = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
web_search_requests: Final = _google_search_query_count(usage_object)
prompt_tokens_details: Final = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
@ -144,7 +147,7 @@ class InteractionsUsageObjectTransformation:
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
completion_tokens_details: Final = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,

43
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -72,7 +72,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
def _get_web_search_requests(server_tool_use: Any) -> int | None:
def get_web_search_requests(server_tool_use: Any) -> int | None:
"""
Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance,
@ -92,6 +92,16 @@ def _get_web_search_requests(server_tool_use: Any) -> int | None:
return getattr(server_tool_use, "web_search_requests", None)
def get_web_search_requests_from_usage(usage: Usage) -> int | None:
"""Read ``web_search_requests`` from a ``Usage``'s ``server_tool_use``.
``Usage`` deletes unset optional fields from ``__dict__`` (see
``SafeAttributeModel``), so direct attribute access can raise
``AttributeError``; ``getattr`` with a default is required here.
"""
return get_web_search_requests(getattr(usage, "server_tool_use", None))
def _is_above_128k(tokens: float) -> bool:
if tokens > 128000:
return True
@ -889,11 +899,22 @@ def generic_cost_per_token(
total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting:
text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens
if has_double_counting:
# cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a
# modality can only bill what the cache did not already cover or the overlap is billed twice
uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
billable_audio: Final = min(audio_tokens, uncached_budget)
billable_image: Final = min(image_tokens, uncached_budget - billable_audio)
billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
prompt_tokens_details["audio_tokens"] = billable_audio
prompt_tokens_details["image_tokens"] = billable_image
prompt_tokens_details["video_tokens"] = billable_video
prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video
elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0:
# Clamp to zero: inconsistent streaming usage
text_tokens = max(text_tokens, 0)
prompt_tokens_details["text_tokens"] = text_tokens
prompt_tokens_details["text_tokens"] = max(
usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
)
(
prompt_base_cost,
@ -1063,15 +1084,17 @@ def get_token_type_cost_breakdown(
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
# Reasoning is billed at the selected tier's reasoning rate for tiered models,
# else at the explicit per-reasoning-token rate when the model defines one,
# otherwise at the standard output-token rate - this mirrors how the total
# completion cost is computed, so the breakdown can never diverge from it.
# else at the service-tier-aware per-reasoning-token rate - this mirrors how the
# total completion cost is computed, so the breakdown can never diverge from it.
tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
flat_reasoning_rate: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
reasoning_rate: Final = (
tiered_reasoning_rate
if tiered_reasoning_rate is not None
else (flat_reasoning_rate if flat_reasoning_rate is not None else completion_base_cost)
else _resolve_reasoning_token_cost(
model_info=model_info,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
)
)
reasoning_cost = float(reasoning_tokens) * reasoning_rate

2
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -178,7 +178,7 @@ def update_response_metadata(
- response._hidden_params["litellm_overhead_time_ms"]
- response.response_time_ms
"""
if result is None:
if result is None or not hasattr(result, "_hidden_params"):
return
metadata: Final = ResponseMetadata(result)

69
litellm/litellm_core_utils/logging_worker.py
View file

@ -4,6 +4,7 @@
import asyncio
import atexit
import contextvars
import inspect
import logging
from collections.abc import Coroutine, Iterator
from typing import Final
@ -53,6 +54,7 @@ class LoggingWorker:
self._queue: asyncio.Queue[LoggingTask] | None = None
self._worker_task: asyncio.Task | None = None
self._running_tasks: set[asyncio.Task] = set()
self._dequeued_tasks: dict[int, LoggingTask] = {} # mutable-ok: refs so flush can rescue never-started tasks
self._sem: asyncio.Semaphore | None = None
self._bound_loop: asyncio.AbstractEventLoop | None = None
self._last_aggressive_clear_time: float = 0.0
@ -61,6 +63,38 @@ class LoggingWorker:
# Register cleanup handler to flush remaining events on exit
atexit.register(self._flush_on_exit)
def _track_dequeued(self, task: LoggingTask) -> None:
self._dequeued_tasks[id(task)] = task
def _untrack_dequeued(self, task: LoggingTask) -> None:
self._dequeued_tasks.pop(id(task), None)
def _unstarted_dequeued_tasks(self) -> tuple[LoggingTask, ...]:
return tuple(
task
for task in self._dequeued_tasks.values()
if inspect.getcoroutinestate(task["coroutine"]) == inspect.CORO_CREATED
)
def _requeue_unstarted_dequeued(self, new_queue: "asyncio.Queue[LoggingTask]") -> int:
revived: Final = self._unstarted_dequeued_tasks()
self._dequeued_tasks.clear()
for index, revived_task in enumerate(revived):
try:
new_queue.put_nowait(revived_task)
except asyncio.QueueFull:
for leftover in revived[index:]:
self._track_dequeued(leftover)
return index
return len(revived)
def _run_coroutine_silently(self, loop: asyncio.AbstractEventLoop, coroutine: Coroutine) -> bool:
try:
loop.run_until_complete(asyncio.wait_for(coroutine, timeout=self.timeout))
except (Exception, asyncio.CancelledError): # noqa: BLE001 # atexit flush must never break the user's program
return False
return True
@staticmethod
def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
"""Pop every task still queued, without awaiting them, so they can be moved to another queue."""
@ -90,10 +124,12 @@ class LoggingWorker:
new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
for carried_task in carried_over:
new_queue.put_nowait(carried_task)
if carried_over:
revived_count: Final = self._requeue_unstarted_dequeued(new_queue)
if carried_over or revived_count:
verbose_logger.warning(
"LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
"LoggingWorker: event loop changed; carried %d pending and revived %d dequeued logging task(s) onto the new loop",
len(carried_over),
revived_count,
)
else:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
@ -129,6 +165,7 @@ class LoggingWorker:
except Exception as e:
verbose_logger.exception("LoggingWorker error: %s", e)
finally:
self._untrack_dequeued(task)
self._queue.task_done()
finally:
# Always release semaphore, even if queue is None
@ -146,6 +183,7 @@ class LoggingWorker:
await self._sem.acquire()
try:
task = await self._queue.get()
self._track_dequeued(task)
# Track each spawned coroutine so we can cancel on shutdown.
processing_task = asyncio.create_task(self._process_log_task(task, self._sem))
self._running_tasks.add(processing_task)
@ -298,9 +336,10 @@ class LoggingWorker:
extracted_tasks: Final = []
for _ in range(items_to_extract):
try:
extracted_tasks.append(self._queue.get_nowait())
extracted_tasks.append(extracted := self._queue.get_nowait())
except asyncio.QueueEmpty:
break
self._track_dequeued(extracted)
return extracted_tasks
@ -318,6 +357,7 @@ class LoggingWorker:
# Add new task to extracted tasks to process directly
if new_task is not None:
self._track_dequeued(new_task)
extracted_tasks.append(new_task)
# Process extracted tasks directly
@ -343,6 +383,7 @@ class LoggingWorker:
# Suppress errors during processing to ensure we keep going
pass
finally:
self._untrack_dequeued(task)
self._queue.task_done()
async def _process_extracted_tasks(self, tasks: list[LoggingTask]) -> None:
@ -486,11 +527,12 @@ class LoggingWorker:
self._safe_log("debug", "[LoggingWorker] atexit: No queue initialized")
return
if self._queue.empty():
unstarted_dequeued: Final = self._unstarted_dequeued_tasks()
if self._queue.empty() and not unstarted_dequeued:
self._safe_log("debug", "[LoggingWorker] atexit: Queue is empty")
return
queue_size: Final = self._queue.qsize()
queue_size: Final = self._queue.qsize() + len(unstarted_dequeued)
self._safe_log("info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events...")
# Create a new event loop since the original is closed
@ -509,6 +551,16 @@ class LoggingWorker:
previous_raise_exceptions: Final = logging.raiseExceptions
logging.raiseExceptions = False
try:
for pending in unstarted_dequeued:
if (
processed >= MAX_ITERATIONS_TO_CLEAR_QUEUE
or loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE
):
break
if self._run_coroutine_silently(loop, pending["coroutine"]):
processed += 1
self._untrack_dequeued(pending)
while not self._queue.empty() and processed < MAX_ITERATIONS_TO_CLEAR_QUEUE:
if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
self._safe_log(
@ -526,11 +578,8 @@ class LoggingWorker:
# Note: We run the coroutine directly, not via create_task,
# since we're in a new event loop context
try:
loop.run_until_complete(task["coroutine"])
processed += 1
except Exception:
# Silent failure to not break user's program
pass
if self._run_coroutine_silently(loop, task["coroutine"]):
processed += 1
finally:
# Clear reference to prevent memory leaks
task = None

6
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -511,9 +511,6 @@ def update_messages_with_model_file_ids(
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
@ -588,9 +585,6 @@ def update_responses_input_with_model_file_ids(
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)

1
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -28,6 +28,7 @@ PTU_ZEROED_PRICING_FIELDS: Final = tuple(f for f in MirroredPricingParams.model_
"cache_creation_input_token_cost_above_1hr",
"cache_creation_input_token_cost_above_200k_tokens",
"cache_read_input_token_cost_above_200k_tokens",
"google_maps_grounding_cost_per_query",
)
# tiered_pricing is emptied rather than zeroed: its tiers outrank the zeros written beside
# them, so a zero here would leave the cost map's tiers billing the traffic the reserved

20
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -330,6 +330,24 @@ class RealTimeStreaming:
except (AttributeError, TypeError):
pass
def _flush_unbilled_transcription_usage(self) -> None:
if self.provider_config is None:
return
usage: Final = self.provider_config.unbilled_usage_on_session_close(self.model)
if usage is None:
return
flush_event: Final = (
cast( # cast-ok: usage-only partial event, the same shape _capture_transcription_usage logs
OpenAIRealtimeEvents,
{
"type": "conversation.item.input_audio_transcription.completed",
"usage": usage,
},
)
)
self.store_message(flush_event)
self._capture_transcription_usage(flush_event)
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
"""Extract function_call items from response.done events for spend logging."""
try:
@ -955,6 +973,7 @@ class RealTimeStreaming:
transcript = event.get("transcript", "")
self._collect_user_input_from_backend_event(cast(dict, event))
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
@ -1068,6 +1087,7 @@ class RealTimeStreaming:
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
finally:
self._flush_unbilled_transcription_usage()
await self.log_messages()
@staticmethod

45
litellm/litellm_core_utils/redact_messages.py
View file

@ -10,6 +10,7 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import litellm
@ -97,16 +98,18 @@ def _redact_function_call(function_call) -> None:
def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, litellm.Choices):
choice.message.content = REDACTED_BY_LITELLM
if hasattr(choice.message, "reasoning_content"):
if choice.message.content is not None:
choice.message.content = REDACTED_BY_LITELLM
if getattr(choice.message, "reasoning_content", None) is not None:
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
elif isinstance(choice, litellm.utils.StreamingChoices):
choice.delta.content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "reasoning_content"):
if choice.delta.content is not None:
choice.delta.content = REDACTED_BY_LITELLM
if getattr(choice.delta, "reasoning_content", None) is not None:
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
@ -117,19 +120,19 @@ def _redact_choice_content(choice):
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
if hasattr(output_item, "text"):
if getattr(output_item, "text", None) is not None:
output_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "content") and isinstance(output_item.content, list):
for content_part in output_item.content:
if hasattr(content_part, "text"):
if getattr(content_part, "text", None) is not None:
content_part.text = REDACTED_BY_LITELLM
# Redact reasoning items in output array
if hasattr(output_item, "type") and output_item.type == "reasoning":
if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
for summary_item in output_item.summary:
if hasattr(summary_item, "text"):
if getattr(summary_item, "text", None) is not None:
summary_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
@ -142,17 +145,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
if not isinstance(output_item, dict):
continue
if "text" in output_item:
if output_item.get("text") is not None:
output_item["text"] = redacted_str
if isinstance(output_item.get("content"), list):
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
if isinstance(content_item, dict) and content_item.get("text") is not None:
content_item["text"] = redacted_str
if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
for summary_item in output_item["summary"]:
if isinstance(summary_item, dict) and "text" in summary_item:
if isinstance(summary_item, dict) and summary_item.get("text") is not None:
summary_item["text"] = redacted_str
if output_item.get("type") == "function_call" and "arguments" in output_item:
@ -189,40 +192,42 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
"""Redact tool call / function_call arguments in a dict-form message or delta."""
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list):
for tool_call in tool_calls:
if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
tool_call["function"]["arguments"] = redacted_str
tool_call["function"]["arguments"] = REDACTED_BY_LITELLM
function_call: Final = message.get("function_call")
if isinstance(function_call, dict) and "arguments" in function_call:
function_call["arguments"] = redacted_str
function_call["arguments"] = REDACTED_BY_LITELLM
def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "reasoning_content" in choice["message"]:
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str
if choice["message"].get("reasoning_content") is not None:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
_redact_tool_calls_dict(choice["message"], redacted_str)
_redact_tool_calls_dict(choice["message"])
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "reasoning_content" in choice["delta"]:
if choice["delta"].get("content") is not None:
choice["delta"]["content"] = redacted_str
if choice["delta"].get("reasoning_content") is not None:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"], redacted_str)
_redact_tool_calls_dict(choice["delta"])
else:
_redact_choice_content(choice)
@ -263,7 +268,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
or (isinstance(result, dict) and ("choices" in result or "output" in result))
):
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
_result: Final = copy.deepcopy(result)
if isinstance(_result, litellm.ModelResponse):

10
litellm/litellm_core_utils/specialty_caches/dynamic_logging_cache.py
View file

@ -13,6 +13,7 @@ import json
from typing import Any, Final
import litellm
from litellm._logging import verbose_logger
from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
from ...caching import InMemoryCache
@ -46,6 +47,15 @@ class LangfuseInMemoryCache(InMemoryCache):
_created_langfuse_logger.Langfuse.flush()
_created_langfuse_logger.Langfuse.shutdown()
# Loggers with a periodic flush task (e.g. NewRelicMetricsLogger) expose
# stop() so eviction actually ends the task instead of leaking it.
_evicted_stop: Final = getattr(self.cache_dict[key], "stop", None)
if callable(_evicted_stop):
try:
_evicted_stop()
except Exception: # noqa: BLE001 # a failing stop() must not block eviction
verbose_logger.debug("DynamicLoggingCache: stop() raised during eviction", exc_info=True)
#########################################################
# Call parent class to remove key from cache
#########################################################

43
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -173,6 +173,27 @@ def attach_cache_creation_token_details(
return prompt_tokens_details.model_copy(update={"cache_creation_token_details": cache_creation_token_details})
def apply_grounding_request_counts(
prompt_tokens_details: PromptTokensDetailsWrapper | None,
web_search_requests: int | None,
google_maps_grounding_requests: int | None,
) -> PromptTokensDetailsWrapper | None:
updates: Final = MappingProxyType(
{
field: value
for field, value in (
("web_search_requests", web_search_requests),
("google_maps_grounding_requests", google_maps_grounding_requests),
)
if value is not None
}
)
if not updates:
return prompt_tokens_details
counted: Final = prompt_tokens_details if prompt_tokens_details is not None else PromptTokensDetailsWrapper()
return counted.model_copy(update=updates)
class ChunkProcessor:
def __init__(self, chunks: list, messages: list | None = None):
self.chunks = self._sort_chunks(chunks)
@ -778,6 +799,7 @@ class ChunkProcessor:
server_tool_use: ServerToolUse | None = None
web_search_requests: int | None = None
google_maps_grounding_requests: int | None = None
completion_tokens_details: CompletionTokensDetails | None = None
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
# Anthropic emits the cache-creation TTL breakdown (5m/1h split) only on
@ -827,6 +849,13 @@ class ChunkProcessor:
)
if chunk_web_search_requests is not None:
web_search_requests = chunk_web_search_requests
chunk_google_maps_grounding_requests: int | None = getattr(
usage_chunk_dict["prompt_tokens_details"],
"google_maps_grounding_requests",
None,
)
if chunk_google_maps_grounding_requests is not None:
google_maps_grounding_requests = chunk_google_maps_grounding_requests
prompt_tokens_details = usage_chunk_dict["prompt_tokens_details"] or prompt_tokens_details
@ -852,6 +881,7 @@ class ChunkProcessor:
cache_read_input_tokens=cache_read_input_tokens,
server_tool_use=server_tool_use,
web_search_requests=web_search_requests,
google_maps_grounding_requests=google_maps_grounding_requests,
completion_tokens_details=completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
cost=cost,
@ -939,6 +969,7 @@ class ChunkProcessor:
server_tool_use: Final[ServerToolUse | None] = calculated_usage_per_chunk["server_tool_use"]
web_search_requests: Final[int | None] = calculated_usage_per_chunk["web_search_requests"]
google_maps_grounding_requests: Final[int | None] = calculated_usage_per_chunk["google_maps_grounding_requests"]
completion_tokens_details: Final[CompletionTokensDetails | None] = calculated_usage_per_chunk[
"completion_tokens_details"
]
@ -998,13 +1029,11 @@ class ChunkProcessor:
if server_tool_use is not None:
returned_usage.server_tool_use = server_tool_use
if web_search_requests is not None:
if returned_usage.prompt_tokens_details is None:
returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper(
web_search_requests=web_search_requests
)
else:
returned_usage.prompt_tokens_details.web_search_requests = web_search_requests
returned_usage.prompt_tokens_details = apply_grounding_request_counts(
returned_usage.prompt_tokens_details,
web_search_requests,
google_maps_grounding_requests,
)
if cost is not None:
setattr(returned_usage, "cost", cost)

15
litellm/llms/__init__.py
View file

@ -14,6 +14,21 @@ if TYPE_CHECKING:
from litellm.types.utils import ModelInfo, Usage
def get_cost_for_google_maps_grounding_request(
custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo"
) -> float | None:
"""
Get the cost of Grounding with Google Maps for a given model. Only Gemini models on the
Gemini API and Vertex AI can populate the Maps grounding counter, so every other provider
returns None.
"""
if custom_llm_provider != "gemini" and not custom_llm_provider.startswith("vertex_ai"):
return None
from .gemini.cost_calculator import cost_per_google_maps_grounding_request
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
"""
Get the cost for a web search request for a given model.

5
litellm/llms/anthropic/chat/handler.py
View file

@ -712,11 +712,14 @@ class ModelResponseIterator:
def _handle_usage(self, anthropic_usage_chunk: dict | UsageDelta) -> Usage:
reasoning_content: Final = "".join(self.reasoning_content_chunks) if self.reasoning_content_chunks else None
return AnthropicConfig().calculate_usage(
usage: Final = AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=reasoning_content,
speed=self.speed,
)
if usage.speed is not None:
self.speed = usage.speed
return usage
def _content_block_delta_helper(
self, chunk: dict

4
litellm/llms/anthropic/chat/transformation.py
View file

@ -2279,6 +2279,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
str | None,
_usage.get("service_tier"),
)
raw_speed: Final = _usage.get("speed")
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
iterations: Final[list[Any] | None] = _usage.get("iterations")
if iterations:
@ -2353,7 +2355,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else None
),
inference_geo=inference_geo,
speed=speed,
speed=resolved_speed,
service_tier=service_tier,
)
return usage

23
litellm/llms/anthropic/common_utils.py
View file

@ -4,7 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import re
from collections.abc import Mapping, Sequence
from collections.abc import Mapping, MutableMapping, Sequence
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Any, Final, Literal
@ -93,8 +93,8 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
"""
Handle Anthropic OAuth token detection and header setup.
If an OAuth token is detected in the Authorization header, extracts it
and sets the required OAuth headers.
If an OAuth token is detected in the Authorization header (any casing),
extracts it and sets the required OAuth headers.
Args:
headers: Request headers dict
@ -104,16 +104,21 @@ def optionally_handle_anthropic_oauth(headers: dict, api_key: str | None) -> tup
Tuple of (updated headers, api_key)
"""
# Check Authorization header (passthrough / forwarded requests)
auth_header: Final = headers.get("authorization", "")
if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.replace("Bearer ", "")
headers.pop("x-api-key", None)
auth_header: Final = next((value for name, value in headers.items() if name.lower() == "authorization"), "")
if auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
api_key = auth_header.removeprefix("Bearer ")
for name in tuple(
header_name for header_name in headers if header_name.lower() in ("x-api-key", "authorization")
):
headers.pop(name)
headers["authorization"] = auth_header
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
return headers, api_key
# Check api_key directly (standard chat/completion flow)
if api_key and api_key.startswith(ANTHROPIC_OAUTH_TOKEN_PREFIX):
headers.pop("x-api-key", None)
for name in tuple(header_name for header_name in headers if header_name.lower() == "x-api-key"):
headers.pop(name)
headers["authorization"] = f"Bearer {api_key}"
headers["anthropic-beta"] = _merge_beta_headers(headers.get("anthropic-beta"), ANTHROPIC_OAUTH_BETA_HEADER)
headers["anthropic-dangerous-direct-browser-access"] = "true"
@ -468,7 +473,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
@staticmethod
def maybe_drop_disabled_thinking(
model: str,
optional_params: dict, # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param, as in _maybe_drop_speed_param
custom_llm_provider: str,
) -> None:
"""Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models

47
litellm/llms/anthropic/cost_calculation.py
View file

@ -8,12 +8,9 @@ from typing import TYPE_CHECKING, Final, Optional
from pydantic import BaseModel, ValidationError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_token_base_cost,
_get_web_search_requests,
calculate_cache_writing_cost,
generic_cost_per_token,
get_provider_specific_geo_multiplier,
parse_prompt_tokens_details,
get_web_search_requests_from_usage,
)
if TYPE_CHECKING:
@ -21,43 +18,6 @@ if TYPE_CHECKING:
import litellm
def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None) -> float:
"""
Return only the cache-related portion of the prompt cost (cache read + cache write).
These costs must NOT be scaled by the ``fast`` speed multiplier because the old
explicit ``fast/`` model entries carried unchanged cache rates while
multiplying only the regular input/output token costs. Regional pricing, by
contrast, uplifts every token type, so the geo multiplier does scale them.
"""
if usage.prompt_tokens_details is None:
return 0.0
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
(
_,
_,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(model_info=model_info, usage=usage, service_tier=service_tier)
cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
if (
prompt_tokens_details["cache_creation_tokens"]
or prompt_tokens_details["cache_creation_token_details"] is not None
):
cache_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
)
return cache_cost
def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -89,8 +49,7 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
)
if speed_multiplier != 1.0:
cache_cost: Final = _compute_cache_only_cost(model_info=model_info, usage=usage, service_tier=service_tier)
prompt_cost = (prompt_cost - cache_cost) * speed_multiplier + cache_cost
prompt_cost *= speed_multiplier
completion_cost *= speed_multiplier
if geo_multiplier != 1.0:
@ -145,7 +104,7 @@ def get_cost_for_anthropic_web_search(
if usage is None:
return 0.0
web_search_requests: Final = _get_web_search_requests(getattr(usage, "server_tool_use", None))
web_search_requests: Final = get_web_search_requests_from_usage(usage)
if web_search_requests is None:
return 0.0

18
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -99,6 +99,7 @@ from litellm.types.llms.anthropic import (
ContextManagementResponse,
MessageBlockDelta,
MessageDelta,
ServerToolUsage,
StreamingContentBlockDeltaType,
UsageDelta,
UsageIteration,
@ -1354,10 +1355,22 @@ class LiteLLMAnthropicMessagesAdapter:
return explicit_value
return cls._first_positive_prompt_tokens_detail_value(usage, ("cache_creation_tokens", "cache_write_tokens"))
@classmethod
def _get_web_search_request_count(cls, usage: Usage) -> int:
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
)
from_server_tool_use: Final = cls._positive_int(get_web_search_requests_from_usage(usage))
if from_server_tool_use > 0:
return from_server_tool_use
return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",))
@classmethod
def _translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta:
cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage)
cache_creation_input_tokens: Final = cls._get_cache_creation_input_tokens(usage)
web_search_requests: Final = cls._get_web_search_request_count(usage)
input_tokens: Final = max(
(usage.prompt_tokens or 0) - cache_read_input_tokens - cache_creation_input_tokens,
0,
@ -1371,6 +1384,11 @@ class LiteLLMAnthropicMessagesAdapter:
usage_delta["cache_creation_input_tokens"] = cache_creation_input_tokens
if cache_read_input_tokens > 0:
usage_delta["cache_read_input_tokens"] = cache_read_input_tokens
if web_search_requests > 0:
return UsageDelta(
**usage_delta,
server_tool_use=ServerToolUsage(web_search_requests=web_search_requests),
)
return usage_delta
@classmethod

4
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -352,8 +352,8 @@ async def _check_summary_model_budget(
)
return False
user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
user_id: Final = getattr(user_api_key_auth, "user_id", None)
user_model_max_budget: Final = user_api_key_auth.user_model_max_budget
user_id: Final = user_api_key_auth.user_id
if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
try:
await model_max_budget_limiter.is_user_within_model_budget(

143
litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
View file

@ -6,13 +6,41 @@ yields every chunk to the caller (preserving real streaming), collects
all bytes, and on stream exhaustion rebuilds the full Anthropic response
to run through agentic completion hooks. If an agentic hook fires, the
follow-up response is chained as Phase 2 of the same iterator.
In hold-back mode (``hold_back=True``) chunks are buffered instead of yielded
live, keepalive pings run whenever no other byte is ready, and then either the
follow-up replaces the message or the buffer replays, except that a tool_use for
a server-fulfilled tool fails the turn rather than reaching a client that cannot
execute it.
"""
import asyncio
import contextlib
import json
from collections.abc import AsyncIterator
from typing import Any, Final, cast
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
b"event: error\n"
b'data: {"type": "error", "error": {"type": "api_error", "message": '
b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
)
def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
try:
return await iterator.__anext__()
except StopAsyncIteration:
return None
# ---------------------------------------------------------------------------
# SSE parsing helpers (module-level to keep the class lean)
@ -156,6 +184,9 @@ class AgenticAnthropicStreamingIterator:
logging_obj: Any,
custom_llm_provider: str,
kwargs: dict,
hold_back: bool = False,
server_fulfilled_tool_names: frozenset[str] = frozenset(),
ping_interval_seconds: float = HOLD_BACK_PING_INTERVAL_SECONDS,
):
self._inner = completion_stream.__aiter__()
self._http_handler = http_handler
@ -166,16 +197,32 @@ class AgenticAnthropicStreamingIterator:
self._logging_obj = logging_obj
self._custom_llm_provider = custom_llm_provider
self._kwargs = kwargs
self._hold_back = hold_back
self._server_fulfilled_tool_names = server_fulfilled_tool_names
self._ping_interval_seconds = ping_interval_seconds
self._collected_bytes: list[bytes] = []
self._stream_exhausted = False
self._hook_processing_done = False
self._follow_up_iterator: AsyncIterator | None = None
self._drain_task: asyncio.Task | None = None
self._hook_task: asyncio.Task | None = None
self._follow_up_chunk_task: asyncio.Task | None = None
self._replay_index = 0
self._error_emitted = False
@property
def has_buffered_provider_output(self) -> bool:
"""Whether provider output was received but withheld from the client behind keepalive pings."""
return self._hold_back and bool(self._collected_bytes)
def __aiter__(self):
return self
async def __anext__(self) -> bytes:
if self._hold_back:
return await self._anext_held_back()
# Phase 1: yield from upstream, collect bytes
if not self._stream_exhausted:
try:
@ -194,11 +241,102 @@ class AgenticAnthropicStreamingIterator:
raise StopAsyncIteration
async def _drain_upstream(self) -> None:
try:
while True:
self._collected_bytes.append(await self._inner.__anext__())
except StopAsyncIteration:
return
async def _completed_within_ping_interval(self, task: asyncio.Task) -> bool:
try:
await asyncio.wait_for(asyncio.shield(task), timeout=self._ping_interval_seconds)
except asyncio.TimeoutError:
return False
return True
async def _anext_held_back(self) -> bytes:
if self._drain_task is None:
self._drain_task = asyncio.create_task(self._drain_upstream())
return STREAM_SSE_KEEPALIVE_PING_BYTES
if not self._stream_exhausted:
if not await self._completed_within_ping_interval(self._drain_task):
return STREAM_SSE_KEEPALIVE_PING_BYTES
self._stream_exhausted = True
if self._hook_task is None:
self._hook_task = asyncio.create_task(self._process_agentic_hooks())
if not await self._completed_within_ping_interval(self._hook_task):
return STREAM_SSE_KEEPALIVE_PING_BYTES
if self._follow_up_iterator is not None:
return await self._next_follow_up_chunk(self._follow_up_iterator)
if self._buffer_holds_server_fulfilled_tool_use():
if self._error_emitted:
raise StopAsyncIteration
self._error_emitted = True
verbose_logger.error(
"AgenticStreamingIterator: hooks did not replace a message containing a server-fulfilled "
"tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
self._model,
)
return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
if self._replay_index < len(self._collected_bytes):
chunk: Final = self._collected_bytes[self._replay_index]
self._replay_index += 1
return chunk
raise StopAsyncIteration
async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
if self._follow_up_chunk_task is None:
self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
return STREAM_SSE_KEEPALIVE_PING_BYTES
chunk: Final = self._follow_up_chunk_task.result()
self._follow_up_chunk_task = None
if chunk is None:
raise StopAsyncIteration
return chunk
def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
if not self._server_fulfilled_tool_names:
return False
started_blocks: Final = (
data.get("content_block")
for event_type, data in _parse_sse_events(b"".join(self._collected_bytes))
if event_type == "content_block_start"
)
return any(
isinstance(block, dict)
and block.get("type") == "tool_use"
and block.get("name") in self._server_fulfilled_tool_names
for block in started_blocks
)
@staticmethod
async def _settle_task(task: asyncio.Task | None) -> None:
if task is None:
return
if task.done():
if not task.cancelled():
task.exception()
return
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
async def aclose(self) -> None:
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
aclose_if_supported,
)
await self._settle_task(self._drain_task)
await self._settle_task(self._hook_task)
await self._settle_task(self._follow_up_chunk_task)
await aclose_if_supported(self._inner)
await aclose_if_supported(self._follow_up_iterator)
@ -217,11 +355,6 @@ class AgenticAnthropicStreamingIterator:
verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
return
[
(f"{b.get('type')}({b.get('name', '')})" if b.get("type") == "tool_use" else b.get("type"))
for b in rebuilt.get("content", [])
]
result: Final = await self._http_handler._call_agentic_completion_hooks(
response=rebuilt,
model=self._model,

21
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,7 @@ from functools import partial
from typing import Any, Final, cast
import litellm
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
flatten_unencrypted_web_search_results_in_anthropic_messages,
@ -21,6 +22,7 @@ from litellm.llms.anthropic.common_utils import (
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@ -382,13 +384,18 @@ async def anthropic_messages(
)
ctx: Final = contextvars.copy_context()
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
try:
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
return await init_response
return init_response
except BaseLLMException as e:
raise exception_type(
model=model,
custom_llm_provider=custom_llm_provider,
original_exception=e,
extra_kwargs=kwargs,
)
def validate_anthropic_api_metadata(metadata: dict | None = None) -> dict | None:

4
litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py
View file

@ -46,6 +46,10 @@ class AnthropicMessagesStreamCacheWriter:
stream._hidden_params if isinstance(stream, AnthropicMessagesStreamingResponse) else _EMPTY_MAPPING
)
@property
def has_buffered_provider_output(self) -> bool:
return getattr(self.stream, "has_buffered_provider_output", False) is True
def __aiter__(self) -> "AnthropicMessagesStreamCacheWriter":
return self

4
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -312,6 +312,10 @@ class AnthropicMessagesStreamingResponse:
self.completion_stream = completion_stream
self._hidden_params = hidden_params
@property
def has_buffered_provider_output(self) -> bool:
return getattr(self.completion_stream, "has_buffered_provider_output", False) is True
def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
return self

17
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -8,6 +8,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -307,10 +308,20 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
# Check for Anthropic OAuth token in Authorization header
headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
if "x-api-key" not in headers and "authorization" not in headers:
header_names: Final = frozenset(name.lower() for name in headers)
if "x-api-key" not in header_names and "authorization" not in header_names:
auth_header: Final = AnthropicModelInfo.get_auth_header(api_key)
if auth_header is not None:
headers.update(auth_header)
if auth_header is None:
raise AuthenticationError(
message=(
"Missing Anthropic API Key - A call is being made to anthropic but no key is set "
"either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` "
"or `ANTHROPIC_AUTH_TOKEN` in your environment vars"
),
llm_provider=self._resolved_provider,
model=model,
)
headers.update(auth_header)
if "anthropic-version" not in headers:
headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
if "content-type" not in headers:

2
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -582,7 +582,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
"type": "json_schema",
"name": "structured_output",
"schema": schema,
"strict": True,
"strict": output_format.get("strict", False),
}
}

23
litellm/llms/azure/realtime/handler.py
View file

@ -4,6 +4,8 @@ This file contains the calling Azure OpenAI's `/openai/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, cast
from litellm._logging import _redact_string, verbose_proxy_logger
@ -30,6 +32,21 @@ async def forward_messages(client_ws: Any, backend_ws: Any):
class AzureOpenAIRealtime(AzureChatCompletion):
@staticmethod
def get_auth_headers(api_key: str | None, azure_ad_token: str | None) -> Mapping[str, str]:
"""
Build the websocket handshake auth headers, preferring a static api-key and falling back to
an Azure AD (Entra ID) bearer token. Never sends both.
"""
if api_key:
return MappingProxyType({"api-key": api_key})
if azure_ad_token:
return MappingProxyType({"Authorization": f"Bearer {azure_ad_token}"})
raise ValueError(
"Missing Azure credentials for the realtime endpoint. Set an api_key, or configure Azure AD auth "
"(azure_ad_token, tenant_id/client_id/client_secret, or a managed identity)"
)
def _construct_url(
self,
api_base: str,
@ -117,13 +134,13 @@ class AzureOpenAIRealtime(AzureChatCompletion):
query_params=query_params,
)
auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
try:
ssl_context: Final = get_shared_realtime_ssl_context()
async with websockets.connect(
url,
additional_headers={
"api-key": api_key,
},
additional_headers=auth_headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
) as backend_ws:

4
litellm/llms/base_llm/realtime/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
@ -70,6 +71,9 @@ class BaseRealtimeConfig(ABC):
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
return None
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None
def transform_session_created_event(
self,
model: str,

7
litellm/llms/bedrock/base_aws_llm.py
View file

@ -1434,9 +1434,12 @@ class BaseAWSLLM:
data: str | bytes,
headers: dict,
api_key: str | None = None,
supports_bearer_token: bool = True,
) -> AWSPreparedRequest:
if api_key is not None:
aws_bearer_token: str | None = api_key
if not supports_bearer_token:
aws_bearer_token: str | None = None
elif api_key is not None:
aws_bearer_token = api_key
else:
aws_bearer_token = get_secret_str("AWS_BEARER_TOKEN_BEDROCK")

30
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -65,6 +65,7 @@ from litellm.types.llms.openai import (
OpenAIMessageContentListBlock,
)
from litellm.types.utils import (
CacheCreationTokenDetails,
ChatCompletionMessageToolCall,
CompletionTokensDetailsWrapper,
Function,
@ -418,12 +419,16 @@ class AmazonConverseConfig(BaseConfig):
Handle the reasoning_effort parameter based on the model type.
- GPT-OSS models: passed through unchanged via additionalModelRequestFields.
- OpenAI GPT-5.x models: mapped to ``reasoning.effort`` via additionalModelRequestFields.
- Nova 2 models: transformed to reasoningConfig.
- Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on
adaptive Claude 4.6 / 4.7).
"""
if "gpt-oss" in model:
optional_params["reasoning_effort"] = reasoning_effort
elif "openai.gpt-5" in model:
reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort}
optional_params["reasoning"] = reasoning
elif self._is_nova_2_model(model):
reasoning_config: Final = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
optional_params.update(reasoning_config)
@ -555,7 +560,7 @@ class AmazonConverseConfig(BaseConfig):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice")
if "gpt-oss" in model:
if "gpt-oss" in model or "openai.gpt-5" in model or "openai.gpt-5" in base_model:
supported_params.append("reasoning_effort")
elif self._is_nova_2_model(model):
# Nova 2 models support reasoning_effort (transformed to reasoningConfig)
@ -903,7 +908,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params["_parallel_tool_use_config"] = {
"tool_choice": {"type": "auto", "disable_parallel_tool_use": not value}
}
if param == "thinking":
if param == "thinking" and "openai.gpt-5" not in model:
if (
isinstance(value, dict)
and value.get("type") == "adaptive"
@ -1803,6 +1808,26 @@ class AmazonConverseConfig(BaseConfig):
thinking_blocks_list.append(_redacted_block)
return thinking_blocks_list
@staticmethod
def _parse_cache_details(usage: ConverseTokenUsageBlock) -> "CacheCreationTokenDetails | None":
"""Split ``cacheDetails`` into 5m/1h buckets, or ``None`` unless the split fully
accounts for ``cacheWriteInputTokens``, since a partial or unrecognized-ttl
breakdown would understate the cache-write cost.
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_CacheDetail.html
"""
cache_details: Final = usage.get("cacheDetails")
if not cache_details:
return None
tokens_5m: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "5m")
tokens_1h: Final = sum(d["inputTokens"] for d in cache_details if d.get("ttl") == "1h")
if tokens_5m + tokens_1h != usage.get("cacheWriteInputTokens", 0):
return None
return CacheCreationTokenDetails(
ephemeral_5m_input_tokens=tokens_5m,
ephemeral_1h_input_tokens=tokens_1h,
)
@staticmethod
def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None:
"""Converse omits thinking tokens from its usage block; they only arrive under
@ -1874,6 +1899,7 @@ class AmazonConverseConfig(BaseConfig):
prompt_tokens_details: Final = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_creation_input_tokens,
cache_creation_token_details=self._parse_cache_details(usage),
text_tokens=raw_input_tokens,
)
estimated_reasoning_tokens: Final = (

30
litellm/llms/bedrock/rerank/handler.py
View file

@ -138,11 +138,6 @@ class BedrockRerankHandler(BaseAWSLLM):
data: dict,
optional_params: dict,
) -> BedrockPreparedRequest:
try:
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
boto3_credentials_info: Final = self._get_boto_credentials_from_optional_params(optional_params, model)
### SET RUNTIME ENDPOINT ###
@ -153,24 +148,21 @@ class BedrockRerankHandler(BaseAWSLLM):
)
proxy_endpoint_url = proxy_endpoint_url.replace("bedrock-runtime", "bedrock-agent-runtime")
proxy_endpoint_url = f"{proxy_endpoint_url}/rerank"
sigv4: Final = SigV4Auth(
boto3_credentials_info.credentials,
"bedrock",
boto3_credentials_info.aws_region_name,
)
# Make POST Request
body: Final = json.dumps(data).encode("utf-8")
body: Final = json.dumps(data).encode("utf-8")
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
request: Final = AWSRequest(method="POST", url=proxy_endpoint_url, data=body, headers=headers)
sigv4.add_auth(request)
if (
extra_headers is not None and "Authorization" in extra_headers
): # prevent sigv4 from overwriting the auth header
request.headers["Authorization"] = extra_headers["Authorization"]
prepped: Final = request.prepare()
prepped: Final = self.get_request_headers(
credentials=boto3_credentials_info.credentials,
aws_region_name=boto3_credentials_info.aws_region_name,
extra_headers=extra_headers,
endpoint_url=proxy_endpoint_url,
data=body,
headers=headers,
supports_bearer_token=False,
)
return BedrockPreparedRequest(
endpoint_url=proxy_endpoint_url,

8
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -243,7 +243,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
@staticmethod
def _agent_message_text(item: "Mapping[str, Any]") -> str:
def _agent_message_text(item: "Mapping[str, object]") -> str:
content: Final = item.get("content")
if not isinstance(content, list):
return ""
@ -254,7 +254,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
)
@classmethod
def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
def _normalize_agent_message_item(cls, item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
text: Final = cls._agent_message_text(item)
if not text:
return None
@ -266,7 +266,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
def _normalize_context_compaction_item(item: "Mapping[str, object]") -> "_RewrittenCompactionItem | None":
encrypted_content: Final = item.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
@ -274,7 +274,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
return rewritten
@staticmethod
def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_RewrittenFunctionCallItem | None":
call_id: Final = item.get("call_id")
if not isinstance(call_id, str) or not call_id:
return None

33
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -1,3 +1,4 @@
import ssl
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Final, cast
@ -18,6 +19,7 @@ from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_ssl_configuration,
)
from litellm.types.llms.openai import FileTypes
from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
@ -56,7 +58,11 @@ class BaseLLMAIOHTTPHandler:
# Create a transport using AsyncHTTPHandler's logic
try:
self.transport = AsyncHTTPHandler._create_aiohttp_transport()
ssl_config: Final = get_ssl_configuration()
self.transport = AsyncHTTPHandler._create_aiohttp_transport(
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
)
self._owns_transport = True
return self.transport
except Exception:
@ -79,20 +85,19 @@ class BaseLLMAIOHTTPHandler:
def _create_client_session_with_transport(self) -> ClientSession:
"""Create a new client session using transport or connector configuration."""
connector: Final = self._get_connector()
if self.transport is None:
connector: Final = self._get_connector()
if connector:
return aiohttp.ClientSession(connector=connector)
if self.transport and hasattr(self.transport, "_get_valid_client_session"):
# Use transport's session creation if available
session = self.transport._get_valid_client_session()
return session
elif connector:
# Use provided connector
session = aiohttp.ClientSession(connector=connector)
return session
else:
# Default session creation
session = aiohttp.ClientSession()
return session
transport: Final = self.transport or self._get_or_create_transport()
if transport is not None and hasattr(transport, "_get_valid_client_session"):
try:
return transport._get_valid_client_session()
except RuntimeError:
pass
return aiohttp.ClientSession()
def _get_async_client_session(self, dynamic_client_session: ClientSession | None = None) -> ClientSession:
if dynamic_client_session:

26
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2268,6 +2268,10 @@ class BaseLLMHTTPHandler:
AgenticAnthropicStreamingIterator,
)
held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
logging_obj=logging_obj,
tools=anthropic_messages_optional_request_params.get("tools"),
)
initial_response = AgenticAnthropicStreamingIterator(
completion_stream=completion_stream,
http_handler=self,
@ -2278,6 +2282,8 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
hold_back=bool(held_back_tool_names),
server_fulfilled_tool_names=held_back_tool_names,
)
return AnthropicMessagesStreamingResponse(
completion_stream=initial_response,
@ -5125,6 +5131,20 @@ class BaseLLMHTTPHandler:
return True
return False
@staticmethod
def _server_fulfilled_tools_in_request(logging_obj: LiteLLMLoggingObj, tools: object) -> frozenset[str]:
"""The request's tools that a registered callback fulfills server-side (e.g. ``headroom_retrieve``)."""
if not isinstance(tools, list) or not tools:
return frozenset()
from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name
return frozenset(
name
for cb in _custom_logger_callbacks(logging_obj)
for name in getattr(cb, "server_fulfilled_tool_names", frozenset())
if has_tool_with_name(tools, name)
)
@staticmethod
def _check_agentic_loop_safety(
tool_calls: object,
@ -5600,10 +5620,9 @@ class BaseLLMHTTPHandler:
kwargs=hook_kwargs,
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_agentic_loop [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)
@ -5625,10 +5644,9 @@ class BaseLLMHTTPHandler:
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
_call_id,
logging_obj.litellm_call_id,
model,
str(e),
)

100
litellm/llms/deepseek/chat/transformation.py
View file

@ -2,16 +2,17 @@
Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
"""
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping, Sequence
from typing import Any, Final, Literal, cast, overload
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
convert_content_list_to_str,
extract_search_results_text,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_reasoning
from litellm.utils import supports_reasoning, supports_vision
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -117,13 +118,98 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]:
"""
DeepSeek does not support content in list format.
DeepSeek vision models accept image_url content blocks in user
messages (https://api-docs.deepseek.com/guides/vision), so those
content lists are forwarded as-is, with any search_results text
appended as a trailing text block. Every other message keeps the
historical string collapse (which also folds search_results text
into string content); a list with no extractable text stays
unchanged, matching what DeepSeek historically received.
"""
messages = handle_messages_with_content_list_to_str_conversion(messages)
forward_images: Final = any(
isinstance(message.get("content"), list) for message in messages
) and supports_vision(model=model, custom_llm_provider="deepseek")
transformed: Final = [ # mutable-ok: provider messages must stay JSON-array lists the base transform mutates
self._forward_or_collapse_content(message=message, forward_images=forward_images) for message in messages
]
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed, model=model, is_async=False)
def _forward_or_collapse_content(self, message: AllMessageValues, forward_images: bool) -> AllMessageValues:
"""
Returns the vision-forwardable message with any search_results text
appended as a text block; every other message keeps the historical
string collapse, which extracts the text from a content list and
folds search_results text into string content.
"""
content: Final = message.get("content")
if (
forward_images
and isinstance(content, list)
and self._is_vision_forwardable_content(message=message, content=content)
):
return self._with_search_results_text_block(message=message, content=content)
collapsed: Final = convert_content_list_to_str(message=message)
if not collapsed or collapsed == content:
return message
collapsed_message: Final = {**message, "content": collapsed} # mutable-ok: wire messages are plain JSON dicts
return cast(AllMessageValues, collapsed_message) # cast-ok: TypedDict spread narrows to dict
def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool:
"""
True only for a user message whose content list holds well-formed
text and image_url blocks with at least one image; a block missing
its payload falls back to the string collapse instead of crashing
or reaching the wire malformed. The model capability gate lives in
the caller.
"""
if message.get("role") != "user":
return False
if not all(self._is_forwardable_block(block) for block in content):
return False
return any(isinstance(block, dict) and block.get("type") == "image_url" for block in content)
@staticmethod
def _is_forwardable_block(block: object) -> bool:
"""A dict block typed text or image_url that carries its payload."""
if not isinstance(block, dict):
return False
block_type: Final = block.get("type")
if block_type == "image_url":
return DeepSeekChatConfig._is_image_url_payload(block.get("image_url"))
if block_type == "text":
return isinstance(block.get("text"), str)
return False
@staticmethod
def _is_image_url_payload(payload: object) -> bool:
"""A url string or an object carrying one, per the OpenAI image_url shape."""
if isinstance(payload, str):
return bool(payload)
if not isinstance(payload, Mapping):
return False
url: Final = payload.get("url")
return isinstance(url, str) and bool(url)
def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues:
"""
Appends the message's search_results text as a trailing text block,
keeping the context that the string collapse used to fold in, and
drops the non-OpenAI search_results key from the wire message.
"""
message_fields: Final = cast(Mapping[str, object], message) # cast-ok: search_results is not on the TypedDicts
search_text: Final = extract_search_results_text(message_fields.get("search_results"))
if not search_text:
return message
forwarded_content: Final = [*content, {"type": "text", "text": search_text}] # mutable-ok: JSON-array content
forwarded: Final = { # mutable-ok: wire messages are plain JSON dicts
**{key: value for key, value in message_fields.items() if key != "search_results"},
"content": forwarded_content,
}
return cast(AllMessageValues, forwarded) # cast-ok: TypedDict spread narrows to dict
def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
"""

10
litellm/llms/fireworks_ai/common_utils.py
View file

@ -13,6 +13,13 @@ class FireworksAIException(BaseLLMException):
def get_fireworks_session_id(litellm_params: dict) -> str | None:
"""
Session id to send as `x-session-affinity`, or None when the caller gave none.
Deliberately does not fall back to `litellm_trace_id`: that is generated per
request (`str(uuid.uuid4())` when absent), so using it pins every request to a
different Fireworks node and prompt caching never hits.
"""
params: Final = litellm_params
for key in ("litellm_session_id", "session_id"):
value = params.get(key)
@ -23,9 +30,6 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None:
value = metadata.get("session_id")
if value:
return str(value)
value = params.get("litellm_trace_id")
if value:
return str(value)
return None

0
litellm/llms/gemini/audio_transcription/__init__.py Normal file
View file

250
litellm/llms/gemini/audio_transcription/transformation.py Normal file
View file

@ -0,0 +1,250 @@
import base64
from collections.abc import Mapping, Sequence
from typing import Final
from httpx import Headers, Response
from litellm.litellm_core_utils.audio_utils.utils import (
normalize_transcription_language_to_bcp47,
process_audio_file,
)
from litellm.llms.base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
from litellm.types.llms.gemini_audio_transcription import (
GeminiTranscriptionAudioInput,
GeminiTranscriptionConfig,
GeminiTranscriptionInteractionRequest,
GeminiTranscriptionInteractionResponse,
GeminiTranscriptionWordAnnotation,
)
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import (
FileTypes,
TranscriptionResponse,
TranscriptionUsageInputTokenDetailsObject,
TranscriptionUsageTokensObject,
)
INTERACTIONS_API_REVISION: Final = "2026-05-20"
WORD_INFO_ANNOTATION_TYPE: Final = "word_info"
class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"""
Maps OpenAI /v1/audio/transcriptions onto the Gemini Interactions API
(POST /v1beta/interactions) for transcription models like
gemini-3.5-transcribe. https://ai.google.dev/gemini-api/docs/transcribe
"""
def get_supported_openai_params(
self, model: str
) -> list[OpenAIAudioTranscriptionOptionalParams]: # mutable-ok: BaseAudioTranscriptionConfig signature
return ["language", "response_format", "timestamp_granularities"] # mutable-ok: base contract returns a list
def map_openai_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
drop_params: bool,
) -> dict: # mutable-ok: BaseAudioTranscriptionConfig signature
supported_params: Final = frozenset(self.get_supported_openai_params(model))
accepted: Final = tuple((k, v) for k, v in non_default_params.items() if k in supported_params)
return dict((*optional_params.items(), *accepted)) # mutable-ok: base contract returns a plain dict
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict | Headers, # mutable-ok: base signature and BaseLLMException take dict | Headers
) -> BaseLLMException:
return GeminiError(status_code=status_code, message=error_message, headers=headers)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict: # mutable-ok: BaseAudioTranscriptionConfig signature
resolved_api_key: Final = GeminiModelInfo.get_api_key(api_key)
if not resolved_api_key:
raise GeminiError(
status_code=401,
message="Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable.",
)
return { # mutable-ok: the http handler passes these headers straight to httpx
**headers,
"Content-Type": "application/json",
"x-goog-api-key": resolved_api_key,
"Api-Revision": INTERACTIONS_API_REVISION,
}
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
stream: bool | None = None,
) -> str:
resolved_api_base: Final = GeminiModelInfo.get_api_base(api_base)
return f"{resolved_api_base}/v1beta/interactions"
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> AudioTranscriptionRequestData:
processed_audio: Final = process_audio_file(audio_file)
audio_input: Final = GeminiTranscriptionAudioInput(
type="audio",
data=base64.b64encode(processed_audio.file_content).decode("utf-8"),
mime_type=processed_audio.content_type,
)
request: Final = _build_interaction_request(
model=model,
audio_input=audio_input,
transcription_config=_build_transcription_config(optional_params),
)
return AudioTranscriptionRequestData(data=dict(request)) # mutable-ok: AudioTranscriptionRequestData wants dict
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
try:
response_json: Final = raw_response.json()
except ValueError:
raise GeminiError(
status_code=raw_response.status_code,
message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}",
)
parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json)
if parsed.status != "completed":
raise GeminiError(
status_code=raw_response.status_code,
message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}",
)
text_contents: Final = tuple(
content
for step in parsed.steps
for content in step.content
if content.type == "text" and content.text is not None
)
response: Final = TranscriptionResponse(text=" ".join(content.text or "" for content in text_contents))
response["task"] = "transcribe"
words: Final = tuple(
word
for content in text_contents
for annotation in content.annotations
if (word := _annotation_to_word(annotation)) is not None
)
if words:
response["words"] = list(words) # mutable-ok: verbose_json words is a JSON array
last_word_end: Final = words[-1].get("end")
if last_word_end is not None:
response["duration"] = last_word_end
if parsed.usage is not None:
audio_tokens: Final = sum(
by_modality.tokens
for by_modality in parsed.usage.input_tokens_by_modality
if by_modality.modality == "audio"
)
response.usage = TranscriptionUsageTokensObject(
type="tokens",
input_tokens=parsed.usage.total_input_tokens,
output_tokens=parsed.usage.total_output_tokens,
total_tokens=parsed.usage.total_tokens,
input_token_details=TranscriptionUsageInputTokenDetailsObject(
audio_tokens=audio_tokens,
text_tokens=parsed.usage.total_input_tokens - audio_tokens,
),
)
return response
_EMPTY_TRANSCRIPTION_CONFIG: Final[GeminiTranscriptionConfig] = {}
_WORD_TIMESTAMP_CONFIG: Final[GeminiTranscriptionConfig] = {
"mode": {
"type": "verbatim",
"timestamp_granularities": ("word",),
"diarization_mode": "speaker",
},
}
def _build_interaction_request(
model: str,
audio_input: GeminiTranscriptionAudioInput,
transcription_config: GeminiTranscriptionConfig,
) -> GeminiTranscriptionInteractionRequest:
if not transcription_config:
bare_request: Final[GeminiTranscriptionInteractionRequest] = {
"model": model.removeprefix("gemini/"),
"input": (audio_input,),
}
return bare_request
configured_request: Final[GeminiTranscriptionInteractionRequest] = {
"model": model.removeprefix("gemini/"),
"input": (audio_input,),
"generation_config": {"transcription_config": transcription_config},
}
return configured_request
def _language_config(language: object) -> GeminiTranscriptionConfig:
if not isinstance(language, str) or not language:
return _EMPTY_TRANSCRIPTION_CONFIG
language_config: Final[GeminiTranscriptionConfig] = {
"language_codes": (normalize_transcription_language_to_bcp47(language),),
}
return language_config
def _timestamp_config(timestamp_granularities: object) -> GeminiTranscriptionConfig:
if isinstance(timestamp_granularities, list) and "word" in timestamp_granularities:
return _WORD_TIMESTAMP_CONFIG
return _EMPTY_TRANSCRIPTION_CONFIG
def _build_transcription_config(optional_params: Mapping[str, object]) -> GeminiTranscriptionConfig:
transcription_config: Final[GeminiTranscriptionConfig] = {
**_language_config(optional_params.get("language")),
**_timestamp_config(optional_params.get("timestamp_granularities")),
}
return transcription_config
def _annotation_to_word(annotation: GeminiTranscriptionWordAnnotation) -> Mapping[str, str | float] | None:
if annotation.type != WORD_INFO_ANNOTATION_TYPE or annotation.text is None:
return None
entries: Final = (
("word", annotation.text),
("start", _parse_offset_seconds(annotation.start_offset)),
("end", _parse_offset_seconds(annotation.end_offset)),
("speaker", annotation.speaker),
)
return {key: value for key, value in entries if value is not None} # mutable-ok: word entries serialize to JSON
def _parse_offset_seconds(offset: str | None) -> float | None:
if offset is None or not offset.endswith("s"):
return None
try:
return float(offset[:-1])
except ValueError:
return None

72
litellm/llms/gemini/cost_calculator.py
View file

@ -39,25 +39,71 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
``model_info`` when available, falling back to $0.035 for models not
yet updated in the pricing JSON.
"""
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
)
from litellm.types.utils import PromptTokensDetailsWrapper
_DEFAULT_COST: Final = 35e-3
search_costs: Final = model_info.get("search_context_cost_per_query") or {}
_cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST)
number_of_web_search_requests = 0
if (
usage is not None
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
):
number_of_web_search_requests = usage.prompt_tokens_details.web_search_requests
requests_from_prompt_details: Final = (
usage.prompt_tokens_details.web_search_requests
if (
usage is not None
and usage.prompt_tokens_details is not None
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
and hasattr(usage.prompt_tokens_details, "web_search_requests")
and usage.prompt_tokens_details.web_search_requests is not None
)
else None
)
requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage)
number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0
# per_prompt billing: clamp to 1 (flat fee per grounded API call)
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
if number_of_web_search_requests > 0 and billing_mode == "per_prompt":
number_of_web_search_requests = 1
billable_requests: Final = (
1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests
)
return _cost * number_of_web_search_requests
return _cost * billable_requests
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT: Final = 25e-3
def google_maps_grounding_requests(usage: "Usage | None") -> int | None:
from litellm.types.utils import PromptTokensDetailsWrapper
details: Final = usage.prompt_tokens_details if usage is not None else None
if not isinstance(details, PromptTokensDetailsWrapper) or not hasattr(details, "google_maps_grounding_requests"):
return None
return details.google_maps_grounding_requests
def cost_per_google_maps_grounding_request(usage: "Usage", model_info: "ModelInfo") -> float:
"""
Calculates the cost of Grounding with Google Maps.
Billing follows ``web_search_billing_unit`` in model_info the same way Google Search grounding
does: ``"per_query"`` (Gemini 3.x) multiplies the executed Maps queries, ``"per_prompt"``
(default, Gemini 2.x) charges one flat fee per grounded prompt.
The rate comes from ``google_maps_grounding_cost_per_query`` in ``model_info``, falling back
to Google's list price for that billing unit when the pricing JSON has no entry yet.
"""
requests: Final = google_maps_grounding_requests(usage)
if not requests or requests <= 0:
return 0.0
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
default_cost: Final = (
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY
if billing_mode == "per_query"
else GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT
)
configured_cost: Final = model_info.get("google_maps_grounding_cost_per_query")
cost: Final = default_cost if configured_cost is None else configured_cost
billed_requests: Final = requests if billing_mode == "per_query" else 1
return cost * billed_requests

116
litellm/llms/gemini/realtime/transformation.py
View file

@ -4,6 +4,7 @@ This file contains the transformation logic for the Gemini realtime API.
import json
from collections import OrderedDict
from collections.abc import Mapping, Sequence
from typing import Any, Final, cast
import litellm
@ -52,6 +53,7 @@ from litellm.types.llms.vertex_ai import (
)
from litellm.types.realtime import (
ALL_DELTA_TYPES,
RealtimeInputAudioTranscriptionUsage,
RealtimeModalityResponseTransformOutput,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
@ -72,6 +74,40 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig()._map_audio_params({"voice": voice})
# Google bills Live transcription at an estimated 25 audio tokens/sec of input and
# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175
PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000
def _base64_decoded_byte_count(data: str) -> int:
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
return max(len(data) * 3 // 4 - padding, 0)
class GeminiRealtimeConfig(BaseRealtimeConfig):
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
@ -81,6 +117,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -282,12 +319,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "voice":
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
vertex_gemini_config = VertexGeminiConfig()
speech_config = vertex_gemini_config._map_audio_params({"voice": value})
speech_config = _gemini_live_speech_config(value)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
@ -366,25 +398,28 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
@staticmethod
def _is_native_audio_model(model: str) -> bool:
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
def _is_text_only_live_model(model: str) -> bool:
return GeminiRealtimeConfig._model_cost_entry(model).get("mode") == "audio_transcription"
@staticmethod
def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
"""Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
normalized: Final = [
def _default_response_modality(model: str) -> GeminiResponseModalities:
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
@staticmethod
def _coerce_response_modalities(model: str, modalities: Sequence[Any]) -> tuple[str, ...]:
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
normalized: Final = tuple(
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
]
if not GeminiRealtimeConfig._is_audio_only_live_model(model):
return normalized
if "TEXT" not in normalized:
return normalized
without_text: Final = [modality for modality in normalized if modality != "TEXT"]
return without_text if without_text else ["AUDIO"]
)
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
return normalized
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
"""Drop fields Gemini Live native-audio rejects on ``setup``."""
generation_config: Final = setup.get("generationConfig")
if isinstance(generation_config, dict):
modalities: Final = generation_config.get("responseModalities")
@ -392,8 +427,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities(
model, modalities
)
if GeminiRealtimeConfig._is_native_audio_model(model):
generation_config.pop("speechConfig", None)
return setup
def _handle_session_update(
@ -425,7 +458,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if session_configuration_request is None:
generation_config: Final = new_overrides.setdefault("generationConfig", {})
generation_config.setdefault("responseModalities", ["AUDIO"])
generation_config.setdefault("responseModalities", [GeminiRealtimeConfig._default_response_modality(model)])
new_overrides.setdefault("inputAudioTranscription", {})
new_overrides["model"] = f"models/{model}"
verbose_logger.debug("Gemini Realtime: Sending initial setup with tools to backend")
@ -547,9 +580,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return self._handle_conversation_item(json_message)
if msg_type == "input_audio_buffer.append":
realtime_input_dict["audio"] = HttpxBlobType(
mimeType=self.get_audio_mime_type(), data=json_message["audio"]
)
audio_b64: Final = json_message["audio"]
if isinstance(audio_b64, str):
self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)
realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64)
realtime_input_dict = cast(
BidiGenerateContentRealtimeInput,
@ -1140,6 +1174,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
raise ValueError(f"Unknown openai event: {key}, value: {value}")
return openai_event
def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
"""Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration."""
if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model):
return None
audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND
self._unbilled_input_audio_bytes = 0
audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND)
output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60)
usage: Final[RealtimeInputAudioTranscriptionUsage] = {
"type": "tokens",
"input_tokens": audio_tokens,
"output_tokens": output_tokens,
"total_tokens": audio_tokens + output_tokens,
"input_token_details": {"text_tokens": 0, "audio_tokens": audio_tokens},
}
return usage
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return self._consume_input_transcription_usage_estimate(model)
def transform_realtime_response(
self,
message: str | bytes,
@ -1179,6 +1233,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(server_content, dict):
input_tx: Final = server_content.get("inputTranscription")
if isinstance(input_tx, dict) and input_tx.get("text"):
transcription_usage: Final = self._consume_input_transcription_usage_estimate(model)
returned_message.append(
cast(
OpenAIRealtimeEvents,
@ -1188,6 +1243,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"transcript": input_tx["text"],
"item_id": f"item_{uuid.uuid4()}",
"content_index": 0,
**({} if transcription_usage is None else {"usage": transcription_usage}),
},
)
)
@ -1224,6 +1280,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
)
# Transcription-only models emit generationComplete with no prior
# modelTurn delta; there is no started OpenAI response to close, so
# drop it and let siblings (turnComplete, usageMetadata) process.
if current_delta_type is None and "modelTurn" not in server_content:
server_content.pop("generationComplete", None)
# Mark transcription-only serverContent as handled so the main loop
# skips it; sibling keys like toolCall are still processed below.
_model_content_keys: Final = {
@ -1572,7 +1634,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
```
"""
response_modalities: Final[list[GeminiResponseModalities]] = ["AUDIO"]
response_modalities: Final[list[GeminiResponseModalities]] = [
GeminiRealtimeConfig._default_response_modality(model)
]
output_audio_transcription: Final = False
# if "audio" in model: ## UNCOMMENT THIS WHEN AUDIO IS SUPPORTED
# output_audio_transcription = True

22
litellm/llms/minimax/messages/transformation.py
View file

@ -2,7 +2,7 @@
MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API
"""
from typing import Final
from typing import Any, Final # noqa: TID251 # override below must mirror the legacy base signature
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
@ -49,6 +49,26 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig):
"""
return api_base or get_secret_str("MINIMAX_API_BASE") or "https://api.minimax.io/anthropic/v1/messages"
def validate_anthropic_messages_environment(
self,
headers: dict, # mutable-ok: mirrors the legacy base override signature
model: str,
messages: list[Any], # mutable-ok: mirrors the legacy base override signature
optional_params: dict, # mutable-ok: mirrors the legacy base override signature
litellm_params: dict, # mutable-ok: mirrors the legacy base override signature
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict, str | None]: # mutable-ok: mirrors the legacy base override signature
return super().validate_anthropic_messages_environment(
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=self.get_api_key(api_key=api_key),
api_base=api_base,
)
def get_complete_url(
self,
api_base: str | None,

81
litellm/llms/together_ai/chat/transformation.py
View file

@ -4,8 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
Docs: https://docs.together.ai/docs/chat-overview
"""
from collections.abc import Container, Coroutine
from types import MappingProxyType
from collections.abc import Callable, Container, Coroutine
from typing import (
Final,
Literal,
@ -17,26 +16,42 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling
from litellm.utils import supports_function_calling, supports_response_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
LITELLM_INTERNAL_ASSISTANT_FIELDS: Final = frozenset({"thinking_blocks", "provider_specific_fields"})
PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs"
def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None:
try:
if check(model):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get(flag) is False:
return False
return None
def _function_calling_verdict(model: str) -> bool | None:
try:
if supports_function_calling(model, custom_llm_provider="together_ai"):
return True
except Exception as e:
verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
return False
return None
return _registry_verdict(
model,
"supports_function_calling",
lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"),
)
def _response_schema_verdict(model: str) -> bool | None:
return _registry_verdict(
model,
"supports_response_schema",
lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"),
)
def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
@ -68,6 +83,32 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
)
def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
if "response_format" not in passed_params:
return False
verdict: Final = _response_schema_verdict(model)
if verdict is True:
return False
if verdict is None:
verbose_logger.warning(
"together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return False
if drop_params or litellm.drop_params:
verbose_logger.warning(
"together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s",
model,
STRUCTURED_OUTPUTS_DOCS_URL,
)
return True
raise UnsupportedParamsError(
status_code=500,
message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
)
def _without_litellm_internal_fields(message: AllMessageValues) -> AllMessageValues:
if message["role"] != "assistant" or LITELLM_INTERNAL_ASSISTANT_FIELDS.isdisjoint(message):
return message
@ -112,18 +153,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
return super()._transform_messages(stripped, model, is_async=True)
return super()._transform_messages(stripped, model, is_async=False)
def get_supported_openai_params(self, model: str) -> list:
supports_fc: Final = _function_calling_verdict(model)
supported_params: Final = super().get_supported_openai_params(model)
if supports_fc is True:
return supported_params
verbose_logger.debug(
"Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
)
return [ # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
param for param in supported_params if param != "response_format"
]
def map_openai_params(
self,
non_default_params: dict,
@ -134,6 +163,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
mapped_openai_params.pop(param)
if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
if _drop_response_format(mapped_openai_params, model, drop_params):
mapped_openai_params.pop("response_format")
return mapped_openai_params

7
litellm/llms/vertex_ai/cost_calculator.py
View file

@ -64,6 +64,7 @@ def cost_per_character(
usage: Usage,
prompt_characters: float | None = None,
completion_characters: float | None = None,
service_tier: str | None = None,
vertex_location: str | None = None,
) -> tuple[float, float]:
"""
@ -74,6 +75,8 @@ def cost_per_character(
- custom_llm_provider: str, "vertex_ai-*"
- prompt_characters: float, the number of input characters
- completion_characters: float, the number of output characters
- service_tier: optional tier derived from Gemini trafficType
("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch).
- vertex_location: the Vertex AI location serving the request; non-global
locations apply the model's regional-endpoint uplift multiplier
@ -92,6 +95,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
else:
try:
@ -123,6 +127,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
## CALCULATE OUTPUT COST
@ -131,6 +136,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
else:
completion_tokens: Final = usage.completion_tokens
@ -162,6 +168,7 @@ def cost_per_character(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)

56
litellm/llms/vertex_ai/gemini/grounding_requests.py Normal file
View file

@ -0,0 +1,56 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final
@dataclass(frozen=True, slots=True)
class GroundingRequests:
web_search_requests: int | None
google_maps_grounding_requests: int | None
def has_billable_grounding(self) -> bool:
return bool(self.web_search_requests or self.google_maps_grounding_requests)
def _chunk_kinds(item: Mapping[str, object]) -> frozenset[str]:
chunks: Final = item.get("groundingChunks")
if not isinstance(chunks, list):
return frozenset()
return frozenset(kind for chunk in chunks if isinstance(chunk, Mapping) for kind in chunk)
def _queries(item: Mapping[str, object]) -> frozenset[str]:
queries: Final = item.get("webSearchQueries")
if not isinstance(queries, list):
return frozenset()
return frozenset(query for query in queries if isinstance(query, str) and query)
def _is_maps_item(item: Mapping[str, object]) -> bool:
return "maps" in _chunk_kinds(item) or bool(item.get("googleMapsWidgetContextToken"))
def _attributes_queries_to_maps(item: Mapping[str, object]) -> bool:
return _is_maps_item(item) and "web" not in _chunk_kinds(item)
def calculate_grounding_requests(grounding_metadata: Sequence[Mapping[str, object]]) -> GroundingRequests:
"""Billable grounding requests across candidates, counting each distinct query once.
Duplicate queries within and across grounding metadata items collapse to the
distinct-query count (#36377), and empty strings are ignored. Maps grounding is
floored at one request whenever a candidate carries maps chunks or a widget token,
since per-prompt billing charges the prompt even when no query is reported.
"""
items: Final = tuple(item for item in grounding_metadata if isinstance(item, Mapping))
web_queries: Final = frozenset(
query for item in items if not _attributes_queries_to_maps(item) for query in _queries(item)
)
maps_queries: Final = frozenset(
query for item in items if _attributes_queries_to_maps(item) for query in _queries(item)
)
has_maps: Final = any(_is_maps_item(item) for item in items)
return GroundingRequests(
web_search_requests=len(web_queries) or None,
google_maps_grounding_requests=max(len(maps_queries), 1) if has_maps else None,
)

44
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -89,6 +89,7 @@ from ..common_utils import (
supports_response_json_schema,
)
from ..vertex_llm_base import VertexBase
from .grounding_requests import calculate_grounding_requests
from .transformation import (
_gemini_convert_messages_with_history,
async_transform_request_body,
@ -1717,14 +1718,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_response: GenerateContentResponseBody | BidiGenerateContentServerMessage,
) -> bool:
"""
Whether the response used Grounding with Google Search, detected via
groundingMetadata.webSearchQueries (an actual web search was performed).
Whether the response used Grounding with Google Search or Grounding with Google Maps,
detected via groundingMetadata.webSearchQueries (an actual web search was performed) or
groundingMetadata.groundingChunks[].maps (a Maps lookup was performed).
Google bills grounding-with-Google-Search retrieved tokens separately (a per-request /
per-query search fee) and excludes them from input token billing, unlike URL context /
File Search / code execution whose tool-use tokens are charged at the input token rate.
URL context also emits groundingMetadata (with groundingChunks but no webSearchQueries),
so presence of groundingMetadata alone is not a sufficient signal.
Google bills both groundings separately (a per-request / per-query fee) and excludes their
retrieved tokens from input token billing, unlike URL context / File Search / code execution
whose tool-use tokens are charged at the input token rate. URL context also emits
groundingMetadata (with web groundingChunks but no webSearchQueries), so presence of
groundingMetadata alone is not a sufficient signal.
See https://ai.google.dev/gemini-api/docs/pricing and
https://github.com/BerriAI/litellm/discussions/33198
"""
@ -1732,7 +1734,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return False
for candidate in completion_response["candidates"] or []:
grounding_metadata, _, _, _ = VertexGeminiConfig._extract_candidate_metadata(candidate)
if VertexGeminiConfig._calculate_web_search_requests(grounding_metadata):
if calculate_grounding_requests(grounding_metadata).has_billable_grounding():
return True
return False
@ -1979,16 +1981,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
@staticmethod
def _calculate_web_search_requests(grounding_metadata: list[dict]) -> int | None:
web_search_requests: int | None = None
return calculate_grounding_requests(grounding_metadata).web_search_requests
if grounding_metadata and isinstance(grounding_metadata, list) and len(grounding_metadata) > 0:
for grounding_metadata_item in grounding_metadata:
web_search_queries = grounding_metadata_item.get("webSearchQueries")
if web_search_queries and web_search_requests:
web_search_requests += len([q for q in web_search_queries if q])
elif web_search_queries:
web_search_requests = len([q for q in web_search_queries if q])
return web_search_requests
@staticmethod
def _set_grounding_usage_counters(usage: Usage, grounding_metadata: Sequence[Mapping[str, object]]) -> None:
grounding_requests: Final = calculate_grounding_requests(grounding_metadata)
details: Final = cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details)
if grounding_requests.web_search_requests is not None:
details.web_search_requests = grounding_requests.web_search_requests
if grounding_requests.google_maps_grounding_requests is not None:
details.google_maps_grounding_requests = grounding_requests.google_maps_grounding_requests
@staticmethod
def _create_streaming_choice(
@ -2454,9 +2456,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
usage: Final = VertexGeminiConfig._calculate_usage(completion_response=completion_response)
web_search_requests: Final = VertexGeminiConfig._calculate_web_search_requests(grounding_metadata)
if web_search_requests is not None:
cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details).web_search_requests = web_search_requests
VertexGeminiConfig._set_grounding_usage_counters(usage, grounding_metadata)
setattr(model_response, "usage", usage)
@ -3221,9 +3221,7 @@ class ModelResponseIterator:
completion_response=processed_chunk,
)
web_search_requests: Final = VertexGeminiConfig._calculate_web_search_requests(grounding_metadata)
if web_search_requests is not None:
cast(PromptTokensDetailsWrapper, usage.prompt_tokens_details).web_search_requests = web_search_requests
VertexGeminiConfig._set_grounding_usage_counters(usage, grounding_metadata)
traffic_type: Final = processed_chunk.get("usageMetadata", {}).get("trafficType")
if traffic_type:

Some files were not shown because too many files have changed in this diff Show more