scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-03 07:23:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 166

chore: merge main and preserve typed logging redaction

Browse source
This commit is contained in:
adc77 2026-10-26 14:28:34 +05:30
commit cbbff2d0a5
628 changed files with 16321 additions and 3585 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

11
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

13
enterprise/enterprise_hooks/aporia_ai.py
View file

@ -15,9 +15,10 @@ sys.path.insert(
) # Adds the parent directory to the system path
import json
import sys
from typing import Any, List, Literal, Optional
from typing import Final, List, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
GUARDRAIL_NAME = "aporia"
_DECODED_JSON: Final = TypeAdapter(object)
class AporiaGuardrail(CustomGuardrail):
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
async def prepare_aporia_request(
self, new_messages: List[dict], response_string: Optional[str] = None
) -> dict:
data: dict[str, Any] = {}
data: dict[str, object] = {}
if new_messages is not None:
data["messages"] = new_messages
if response_string is not None:
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
action: str = _json_response.get(
_json_response: Final = decoded_json_object(
_DECODED_JSON.validate_python(response.json())
)
action: Final = _json_response.get(
"action"
) # possible values are modify, passthrough, block, rephrase
if action == "block":

17
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -7,9 +7,10 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Literal, Optional
from typing import Final, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import EndUserRepository
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_BlockedUserList(CustomLogger):
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
)
if end_user_cache_obj is None and self.prisma_client is not None:
# check db
end_user_obj = (
await self.prisma_client.db.litellm_endusertable.find_unique(
where={"user_id": user}
)
)
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
end_user_obj = await EndUserRepository(
self.prisma_client
).table.find_unique(where={"user_id": end_user_id})
if end_user_obj is None: # user not in db - assume not blocked
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
if end_user_obj is not None and end_user_obj.blocked is True:
if end_user_obj.blocked is True:
raise HTTPException(
status_code=400,
detail={

13
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -6,7 +6,10 @@
# +-----------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
from typing import Final
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
f"{category}_confidence_threshold",
default_confidence_threshold,
)
set_confidence_value = getattr(
self,
f"{category}_confidence_threshold",
set_confidence_value = _THRESHOLD_VALUE.validate_python(
getattr(
self,
f"{category}_confidence_threshold",
)
)
verbose_proxy_logger.info(
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"

7
enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py
View file

@ -1,15 +1,20 @@
# this is an example endpoint to receive data from litellm
from typing import Final
from fastapi import FastAPI, HTTPException, Request
from pydantic import TypeAdapter
app = FastAPI()
_LOGGED_EVENT: Final = TypeAdapter(object)
@app.post("/log-event")
async def log_event(request: Request):
try:
print("Received /log-event request") # noqa
# Assuming the incoming request has JSON data
data = await request.json()
data: Final = _LOGGED_EVENT.validate_python(await request.json())
print("Received request data:") # noqa
print(data) # noqa

18
enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py
View file

@ -8,25 +8,30 @@
## This provides an LLM Guard Integration for content moderation on the proxy
import asyncio
from typing import Final, Optional
from collections.abc import Mapping
from typing import Final
import aiohttp
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import CallTypesLiteral
_DECODED_JSON: Final = TypeAdapter(object)
class _ENTERPRISE_LLMGuard(CustomLogger):
# Class variables or attributes
def __init__(
self,
mock_testing: bool = False,
mock_redacted_text: Optional[dict] = None,
mock_redacted_text: Mapping[str, object] | None = None,
):
self.mock_redacted_text = mock_redacted_text
self.llm_guard_mode = litellm.llm_guard_mode
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
async with session.post(
analyze_url, json={"prompt": text}
) as response:
redacted_text = await response.json()
redacted_text = _DECODED_JSON.validate_python(
await response.json()
)
verbose_proxy_logger.debug(
f"LLM Guard: Received response - {redacted_text}"
)
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
"error": f"Invalid content moderation response: {redacted_text}"
},
)
if redacted_text.get("is_valid", None) is False:
redacted_response: Final = decoded_json_object(redacted_text)
if redacted_response.get("is_valid", None) is False:
raise HTTPException(
status_code=400,
detail={"error": "Violated content safety policy"},
)
sanitized_prompt = redacted_text.get("sanitized_prompt")
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
except Exception as e:
verbose_proxy_logger.exception(

4
enterprise/litellm_enterprise/proxy/auth/route_checks.py
View file

@ -2,6 +2,8 @@ import os
from fastapi import HTTPException, status
from litellm.proxy._types import MODEL_LIST_ROUTES
class EnterpriseRouteChecks:
@staticmethod
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
# Routes that should remain accessible even when LLM API endpoints are disabled.
# These are read-only model listing routes needed by the Admin UI.
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
@staticmethod
def should_call_route(route: str):

4
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -1040,8 +1040,8 @@ class CheckBatchCost:
from litellm.proxy.proxy_server import proxy_config
from litellm.secret_managers.main import str_to_bool
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
api_key: Final = job.api_key if hasattr(job, "api_key") else None
team_id: Final = job.team_id if hasattr(job, "team_id") else None
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
try:
key_row: Final = (

27
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -44,6 +44,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -133,7 +134,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _SupportsKeyedGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return holders if holders or "input_file_id" in request else healthy_deployments
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
# model_info may be at top-level or nested under litellm_metadata
# (batch/file operations use litellm_metadata)
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
)
for accessor_key in accessor_keys:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
)
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
"""
Get model-specific file IDs for a list of proxy file IDs.
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Optional[Router],
_create_file_request: CreateFileRequest,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> List[OpenAIFileObject]:
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
create_file_request: CreateFileRequest,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_objects: List[OpenAIFileObject],
create_file_request: CreateFileRequest,
internal_usage_cache: InternalUsageCache,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
target_model_names_list: List[str],
) -> OpenAIFileObject:
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Continue with other files even if one fails
continue
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
def _get_content_type_from_file_object(self, file_object: object) -> str:
"""
Determine content type from file object.

6
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
llm_router: Router,
model: str,
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""

13
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -73,6 +74,16 @@ def _verification_token_table(
return VerificationTokenRepository(prisma_client).table
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
table_name = "litellm_verificationtoken"
def _writer_verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
@ -849,7 +860,7 @@ async def update_project(
else None
)
if current_project is not None and data.team_id != current_project.team_id:
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
where={
"project_id": data.project_id,
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],

8
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -14,6 +14,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -37,6 +38,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
class ManagedVectorStoreRow(Protocol):
@ -348,11 +350,11 @@ async def update_vector_store(
raise HTTPException(status_code=500, detail="Database not connected")
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id: Final[str] = update_data.pop("vector_store_id")
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
vector_store_id: Final = data.vector_store_id
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
)
updated = await managed_vector_store_table(prisma_client).update(

11
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer

21
litellm/__init__.py
View file

@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
argilla_transformation_object: Mapping[str, str] | None = None
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
proxy_auth: _ProxyAuthHandlerType | None = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
@ -682,6 +682,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.add(key)
elif value.get("litellm_provider") == "opencode_go":
opencode_go_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
@ -1137,6 +1140,7 @@ model_list = list(
| openinfer_models
| v0_models
| morph_models
| opencode_go_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
@ -1254,6 +1258,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
@ -2289,6 +2294,10 @@ if TYPE_CHECKING:
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
from collections.abc import Awaitable
from litellm.types.utils import ModelResponse as _ModelResponseType
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -2312,7 +2321,7 @@ if TYPE_CHECKING:
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
@ -2610,8 +2619,10 @@ def __getattr__(name: str) -> Any:
cancel_run,
delete_run,
)
from types import MappingProxyType
return locals()[name]
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

4
litellm/_lazy_imports.py
View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

20
litellm/a2a_protocol/cost_calculator.py
View file

@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
to define custom costs per agent query or per token.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -15,6 +18,15 @@ else:
LitellmLoggingObject = Any
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is None:
return None
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(response_cost)
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
class A2ACostCalculator:
@staticmethod
def calculate_a2a_cost(
@ -45,9 +57,9 @@ class A2ACostCalculator:
model_call_details: Final = litellm_logging_obj.model_call_details
# Check if user set a custom response cost (backward compatibility)
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is not None:
return float(response_cost)
custom_response_cost: Final = _custom_response_cost(model_call_details)
if custom_response_cost is not None:
return custom_response_cost
# Get litellm_params for cost parameters
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}

27
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -84,14 +84,23 @@ def _filter_reserved_headers(
return filtered or None
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
if principal is None or isinstance(principal, str):
return principal
if not principal:
return None
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
def _request_scoped_runtime_session_id(
params: Mapping[str, object],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
)
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
model: Final = litellm_params.get("model", "")
if isinstance(model, str):
return model
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model: Final = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
model: Final = _agentcore_model_name(litellm_params)
agentcore_model: Final = model.removeprefix("bedrock/")
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}

40
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from itertools import chain
from typing import Final
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.openai.common_utils import decoded_json_object
def _a2a_text_part(part: object) -> Sequence[object]:
part_fields: Final = as_str_mapping(part)
if not isinstance(part_fields, dict):
return ()
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
return (part_fields["text"],)
return ()
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
class WatsonxOrchestrateTransformation:
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message: Final = params.get("message", {})
parts: Final = message.get("parts", [])
texts: Final = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
message: Final = decoded_json_object(params.get("message", {}))
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
return _join_with_spaces(texts) or ""
@staticmethod
def build_wxo_run_body(

21
litellm/a2a_protocol/utils.py
View file

@ -4,15 +4,20 @@ Utility functions for A2A protocol.
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
class A2ARequestUtils:
"""Utility class for A2A request/response processing."""
@ -74,7 +79,7 @@ class A2ARequestUtils:
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
) -> Any:
) -> object:
"""
Extract the input message from an A2A request.
@ -84,7 +89,7 @@ class A2ARequestUtils:
Returns:
The message object/dict or None
"""
params: Final = getattr(request, "params", None)
params: Final[object] = getattr(request, "params", None)
if params is None:
return None
return getattr(params, "message", None)
@ -127,10 +132,12 @@ class A2ARequestUtils:
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
# Count output tokens

13
litellm/anthropic_beta_headers_manager.py
View file

@ -45,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
class GetAnthropicBetaHeadersConfig:
@ -76,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
def _check_is_valid_dict(fetched_config: object) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
@ -110,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
"""
Validate the integrity of a fetched beta headers config.
@ -119,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
"""
Fetch the beta headers config from a remote URL.
@ -128,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return _DECODED_JSON.validate_python(response.json())
def get_beta_headers_config(url: str) -> dict:
@ -161,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
content, dict
):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
url,

11
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
import json
from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
- Generic: {"message": "..."}
- Plain strings
"""
parsed: Final = safe_json_loads(raw_message)
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
if isinstance(parsed, dict):
# Bedrock format
if "detail" in parsed and isinstance(parsed["detail"], dict):
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
AnthropicErrorResponse dict
"""
# Try to parse as JSON once
parsed: dict | None = safe_json_loads(raw_message)
if not isinstance(parsed, dict):
parsed = None
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):

24
litellm/assistants/main.py
View file

@ -77,7 +77,7 @@ def get_assistants(
api_base: str | None = None,
api_version: str | None = None,
**kwargs,
) -> SyncCursorPage[Assistant]:
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
if aget_assistants is not None and not isinstance(aget_assistants, bool):
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
@ -100,7 +100,7 @@ def get_assistants(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[Assistant] | None = None
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -564,7 +564,7 @@ def create_thread(
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
client: OpenAI | None = None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""
- get the llm provider
- if openai - route it there
@ -610,7 +610,7 @@ def create_thread(
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -735,7 +735,7 @@ def get_thread(
thread_id: str,
client=None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""Get the thread object, given a thread_id"""
aget_thread: Final = kwargs.pop("aget_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -757,7 +757,7 @@ def get_thread(
timeout = 600.0
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -903,7 +903,7 @@ def add_message(
metadata: dict | None = None,
client=None,
**kwargs,
) -> OpenAIMessage:
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
### COMMON OBJECTS ###
a_add_message: Final = kwargs.pop("a_add_message", None)
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
@ -935,7 +935,7 @@ def add_message(
timeout = 600.0
api_key: str | None = None
api_base: str | None = None
response: OpenAIMessage | None = None
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -1065,7 +1065,7 @@ def get_messages(
thread_id: str,
client: Any | None = None,
**kwargs,
) -> SyncCursorPage[OpenAIMessage]:
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
aget_messages: Final = kwargs.pop("aget_messages", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
@ -1086,7 +1086,7 @@ def get_messages(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[OpenAIMessage] | None = None
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
api_key: str | None = None
api_base: str | None = None
if custom_llm_provider == "openai":
@ -1256,7 +1256,7 @@ def run_thread(
client: object | None = None,
event_handler: AssistantEventHandler | None = None, # for stream=True calls
**kwargs,
) -> Run:
) -> Run | Coroutine[None, None, Run]:
"""Run a given thread + assistant."""
arun_thread: Final = kwargs.pop("arun_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -1278,7 +1278,7 @@ def run_thread(
elif timeout is None:
timeout = 600.0
response: Run | None = None
response: Run | Coroutine[None, None, Run] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there

6
litellm/caching/_embedding_router.py
View file

@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
@ -25,12 +25,12 @@ if TYPE_CHECKING:
def resolve_embedding_router(
embedding_model: str,
llm_router: Router | None,
llm_model_list: list[dict[str, Any]] | None,
llm_model_list: Sequence[Mapping[str, object]] | None,
) -> Router | None:
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
if llm_router is None:
return None
router_model_names: Final[list[str]] = (
router_model_names: Final[Sequence[object]] = (
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
)
if embedding_model in router_model_names:

29
litellm/caching/affinity_cache.py
View file

@ -24,6 +24,16 @@ if current == false then
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if ARGV[4] then
local legacy = current
if decoded and type(stored) == 'string' then legacy = stored end
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
if eligible['model_id'] == legacy then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
eligible_existing: Final = existing in (eligible_values or ()) or (
preserve_legacy_model_ids
and isinstance(existing, str)
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
)
winner: Final = existing if existing is not None and eligible_existing else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
@ -94,6 +110,7 @@ async def claim_affinity_pin(
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
preserve_legacy_model_ids: bool = False,
) -> object:
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
@ -112,6 +129,7 @@ async def claim_affinity_pin(
if eligible_values is not None
else ()
),
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
@ -124,4 +142,11 @@ async def claim_affinity_pin(
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
return claim_affinity_pin_in_memory(
cache,
cache_key,
pin_value,
ttl_seconds,
eligible_values=eligible_values,
preserve_legacy_model_ids=preserve_legacy_model_ids,
)

8
litellm/caching/azure_blob_cache.py
View file

@ -13,10 +13,14 @@ import json
from contextlib import suppress
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import print_verbose, verbose_logger
from .base_cache import BaseCache
_DECODED_BLOB: Final = TypeAdapter(object)
class AzureBlobCache(BaseCache):
def __init__(self, account_url, container) -> None:
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
print_verbose(f"Get Azure Blob Cache: key: {key}")
as_bytes: Final = self.container_client.download_blob(key).readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
blob: Final = await self.async_container_client.download_blob(key)
as_bytes: Final = await blob.readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
key,

9
litellm/caching/base_cache.py
View file

@ -9,14 +9,7 @@ Has 4 methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
else:
Span = Any
from typing import Final
class BaseCache(ABC):

8
litellm/caching/disk_cache.py
View file

@ -1,15 +1,19 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response)
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response

6
litellm/caching/redis_cache.py
View file

@ -1738,7 +1738,9 @@ class RedisCache(BaseCache):
return key_value_dict
@_redis_circuit_breaker_guard
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
async def async_get_cache(
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
):
from redis.asyncio import Redis
_redis_client: Final[Redis] = self.init_async_client()
@ -1781,6 +1783,8 @@ class RedisCache(BaseCache):
)
)
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
if raise_on_error:
raise
_record_swallowed_redis_failure(self._circuit_breaker, e)
@_redis_circuit_breaker_guard

4
litellm/constants.py
View file

@ -480,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
@ -861,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"heroku",
"oci",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"vercel_ai_gateway",
@ -1116,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
"v0",
"helicone",
"morph",
"opencode_go",
"lambda_ai",
"inception",
"hyperbolic",
@ -1842,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"

75
litellm/cost_calculator.py
View file

@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
@ -1200,12 +1201,16 @@ def get_usage_object(
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
return Usage(**usage_obj.model_dump())
return _usage_from_model(usage_obj)
else:
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
return None
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
match usage:
case DecisionsUsage():
@ -1546,6 +1551,8 @@ def completion_cost(
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
if pricing_provider is not None:
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
@ -1686,7 +1693,7 @@ def completion_cost(
setattr(
completion_response,
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
_usage_from_model(_usage_for_dump),
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
@ -2879,6 +2886,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -2964,7 +2972,12 @@ def batch_cost_calculator(
total_completion_cost = 0.0
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
usage,
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
@ -3009,10 +3022,13 @@ def _batch_prompt_cost(
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
@ -3026,13 +3042,27 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,
details["cache_creation_token_details"],
input_rate if cache_creation_rate is None else cache_creation_rate,
cache_creation_rates_by_ttl,
)
if bills_cache_writes
else 0.0
)
)
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
@ -3667,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
return None
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
return info.get("output_cost_per_token")
case _:
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3676,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
if declared is None:
return None
return next(
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
(
float(value)
for key in keys
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
),
None,
)
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)

2
litellm/exceptions.py
View file

@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.message = _saved_message
self.args = (_saved_message,)
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
if original_headers is not None:
self.litellm_response_headers = original_headers

20
litellm/experimental_mcp_client/tools.py
View file

@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
@ -20,6 +21,8 @@ from litellm.constants import (
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
_DECODED_JSON: Final = TypeAdapter(object)
########################################################
# List MCP Tool functions
@ -200,15 +203,18 @@ async def call_mcp_tool(
return tool_result
def _decode_json_arguments(arguments: str) -> object:
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}
def _get_function_arguments(function: FunctionDefinition) -> dict:
"""Helper to safely get and parse function arguments."""
arguments = function.get("arguments", {})
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except json.JSONDecodeError:
arguments = {}
return arguments if isinstance(arguments, dict) else {}
arguments: Final = function.get("arguments", {})
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
return decoded if isinstance(decoded, dict) else {}
def transform_openai_tool_call_request_to_mcp_tool_call_request(

14
litellm/files/main.py
View file

@ -9,7 +9,7 @@ import asyncio
import contextvars
import time
import uuid as uuid_module
from collections.abc import Coroutine
from collections.abc import Awaitable, Coroutine
from functools import partial
from typing import Final, Literal, cast
@ -293,6 +293,14 @@ def create_file(
raise e
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
return await pending
async def _await_deleted_file(pending: Awaitable[object]) -> object:
return await pending
@client
async def afile_retrieve(
file_id: str,
@ -325,7 +333,7 @@ async def afile_retrieve(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_retrieved_file(init_response)
else:
response = init_response
@ -505,7 +513,7 @@ async def afile_delete(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_deleted_file(init_response)
else:
response = init_response

4
litellm/files/streaming.py
View file

@ -1,7 +1,7 @@
import datetime
import traceback
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, cast
import anyio
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
merged_hidden_params: Final = cast(
"StandardLoggingHiddenParams",
{
**cast(dict[str, Any], payload.get("hidden_params") or {}),
**(payload.get("hidden_params") or {}),
**self._hidden_params,
},
)

3
litellm/harness/handlers/base.py
View file

@ -4,7 +4,6 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
from litellm.harness.context import SessionContext
from litellm.harness.errors import CapabilityUnsupported
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
"""Continue the runtime's own session on the next turn."""
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")

4
litellm/harness/handlers/cli_handler.py
View file

@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
async def private_dir_for(sandbox: Sandbox) -> str:
tempdir = getattr(sandbox, "tempdir", None)
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
if tempdir is None:
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
path: str = await tempdir()
path: Final = await tempdir()
return path

2
litellm/harness/handlers/deepagents_handler.py
View file

@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
async def history(
self, ctx: SessionContext
) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
agent, deps = self._require_agent()
snapshot = await agent.aget_state(self._run_config(ctx, None))
messages = (snapshot.values or MappingProxyType({})).get("messages") or ()

11
litellm/harness/runtime.py
View file

@ -12,11 +12,10 @@ import inspect
import logging
import os
import uuid
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
Any,
Final,
get_args,
)
@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
queue.put_nowait(end)
async def _await_decision(pending: Awaitable[object]) -> object:
return await pending
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
try:
if inspect.iscoroutinefunction(handler):
decision: object = await handler(approval)
decision: object = await _await_decision(handler(approval))
else:
decision = await asyncio.to_thread(handler, approval)
if inspect.isawaitable(decision):
@ -749,7 +752,7 @@ class AsyncSession:
async def history(
self,
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
if not self.harness_config.capabilities.history:
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
await self._ensure_ready()

11
litellm/images/main.py
View file

@ -3,7 +3,7 @@ import contextvars
import importlib
from collections.abc import Coroutine
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, overload
from pydantic import TypeAdapter, ValidationError
@ -70,16 +70,17 @@ from litellm.utils import (
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
"""Get ImageEditRequestUtils, loading it lazily if needed."""
global _ImageEditRequestUtils_cache
if _ImageEditRequestUtils_cache is None:
# Access via module to trigger __getattr__ if not cached
module: Final = importlib.import_module(__name__)
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
from litellm.images import main as images_main
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
return _ImageEditRequestUtils_cache

14
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -115,14 +115,16 @@ class AlertingHangingRequestCheck:
if hanging_request_data.alerted:
continue
request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
# this means the request status was either success or fail
# and is not hanging
if request_status is not None:
if (
await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
is not None
):
# clear this request from hanging request cache since the request was either success or failed
self.hanging_request_cache._remove_key(
key=request_id,

5
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -56,8 +56,9 @@ async def fetch_user_spend_rows(
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:

23
litellm/integrations/agentops/agentops.py
View file

@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
import os
from dataclasses import dataclass
from typing import Any, Final
from typing import Final
from pydantic import TypeAdapter
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import get_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
@dataclass
class AgentOpsConfig:
@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry):
project_id = None
if config.api_key:
try:
response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint)
jwt_token = response.get("token")
project_id = response.get("project_id")
response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint))
if response is not None:
jwt_token = response.get("token")
project_id = response.get("project_id")
except Exception:
pass
@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry):
"telemetry.sdk.name": "agentops",
}
if project_id:
resource_attrs["project.id"] = project_id
self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs
self.resource_attributes = resource_attrs
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]:
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object:
"""
Fetch JWT authentication token from AgentOps API
@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry):
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return response.json()
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()

2
litellm/integrations/arize/arize.py
View file

@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
primary ``otel`` callback which handles proxy-level parent spans.
"""
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
def set_attributes(self, span: Span, kwargs, response_obj: object):
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
@staticmethod

29
litellm/integrations/braintrust_logging.py
View file

@ -3,9 +3,10 @@
import os
from datetime import datetime
from typing import Final
from typing import Final, Protocol
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import verbose_logger
@ -24,6 +25,18 @@ from litellm.utils import print_verbose
API_BASE: Final = "https://api.braintrustdata.com/v1"
class _BraintrustProjectBody(TypedDict):
id: ReadOnly[str]
class _BraintrustProjectResponse(Protocol):
def json(self) -> _BraintrustProjectBody: ...
def _project_id(response: _BraintrustProjectResponse) -> str:
return response.json()["id"]
def get_utc_datetime():
import datetime as dt
@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger):
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def create_sync_default_project_and_experiment(self):
project: Final = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")

7
litellm/integrations/cloudzero/cz_stream_api.py
View file

@ -19,8 +19,9 @@
"""Output modules for writing CBF data to various destinations."""
import zoneinfo
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
import polars as pl
@ -188,7 +189,7 @@ class CloudZeroStreamer:
)
raise
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]:
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]:
"""Prepare batch payload according to CloudZero AnyCost API format."""
# Convert batch_date to month for the API (YYYY-MM format)
try:
@ -205,7 +206,7 @@ class CloudZeroStreamer:
if record:
data_records.append(record)
payload: Final = {"month": month_str, "operation": operation, "data": data_records}
payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records}
return payload

7
litellm/integrations/custom_guardrail.py
View file

@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
self,
violation_message: str,
request_data: dict[str, Any],
detection_info: dict[str, Any] | None = None,
detection_info: Mapping[str, object] | None = None,
original_response: object = None,
) -> None:
"""
@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
guardrail_mode = GuardrailMode(
tags=copy.deepcopy(self.event_hook.tags),
default=copy.deepcopy(self.event_hook.default),
)
else:
guardrail_mode = self.event_hook

3
litellm/integrations/custom_secret_manager.py
View file

@ -37,7 +37,6 @@ Usage:
"""
from abc import abstractmethod
from typing import Any
import httpx
@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager):
optional_params: dict | None = None,
timeout: float | httpx.Timeout | None = None,
tags: dict | list | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Asynchronously write a secret to your custom secret manager.

4
litellm/integrations/dotprompt/__init__.py
View file

@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
prompt_file: Final = getattr(litellm_params, "prompt_file", None)
# Handle dotprompt_content from database
dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None)
dotprompt_content: Final = (
litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None
)
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content)

15
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -7,6 +7,8 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.types.llms.openai import AllMessageValues
@ -18,7 +20,12 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
from .prompt_manager import PromptManager, PromptTemplate
from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate
class _PromptJsonData(TypedDict, total=False):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
class DotpromptManager(CustomPromptManagement):
@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement):
if self._prompt_manager:
self._prompt_manager.reload_prompts()
def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None:
def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None:
"""Add a prompt from JSON data."""
content: Final = json_data.get("content", "")
metadata: Final = json_data.get("metadata", {})
self.prompt_manager.add_prompt(prompt_id, content, metadata)
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None:
"""Load multiple prompts from JSON data."""
self.prompt_manager.load_prompts_from_json_data(prompts_data)
def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all prompts in JSON format."""
return self.prompt_manager.get_all_prompts_as_json()

29
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
class _PromptFileJson(TypedDict):
"""JSON form of a .prompt file: rendered template text plus its frontmatter."""
@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict):
metadata: ReadOnly[NotRequired[dict[str, object]]]
class PromptJsonEntry(TypedDict):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
@ -358,20 +365,26 @@ class PromptManager:
return f"---\n{frontmatter_yaml}---\n{content}"
def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]:
def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all loaded prompts in JSON format.
Returns:
Dictionary mapping prompt_id to prompt data
"""
result: Final = {}
for prompt_id, template in self.prompts.items():
result[prompt_id] = {
return {
prompt_id: {
"content": template.content,
"metadata": template.metadata,
}
return result
for prompt_id, template in self.prompts.items()
}
def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts)."""
self._load_prompts_from_json(prompt_data)
def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts). Entries that are not
mappings are skipped, the same as a malformed entry inside the loader."""
entries: Final = {
prompt_id: dict(entry)
for prompt_id, raw in prompt_data.items()
if (entry := as_str_mapping(raw)) is not None
}
self._load_prompts_from_json(entries)

5
litellm/integrations/focus/destinations/factory.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from typing import Any, Final
from .base import FocusDestination
@ -20,7 +21,7 @@ class FocusDestinationFactory:
*,
provider: str,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, object] | None = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower: Final = provider.lower()
@ -39,7 +40,7 @@ class FocusDestinationFactory:
def _resolve_config(
*,
provider: str,
overrides: dict[str, object],
overrides: Mapping[str, object],
) -> dict[str, Any]:
if provider == "s3":
resolved = {

8
litellm/integrations/focus/export_engine.py
View file

@ -2,8 +2,8 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
from collections.abc import Callable, Mapping
from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast
import polars as pl
@ -24,7 +24,7 @@ class FocusExportEngine:
provider: str,
export_format: str,
prefix: str,
destination_config: dict[str, Any] | None = None,
destination_config: Mapping[str, object] | None = None,
) -> None:
self.provider = provider
self.export_format = export_format
@ -77,7 +77,7 @@ class FocusExportEngine:
return FocusParquetSerializer()
raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]:
data: Final = await self.database.get_usage_data(limit=limit)
normalized: Final = self.transformer.transform(data)

8
litellm/integrations/focus/focus_logger.py
View file

@ -6,6 +6,8 @@ import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
from typing_extensions import TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT: Final = 500
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
pass
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
@ -40,7 +46,7 @@ class FocusLogger(CustomLogger):
interval_seconds: int | None = None,
prefix: str | None = None,
destination_config: dict[str, Any] | None = None,
**kwargs: Any,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()

8
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
"""
return f"{date_str}/batch-{batch_id}.ndjson"
def _get_config_key(self, kwargs: dict[str, Any]) -> str:
def _get_config_key(self, kwargs: Mapping[str, object]) -> str:
"""
Extract a synchronous grouping key from kwargs to group items by GCS config.
This allows us to batch items with the same bucket/credentials together.
@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key()
for logging purposes.
"""
standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {}
standard_callback_dynamic_params: Final = (
as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {}
)
bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default"
path_service_account: Final = (

3
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -9,6 +9,7 @@ Usage:
"""
import asyncio
import os
from collections.abc import AsyncIterable, Iterable
from typing import Final
@ -40,7 +41,7 @@ _mocks_initialized = False
# Default mock latency in seconds (simulates network round-trip)
# Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE
_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None):

8
litellm/integrations/gcs_pubsub/pub_sub.py
View file

@ -12,6 +12,8 @@ import os
import traceback
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
_DECODED_JSON: Final = TypeAdapter(object)
class GcsPubSubLogger(CustomBatchLogger):
def __init__(
@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger):
finally:
self.log_queue.clear()
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None:
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object:
"""
Publish message to Google Cloud Pub/Sub using REST API
@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger):
raise Exception(f"Failed to publish message: {response.text}")
verbose_logger.debug("Pub/Sub response: %s", response.text)
return response.json()
return _DECODED_JSON.validate_python(response.json())
except Exception as e:
verbose_logger.error("Pub/Sub publish error: %s", str(e))

5
litellm/integrations/langfuse/langfuse.py
View file

@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import (
should_use_langfuse_mock,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
filter_exceptions_from_params,
reconstruct_model_name,
safe_deep_copy,
@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span(
*,
tracing: "LangfuseTracing",
parent: "LangfuseObservation",
enrichments: Mapping[str, Any],
enrichments: Mapping[str, object],
):
"""Logs provider-specific information as spans under the generation."""
_hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None)
_hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None))
if _hidden_params is None:
return

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
@staticmethod
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj):
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj):
"""
Sets Langfuse specific metadata attributes onto the OTEL span.
@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params"))
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")

12
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -2,6 +2,7 @@
Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
"""
from collections.abc import Mapping
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) ->
in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id)
def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]:
return {k: v for k, v in config.items() if k != "model"}
class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger):
def __init__(
self,
@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return compiled_prompt
def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict:
config: Final = langfuse_prompt_client.config
optional_params: Final = {}
for k, v in config.items():
if k != "model":
optional_params[k] = v
return optional_params
return dict(_params_without_model(langfuse_prompt_client.config))
async def async_get_chat_completion_prompt(
self,

11
litellm/integrations/langsmith.py
View file

@ -7,10 +7,11 @@ import random
import traceback
import types
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa
_RUN_RESPONSE: Final = TypeAdapter(object)
class _DynamicParamsKwargs(TypedDict, total=False):
standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None]
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float:
def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float:
standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
"standard_callback_dynamic_params", None
)
@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger):
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject:
def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject:
"""
Handles key/team based logging

2
litellm/integrations/langtrace.py
View file

@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

4
litellm/integrations/levo/levo.py
View file

@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any, Final
from litellm.integrations.opentelemetry import OpenTelemetry
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import (
OpenTelemetryConfig as _OpenTelemetryConfig,
)
@ -13,11 +11,9 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = _Span | Any
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
class LevoConfig:

5
litellm/integrations/logfire_logger.py
View file

@ -3,8 +3,9 @@
import os
import traceback
from collections.abc import Mapping
from enum import Enum
from typing import Any, Final, NamedTuple
from typing import Final, NamedTuple
from typing_extensions import LiteralString
@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
class SpanConfig(NamedTuple):
message_template: LiteralString
span_data: dict[str, Any]
span_data: Mapping[str, object]
class LogfireLevel(str, Enum):

19
litellm/integrations/newrelic/newrelic.py
View file

@ -49,11 +49,12 @@ import time
import uuid
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.newrelic import NewRelicInitParams
@ -65,6 +66,10 @@ except ImportError:
_newrelic_agent = None
class _SupportsTimestamp(Protocol):
def timestamp(self) -> float: ...
class NewRelicLogger(CustomLogger):
"""
New Relic logger for LiteLLM to send AI monitoring events.
@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger):
verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e)
self.enabled = False
def _get_newrelic_params(self) -> dict:
def _get_newrelic_params(self) -> Mapping[str, object]:
"""
Get the newrelic_params from litellm.newrelic_params
These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging
"""
dict_newrelic_params: dict = {}
dict_newrelic_params: Mapping[str, object] = {}
if litellm.newrelic_params is not None:
if isinstance(litellm.newrelic_params, NewRelicInitParams):
dict_newrelic_params = litellm.newrelic_params.model_dump()
@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger):
return choices[0].get("finish_reason") or "unknown"
return "unknown"
def _to_epoch_ms(self, t: Any) -> float:
def _to_epoch_ms(self, t: datetime | float) -> float:
"""Convert a datetime or float timestamp to epoch milliseconds."""
if hasattr(t, "timestamp"):
if has_attributes(t, _SupportsTimestamp, "timestamp"):
return t.timestamp() * 1000.0
return float(t) * 1000.0
@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger):
sequence = 0
# Extract timestamps, preferring StandardLoggingPayload
start_time = None
start_time: datetime | float | None = None
if standard_logging_object:
start_time = standard_logging_object.get("startTime")
if not start_time:
start_time = kwargs.get("start_time")
end_time = None
end_time: datetime | float | None = None
if standard_logging_object:
end_time = standard_logging_object.get("endTime")
if not end_time:

2
litellm/integrations/newrelic/newrelic_metrics.py
View file

@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger):
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
await asyncio.sleep(1 << _pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)

4
litellm/integrations/opik/opik_payload_builder/payload_builders.py
View file

@ -57,9 +57,7 @@ def build_span_payload(
span_id: Final = utils.create_uuid7()
model: Final[str] = response_obj.get("model", "unknown-model")
obj_type: Final[str] = response_obj.get("object", "unknown-object")
created: Final[int] = response_obj.get("created", 0)
span_name: Final = f"{model}_{obj_type}_{created}"
span_name: Final = f"{model}_{response_obj.get('object', 'unknown-object')}_{response_obj.get('created', 0)}"
_logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id)

3
litellm/integrations/otel/model/config.py
View file

@ -130,7 +130,8 @@ class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
return self._env_settings.get_field_value(field, field_name)
def __call__(self) -> dict[str, object]:
return {key: value for key, value in self._env_settings().items() if key != "excluded_services"}
settings: Final = self._env_settings()
return {key: settings[key] for key in settings if key != "excluded_services"}
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str
return (None if echoes_trace_id else explicit) or trace.session_id or None
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None:
"""Seconds from the upstream request being issued (``api_call_start_time``)
to the first streamed chunk (``completion_start_time``); ``None`` for
non-streaming calls, where ``completion_start_time`` is backfilled with the
@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {})
if not optional_params.get("stream"):
return None
api_call_start: Final = to_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = to_seconds(kwargs.get("completion_start_time"))
api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time"))
if api_call_start is None or completion_start is None:
return None
return completion_start - api_call_start

15
litellm/integrations/otel/model/payloads.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from enum import Enum
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
from urllib.parse import urlsplit
from typing_extensions import ReadOnly, TypedDict
@ -716,6 +716,15 @@ def _json_or_none(value: object) -> str | None:
return None
class _EnumValue(Protocol):
@property
def value(self) -> object: ...
def _enum_value(item: _EnumValue) -> object:
return item.value
def _guardrail_mode_str(value: object) -> str | None:
"""Normalize ``guardrail_mode`` to a stable string.
@ -732,12 +741,12 @@ def _guardrail_mode_str(value: object) -> str | None:
for item in value:
if item is None:
continue
part = as_str(item.value) if isinstance(item, Enum) else as_str(item)
part = as_str(_enum_value(item)) if isinstance(item, Enum) else as_str(item)
if part:
parts.append(part)
return ",".join(parts) or None
if isinstance(value, Enum):
return as_str(value.value)
return as_str(_enum_value(value))
return as_str(value)

8
litellm/integrations/otel/plumbing/metrics.py
View file

@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
response_cost: ReadOnly[float | None]
completion_start_time: ReadOnly[datetime | float | str | None]
api_call_start_time: ReadOnly[datetime | float | str | None]
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
@ -414,15 +415,12 @@ class GenAIMetricRecorder:
if generation_time > 0:
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
api_call_start_time: Final = kwargs.get("api_call_start_time")
if api_call_start_time is None:
return
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
api_call_start: Final = to_seconds(api_call_start_time)
end_ts: Final = to_seconds(_end_time)
end_ts: Final = to_seconds(kwargs.get("end_time") or end_time)
if api_call_start is None or end_ts is None:
return
duration: Final = end_ts - api_call_start

5
litellm/integrations/sqs.py
View file

@ -13,6 +13,8 @@ import re
import traceback
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import (
@ -32,6 +34,7 @@ from litellm.types.utils import StandardLoggingPayload
from .custom_batch_logger import CustomBatchLogger
_DECODED_JSON: Final = TypeAdapter(object)
_BASE64_INLINE_PATTERN: Final = re.compile(
r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+",
re.MULTILINE,
@ -270,7 +273,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM):
if self.sqs_queue_url is None:
raise ValueError("sqs_queue_url not set")
json_data: Final = json.loads(safe_dumps(payload))
json_data: Final = _DECODED_JSON.validate_python(json.loads(safe_dumps(payload)))
if self.app_crypto:
aad_bytes: Final = self.sqs_app_encryption_aad.encode("utf-8") if self.sqs_app_encryption_aad else None
encrypted: Final = self.app_crypto.encrypt_json(json_data, aad=aad_bytes)

5
litellm/interactions/litellm_responses_transformation/handler.py
View file

@ -116,10 +116,7 @@ class LiteLLMResponsesInteractionsHandler:
"""Async handler for interactions API requests."""
# Call litellm.aresponses()
# Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
aresponses_fn: Final[
Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]]
] = vars(litellm)["aresponses"]
responses_response: Final = await aresponses_fn(
responses_response: Final = await litellm.aresponses(
**responses_request,
)

9
litellm/litellm_core_utils/app_crypto.py
View file

@ -4,6 +4,9 @@ import os
from typing import Final
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from pydantic import TypeAdapter
_DECODED_JSON: Final = TypeAdapter(object)
class AppCrypto:
@ -12,7 +15,7 @@ class AppCrypto:
raise ValueError("Master key must be 32 bytes for AES-256-GCM")
self.key = master_key
def encrypt_json(self, data: dict, aad: bytes | None = None) -> dict:
def encrypt_json(self, data: object, aad: bytes | None = None) -> dict:
aes: Final = AESGCM(self.key)
nonce: Final = os.urandom(12)
plaintext: Final = json.dumps(data).encode("utf-8")
@ -24,10 +27,10 @@ class AppCrypto:
"tag": base64.b64encode(tag).decode(),
}
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> dict:
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> object:
aes: Final = AESGCM(self.key)
nonce: Final = base64.b64decode(enc["nonce"])
ct: Final = base64.b64decode(enc["ciphertext"])
tag: Final = base64.b64decode(enc["tag"])
data: Final = aes.decrypt(nonce, ct + tag, aad)
return json.loads(data.decode())
return _DECODED_JSON.validate_python(json.loads(data.decode()))

6
litellm/litellm_core_utils/cloud_storage_security.py
View file

@ -3,10 +3,11 @@ import re
from collections.abc import Mapping, Sequence
from itertools import accumulate, repeat
from types import MappingProxyType
from typing import Any, Final, cast
from typing import Final
from urllib.parse import quote, unquote
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/"
BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-"
@ -141,7 +142,8 @@ def should_allow_legacy_cloud_file_ids(
if isinstance(litellm_params, Mapping):
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
value = cast(Mapping[str, Any], trusted_model_credentials).get("allow_legacy_cloud_file_ids")
trusted_mapping: Final = as_str_mapping(trusted_model_credentials)
value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None
if isinstance(value, bool):
return value

112
litellm/litellm_core_utils/core_helpers.py
View file

@ -5,10 +5,11 @@ import logging
import re
from collections.abc import Collection, Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
@ -58,6 +59,13 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:
return mapping
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value)
@runtime_checkable
class _SupportsIter(Protocol):
def __iter__(self) -> Iterator[object]: ...
@ -72,6 +80,24 @@ def attribute_or_none(source: object, name: str) -> object:
return getattr(source, name, None)
_Shape: Final = TypeVar("_Shape")
def has_attributes(
value: object,
shape: type[_Shape],
*names: str,
) -> TypeIs[_Shape]: # guard-ok: hasattr sees what __getattr__ serves, which Protocol isinstance skips on 3.12+
return all(hasattr(value, name) for name in names)
def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shape:
if has_attributes(value, shape, *names):
return value
missing: Final = next(name for name in names if not hasattr(value, name))
raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value)
def iter_items(value: object) -> Iterable[object]:
if isinstance(value, _SupportsIter):
return value
@ -634,11 +660,14 @@ def preserve_upstream_non_openai_attributes(
for key in candidate_keys
)
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
for key, value in candidate_values:
setattr(model_response, key, value)
_assign_attributes(model_response, candidate_values)
return
for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items():
setattr(model_response, key, value)
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
def safe_deep_copy(data):
@ -764,6 +793,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
Filtered data structure with Exception and callable objects removed, or None if the
entire input was an Exception or callable
"""
return _filter_exceptions_from_params(data, max_depth)
def _filter_exceptions_from_params(data: object, max_depth: int) -> object:
if max_depth <= 0:
return data
@ -785,7 +818,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
continue
try:
filtered = filter_exceptions_from_params(v, max_depth - 1)
filtered = _filter_exceptions_from_params(v, max_depth - 1)
if filtered is not None:
result[k] = filtered
except Exception:
@ -799,7 +832,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
continue
try:
filtered = filter_exceptions_from_params(item, max_depth - 1)
filtered = _filter_exceptions_from_params(item, max_depth - 1)
if filtered is not None:
result_list.append(filtered)
except Exception:
@ -895,6 +928,71 @@ class _CarriesHiddenParams(Protocol):
def hidden_params(self, hidden_params: dict[str, object]) -> None: ... # mutable-ok: API requires mutation
_USAGE_BLOCK_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _usage_field(usage: object, name: str) -> object:
if isinstance(usage, Mapping):
return _USAGE_BLOCK_ADAPTER.validate_python(usage).get(name)
return getattr(usage, name, None)
def _cost_number(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float):
return None
return float(value)
def _reported_cost_value(value: object) -> float | None:
"""A reported cost is a number, or a breakdown whose total lives under ``total_cost``."""
raw: Final[object] = value
if isinstance(value, Mapping):
return _cost_number(_usage_field(raw, "total_cost"))
return _cost_number(value)
def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None:
additional_headers: Final[object] = response.hidden_params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER))
def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None:
"""Replace the cost an earlier builder stamped so every reader of the hidden params agrees with the header."""
set_response_cost_in_hidden_params(response, cost)
hidden_params: Final = response.hidden_params
hidden_params["response_cost"] = cost
def chunk_usage(chunk: object) -> object:
"""Stream chunks reach the builders both as mappings and as ModelResponseStream objects."""
if isinstance(chunk, Mapping):
return _USAGE_BLOCK_ADAPTER.validate_python(chunk).get("usage")
return getattr(chunk, "usage", None)
_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
"""Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
return None
return provider_reported_usage_cost(usage, custom_llm_provider)
def provider_reported_usage_cost(usage: object, custom_llm_provider: str | None) -> float | None:
"""The request cost a provider states on its usage block, as a number or a breakdown with ``total_cost``.
OpenCode Go reports the model's list price (peak hours and long-context tiers included) under
``usage.opencode.list_cost``, while its ``usage.cost`` is what hit the prepaid balance, 0 inside the plan.
"""
if custom_llm_provider == "opencode_go":
return _reported_cost_value(_usage_field(_usage_field(usage, "opencode"), "list_cost"))
return _reported_cost_value(_usage_field(usage, "cost"))
def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: float | None) -> None:
"""Record a provider-reported cost where the cost calculator looks before the price map."""
if cost is None:

2
litellm/litellm_core_utils/cost_discount.py
View file

@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey:
def _literal_length(pattern: str) -> int:
return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?")
return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?")
def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]:

2
litellm/litellm_core_utils/default_encoding.py
View file

@ -48,5 +48,5 @@ for attempt in range(_max_retries):
# Last attempt, re-raise the exception
raise
# Exponential backoff with jitter to reduce collision probability
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1)
time.sleep(delay)

4
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -181,7 +181,7 @@ def _delete_nested_value_custom(
def delete_nested_value(
data: dict[str, Any],
data: Mapping[str, object],
path: str,
depth: int = 0,
max_depth: int = 20,
@ -214,7 +214,7 @@ def delete_nested_value(
"""
import copy
result: Final = copy.deepcopy(data)
result: Final = copy.deepcopy(data if isinstance(data, dict) else dict(data))
try:
# Parse path into segments

4
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -16,7 +16,7 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.types.utils import LlmProviders
@ -169,7 +169,7 @@ def get_error_message(error_obj) -> str | None:
return None
if hasattr(error_obj, "body"):
_error_obj_body: Final = getattr(error_obj, "body")
_error_obj_body: Final = attribute_or_none(error_obj, "body")
if isinstance(_error_obj_body, dict):
# OpenAI-style: {"message": "...", "type": "...", ...}
if _error_obj_body.get("message"):

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -902,6 +902,10 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.MorphChatConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "opencode_go":
from litellm.llms.opencode.common_utils import go_api_base, go_api_key
return model, custom_llm_provider, go_api_key(api_key), go_api_base(api_base, model)
elif custom_llm_provider == "lambda_ai":
(
api_base,

6
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -43,6 +43,7 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
_DECODED_JSON: Final = TypeAdapter(object)
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
@ -314,7 +315,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
@ -383,7 +384,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
etag_values: Final = response.headers.get_list("etag")
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
def _next_retry_wait(

3
litellm/litellm_core_utils/hidden_params.py
View file

@ -1,4 +1,5 @@
from collections.abc import Iterator, Mapping, MutableMapping
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
@ -52,7 +53,7 @@ class HiddenParamsModelView(MutableMapping[str, object]):
def stored_attribute(obj: object, name: str) -> object:
if not isinstance(obj, BaseModel):
return getattr(obj, name, None)
values: Final[Mapping[str, object]] = vars(obj)
values: Final[Mapping[str, object]] = MappingProxyType[str, object](vars(obj))
if name in values:
return values[name]
extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {}

2
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -28,7 +28,7 @@ def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
def iter_client_callback_metadata_dicts(
kwargs: dict[str, Any],
kwargs: Mapping[str, object],
) -> Iterator[tuple[str, dict[str, Any]]]:
litellm_params: Final = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):

90
litellm/litellm_core_utils/litellm_logging.py
View file

@ -78,6 +78,8 @@ from litellm.litellm_core_utils.classifier_logging import (
from litellm.litellm_core_utils.core_helpers import (
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
native_messages_reported_cost,
provider_reported_usage_cost,
proxy_stamped_kubernetes_pod_routing,
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
@ -559,6 +561,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
def _provider_response_id(source: object) -> str | None:
@ -1174,7 +1177,7 @@ class Logging(LiteLLMLoggingBaseClass):
automatically extracts metadata/litellm_metadata from kwargs,
so callers don't need to manually plumb them into litellm_params.
"""
base_litellm_params: Final[dict[str, Any]] = {}
base_litellm_params: Final[dict[str, object]] = {}
if isinstance(kwargs.get("metadata"), dict):
base_litellm_params["metadata"] = kwargs["metadata"].copy()
@ -1191,10 +1194,11 @@ class Logging(LiteLLMLoggingBaseClass):
lp_metadata: Final = litellm_params.pop("metadata", None)
base_litellm_params.update(litellm_params)
if lp_metadata and isinstance(lp_metadata, dict):
base_litellm_params.setdefault("metadata", {})
for k, v in lp_metadata.items():
if k not in base_litellm_params["metadata"]:
base_litellm_params["metadata"][k] = v
merged_metadata: Final = base_litellm_params.setdefault("metadata", {})
if isinstance(merged_metadata, dict):
for k, v in lp_metadata.items():
if k not in merged_metadata:
merged_metadata[k] = v
self.update_environment_variables(
litellm_params=base_litellm_params,
@ -1796,7 +1800,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.exception(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
)
original_response = redact_message_input_output_from_logging(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=original_response,
)
@ -2026,10 +2030,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details.pop("response_cost_failure_debug_information", None)
if cache_hit is None:
cache_hit = self.model_call_details.get("cache_hit", False)
resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
if cache_hit is True:
if resolved_cache_hit is True:
return 0.0
if is_unbilled_non_inference_call(
@ -2081,7 +2084,7 @@ class Logging(LiteLLMLoggingBaseClass):
model_value: Final = litellm_model_name or self.model
cost_cache_hit_value: Final = (
self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit
)
cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided
bool | None, cost_cache_hit_value
@ -2515,8 +2518,10 @@ class Logging(LiteLLMLoggingBaseClass):
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
@ -3305,7 +3310,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
@ -3326,9 +3331,11 @@ class Logging(LiteLLMLoggingBaseClass):
)
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
)
)
## LOGGING HOOK ##
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
@ -3594,7 +3601,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
response_obj=call_details_view["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -4017,9 +4024,11 @@ class Logging(LiteLLMLoggingBaseClass):
self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
)
redacted_result: Final = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
redacted_result: Final = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
## LOGGING HOOK ##
@ -4354,9 +4363,11 @@ class Logging(LiteLLMLoggingBaseClass):
result: object = None # result sent to all loggers, init this to None incase it's not created
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
self.mark_logging_complete(event_type="sync_failure")
for callback in callbacks:
@ -4840,7 +4851,12 @@ class Logging(LiteLLMLoggingBaseClass):
):
## return unified Usage object
if isinstance(result.response.usage, ResponseAPIUsage):
set_response_cost_in_hidden_params(result.response, result.response.usage.cost)
set_response_cost_in_hidden_params(
result.response,
provider_reported_usage_cost(
result.response.usage, self.model_call_details.get("custom_llm_provider")
),
)
transformed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(
result.response.usage
)
@ -4906,8 +4922,11 @@ class Logging(LiteLLMLoggingBaseClass):
bridge is replaced by the ModelResponse its spend row is built from, anything else is
priced as is."""
served: Final[object] = result
if self.bridged_model_response is None or not isinstance(result, dict):
return result
call_type: Final[object] = getattr(self, "call_type", None)
if not isinstance(result, dict):
return served
if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
return served
return self._anthropic_messages_logged_response(result=served)
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
@ -4950,7 +4969,13 @@ class Logging(LiteLLMLoggingBaseClass):
if isinstance(result, ResponsesAPIResponse):
return self._translate_responses_api_response_to_model_response(result)
provider_response_id: Final = _provider_response_id(result)
raw_body: Final[object] = result
served_body: Final = _LITELLM_PARAMS.validate_python(raw_body) if isinstance(result, dict) else None
reported_cost: Final = native_messages_reported_cost(
served_body.get("usage") if served_body is not None else None,
self.model_call_details.get("custom_llm_provider"),
)
provider_response_id: Final = _provider_response_id(raw_body)
bridged: Final = self.bridged_model_response
if bridged is not None and isinstance(result, dict):
return bridged if provider_response_id is None else bridged.model_copy(update={"id": provider_response_id})
@ -4975,7 +5000,7 @@ class Logging(LiteLLMLoggingBaseClass):
else:
from litellm.types.llms.anthropic import AnthropicResponse
pydantic_result: Final = AnthropicResponse.model_validate(result)
pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})})
import httpx
result = litellm.AnthropicConfig().transform_parsed_response(
@ -4988,6 +5013,7 @@ class Logging(LiteLLMLoggingBaseClass):
json_mode=None,
speed=self.optional_params.get("speed") if self.optional_params else None,
)
set_response_cost_in_hidden_params(result, reported_cost)
return result
def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
@ -5045,7 +5071,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
import httpx
httpx_response: Final = self.model_call_details.get("httpx_response", None)
httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
@ -5075,7 +5101,7 @@ class Logging(LiteLLMLoggingBaseClass):
The response object with usage added if available
"""
# Get usage from model_call_details (set by asend_message)
usage: Final = self.model_call_details.get("usage")
usage: Final[Usage | None] = self.model_call_details.get("usage")
if usage is None:
return result
@ -5908,7 +5934,9 @@ def build_custom_logger_compatible_class(
return callback
# Get global BitBucket config
bitbucket_config: Final = getattr(litellm, "global_bitbucket_config", None)
bitbucket_config: Final = (
litellm.global_bitbucket_config if hasattr(litellm, "global_bitbucket_config") else None
)
if bitbucket_config is None:
raise ValueError("BitBucket configuration not found. Please set litellm.global_bitbucket_config first.")

42
litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py
View file

@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
reservation logic so neither has to depend on the other.
"""
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, SupportsFloat
from pydantic import TypeAdapter, ValidationError
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
def _coerce_cost_per_token(value: float | str | None) -> float:
def _coerce_cost_per_token(value: object) -> float:
"""
Coerce a per-token cost into a float.
Model cost values loaded from YAML config may arrive as strings (e.g.
scientific notation like "4e-07"), which would break arithmetic.
"""
if value is None:
return 0.0
if isinstance(value, str):
try:
return float(value)
except ValueError:
return 0.0
return float(value)
if isinstance(value, SupportsFloat):
return float(value)
return 0.0
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
try:
return _TIER_RANGE.validate_python(tier.get("range"))
except ValidationError:
return None
def select_tier_for_input(
tiered_pricing: list[dict],
tiered_pricing: Sequence[Mapping[str, object]],
input_tokens: int,
) -> dict | None:
) -> Mapping[str, object] | None:
"""
Select the pricing tier for a request based on its total input token count.
@ -44,19 +56,21 @@ def select_tier_for_input(
if not tiered_pricing or input_tokens <= 0:
return None
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
if not valid_tiers:
ranged_tiers: Final = sorted(
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
key=lambda ranged: ranged[0][0],
)
if not ranged_tiers:
return None
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
if matching:
return matching[0]
return valid_tiers[-1]
return ranged_tiers[-1][1]
def tier_rate(
tier: dict,
tier: Mapping[str, object],
cost_key: str,
fallback_cost_key: str | None = None,
) -> float:
@ -68,4 +82,6 @@ def tier_rate(
primary: Final = tier.get(cost_key)
if primary is not None:
return _coerce_cost_per_token(primary)
if fallback_cost_key is None:
return 0.0
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))

2
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -450,7 +450,7 @@ class StandardBuiltInToolCostTracking:
def _get_code_interpreter_cost(
model_info: ModelInfo | None,
custom_llm_provider: str | None,
standard_built_in_tools_params: StandardBuiltInToolsParams,
standard_built_in_tools_params: Mapping[str, object],
) -> float:
"""Calculate code interpreter cost."""
code_interpreter_sessions: Final = standard_built_in_tools_params.get("code_interpreter_sessions", None)

289
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -3,7 +3,7 @@
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, TypedDict, cast
@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{
"5m": _BASE_CACHE_WRITE_RATE_KEY,
"30m": _BASE_CACHE_WRITE_RATE_KEY,
"1hr": "cache_creation_input_token_cost_above_1hr",
"12hr": "cache_creation_input_token_cost_above_12hr",
}
)
_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY}
)
_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
@ -88,7 +104,8 @@ _BATCH_RATE_PREFIXES: Final = (
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
)
_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$")
_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values())
_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$")
_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX)
@ -298,6 +315,7 @@ class BatchCostRates:
output: float | None
cache_read: float | None
cache_creation: float | None
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
@ -353,10 +371,31 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive),
cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive),
cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive),
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: rate
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None
}
),
)
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None:
tiered_pricing: Final = model_info.get("tiered_pricing")
if not isinstance(tiered_pricing, list) or not tiered_pricing:
return None
@ -376,7 +415,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None:
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
"""
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
@ -391,22 +430,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float,
if tier is None:
return None
cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
completion_cost: Final = (
tier_rate(tier, "output_cost_per_token")
if "output_cost_per_token" in tier
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
)
return (
tier_rate(tier, "input_cost_per_token"),
completion_cost,
cache_creation_cost,
tier_rate(tier, "cache_creation_input_token_cost_above_1hr")
if "cache_creation_input_token_cost_above_1hr" in tier
and tier["cache_creation_input_token_cost_above_1hr"] is not None
else cache_creation_cost,
tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
rates: Final = TokenRates(
input_rate=tier_rate(tier, "input_cost_per_token"),
output_rate=completion_cost,
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=cache_creation_rate,
reasoning_rate=None,
)
return rates, tier
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
@ -532,19 +569,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
return False
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def resolve_explicit_cache_read_cost_rate(
model_info: ModelInfo,
usage: Usage,
@ -649,7 +673,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
if off_peak is None:
return rates
off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
return TokenRates(
return replace(
rates,
input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
@ -661,32 +686,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
def _apply_off_peak_to_base_costs(
model_info: ModelInfo,
current_time: datetime | None,
base_costs: tuple[float, float, float, float | None, float],
) -> tuple[float, float, float, float, float]:
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
present one passes through untouched and an absent one resolves to the applied
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
base_rates: TokenRates,
tier: Mapping[str, object],
) -> TokenRates:
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
_resolve_billed_reasoning_rate.
"""
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=prompt,
output_rate=completion,
cache_read_rate=cache_read,
cache_creation_rate=cache_creation,
reasoning_rate=None,
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
return replace(
rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
}
),
)
return (
rates.input_rate,
rates.output_rate,
rates.cache_creation_rate,
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
rates.cache_read_rate,
)
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
def _get_token_base_cost(
@ -696,7 +717,7 @@ def _get_token_base_cost(
current_time: datetime | None = None,
*,
threshold_is_inclusive: bool = False,
) -> tuple[float, float, float, float, float]:
) -> TokenRates:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@ -709,15 +730,12 @@ def _get_token_base_cost(
An absent cache-creation or cache-read rate always resolves to the resolved input
rate, the way the tiered table and custom deployment pricing already do, since a
provider that publishes no cache price bills cached tokens as ordinary input. An
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
explicit 0.0 stays a real price for all of them.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
if tiered_base_costs is not None:
return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs)
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
# Get service tier aware cost keys
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
@ -734,13 +752,13 @@ def _get_token_base_cost(
# For image generation models that don't have output_cost_per_token,
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
if completion_base_cost == 0.0 or completion_base_cost is None:
if completion_base_cost == 0.0:
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
if output_image_cost is not None:
completion_base_cost = output_image_cost
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
@ -811,14 +829,6 @@ def _get_token_base_cost(
if service_tier
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
)
cache_creation_1hr_tiered_key = (
get_service_tier_cost_key(
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
)
cache_read_tiered_key = (
get_service_tier_cost_key(
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
@ -830,10 +840,19 @@ def _get_token_base_cost(
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
cache_creation_keys_by_ttl = MappingProxyType(
{
ttl: _first_present_key(
model_info,
get_service_tier_cost_key(
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
service_tier,
),
key,
)
for ttl, key in cache_creation_keys_by_ttl.items()
}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
except (IndexError, ValueError):
continue
@ -849,16 +868,26 @@ def _get_token_base_cost(
resolved_cache_creation_cost: Final = (
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
)
return _apply_off_peak_to_base_costs(
base_rates: Final = apply_off_peak_pricing(
model_info,
current_time,
(
prompt_base_cost,
completion_base_cost,
resolved_cache_creation_cost,
cache_creation_cost_above_1hr,
resolved_cache_read_cost,
TokenRates(
input_rate=prompt_base_cost,
output_rate=completion_base_cost,
cache_read_rate=resolved_cache_read_cost,
cache_creation_rate=resolved_cache_creation_cost,
reasoning_rate=None,
),
)
return replace(
base_rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: base_rates.cache_creation_rate
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
else rate
for ttl, key in cache_creation_keys_by_ttl.items()
}
),
)
@ -975,25 +1004,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: CacheCreationTokenDetails | None,
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
cache_creation_cost_by_ttl: Mapping[str, float],
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts.
Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens
cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
if cache_creation_token_details is None:
return cache_creation_tokens * cache_creation_cost
tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType(
{
"5m": cache_creation_token_details.ephemeral_5m_input_tokens,
"30m": cache_creation_token_details.ephemeral_30m_input_tokens,
"1hr": cache_creation_token_details.ephemeral_1h_input_tokens,
"12hr": cache_creation_token_details.ephemeral_12h_input_tokens,
}
)
unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0)
return unattributed_tokens * cache_creation_cost + sum(
(tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost)
for ttl, tokens in tokens_by_ttl.items()
)
class PromptTokensDetailsResult(TypedDict):
@ -1178,7 +1210,7 @@ def _calculate_input_cost(
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
cache_creation_cost_by_ttl: Mapping[str, float],
service_tier: str | None = None,
) -> float:
"""
@ -1248,8 +1280,8 @@ def _calculate_input_cost(
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
)
### CHARACTER COST
@ -1510,34 +1542,31 @@ def generic_cost_per_token(
)
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=resolved_model_info,
usage=usage,
service_tier=service_tier,
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
completion_base_cost: Final = base_costs.output_rate
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=resolved_model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
)
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=resolved_model_info,
prompt_base_cost=prompt_base_cost,
prompt_base_cost=base_costs.input_rate,
cache_read_cost=effective_cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=base_costs.cache_creation_rate,
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
service_tier=service_tier,
)
@ -1640,7 +1669,7 @@ class BilledTokenRates:
cache_read_input_token_cost: float
cache_read_input_audio_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
output_cost_per_reasoning_token: float
def scaled(self, multiplier: float) -> "BilledTokenRates":
@ -1652,7 +1681,9 @@ class BilledTokenRates:
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
cache_creation_input_token_cost_by_ttl=MappingProxyType(
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
),
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
)
@ -1702,7 +1733,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
cache_read_input_token_cost=cache_read_rate,
cache_read_input_audio_token_cost=cache_read_rate,
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
cache_creation_input_token_cost_by_ttl=MappingProxyType({}),
output_cost_per_reasoning_token=output_rate,
)
@ -1717,13 +1748,7 @@ def _cost_map_billed_rates(
current_time: datetime | None,
) -> BilledTokenRates:
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost_rate,
cache_creation_cost_above_1hr_rate,
cache_read_cost_rate,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier=service_tier,
@ -1737,14 +1762,14 @@ def _cost_map_billed_rates(
service_tier=service_tier,
)
effective_cache_read_cost_rate: Final = (
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
)
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
completion_base_cost=base_costs.output_rate,
current_time=billing_time,
)
audio_cache_read_rate: Final = get_cost_per_unit(
@ -1759,14 +1784,14 @@ def _cost_map_billed_rates(
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
)
return BilledTokenRates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
input_cost_per_token=base_costs.input_rate,
output_cost_per_token=base_costs.output_rate,
cache_read_input_token_cost=effective_cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
cache_creation_input_token_cost=base_costs.cache_creation_rate,
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
output_cost_per_reasoning_token=reasoning_rate,
).scaled(multiplier)
@ -1854,8 +1879,8 @@ def get_token_type_cost_breakdown(
else calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
cache_creation_cost=rates.cache_creation_input_token_cost,
cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl,
)
)
return TokenTypeCostBreakdown(
@ -1886,7 +1911,7 @@ def calculate_prompt_caching_savings(
biller saw them rather than at the later spend write.
"""
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info_with_cache_read_default,
usage=usage,
service_tier=service_tier,
@ -1899,9 +1924,11 @@ def calculate_prompt_caching_savings(
current_time=billed_at,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
write_rate: Final = cache_creation_cost or prompt_base_cost
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
prompt_base_cost: Final = base_costs.input_rate
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
)
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
@ -1909,7 +1936,9 @@ def calculate_prompt_caching_savings(
cache_creation_details: Final = (
CacheCreationTokenDetails(
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0),
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0),
)
if details is not None
else None
@ -1918,8 +1947,10 @@ def calculate_prompt_caching_savings(
write_premium: Final = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_details,
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
cache_creation_cost=write_rate - prompt_base_cost,
cache_creation_cost_by_ttl=MappingProxyType(
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
),
)
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
model_info, vertex_location

29
litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py
View file

@ -5,6 +5,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
prompt_details: Final = parse_prompt_tokens_details(usage)
prompt_audio: Final = prompt_details["audio_tokens"]
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
prompt_text: Final = max(
_tokens(usage.prompt_tokens)
- prompt_details["cache_hit_tokens"]
- prompt_audio
- prompt_details["image_tokens"]
- prompt_details["video_tokens"],
0,
)
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
components: Final = (
(_TEXT_INPUT_RATE, prompt_text),
(_AUDIO_INPUT_RATE, prompt_audio),
(_CACHE_READ_INPUT_RATE, cached_text),
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
(_TEXT_OUTPUT_RATE, completion_text),
(_AUDIO_OUTPUT_RATE, completion_audio),
)
return tuple(key for key, count in components if count > 0)
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
return _TEXT_INPUT_RATE
return key
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
try:
return _NESTED_PRICING.validate_python(value)
@ -89,7 +111,8 @@ def diagnose_zero_cost(
pricing_entry: Mapping[str, object],
calculation_failed: bool,
) -> StandardLoggingZeroCostDiagnostic | None:
used_keys: Final = used_pricing_keys(usage)
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
if not used_keys:
return None
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)

5
litellm/litellm_core_utils/llm_judge.py
View file

@ -21,6 +21,7 @@ if TYPE_CHECKING:
JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
def default_router_provider() -> Router | None:
@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars
text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload
parsed: object
try:
parsed = json.loads(text)
parsed = _DECODED_JSON.validate_python(json.loads(text))
except json.JSONDecodeError:
start: Final = text.find("{")
end: Final = text.rfind("}")
if start == -1 or end <= start:
raise
parsed = json.loads(text[start : end + 1])
parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1]))
if not isinstance(parsed, dict):
raise ValueError("judge response is not a JSON object")
return {str(k): v for k, v in parsed.items()}

8
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -1,5 +1,5 @@
import threading
from collections.abc import Callable, Iterable, Iterator
from collections.abc import Callable, Iterable, Iterator, Mapping
from types import MethodType
from typing import TYPE_CHECKING, Final
@ -20,6 +20,10 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
def _instance_attributes(target: object) -> Mapping[str, object]:
return vars(target)
class LoggingCallbackManager:
"""
A centralized class that allows easy add / remove callbacks for litellm.
@ -341,7 +345,7 @@ class LoggingCallbackManager:
key_parts: Final = [type(custom_logger).__name__]
# Add only fundamental type instance variables to the key
for attr_name, attr_value in vars(custom_logger).items():
for attr_name, attr_value in _instance_attributes(custom_logger).items():
if not attr_name.startswith("_"): # Skip private attributes
if isinstance(attr_value, (str, bool, int)):
key_parts.append(f"{attr_name}={attr_value}")

20
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
@ -1527,7 +1528,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int:
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
return (1 << optional) * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
@ -2839,7 +2840,7 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
break
try:
obj, end_idx = decoder.raw_decode(raw, idx)
decoded = decoder.raw_decode(raw, idx)
except json.JSONDecodeError as e:
verbose_logger.warning(
"split_concatenated_json_objects: discarding unparseable tool-call "
@ -2849,13 +2850,14 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
e,
)
break
if isinstance(obj, dict):
results.append(obj)
obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if obj is not None:
results.append(dict(obj))
else:
# Non-dict JSON value – wrap in empty dict (Bedrock requires
# toolUse.input to be an object).
results.append({})
idx = end_idx
idx = decoded[1]
return results
@ -2887,14 +2889,16 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ..
if idx >= length:
break
try:
obj, end_idx = decoder.raw_decode(stripped, idx)
decoded = decoder.raw_decode(stripped, idx)
except json.JSONDecodeError:
return ()
if not isinstance(obj, dict):
decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if decoded_object is None:
return ()
idx = end_idx
idx = decoded[1]
if exceeded:
continue
obj = dict(decoded_object)
count += 1
if not kept:
kept = (obj,)

168
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -21,6 +21,7 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
@ -1021,11 +1022,12 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
tool_function = get_attribute_or_key(tool, "function")
tool_name = get_attribute_or_key(tool_function, "name")
tool_arguments = get_attribute_or_key(tool_function, "arguments")
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
parsed_args = _DECODED_JSON.validate_python(
parse_tool_call_arguments(tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke")
)
if isinstance(parsed_args, dict):
parameters = "".join(f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items())
parsed_mapping = as_str_mapping(parsed_args)
if parsed_mapping is not None:
parameters = "".join(f"<{param}>{val}</{param}>\n" for param, val in parsed_mapping.items())
else:
parameters = f"<result>{parsed_args}</result>\n"
invokes += f"<invoke>\n<tool_name>{tool_name}</tool_name>\n<parameters>\n{parameters}</parameters>\n</invoke>\n"
@ -2203,15 +2205,97 @@ def anthropic_process_openai_file_message(
_EMPTY_TEXT_PLACEHOLDER: Final = "[System: Empty message content sanitised to satisfy protocol]"
def _dict_block(block: object) -> Mapping[str, object] | None:
if not isinstance(block, dict):
return None
return cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance; the converters only read dict blocks
def _is_empty_text_block(block: object) -> bool:
match _dict_block(block):
case {"type": "text", "text": str() as text}:
return not text.strip()
case {"type": "text"}:
return True
case _:
return False
def _with_placeholder_text(block: object) -> object:
fields: Final = _dict_block(block)
if fields is None or not _is_empty_text_block(fields):
return block
return {**fields, "text": _EMPTY_TEXT_PLACEHOLDER}
def _survives_assistant_conversion(block: object) -> bool:
"""Whether ``anthropic_messages_pt`` keeps this list-content block of an assistant turn.
Mirrors the assistant converter: a ``thinking`` / ``redacted_thinking`` block is kept
unless it is unsignable (a signed block is kept even with empty text), and
``server_tool_use`` and ``*_tool_result`` blocks pass through. A redacted block with
blank data is dropped under ``drop_params``, which is not known here, so it never counts.
"""
fields: Final = _dict_block(block)
if fields is None:
return False
if (thinking_block := _as_thinking_block(fields)) is not None:
return not (is_unsignable_thinking_block(thinking_block) or has_blank_redacted_data(thinking_block))
match fields:
case {"type": "server_tool_use"}:
return True
case {"type": str() as block_type}:
return block_type.endswith("_tool_result")
case _:
return False
def _tool_call_survives_conversion(tool_call: object) -> bool:
"""Whether ``convert_to_anthropic_tool_invoke`` can emit a block for this tool call.
Non-function calls are skipped there, and a tool search call is dropped when its
result is not replayed, which depends on the rest of the conversation, so neither counts.
"""
if message_field(tool_call, "type") != "function":
return False
match message_field(tool_call, "id"), message_field(message_field(tool_call, "function"), "name"):
case (str() as tool_id, str() as tool_name):
return not is_anthropic_tool_search_use(tool_id, tool_name)
case _:
return True
def _carries_non_text_content(message: AllMessageValues) -> bool:
if any(_tool_call_survives_conversion(tool_call) for tool_call in parts_of(message_field(message, "tool_calls"))):
return True
content: Final = message_field(message, "content")
blocks: Final = parts_of(content)
return isinstance(content, list) and any(_survives_assistant_conversion(block) for block in blocks)
def _with_content(message: AllMessageValues, content: object, action: str) -> AllMessageValues:
sanitized: Final = cast(AllMessageValues, {**message, "content": content})
verbose_logger.debug("_sanitize_empty_text_content: %s in %s message", action, message_field(message, "role"))
return sanitized
def _sanitize_empty_text_content(
message: AllMessageValues,
) -> AllMessageValues:
"""
Case C: Sanitize empty text content
- Replace empty or whitespace-only text content with a placeholder message.
- Handles both string content and list-of-blocks content (rewriting only
the empty text blocks in place; non-text blocks like images are left
untouched).
Anthropic rejects empty text blocks, so empty or whitespace-only text must never be
forwarded verbatim. What replaces it depends on what else the turn carries.
Assistant turns that already carry substantive content (``tool_calls``, or non-text
blocks such as ``server_tool_use``) get the empty text dropped. Anthropic accepts an
assistant turn whose content is just ``tool_use``, so there is nothing to substitute, and
substituting anyway leaks a protocol diagnostic into the conversation as though the
assistant had said it, which the model then echoes on later turns.
Every other empty turn would otherwise end up with no content at all, which Anthropic
also rejects, so the placeholder is still substituted there.
Returns:
The message with sanitized content if needed, otherwise the original message
@ -2219,42 +2303,26 @@ def _sanitize_empty_text_content(
if message.get("role") not in ["user", "assistant"]:
return message
content: Final = message.get("content")
content: Final = message_field(message, "content")
drop_empty_text: Final = message_field(message, "role") == "assistant" and _carries_non_text_content(message)
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = _EMPTY_TEXT_PLACEHOLDER
verbose_logger.debug(
"_sanitize_empty_text_content: Replaced empty text content in %s message", message.get("role")
)
if content.strip():
return message
if drop_empty_text:
return _with_content(message, None, "Dropped empty text content")
return _with_content(message, _EMPTY_TEXT_PLACEHOLDER, "Replaced empty text content")
blocks: Final = parts_of(content)
if not isinstance(content, list) or not any(_is_empty_text_block(block) for block in blocks):
return message
if isinstance(content, list):
# Walk the blocks and rewrite any empty text blocks. We rewrite (rather
# than drop) so callers don't end up with an entirely empty content
# list, which Anthropic also rejects.
new_blocks: Final[list[Any]] = []
rewrote_any = False
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text")
if not isinstance(text, str) or not text or not text.strip():
new_block = dict(block)
new_block["text"] = _EMPTY_TEXT_PLACEHOLDER
new_blocks.append(new_block)
rewrote_any = True
continue
new_blocks.append(block)
if drop_empty_text:
kept: Final = [block for block in blocks if not _is_empty_text_block(block)]
return _with_content(message, kept, "Dropped empty text block(s)")
if rewrote_any:
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = new_blocks
verbose_logger.debug(
"_sanitize_empty_text_content: Replaced empty text block(s) in %s message", message.get("role")
)
return message
rewritten: Final = [_with_placeholder_text(block) for block in blocks]
return _with_content(message, rewritten, "Replaced empty text block(s)")
def _add_missing_tool_results(
@ -2342,7 +2410,7 @@ def _add_missing_tool_results(
else:
tool_name = getattr(function, "name", "unknown_tool")
else:
function = getattr(tool_call, "function", None)
function = attribute_or_none(tool_call, "function")
if function:
tool_name = getattr(function, "name", "unknown_tool")
break
@ -3184,10 +3252,14 @@ def anthropic_messages_pt(
def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]:
ext_list = re.findall(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
if strip:
ext_list = [e.strip() for e in ext_list]
ext_list = list(_stripped(ext_list))
return ext_list
def _stripped(values: Iterable[str]) -> Iterator[str]:
return (value.strip() for value in values)
def contains_tag(tag: str, string: str) -> bool:
return bool(re.search(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL))
@ -4061,13 +4133,15 @@ def _convert_to_bedrock_tool_call_invoke(
arguments_dict = {}
else:
try:
arguments_dict = json.loads(arguments)
decoded_arguments = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments)))
# Ensure arguments_dict is always a dict
# (Bedrock requires toolUse.input to be an object).
# Some providers return arguments: '""' which
# json.loads decodes to a bare string.
if not isinstance(arguments_dict, dict):
if decoded_arguments is None:
arguments_dict = {}
else:
arguments_dict = dict(decoded_arguments)
except json.JSONDecodeError:
# The model may return multiple JSON objects
# concatenated in a single arguments string, e.g.
@ -6028,7 +6102,11 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
return ({},)
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
try:
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
parsed: Final = as_str_mapping(
_DECODED_JSON.validate_python(
parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
)
)
except ValueError as e:
salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw)
if salvaged:
@ -6041,7 +6119,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
return salvaged
verbose_logger.warning("Failed to parse tool call arguments: %s", e)
return ({},)
return (parsed,) if isinstance(parsed, dict) else ({},)
return (dict(parsed),) if parsed is not None else ({},)
def _choice_tool_calls(choice: object) -> tuple[object, ...]:

2
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -482,7 +482,7 @@ def _runs_by_anchor(
) -> Mapping[int, tuple[tuple[int, ...], ...]]:
anchored: Final = tuple(sorted(((anchor, run) for run, anchor in anchors if anchor is not None), key=itemgetter(0)))
return MappingProxyType(
{anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=itemgetter(0))}
{anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=lambda entry: entry[0])}
)

8
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,4 +1,4 @@
from collections.abc import Mapping, Sequence
from collections.abc import ItemsView, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from typing import Any, Final
@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset(
)
def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]:
return data.items()
class SensitiveDataMasker:
def __init__(
self,
@ -127,7 +131,7 @@ class SensitiveDataMasker:
return data
masked_data: Final[dict[str, object]] = {}
for k, v in data.items():
for k, v in _object_items(data):
try:
key_is_sensitive = self.is_sensitive_key(k, excluded_keys)
if isinstance(v, Mapping):

43
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
@ -425,7 +426,7 @@ class ChunkProcessor:
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
@ -1226,6 +1227,7 @@ def concatenate_base64_list(base64_strings: Sequence[str]) -> str:
_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]])
_CHOICE_FIELD_VALUE: Final = TypeAdapter(object)
def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse:
@ -1351,14 +1353,27 @@ def stream_chunk_builder_text_completion(
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return False
return None
provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider")
return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
return provider
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
if _logged_provider(logging_obj) == LlmProviders.OPENCODE_GO.value:
# usage.opencode only survives on the chunks, the assembled Usage keeps declared fields
reported: Final = (
provider_reported_usage_cost(chunk_usage(chunk), LlmProviders.OPENCODE_GO.value)
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
@ -1396,8 +1411,10 @@ def _set_hidden_response_cost(response: ModelResponse, response_cost: float) ->
hidden_params["response_cost"] = response_cost
def _set_stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj)
def _set_stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks)
if response_cost is None:
return
_set_hidden_response_cost(response, response_cost)
@ -1549,7 +1566,11 @@ def stream_chunk_builder(
break
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
_set_stream_builder_response_cost(
response,
logging_obj,
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
@ -1733,7 +1754,11 @@ def stream_chunk_builder(
break
_stamp_streaming_usage_cost(usage, response, logging_obj)
_set_stream_builder_response_cost(response, logging_obj)
_set_stream_builder_response_cost(
response,
logging_obj,
chunks, # pyright: ignore[reportUnknownArgumentType] # legacy chunk list is untyped
)
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response

109
litellm/litellm_core_utils/streaming_handler.py
View file

@ -24,6 +24,7 @@ import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
@ -37,7 +38,6 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import OpenAIChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CacheCreationTokenDetails,
@ -73,7 +73,9 @@ _SYNC_ITER_EXHAUSTED: Final = object()
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
_CompleteStreamingResponse = ModelResponse | TextCompletionResponse | ModelResponseStream | None
_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.OPENROUTER.value})
_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset(
{LlmProviders.OPENROUTER.value, LlmProviders.OPENCODE_GO.value}
)
def _next_sync_or_exhausted(it: Iterator[object]) -> object:
@ -111,14 +113,15 @@ def _prepend_upstream_choice_extras(
choice.__pydantic_fields_set__.update(upstream_extras)
def _holds_only_scalars(model: BaseModel) -> bool:
fields: Final[Iterator[tuple[str, object]]] = iter(model)
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
def _delta_from_upstream(upstream_delta: BaseModel | None) -> Delta | None:
def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None:
if upstream_delta is None:
return Delta()
if not isinstance(upstream_delta, BaseModel):
return None
try:
return Delta(**dict(iter(upstream_delta)))
except Exception:
@ -189,6 +192,33 @@ class _DeltaDumpDict(TypedDict):
tool_calls: NotRequired[Sequence[Mapping[str, object]]]
class _UpstreamFunctionCallLike(Protocol):
arguments: str | None
class _UpstreamToolCallLike(Protocol):
function: _UpstreamFunctionCallLike
class _UpstreamDeltaLike(Protocol):
function_call: _UpstreamFunctionCallLike | None
tool_calls: Sequence[_UpstreamToolCallLike] | None
def __iter__(self) -> Iterator[tuple[str, object]]: ...
def model_dump(self) -> _DeltaDumpDict: ...
class _UpstreamChoiceLike(Protocol):
delta: _UpstreamDeltaLike | None
class _UpstreamChunkLike(Protocol):
id: str
choices: Sequence[_UpstreamChoiceLike]
system_fingerprint: str | None
class _TextCompletionChoiceLike(Protocol):
text: str
finish_reason: str | None
@ -920,7 +950,7 @@ class CustomStreamWrapper:
def copy_model_response_level_provider_specific_fields(
self,
original_chunk: ModelResponseStream | OpenAIChatCompletionChunk,
original_chunk: object,
model_response: ModelResponseStream,
) -> ModelResponseStream:
"""
@ -949,8 +979,8 @@ class CustomStreamWrapper:
or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
or (
"tool_calls" in model_response.choices[0].delta
and model_response.choices[0].delta["tool_calls"] is not None
and len(model_response.choices[0].delta["tool_calls"]) > 0
and model_response.choices[0].delta.tool_calls is not None
and len(model_response.choices[0].delta.tool_calls) > 0
)
or (
"function_call" in model_response.choices[0].delta
@ -1115,7 +1145,8 @@ class CustomStreamWrapper:
## check if openai/azure chunk
original_chunk: Final = response_obj.get("original_chunk", None)
if original_chunk:
if len(original_chunk.choices) > 0:
upstream_chunk: Final[_UpstreamChunkLike] = original_chunk
if len(upstream_chunk.choices) > 0:
original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access
object,
original_chunk,
@ -1151,7 +1182,7 @@ class CustomStreamWrapper:
)
else:
choices: Final = []
for choice in original_chunk.choices:
for choice in upstream_chunk.choices:
try:
if isinstance(choice, BaseModel):
choice_json = choice.model_dump()
@ -1164,11 +1195,11 @@ class CustomStreamWrapper:
setattr(model_response, "choices", choices)
else:
return
model_response.system_fingerprint = original_chunk.system_fingerprint
model_response.system_fingerprint = upstream_chunk.system_fingerprint
setattr(
model_response,
"citations",
stored_attribute(original_chunk, "citations"),
stored_attribute(upstream_chunk, "citations"),
)
preserve_upstream_non_openai_attributes(
model_response=model_response,
@ -1611,7 +1642,9 @@ class CustomStreamWrapper:
model_response.model = self.model
## FUNCTION CALL PARSING
original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None
original_chunk: Final[_UpstreamChunkLike | None] = (
response_obj.get("original_chunk") if response_obj is not None else None
)
if (
original_chunk is not None
): # function / tool calling branch - only set for openai/azure compatible endpoints
@ -1623,31 +1656,19 @@ class CustomStreamWrapper:
original_chunk, model_response
)
if original_chunk.choices and len(original_chunk.choices) > 0:
delta = original_chunk.choices[0].delta
delta: Final = original_chunk.choices[0].delta
if delta is not None and (delta.function_call is not None or delta.tool_calls is not None):
try:
model_response.system_fingerprint = original_chunk.system_fingerprint
## AZURE - check if arguments is not None
if original_chunk.choices[0].delta.function_call is not None:
if (
getattr(
original_chunk.choices[0].delta.function_call,
"arguments",
)
is None
):
original_chunk.choices[0].delta.function_call.arguments = ""
elif original_chunk.choices[0].delta.tool_calls is not None:
if isinstance(original_chunk.choices[0].delta.tool_calls, list):
for t in original_chunk.choices[0].delta.tool_calls:
if hasattr(t, "functions") and hasattr(t.functions, "arguments"):
if (
getattr(
t.function,
"arguments",
)
is None
):
if delta.function_call is not None:
if delta.function_call.arguments is None:
delta.function_call.arguments = ""
elif delta.tool_calls is not None:
if isinstance(delta.tool_calls, list):
for t in delta.tool_calls:
if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"):
if t.function.arguments is None:
t.function.arguments = ""
_json_delta: Final[_DeltaDumpDict] = delta.model_dump()
if "role" not in _json_delta or _json_delta["role"] is None:
@ -1886,7 +1907,7 @@ class CustomStreamWrapper:
else:
asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit))
## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler
litellm_params: Final = self.logging_obj.model_call_details.get("litellm_params", {})
litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {})
if self.logging_obj.is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
@ -1935,11 +1956,17 @@ class CustomStreamWrapper:
def _propagate_usage_cost_to_hidden_params(
response: "ModelResponse",
custom_llm_provider: str | None,
chunks: Sequence[object] = (),
) -> None:
"""The builder's usage only keeps declared fields, so provider blocks like usage.opencode are read off the chunks."""
if custom_llm_provider not in _USAGE_COST_HEADER_PROVIDERS:
return
_usage: Final[Usage | None] = getattr(response, "usage", None)
_cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None))
usages: Final = (
getattr(response, "usage", None),
*(chunk_usage(chunk) for chunk in reversed(chunks)),
)
costs: Final = (provider_reported_usage_cost(usage, custom_llm_provider) for usage in usages)
_cost: Final = next((cost for cost in costs if cost is not None), None)
if _cost is not None:
if "additional_headers" not in response.hidden_params:
response.hidden_params["additional_headers"] = {}
@ -2081,7 +2108,9 @@ class CustomStreamWrapper:
complete_streaming_response: Final = self._build_complete_streaming_response()
response = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
self._propagate_usage_cost_to_hidden_params(
complete_streaming_response, self.custom_llm_provider, self.chunks
)
setattr(
response,
@ -2300,7 +2329,9 @@ class CustomStreamWrapper:
complete_streaming_response: Final = await self._async_build_complete_streaming_response()
response: Final = self._new_usage_chunk()
if complete_streaming_response is not None:
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
self._propagate_usage_cost_to_hidden_params(
complete_streaming_response, self.custom_llm_provider, self.chunks
)
setattr(
response,

4
litellm/litellm_core_utils/token_counter.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
@ -247,7 +247,7 @@ def get_image_dimensions(
client: Final = get_httpx_client()
response: Final[httpx.Response] = safe_get(client, data)
max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
content_length: Final[str | None] = response.headers.get("Content-Length")
content_length: Final[str | None] = header_value(response.headers, "Content-Length")
if content_length is not None and int(content_length) > max_bytes:
pass # skip download; img_data stays None
else:

Some files were not shown because too many files have changed in this diff Show more