scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-27 06:13:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 72

refactor(typing): clear 1181 basedpyright Any errors across 431 files (#47789)

Browse source 
* refactor(typing): narrow Any sources across providers, proxy and guardrails (wave 1)

* refactor(typing): narrow Any sources across hubs, providers and proxy (wave 2)

* refactor(typing): validate RunwayML task JSON and let dotprompt loaders accept exported prompts

* fix: restore pass-through behavior on paths the typing pass tightened

MCP client IP extraction trusts X-Forwarded-For only when use_x_forwarded_for is True, dotprompt skips a malformed JSON prompt entry instead of raising, RunwayML task payloads pass through unvalidated, and a malformed provider body raises the same AttributeError as before at the CompactifAI, Gemini, and mock transport parse sites

* fix(runwayml): keep a malformed 2xx task body on the 500 path

* fix(edenai,cli): keep a non-object body in the error and fail on a malformed migration report

EdenAI's embedding and image generation parse dropped a non-object body from the error
they raised, so the message no longer named what the provider sent. A helper now raises
with the body in the message, and both paths have a regression test.

`lite encryption migrate` exited 0 with no message when the proxy's report was not an
object or its residual_legacy was not a number, where base crashed. It now fails with a
message naming the value.

The CompactifAI and Gemini image-edit non-object body tests accept TypeError or
AttributeError, which both base and this branch raise, so they guard against a
ValidationError instead of pinning one class. RunwayML's async parse gets the same
non-object body test as the sync one.

* fix: keep base's error classes, duck-typed checks and fail-closed guardrails the typing pass changed

* test: annotate the new regression tests' locals with Final

* test: annotate the remaining new test locals with Final
This commit is contained in:
Mateo Wang 2026-10-24 11:31:09 -07:00 • committed by GitHub
parent 0820645690
commit 464932b1d1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
455 changed files with 3670 additions and 1546 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
enterprise/enterprise_hooks/aporia_ai.py
View file

@ -15,9 +15,10 @@ sys.path.insert(
) # Adds the parent directory to the system path
import json
import sys
from typing import Any, List, Literal, Optional
from typing import Final, List, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
GUARDRAIL_NAME = "aporia"
_DECODED_JSON: Final = TypeAdapter(object)
class AporiaGuardrail(CustomGuardrail):
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
async def prepare_aporia_request(
self, new_messages: List[dict], response_string: Optional[str] = None
) -> dict:
data: dict[str, Any] = {}
data: dict[str, object] = {}
if new_messages is not None:
data["messages"] = new_messages
if response_string is not None:
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
if response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
action: str = _json_response.get(
_json_response: Final = decoded_json_object(
_DECODED_JSON.validate_python(response.json())
)
action: Final = _json_response.get(
"action"
) # possible values are modify, passthrough, block, rephrase
if action == "block":

17
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -7,9 +7,10 @@
## This accepts a list of user id's for whom calls will be rejected
from typing import Literal, Optional
from typing import Final, Literal, Optional
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._internal_context import with_service_target
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import EndUserRepository
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_BlockedUserList(CustomLogger):
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
)
if end_user_cache_obj is None and self.prisma_client is not None:
# check db
end_user_obj = (
await self.prisma_client.db.litellm_endusertable.find_unique(
where={"user_id": user}
)
)
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
end_user_obj = await EndUserRepository(
self.prisma_client
).table.find_unique(where={"user_id": end_user_id})
if end_user_obj is None: # user not in db - assume not blocked
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
if end_user_obj is not None and end_user_obj.blocked is True:
if end_user_obj.blocked is True:
raise HTTPException(
status_code=400,
detail={

13
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -6,7 +6,10 @@
# +-----------------------------------------------+
# Thank you users! We ❤️ you! - Krrish & Ishaan
from typing import Final
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
f"{category}_confidence_threshold",
default_confidence_threshold,
)
set_confidence_value = getattr(
self,
f"{category}_confidence_threshold",
set_confidence_value = _THRESHOLD_VALUE.validate_python(
getattr(
self,
f"{category}_confidence_threshold",
)
)
verbose_proxy_logger.info(
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"

7
enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py
View file

@ -1,15 +1,20 @@
# this is an example endpoint to receive data from litellm
from typing import Final
from fastapi import FastAPI, HTTPException, Request
from pydantic import TypeAdapter
app = FastAPI()
_LOGGED_EVENT: Final = TypeAdapter(object)
@app.post("/log-event")
async def log_event(request: Request):
try:
print("Received /log-event request") # noqa
# Assuming the incoming request has JSON data
data = await request.json()
data: Final = _LOGGED_EVENT.validate_python(await request.json())
print("Received request data:") # noqa
print(data) # noqa

18
enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py
View file

@ -8,25 +8,30 @@
## This provides an LLM Guard Integration for content moderation on the proxy
import asyncio
from typing import Final, Optional
from collections.abc import Mapping
from typing import Final
import aiohttp
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.openai.common_utils import decoded_json_object
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import CallTypesLiteral
_DECODED_JSON: Final = TypeAdapter(object)
class _ENTERPRISE_LLMGuard(CustomLogger):
# Class variables or attributes
def __init__(
self,
mock_testing: bool = False,
mock_redacted_text: Optional[dict] = None,
mock_redacted_text: Mapping[str, object] | None = None,
):
self.mock_redacted_text = mock_redacted_text
self.llm_guard_mode = litellm.llm_guard_mode
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
async with session.post(
analyze_url, json={"prompt": text}
) as response:
redacted_text = await response.json()
redacted_text = _DECODED_JSON.validate_python(
await response.json()
)
verbose_proxy_logger.debug(
f"LLM Guard: Received response - {redacted_text}"
)
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
"error": f"Invalid content moderation response: {redacted_text}"
},
)
if redacted_text.get("is_valid", None) is False:
redacted_response: Final = decoded_json_object(redacted_text)
if redacted_response.get("is_valid", None) is False:
raise HTTPException(
status_code=400,
detail={"error": "Violated content safety policy"},
)
sanitized_prompt = redacted_text.get("sanitized_prompt")
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
except Exception as e:
verbose_proxy_logger.exception(

4
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -1040,8 +1040,8 @@ class CheckBatchCost:
from litellm.proxy.proxy_server import proxy_config
from litellm.secret_managers.main import str_to_bool
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
api_key: Final = job.api_key if hasattr(job, "api_key") else None
team_id: Final = job.team_id if hasattr(job, "team_id") else None
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
try:
key_row: Final = (

27
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -44,6 +44,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -133,7 +134,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
class _SupportsKeyedGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return holders if holders or "input_file_id" in request else healthy_deployments
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
) -> Optional[dict]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
# model_info may be at top-level or nested under litellm_metadata
# (batch/file operations use litellm_metadata)
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
)
for accessor_key in accessor_keys:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
)
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
"""
Get model-specific file IDs for a list of proxy file IDs.
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Optional[Router],
_create_file_request: CreateFileRequest,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> List[OpenAIFileObject]:
if llm_router is None:
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
create_file_request: CreateFileRequest,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_objects: List[OpenAIFileObject],
create_file_request: CreateFileRequest,
internal_usage_cache: InternalUsageCache,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
target_model_names_list: List[str],
) -> OpenAIFileObject:
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Continue with other files even if one fails
continue
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
def _get_content_type_from_file_object(self, file_object: object) -> str:
"""
Determine content type from file object.

6
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.proxy.utils import PrismaClient as _PrismaClient
Span = Union[_Span, Any]
Span = _Span
InternalUsageCache = _InternalUsageCache
PrismaClient = _PrismaClient
else:
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
llm_router: Router,
model: str,
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
) -> VectorStoreCreateResponse:
"""
Create a vector store for a specific model.
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
create_request: VectorStoreCreateOptionalRequestParams,
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
) -> VectorStoreCreateResponse:
"""

13
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
@ -73,6 +74,16 @@ def _verification_token_table(
return VerificationTokenRepository(prisma_client).table
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
table_name = "litellm_verificationtoken"
def _writer_verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
@ -849,7 +860,7 @@ async def update_project(
else None
)
if current_project is not None and data.team_id != current_project.team_id:
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
where={
"project_id": data.project_id,
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],

8
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -14,6 +14,7 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -37,6 +38,7 @@ if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
class ManagedVectorStoreRow(Protocol):
@ -348,11 +350,11 @@ async def update_vector_store(
raise HTTPException(status_code=500, detail="Database not connected")
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id: Final[str] = update_data.pop("vector_store_id")
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
vector_store_id: Final = data.vector_store_id
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
)
updated = await managed_vector_store_table(prisma_client).update(

16
litellm/__init__.py
View file

@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
argilla_transformation_object: Optional[Dict[str, Any]] = None
argilla_transformation_object: Mapping[str, str] | None = None
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
proxy_auth: _ProxyAuthHandlerType | None = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
# New format: { "displayName": { "url": "...", "index": 0 } }
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
@ -2289,6 +2289,10 @@ if TYPE_CHECKING:
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
from collections.abc import Awaitable
from litellm.types.utils import ModelResponse as _ModelResponseType
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -2312,7 +2316,7 @@ if TYPE_CHECKING:
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
@ -2610,8 +2614,10 @@ def __getattr__(name: str) -> Any:
cancel_run,
delete_run,
)
from types import MappingProxyType
return locals()[name]
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

4
litellm/_lazy_imports.py
View file

@ -19,7 +19,7 @@ import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = cast(Any, "litellm_module_level_client")
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,

20
litellm/a2a_protocol/cost_calculator.py
View file

@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
to define custom costs per agent query or per token.
"""
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
from typing_extensions import Buffer
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -15,6 +18,15 @@ else:
LitellmLoggingObject = Any
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is None:
return None
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
return float(response_cost)
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
class A2ACostCalculator:
@staticmethod
def calculate_a2a_cost(
@ -45,9 +57,9 @@ class A2ACostCalculator:
model_call_details: Final = litellm_logging_obj.model_call_details
# Check if user set a custom response cost (backward compatibility)
response_cost: Final = model_call_details.get("response_cost", None)
if response_cost is not None:
return float(response_cost)
custom_response_cost: Final = _custom_response_cost(model_call_details)
if custom_response_cost is not None:
return custom_response_cost
# Get litellm_params for cost parameters
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}

27
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py
View file

@ -84,14 +84,23 @@ def _filter_reserved_headers(
return filtered or None
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
if principal is None or isinstance(principal, str):
return principal
if not principal:
return None
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
def _request_scoped_runtime_session_id(
params: Mapping[str, object],
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str | None:
context_id: Final = get_session_id_from_a2a_params(params)
if not isinstance(context_id, str) or not context_id:
return None
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
def _validate_runtime_session_id(session_id: str, model: str) -> str:
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
)
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
model: Final = litellm_params.get("model", "")
if isinstance(model, str):
return model
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model: Final = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
model: Final = _agentcore_model_name(litellm_params)
agentcore_model: Final = model.removeprefix("bedrock/")
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}

40
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
"""
import asyncio
from collections.abc import AsyncIterator, Mapping
from typing import Any, Final
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from itertools import chain
from typing import Final
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.llms.openai.common_utils import decoded_json_object
def _a2a_text_part(part: object) -> Sequence[object]:
part_fields: Final = as_str_mapping(part)
if not isinstance(part_fields, dict):
return ()
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
return (part_fields["text"],)
return ()
def _join_with_spaces(texts: Sequence[object]) -> str:
strings: Final = [text for text in texts if isinstance(text, str)]
if len(strings) == len(texts):
return " ".join(strings)
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
class WatsonxOrchestrateTransformation:
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message: Final = params.get("message", {})
parts: Final = message.get("parts", [])
texts: Final = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
message: Final = decoded_json_object(params.get("message", {}))
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
return _join_with_spaces(texts) or ""
@staticmethod
def build_wxo_run_body(

21
litellm/a2a_protocol/utils.py
View file

@ -4,15 +4,20 @@ Utility functions for A2A protocol.
import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
class A2ARequestUtils:
"""Utility class for A2A request/response processing."""
@ -74,7 +79,7 @@ class A2ARequestUtils:
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
) -> Any:
) -> object:
"""
Extract the input message from an A2A request.
@ -84,7 +89,7 @@ class A2ARequestUtils:
Returns:
The message object/dict or None
"""
params: Final = getattr(request, "params", None)
params: Final[object] = getattr(request, "params", None)
if params is None:
return None
return getattr(params, "message", None)
@ -127,10 +132,12 @@ class A2ARequestUtils:
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message = A2ARequestUtils.get_input_message_from_request(request)
if input_message is not None and hasattr(input_message, "model_dump"):
input_message = input_message.model_dump(mode="json")
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
# Count output tokens

13
litellm/anthropic_beta_headers_manager.py
View file

@ -45,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
_DECODED_JSON: Final = TypeAdapter(object)
class GetAnthropicBetaHeadersConfig:
@ -76,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
}
@staticmethod
def _check_is_valid_dict(fetched_config: dict) -> bool:
def _check_is_valid_dict(fetched_config: object) -> bool:
"""Check if fetched config is a non-empty dict with expected structure."""
if not isinstance(fetched_config, dict):
verbose_logger.warning(
@ -110,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
return True
@classmethod
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
"""
Validate the integrity of a fetched beta headers config.
@ -119,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
return cls._check_is_valid_dict(fetched_config)
@staticmethod
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
"""
Fetch the beta headers config from a remote URL.
@ -128,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
"""
response: Final = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return _DECODED_JSON.validate_python(response.json())
def get_beta_headers_config(url: str) -> dict:
@ -161,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
# Validate the fetched config
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
content, dict
):
verbose_logger.warning(
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
url,

11
litellm/anthropic_interface/exceptions/exception_mapping_utils.py
View file

@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
import json
from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
# HTTP status code -> Anthropic error type
# Source: https://docs.anthropic.com/en/api/errors
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
- Generic: {"message": "..."}
- Plain strings
"""
parsed: Final = safe_json_loads(raw_message)
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
if isinstance(parsed, dict):
# Bedrock format
if "detail" in parsed and isinstance(parsed["detail"], dict):
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
AnthropicErrorResponse dict
"""
# Try to parse as JSON once
parsed: dict | None = safe_json_loads(raw_message)
if not isinstance(parsed, dict):
parsed = None
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
# If parsed and already in Anthropic format - passthrough
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):

24
litellm/assistants/main.py
View file

@ -77,7 +77,7 @@ def get_assistants(
api_base: str | None = None,
api_version: str | None = None,
**kwargs,
) -> SyncCursorPage[Assistant]:
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
if aget_assistants is not None and not isinstance(aget_assistants, bool):
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
@ -100,7 +100,7 @@ def get_assistants(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[Assistant] | None = None
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -564,7 +564,7 @@ def create_thread(
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
client: OpenAI | None = None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""
- get the llm provider
- if openai - route it there
@ -610,7 +610,7 @@ def create_thread(
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -735,7 +735,7 @@ def get_thread(
thread_id: str,
client=None,
**kwargs,
) -> Thread:
) -> Thread | Coroutine[None, None, Thread]:
"""Get the thread object, given a thread_id"""
aget_thread: Final = kwargs.pop("aget_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -757,7 +757,7 @@ def get_thread(
timeout = 600.0
api_base: str | None = None
api_key: str | None = None
response: Thread | None = None
response: Thread | Coroutine[None, None, Thread] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -903,7 +903,7 @@ def add_message(
metadata: dict | None = None,
client=None,
**kwargs,
) -> OpenAIMessage:
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
### COMMON OBJECTS ###
a_add_message: Final = kwargs.pop("a_add_message", None)
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
@ -935,7 +935,7 @@ def add_message(
timeout = 600.0
api_key: str | None = None
api_base: str | None = None
response: OpenAIMessage | None = None
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@ -1065,7 +1065,7 @@ def get_messages(
thread_id: str,
client: Any | None = None,
**kwargs,
) -> SyncCursorPage[OpenAIMessage]:
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
aget_messages: Final = kwargs.pop("aget_messages", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
litellm_params_dict: Final = get_litellm_params(**kwargs)
@ -1086,7 +1086,7 @@ def get_messages(
elif timeout is None:
timeout = 600.0
response: SyncCursorPage[OpenAIMessage] | None = None
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
api_key: str | None = None
api_base: str | None = None
if custom_llm_provider == "openai":
@ -1256,7 +1256,7 @@ def run_thread(
client: object | None = None,
event_handler: AssistantEventHandler | None = None, # for stream=True calls
**kwargs,
) -> Run:
) -> Run | Coroutine[None, None, Run]:
"""Run a given thread + assistant."""
arun_thread: Final = kwargs.pop("arun_thread", None)
optional_params: Final = GenericLiteLLMParams(**kwargs)
@ -1278,7 +1278,7 @@ def run_thread(
elif timeout is None:
timeout = 600.0
response: Run | None = None
response: Run | Coroutine[None, None, Run] | None = None
if custom_llm_provider == "openai":
api_base = (
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there

6
litellm/caching/_embedding_router.py
View file

@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
@ -25,12 +25,12 @@ if TYPE_CHECKING:
def resolve_embedding_router(
embedding_model: str,
llm_router: Router | None,
llm_model_list: list[dict[str, Any]] | None,
llm_model_list: Sequence[Mapping[str, object]] | None,
) -> Router | None:
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
if llm_router is None:
return None
router_model_names: Final[list[str]] = (
router_model_names: Final[Sequence[object]] = (
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
)
if embedding_model in router_model_names:

8
litellm/caching/azure_blob_cache.py
View file

@ -13,10 +13,14 @@ import json
from contextlib import suppress
from typing import Final
from pydantic import TypeAdapter
from litellm._logging import print_verbose, verbose_logger
from .base_cache import BaseCache
_DECODED_BLOB: Final = TypeAdapter(object)
class AzureBlobCache(BaseCache):
def __init__(self, account_url, container) -> None:
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
print_verbose(f"Get Azure Blob Cache: key: {key}")
as_bytes: Final = self.container_client.download_blob(key).readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
blob: Final = await self.async_container_client.download_blob(key)
as_bytes: Final = await blob.readall()
as_str: Final = as_bytes.decode("utf-8")
cached_response: Final[object] = json.loads(as_str)
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
verbose_logger.debug(
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
key,

2
litellm/caching/base_cache.py
View file

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

8
litellm/caching/disk_cache.py
View file

@ -1,15 +1,19 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = json.loads(original_cached_response)
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response

44
litellm/cost_calculator.py
View file

@ -1200,12 +1200,16 @@ def get_usage_object(
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
return Usage(**usage_obj.model_dump())
return _usage_from_model(usage_obj)
else:
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
return None
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
match usage:
case DecisionsUsage():
@ -1686,7 +1690,7 @@ def completion_cost(
setattr(
completion_response,
"usage",
litellm.Usage(**_usage_for_dump.model_dump()),
_usage_from_model(_usage_for_dump),
)
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
_usage = _decisions_usage(usage_obj).model_dump()
@ -3035,8 +3039,11 @@ def _batch_prompt_cost(
)
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
@ -3671,7 +3678,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
return None
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
return info.get("output_cost_per_token")
case _:
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3680,12 +3706,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
if declared is None:
return None
return next(
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
(
float(value)
for key in keys
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
),
None,
)
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)

2
litellm/exceptions.py
View file

@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.message = _saved_message
self.args = (_saved_message,)
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
if original_headers is not None:
self.litellm_response_headers = original_headers

20
litellm/experimental_mcp_client/tools.py
View file

@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
from openai.types.chat import ChatCompletionToolParam
from openai.types.responses.function_tool_param import FunctionToolParam
from openai.types.shared_params.function_definition import FunctionDefinition
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import (
@ -20,6 +21,8 @@ from litellm.constants import (
from litellm.types.llms.anthropic import AnthropicMessagesTool
from litellm.types.utils import ChatCompletionMessageToolCall
_DECODED_JSON: Final = TypeAdapter(object)
########################################################
# List MCP Tool functions
@ -200,15 +203,18 @@ async def call_mcp_tool(
return tool_result
def _decode_json_arguments(arguments: str) -> object:
try:
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}
def _get_function_arguments(function: FunctionDefinition) -> dict:
"""Helper to safely get and parse function arguments."""
arguments = function.get("arguments", {})
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except json.JSONDecodeError:
arguments = {}
return arguments if isinstance(arguments, dict) else {}
arguments: Final = function.get("arguments", {})
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
return decoded if isinstance(decoded, dict) else {}
def transform_openai_tool_call_request_to_mcp_tool_call_request(

14
litellm/files/main.py
View file

@ -9,7 +9,7 @@ import asyncio
import contextvars
import time
import uuid as uuid_module
from collections.abc import Coroutine
from collections.abc import Awaitable, Coroutine
from functools import partial
from typing import Final, Literal, cast
@ -293,6 +293,14 @@ def create_file(
raise e
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
return await pending
async def _await_deleted_file(pending: Awaitable[object]) -> object:
return await pending
@client
async def afile_retrieve(
file_id: str,
@ -325,7 +333,7 @@ async def afile_retrieve(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_retrieved_file(init_response)
else:
response = init_response
@ -505,7 +513,7 @@ async def afile_delete(
func_with_context: Final = partial(ctx.run, func)
init_response: Final = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
response = await _await_deleted_file(init_response)
else:
response = init_response

4
litellm/files/streaming.py
View file

@ -1,7 +1,7 @@
import datetime
import traceback
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Any, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, cast
import anyio
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
merged_hidden_params: Final = cast(
"StandardLoggingHiddenParams",
{
**cast(dict[str, Any], payload.get("hidden_params") or {}),
**(payload.get("hidden_params") or {}),
**self._hidden_params,
},
)

3
litellm/harness/handlers/base.py
View file

@ -4,7 +4,6 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
from litellm.harness.context import SessionContext
from litellm.harness.errors import CapabilityUnsupported
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
"""Continue the runtime's own session on the next turn."""
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")

4
litellm/harness/handlers/cli_handler.py
View file

@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
async def private_dir_for(sandbox: Sandbox) -> str:
tempdir = getattr(sandbox, "tempdir", None)
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
if tempdir is None:
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
path: str = await tempdir()
path: Final = await tempdir()
return path

2
litellm/harness/handlers/deepagents_handler.py
View file

@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
async def history(
self, ctx: SessionContext
) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
agent, deps = self._require_agent()
snapshot = await agent.aget_state(self._run_config(ctx, None))
messages = (snapshot.values or MappingProxyType({})).get("messages") or ()

11
litellm/harness/runtime.py
View file

@ -12,11 +12,10 @@ import inspect
import logging
import os
import uuid
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
Any,
Final,
get_args,
)
@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
queue.put_nowait(end)
async def _await_decision(pending: Awaitable[object]) -> object:
return await pending
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
try:
if inspect.iscoroutinefunction(handler):
decision: object = await handler(approval)
decision: object = await _await_decision(handler(approval))
else:
decision = await asyncio.to_thread(handler, approval)
if inspect.isawaitable(decision):
@ -749,7 +752,7 @@ class AsyncSession:
async def history(
self,
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
if not self.harness_config.capabilities.history:
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
await self._ensure_ready()

11
litellm/images/main.py
View file

@ -3,7 +3,7 @@ import contextvars
import importlib
from collections.abc import Coroutine
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, overload
from pydantic import TypeAdapter, ValidationError
@ -70,16 +70,17 @@ from litellm.utils import (
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
"""Get ImageEditRequestUtils, loading it lazily if needed."""
global _ImageEditRequestUtils_cache
if _ImageEditRequestUtils_cache is None:
# Access via module to trigger __getattr__ if not cached
module: Final = importlib.import_module(__name__)
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
from litellm.images import main as images_main
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
return _ImageEditRequestUtils_cache

14
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -115,14 +115,16 @@ class AlertingHangingRequestCheck:
if hanging_request_data.alerted:
continue
request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
# this means the request status was either success or fail
# and is not hanging
if request_status is not None:
if (
await proxy_logging_obj.internal_usage_cache.async_get_cache(
key=f"request_status:{hanging_request_data.request_id}",
litellm_parent_otel_span=None,
local_only=True,
)
is not None
):
# clear this request from hanging request cache since the request was either success or failed
self.hanging_request_cache._remove_key(
key=request_id,

5
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -56,8 +56,9 @@ async def fetch_user_spend_rows(
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:

23
litellm/integrations/agentops/agentops.py
View file

@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
import os
from dataclasses import dataclass
from typing import Any, Final
from typing import Final
from pydantic import TypeAdapter
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import get_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
@dataclass
class AgentOpsConfig:
@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry):
project_id = None
if config.api_key:
try:
response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint)
jwt_token = response.get("token")
project_id = response.get("project_id")
response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint))
if response is not None:
jwt_token = response.get("token")
project_id = response.get("project_id")
except Exception:
pass
@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry):
"telemetry.sdk.name": "agentops",
}
if project_id:
resource_attrs["project.id"] = project_id
self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs
self.resource_attributes = resource_attrs
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]:
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object:
"""
Fetch JWT authentication token from AgentOps API
@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry):
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return response.json()
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()

2
litellm/integrations/arize/arize.py
View file

@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
primary ``otel`` callback which handles proxy-level parent spans.
"""
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
def set_attributes(self, span: Span, kwargs, response_obj: object):
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
@staticmethod

29
litellm/integrations/braintrust_logging.py
View file

@ -3,9 +3,10 @@
import os
from datetime import datetime
from typing import Final
from typing import Final, Protocol
import httpx
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm import verbose_logger
@ -24,6 +25,18 @@ from litellm.utils import print_verbose
API_BASE: Final = "https://api.braintrustdata.com/v1"
class _BraintrustProjectBody(TypedDict):
id: ReadOnly[str]
class _BraintrustProjectResponse(Protocol):
def json(self) -> _BraintrustProjectBody: ...
def _project_id(response: _BraintrustProjectResponse) -> str:
return response.json()["id"]
def get_utc_datetime():
import datetime as dt
@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger):
headers=self.headers,
json={"name": project_name},
)
project_dict: Final = response.json()
project_id: Final = project_dict["id"]
project_id: Final = _project_id(response)
self._project_id_cache[project_name] = project_id
return project_id
except httpx.HTTPStatusError as e:
@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger):
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def create_sync_default_project_and_experiment(self):
project: Final = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
project_dict: Final = project.json()
self.default_project_id = project_dict["id"]
self.default_project_id = _project_id(project)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")

7
litellm/integrations/cloudzero/cz_stream_api.py
View file

@ -19,8 +19,9 @@
"""Output modules for writing CBF data to various destinations."""
import zoneinfo
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
import polars as pl
@ -188,7 +189,7 @@ class CloudZeroStreamer:
)
raise
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]:
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]:
"""Prepare batch payload according to CloudZero AnyCost API format."""
# Convert batch_date to month for the API (YYYY-MM format)
try:
@ -205,7 +206,7 @@ class CloudZeroStreamer:
if record:
data_records.append(record)
payload: Final = {"month": month_str, "operation": operation, "data": data_records}
payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records}
return payload

7
litellm/integrations/custom_guardrail.py
View file

@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
self,
violation_message: str,
request_data: dict[str, Any],
detection_info: dict[str, Any] | None = None,
detection_info: Mapping[str, object] | None = None,
original_response: object = None,
) -> None:
"""
@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger):
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
guardrail_mode = GuardrailMode(
tags=copy.deepcopy(self.event_hook.tags),
default=copy.deepcopy(self.event_hook.default),
)
else:
guardrail_mode = self.event_hook

3
litellm/integrations/custom_secret_manager.py
View file

@ -37,7 +37,6 @@ Usage:
"""
from abc import abstractmethod
from typing import Any
import httpx
@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager):
optional_params: dict | None = None,
timeout: float | httpx.Timeout | None = None,
tags: dict | list | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Asynchronously write a secret to your custom secret manager.

4
litellm/integrations/dotprompt/__init__.py
View file

@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
prompt_file: Final = getattr(litellm_params, "prompt_file", None)
# Handle dotprompt_content from database
dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None)
dotprompt_content: Final = (
litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None
)
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content)

15
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -7,6 +7,8 @@ import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing_extensions import ReadOnly, TypedDict
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.types.llms.openai import AllMessageValues
@ -18,7 +20,12 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
from .prompt_manager import PromptManager, PromptTemplate
from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate
class _PromptJsonData(TypedDict, total=False):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
class DotpromptManager(CustomPromptManagement):
@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement):
if self._prompt_manager:
self._prompt_manager.reload_prompts()
def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None:
def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None:
"""Add a prompt from JSON data."""
content: Final = json_data.get("content", "")
metadata: Final = json_data.get("metadata", {})
self.prompt_manager.add_prompt(prompt_id, content, metadata)
def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None:
def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None:
"""Load multiple prompts from JSON data."""
self.prompt_manager.load_prompts_from_json_data(prompts_data)
def get_prompts_as_json(self) -> dict[str, dict[str, object]]:
def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all prompts in JSON format."""
return self.prompt_manager.get_all_prompts_as_json()

29
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
class _PromptFileJson(TypedDict):
"""JSON form of a .prompt file: rendered template text plus its frontmatter."""
@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict):
metadata: ReadOnly[NotRequired[dict[str, object]]]
class PromptJsonEntry(TypedDict):
content: ReadOnly[str]
metadata: ReadOnly[dict[str, object]]
def strip_version_suffix(prompt_id: str) -> str | None:
base, separator, version = prompt_id.rpartition(".v")
if separator and base and version.isdigit():
@ -358,20 +365,26 @@ class PromptManager:
return f"---\n{frontmatter_yaml}---\n{content}"
def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]:
def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]:
"""Get all loaded prompts in JSON format.
Returns:
Dictionary mapping prompt_id to prompt data
"""
result: Final = {}
for prompt_id, template in self.prompts.items():
result[prompt_id] = {
return {
prompt_id: {
"content": template.content,
"metadata": template.metadata,
}
return result
for prompt_id, template in self.prompts.items()
}
def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts)."""
self._load_prompts_from_json(prompt_data)
def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None:
"""Load additional prompts from JSON data (merges with existing prompts). Entries that are not
mappings are skipped, the same as a malformed entry inside the loader."""
entries: Final = {
prompt_id: dict(entry)
for prompt_id, raw in prompt_data.items()
if (entry := as_str_mapping(raw)) is not None
}
self._load_prompts_from_json(entries)

5
litellm/integrations/focus/destinations/factory.py
View file

@ -3,6 +3,7 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from typing import Any, Final
from .base import FocusDestination
@ -20,7 +21,7 @@ class FocusDestinationFactory:
*,
provider: str,
prefix: str,
config: dict[str, Any] | None = None,
config: Mapping[str, object] | None = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower: Final = provider.lower()
@ -39,7 +40,7 @@ class FocusDestinationFactory:
def _resolve_config(
*,
provider: str,
overrides: dict[str, object],
overrides: Mapping[str, object],
) -> dict[str, Any]:
if provider == "s3":
resolved = {

8
litellm/integrations/focus/export_engine.py
View file

@ -2,8 +2,8 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast
from collections.abc import Callable, Mapping
from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast
import polars as pl
@ -24,7 +24,7 @@ class FocusExportEngine:
provider: str,
export_format: str,
prefix: str,
destination_config: dict[str, Any] | None = None,
destination_config: Mapping[str, object] | None = None,
) -> None:
self.provider = provider
self.export_format = export_format
@ -77,7 +77,7 @@ class FocusExportEngine:
return FocusParquetSerializer()
raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.")
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]:
async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]:
data: Final = await self.database.get_usage_data(limit=limit)
normalized: Final = self.transformer.transform(data)

8
litellm/integrations/focus/focus_logger.py
View file

@ -6,6 +6,8 @@ import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, ClassVar, Final, cast
from typing_extensions import TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT: Final = 500
class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
pass
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
@ -40,7 +46,7 @@ class FocusLogger(CustomLogger):
interval_seconds: int | None = None,
prefix: str | None = None,
destination_config: dict[str, Any] | None = None,
**kwargs: Any,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()

8
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload
@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
"""
return f"{date_str}/batch-{batch_id}.ndjson"
def _get_config_key(self, kwargs: dict[str, Any]) -> str:
def _get_config_key(self, kwargs: Mapping[str, object]) -> str:
"""
Extract a synchronous grouping key from kwargs to group items by GCS config.
This allows us to batch items with the same bucket/credentials together.
@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key()
for logging purposes.
"""
standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {}
standard_callback_dynamic_params: Final = (
as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {}
)
bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default"
path_service_account: Final = (

3
litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py
View file

@ -9,6 +9,7 @@ Usage:
"""
import asyncio
import os
from collections.abc import AsyncIterable, Iterable
from typing import Final
@ -40,7 +41,7 @@ _mocks_initialized = False
# Default mock latency in seconds (simulates network round-trip)
# Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE
_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0
async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None):

8
litellm/integrations/gcs_pubsub/pub_sub.py
View file

@ -12,6 +12,8 @@ import os
import traceback
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
_DECODED_JSON: Final = TypeAdapter(object)
class GcsPubSubLogger(CustomBatchLogger):
def __init__(
@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger):
finally:
self.log_queue.clear()
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None:
async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object:
"""
Publish message to Google Cloud Pub/Sub using REST API
@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger):
raise Exception(f"Failed to publish message: {response.text}")
verbose_logger.debug("Pub/Sub response: %s", response.text)
return response.json()
return _DECODED_JSON.validate_python(response.json())
except Exception as e:
verbose_logger.error("Pub/Sub publish error: %s", str(e))

5
litellm/integrations/langfuse/langfuse.py
View file

@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import (
should_use_langfuse_mock,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
filter_exceptions_from_params,
reconstruct_model_name,
safe_deep_copy,
@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span(
*,
tracing: "LangfuseTracing",
parent: "LangfuseObservation",
enrichments: Mapping[str, Any],
enrichments: Mapping[str, object],
):
"""Logs provider-specific information as spans under the generation."""
_hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None)
_hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None))
if _hidden_params is None:
return

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
serialize_langfuse_datetime,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
@staticmethod
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj):
def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj):
"""
Sets Langfuse specific metadata attributes onto the OTEL span.
@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry):
from litellm.integrations.arize._utils import safe_set_attribute
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params"))
langfuse_environment: Final = (
dynamic_params.get("langfuse_environment") if dynamic_params else None
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")

12
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -2,6 +2,7 @@
Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
"""
from collections.abc import Mapping
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) ->
in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id)
def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]:
return {k: v for k, v in config.items() if k != "model"}
class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger):
def __init__(
self,
@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return compiled_prompt
def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict:
config: Final = langfuse_prompt_client.config
optional_params: Final = {}
for k, v in config.items():
if k != "model":
optional_params[k] = v
return optional_params
return dict(_params_without_model(langfuse_prompt_client.config))
async def async_get_chat_completion_prompt(
self,

11
litellm/integrations/langsmith.py
View file

@ -7,10 +7,11 @@ import random
import traceback
import types
from datetime import datetime, timezone
from typing import Any, Final
from typing import Final
import httpx
from pydantic import BaseModel, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa
_RUN_RESPONSE: Final = TypeAdapter(object)
class _DynamicParamsKwargs(TypedDict, total=False):
standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None]
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float:
def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float:
standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get(
"standard_callback_dynamic_params", None
)
@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger):
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject:
def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject:
"""
Handles key/team based logging

2
litellm/integrations/langtrace.py
View file

@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span | Any
Span = _Span
else:
Span = Any

2
litellm/integrations/levo/levo.py
View file

@ -13,7 +13,7 @@ if TYPE_CHECKING:
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = _Span | Any
Span = _Span
else:
Protocol = Any
OpenTelemetryConfig = Any

5
litellm/integrations/logfire_logger.py
View file

@ -3,8 +3,9 @@
import os
import traceback
from collections.abc import Mapping
from enum import Enum
from typing import Any, Final, NamedTuple
from typing import Final, NamedTuple
from typing_extensions import LiteralString
@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
class SpanConfig(NamedTuple):
message_template: LiteralString
span_data: dict[str, Any]
span_data: Mapping[str, object]
class LogfireLevel(str, Enum):

19
litellm/integrations/newrelic/newrelic.py
View file

@ -49,11 +49,12 @@ import time
import uuid
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.newrelic import NewRelicInitParams
@ -65,6 +66,10 @@ except ImportError:
_newrelic_agent = None
class _SupportsTimestamp(Protocol):
def timestamp(self) -> float: ...
class NewRelicLogger(CustomLogger):
"""
New Relic logger for LiteLLM to send AI monitoring events.
@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger):
verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e)
self.enabled = False
def _get_newrelic_params(self) -> dict:
def _get_newrelic_params(self) -> Mapping[str, object]:
"""
Get the newrelic_params from litellm.newrelic_params
These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging
"""
dict_newrelic_params: dict = {}
dict_newrelic_params: Mapping[str, object] = {}
if litellm.newrelic_params is not None:
if isinstance(litellm.newrelic_params, NewRelicInitParams):
dict_newrelic_params = litellm.newrelic_params.model_dump()
@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger):
return choices[0].get("finish_reason") or "unknown"
return "unknown"
def _to_epoch_ms(self, t: Any) -> float:
def _to_epoch_ms(self, t: datetime | float) -> float:
"""Convert a datetime or float timestamp to epoch milliseconds."""
if hasattr(t, "timestamp"):
if has_attributes(t, _SupportsTimestamp, "timestamp"):
return t.timestamp() * 1000.0
return float(t) * 1000.0
@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger):
sequence = 0
# Extract timestamps, preferring StandardLoggingPayload
start_time = None
start_time: datetime | float | None = None
if standard_logging_object:
start_time = standard_logging_object.get("startTime")
if not start_time:
start_time = kwargs.get("start_time")
end_time = None
end_time: datetime | float | None = None
if standard_logging_object:
end_time = standard_logging_object.get("endTime")
if not end_time:

2
litellm/integrations/newrelic/newrelic_metrics.py
View file

@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger):
if not self.log_queue:
return
if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
await asyncio.sleep(2**_pass)
await asyncio.sleep(1 << _pass)
async with self.flush_lock:
tried_ids: Final = frozenset(id(record) for record in attempted)
survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)

4
litellm/integrations/opik/opik_payload_builder/payload_builders.py
View file

@ -57,9 +57,7 @@ def build_span_payload(
span_id: Final = utils.create_uuid7()
model: Final[str] = response_obj.get("model", "unknown-model")
obj_type: Final[str] = response_obj.get("object", "unknown-object")
created: Final[int] = response_obj.get("created", 0)
span_name: Final = f"{model}_{obj_type}_{created}"
span_name: Final = f"{model}_{response_obj.get('object', 'unknown-object')}_{response_obj.get('created', 0)}"
_logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id)

3
litellm/integrations/otel/model/config.py
View file

@ -130,7 +130,8 @@ class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
return self._env_settings.get_field_value(field, field_name)
def __call__(self) -> dict[str, object]:
return {key: value for key, value in self._env_settings().items() if key != "excluded_services"}
settings: Final = self._env_settings()
return {key: settings[key] for key in settings if key != "excluded_services"}
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str
return (None if echoes_trace_id else explicit) or trace.session_id or None
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None:
"""Seconds from the upstream request being issued (``api_call_start_time``)
to the first streamed chunk (``completion_start_time``); ``None`` for
non-streaming calls, where ``completion_start_time`` is backfilled with the
@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {})
if not optional_params.get("stream"):
return None
api_call_start: Final = to_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = to_seconds(kwargs.get("completion_start_time"))
api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time"))
completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time"))
if api_call_start is None or completion_start is None:
return None
return completion_start - api_call_start

15
litellm/integrations/otel/model/payloads.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from enum import Enum
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast
from urllib.parse import urlsplit
from typing_extensions import ReadOnly, TypedDict
@ -716,6 +716,15 @@ def _json_or_none(value: object) -> str | None:
return None
class _EnumValue(Protocol):
@property
def value(self) -> object: ...
def _enum_value(item: _EnumValue) -> object:
return item.value
def _guardrail_mode_str(value: object) -> str | None:
"""Normalize ``guardrail_mode`` to a stable string.
@ -732,12 +741,12 @@ def _guardrail_mode_str(value: object) -> str | None:
for item in value:
if item is None:
continue
part = as_str(item.value) if isinstance(item, Enum) else as_str(item)
part = as_str(_enum_value(item)) if isinstance(item, Enum) else as_str(item)
if part:
parts.append(part)
return ",".join(parts) or None
if isinstance(value, Enum):
return as_str(value.value)
return as_str(_enum_value(value))
return as_str(value)

8
litellm/integrations/otel/plumbing/metrics.py
View file

@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False):
response_cost: ReadOnly[float | None]
completion_start_time: ReadOnly[datetime | float | str | None]
api_call_start_time: ReadOnly[datetime | float | str | None]
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
@ -414,15 +415,12 @@ class GenAIMetricRecorder:
if generation_time > 0:
self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs)
def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None:
def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None:
api_call_start_time: Final = kwargs.get("api_call_start_time")
if api_call_start_time is None:
return
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
api_call_start: Final = to_seconds(api_call_start_time)
end_ts: Final = to_seconds(_end_time)
end_ts: Final = to_seconds(kwargs.get("end_time") or end_time)
if api_call_start is None or end_ts is None:
return
duration: Final = end_ts - api_call_start

5
litellm/integrations/sqs.py
View file

@ -13,6 +13,8 @@ import re
import traceback
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import (
@ -32,6 +34,7 @@ from litellm.types.utils import StandardLoggingPayload
from .custom_batch_logger import CustomBatchLogger
_DECODED_JSON: Final = TypeAdapter(object)
_BASE64_INLINE_PATTERN: Final = re.compile(
r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+",
re.MULTILINE,
@ -270,7 +273,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM):
if self.sqs_queue_url is None:
raise ValueError("sqs_queue_url not set")
json_data: Final = json.loads(safe_dumps(payload))
json_data: Final = _DECODED_JSON.validate_python(json.loads(safe_dumps(payload)))
if self.app_crypto:
aad_bytes: Final = self.sqs_app_encryption_aad.encode("utf-8") if self.sqs_app_encryption_aad else None
encrypted: Final = self.app_crypto.encrypt_json(json_data, aad=aad_bytes)

5
litellm/interactions/litellm_responses_transformation/handler.py
View file

@ -116,10 +116,7 @@ class LiteLLMResponsesInteractionsHandler:
"""Async handler for interactions API requests."""
# Call litellm.aresponses()
# Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
aresponses_fn: Final[
Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]]
] = vars(litellm)["aresponses"]
responses_response: Final = await aresponses_fn(
responses_response: Final = await litellm.aresponses(
**responses_request,
)

9
litellm/litellm_core_utils/app_crypto.py
View file

@ -4,6 +4,9 @@ import os
from typing import Final
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from pydantic import TypeAdapter
_DECODED_JSON: Final = TypeAdapter(object)
class AppCrypto:
@ -12,7 +15,7 @@ class AppCrypto:
raise ValueError("Master key must be 32 bytes for AES-256-GCM")
self.key = master_key
def encrypt_json(self, data: dict, aad: bytes | None = None) -> dict:
def encrypt_json(self, data: object, aad: bytes | None = None) -> dict:
aes: Final = AESGCM(self.key)
nonce: Final = os.urandom(12)
plaintext: Final = json.dumps(data).encode("utf-8")
@ -24,10 +27,10 @@ class AppCrypto:
"tag": base64.b64encode(tag).decode(),
}
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> dict:
def decrypt_json(self, enc: dict, aad: bytes | None = None) -> object:
aes: Final = AESGCM(self.key)
nonce: Final = base64.b64decode(enc["nonce"])
ct: Final = base64.b64decode(enc["ciphertext"])
tag: Final = base64.b64decode(enc["tag"])
data: Final = aes.decrypt(nonce, ct + tag, aad)
return json.loads(data.decode())
return _DECODED_JSON.validate_python(json.loads(data.decode()))

6
litellm/litellm_core_utils/cloud_storage_security.py
View file

@ -3,10 +3,11 @@ import re
from collections.abc import Mapping, Sequence
from itertools import accumulate, repeat
from types import MappingProxyType
from typing import Any, Final, cast
from typing import Final
from urllib.parse import quote, unquote
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import as_str_mapping
VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/"
BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-"
@ -141,7 +142,8 @@ def should_allow_legacy_cloud_file_ids(
if isinstance(litellm_params, Mapping):
trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
value = cast(Mapping[str, Any], trusted_model_credentials).get("allow_legacy_cloud_file_ids")
trusted_mapping: Final = as_str_mapping(trusted_model_credentials)
value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None
if isinstance(value, bool):
return value

47
litellm/litellm_core_utils/core_helpers.py
View file

@ -5,10 +5,11 @@ import logging
import re
from collections.abc import Collection, Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
import httpx
from pydantic import TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads
import litellm
from litellm._logging import verbose_logger
@ -58,6 +59,13 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
return value if isinstance(value, Mapping) else None
def decoded_json_object(value: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(value)
if mapping is not None:
return mapping
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value)
@runtime_checkable
class _SupportsIter(Protocol):
def __iter__(self) -> Iterator[object]: ...
@ -72,6 +80,24 @@ def attribute_or_none(source: object, name: str) -> object:
return getattr(source, name, None)
_Shape: Final = TypeVar("_Shape")
def has_attributes(
value: object,
shape: type[_Shape],
*names: str,
) -> TypeIs[_Shape]: # guard-ok: hasattr sees what __getattr__ serves, which Protocol isinstance skips on 3.12+
return all(hasattr(value, name) for name in names)
def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shape:
if has_attributes(value, shape, *names):
return value
missing: Final = next(name for name in names if not hasattr(value, name))
raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value)
def iter_items(value: object) -> Iterable[object]:
if isinstance(value, _SupportsIter):
return value
@ -634,11 +660,14 @@ def preserve_upstream_non_openai_attributes(
for key in candidate_keys
)
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
for key, value in candidate_values:
setattr(model_response, key, value)
_assign_attributes(model_response, candidate_values)
return
for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items():
setattr(model_response, key, value)
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
for key, value in values:
setattr(target, key, value)
def safe_deep_copy(data):
@ -764,6 +793,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
Filtered data structure with Exception and callable objects removed, or None if the
entire input was an Exception or callable
"""
return _filter_exceptions_from_params(data, max_depth)
def _filter_exceptions_from_params(data: object, max_depth: int) -> object:
if max_depth <= 0:
return data
@ -785,7 +818,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
continue
try:
filtered = filter_exceptions_from_params(v, max_depth - 1)
filtered = _filter_exceptions_from_params(v, max_depth - 1)
if filtered is not None:
result[k] = filtered
except Exception:
@ -799,7 +832,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any:
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
continue
try:
filtered = filter_exceptions_from_params(item, max_depth - 1)
filtered = _filter_exceptions_from_params(item, max_depth - 1)
if filtered is not None:
result_list.append(filtered)
except Exception:

2
litellm/litellm_core_utils/cost_discount.py
View file

@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey:
def _literal_length(pattern: str) -> int:
return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?")
return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?")
def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]:

2
litellm/litellm_core_utils/default_encoding.py
View file

@ -48,5 +48,5 @@ for attempt in range(_max_retries):
# Last attempt, re-raise the exception
raise
# Exponential backoff with jitter to reduce collision probability
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1)
time.sleep(delay)

2
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -181,7 +181,7 @@ def _delete_nested_value_custom(
def delete_nested_value(
data: dict[str, Any],
data: dict[str, object],
path: str,
depth: int = 0,
max_depth: int = 20,

4
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -16,7 +16,7 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
from litellm.types.utils import LlmProviders
@ -169,7 +169,7 @@ def get_error_message(error_obj) -> str | None:
return None
if hasattr(error_obj, "body"):
_error_obj_body: Final = getattr(error_obj, "body")
_error_obj_body: Final = attribute_or_none(error_obj, "body")
if isinstance(_error_obj_body, dict):
# OpenAI-style: {"message": "...", "type": "...", ...}
if _error_obj_body.get("message"):

6
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -43,6 +43,7 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
_DECODED_JSON: Final = TypeAdapter(object)
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
@ -314,7 +315,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
@ -383,7 +384,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
etag_values: Final = response.headers.get_list("etag")
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
def _next_retry_wait(

3
litellm/litellm_core_utils/hidden_params.py
View file

@ -1,4 +1,5 @@
from collections.abc import Iterator, Mapping, MutableMapping
from types import MappingProxyType
from typing import Final, Protocol, cast
from pydantic import BaseModel
@ -52,7 +53,7 @@ class HiddenParamsModelView(MutableMapping[str, object]):
def stored_attribute(obj: object, name: str) -> object:
if not isinstance(obj, BaseModel):
return getattr(obj, name, None)
values: Final[Mapping[str, object]] = vars(obj)
values: Final[Mapping[str, object]] = MappingProxyType[str, object](vars(obj))
if name in values:
return values[name]
extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {}

2
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -28,7 +28,7 @@ def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
def iter_client_callback_metadata_dicts(
kwargs: dict[str, Any],
kwargs: Mapping[str, object],
) -> Iterator[tuple[str, dict[str, Any]]]:
litellm_params: Final = kwargs.get("litellm_params")
if isinstance(litellm_params, dict):

63
litellm/litellm_core_utils/litellm_logging.py
View file

@ -556,6 +556,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
def _provider_response_id(source: object) -> str | None:
@ -1171,7 +1172,7 @@ class Logging(LiteLLMLoggingBaseClass):
automatically extracts metadata/litellm_metadata from kwargs,
so callers don't need to manually plumb them into litellm_params.
"""
base_litellm_params: Final[dict[str, Any]] = {}
base_litellm_params: Final[dict[str, object]] = {}
if isinstance(kwargs.get("metadata"), dict):
base_litellm_params["metadata"] = kwargs["metadata"].copy()
@ -1188,10 +1189,11 @@ class Logging(LiteLLMLoggingBaseClass):
lp_metadata: Final = litellm_params.pop("metadata", None)
base_litellm_params.update(litellm_params)
if lp_metadata and isinstance(lp_metadata, dict):
base_litellm_params.setdefault("metadata", {})
for k, v in lp_metadata.items():
if k not in base_litellm_params["metadata"]:
base_litellm_params["metadata"][k] = v
merged_metadata: Final = base_litellm_params.setdefault("metadata", {})
if isinstance(merged_metadata, dict):
for k, v in lp_metadata.items():
if k not in merged_metadata:
merged_metadata[k] = v
self.update_environment_variables(
litellm_params=base_litellm_params,
@ -1793,7 +1795,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.exception(
"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
)
original_response = redact_message_input_output_from_logging(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=original_response,
)
@ -2023,10 +2025,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details.pop("response_cost_failure_debug_information", None)
if cache_hit is None:
cache_hit = self.model_call_details.get("cache_hit", False)
resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
if cache_hit is True:
if resolved_cache_hit is True:
return 0.0
if is_unbilled_non_inference_call(
@ -2078,7 +2079,7 @@ class Logging(LiteLLMLoggingBaseClass):
model_value: Final = litellm_model_name or self.model
cost_cache_hit_value: Final = (
self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit
self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit
)
cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided
bool | None, cost_cache_hit_value
@ -2512,8 +2513,10 @@ class Logging(LiteLLMLoggingBaseClass):
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
@ -3302,7 +3305,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
@ -3323,9 +3326,11 @@ class Logging(LiteLLMLoggingBaseClass):
)
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=call_details,
result=result,
)
)
## LOGGING HOOK ##
hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
@ -3591,7 +3596,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
response_obj=call_details_view["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -3914,9 +3919,11 @@ class Logging(LiteLLMLoggingBaseClass):
self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
)
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
## LOGGING HOOK ##
@ -4261,9 +4268,11 @@ class Logging(LiteLLMLoggingBaseClass):
result: object = None # result sent to all loggers, init this to None incase it's not created
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
result = _REDACTED_RESULT.validate_python(
redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
result=result,
)
)
self.mark_logging_complete(event_type="sync_failure")
for callback in callbacks:
@ -4952,7 +4961,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
import httpx
httpx_response: Final = self.model_call_details.get("httpx_response", None)
httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
@ -4982,7 +4991,7 @@ class Logging(LiteLLMLoggingBaseClass):
The response object with usage added if available
"""
# Get usage from model_call_details (set by asend_message)
usage: Final = self.model_call_details.get("usage")
usage: Final[Usage | None] = self.model_call_details.get("usage")
if usage is None:
return result
@ -5815,7 +5824,9 @@ def build_custom_logger_compatible_class(
return callback
# Get global BitBucket config
bitbucket_config: Final = getattr(litellm, "global_bitbucket_config", None)
bitbucket_config: Final = (
litellm.global_bitbucket_config if hasattr(litellm, "global_bitbucket_config") else None
)
if bitbucket_config is None:
raise ValueError("BitBucket configuration not found. Please set litellm.global_bitbucket_config first.")

2
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -450,7 +450,7 @@ class StandardBuiltInToolCostTracking:
def _get_code_interpreter_cost(
model_info: ModelInfo | None,
custom_llm_provider: str | None,
standard_built_in_tools_params: StandardBuiltInToolsParams,
standard_built_in_tools_params: Mapping[str, object],
) -> float:
"""Calculate code interpreter cost."""
code_interpreter_sessions: Final = standard_built_in_tools_params.get("code_interpreter_sessions", None)

5
litellm/litellm_core_utils/llm_judge.py
View file

@ -21,6 +21,7 @@ if TYPE_CHECKING:
JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
def default_router_provider() -> Router | None:
@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars
text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload
parsed: object
try:
parsed = json.loads(text)
parsed = _DECODED_JSON.validate_python(json.loads(text))
except json.JSONDecodeError:
start: Final = text.find("{")
end: Final = text.rfind("}")
if start == -1 or end <= start:
raise
parsed = json.loads(text[start : end + 1])
parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1]))
if not isinstance(parsed, dict):
raise ValueError("judge response is not a JSON object")
return {str(k): v for k, v in parsed.items()}

8
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -1,5 +1,5 @@
import threading
from collections.abc import Callable, Iterable, Iterator
from collections.abc import Callable, Iterable, Iterator, Mapping
from types import MethodType
from typing import TYPE_CHECKING, Final
@ -20,6 +20,10 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {}
_callback_lists_lock: Final = threading.RLock()
def _instance_attributes(target: object) -> Mapping[str, object]:
return vars(target)
class LoggingCallbackManager:
"""
A centralized class that allows easy add / remove callbacks for litellm.
@ -341,7 +345,7 @@ class LoggingCallbackManager:
key_parts: Final = [type(custom_logger).__name__]
# Add only fundamental type instance variables to the key
for attr_name, attr_value in vars(custom_logger).items():
for attr_name, attr_value in _instance_attributes(custom_logger).items():
if not attr_name.startswith("_"): # Skip private attributes
if isinstance(attr_value, (str, bool, int)):
key_parts.append(f"{attr_name}={attr_value}")

20
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
@ -1527,7 +1528,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int:
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
return (1 << optional) * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
@ -2839,7 +2840,7 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
break
try:
obj, end_idx = decoder.raw_decode(raw, idx)
decoded = decoder.raw_decode(raw, idx)
except json.JSONDecodeError as e:
verbose_logger.warning(
"split_concatenated_json_objects: discarding unparseable tool-call "
@ -2849,13 +2850,14 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]:
e,
)
break
if isinstance(obj, dict):
results.append(obj)
obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if obj is not None:
results.append(dict(obj))
else:
# Non-dict JSON value – wrap in empty dict (Bedrock requires
# toolUse.input to be an object).
results.append({})
idx = end_idx
idx = decoded[1]
return results
@ -2887,14 +2889,16 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], ..
if idx >= length:
break
try:
obj, end_idx = decoder.raw_decode(stripped, idx)
decoded = decoder.raw_decode(stripped, idx)
except json.JSONDecodeError:
return ()
if not isinstance(obj, dict):
decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0]))
if decoded_object is None:
return ()
idx = end_idx
idx = decoded[1]
if exceeded:
continue
obj = dict(decoded_object)
count += 1
if not kept:
kept = (obj,)

32
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -21,6 +21,7 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
@ -1021,11 +1022,12 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
tool_function = get_attribute_or_key(tool, "function")
tool_name = get_attribute_or_key(tool_function, "name")
tool_arguments = get_attribute_or_key(tool_function, "arguments")
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
parsed_args = _DECODED_JSON.validate_python(
parse_tool_call_arguments(tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke")
)
if isinstance(parsed_args, dict):
parameters = "".join(f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items())
parsed_mapping = as_str_mapping(parsed_args)
if parsed_mapping is not None:
parameters = "".join(f"<{param}>{val}</{param}>\n" for param, val in parsed_mapping.items())
else:
parameters = f"<result>{parsed_args}</result>\n"
invokes += f"<invoke>\n<tool_name>{tool_name}</tool_name>\n<parameters>\n{parameters}</parameters>\n</invoke>\n"
@ -2342,7 +2344,7 @@ def _add_missing_tool_results(
else:
tool_name = getattr(function, "name", "unknown_tool")
else:
function = getattr(tool_call, "function", None)
function = attribute_or_none(tool_call, "function")
if function:
tool_name = getattr(function, "name", "unknown_tool")
break
@ -3184,10 +3186,14 @@ def anthropic_messages_pt(
def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]:
ext_list = re.findall(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL)
if strip:
ext_list = [e.strip() for e in ext_list]
ext_list = list(_stripped(ext_list))
return ext_list
def _stripped(values: Iterable[str]) -> Iterator[str]:
return (value.strip() for value in values)
def contains_tag(tag: str, string: str) -> bool:
return bool(re.search(f"<{tag}>(.+?)</{tag}>", string, re.DOTALL))
@ -4061,13 +4067,15 @@ def _convert_to_bedrock_tool_call_invoke(
arguments_dict = {}
else:
try:
arguments_dict = json.loads(arguments)
decoded_arguments = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments)))
# Ensure arguments_dict is always a dict
# (Bedrock requires toolUse.input to be an object).
# Some providers return arguments: '""' which
# json.loads decodes to a bare string.
if not isinstance(arguments_dict, dict):
if decoded_arguments is None:
arguments_dict = {}
else:
arguments_dict = dict(decoded_arguments)
except json.JSONDecodeError:
# The model may return multiple JSON objects
# concatenated in a single arguments string, e.g.
@ -6028,7 +6036,11 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
return ({},)
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
try:
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
parsed: Final = as_str_mapping(
_DECODED_JSON.validate_python(
parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
)
)
except ValueError as e:
salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw)
if salvaged:
@ -6041,7 +6053,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str)
return salvaged
verbose_logger.warning("Failed to parse tool call arguments: %s", e)
return ({},)
return (parsed,) if isinstance(parsed, dict) else ({},)
return (dict(parsed),) if parsed is not None else ({},)
def _choice_tool_calls(choice: object) -> tuple[object, ...]:

2
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -482,7 +482,7 @@ def _runs_by_anchor(
) -> Mapping[int, tuple[tuple[int, ...], ...]]:
anchored: Final = tuple(sorted(((anchor, run) for run, anchor in anchors if anchor is not None), key=itemgetter(0)))
return MappingProxyType(
{anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=itemgetter(0))}
{anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=lambda entry: entry[0])}
)

8
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,4 +1,4 @@
from collections.abc import Mapping, Sequence
from collections.abc import ItemsView, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, field
from typing import Any, Final
@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset(
)
def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]:
return data.items()
class SensitiveDataMasker:
def __init__(
self,
@ -127,7 +131,7 @@ class SensitiveDataMasker:
return data
masked_data: Final[dict[str, object]] = {}
for k, v in data.items():
for k, v in _object_items(data):
try:
key_is_sensitive = self.is_sensitive_key(k, excluded_keys)
if isinstance(v, Mapping):

3
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -425,7 +425,7 @@ class ChunkProcessor:
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
@ -1226,6 +1226,7 @@ def concatenate_base64_list(base64_strings: Sequence[str]) -> str:
_CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]])
_CHOICE_FIELD_VALUE: Final = TypeAdapter(object)
def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse:

86
litellm/litellm_core_utils/streaming_handler.py
View file

@ -37,7 +37,6 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import OpenAIChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CacheCreationTokenDetails,
@ -111,14 +110,15 @@ def _prepend_upstream_choice_extras(
choice.__pydantic_fields_set__.update(upstream_extras)
def _holds_only_scalars(model: BaseModel) -> bool:
fields: Final[Iterator[tuple[str, object]]] = iter(model)
def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool:
return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields)
def _delta_from_upstream(upstream_delta: BaseModel | None) -> Delta | None:
def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None:
if upstream_delta is None:
return Delta()
if not isinstance(upstream_delta, BaseModel):
return None
try:
return Delta(**dict(iter(upstream_delta)))
except Exception:
@ -189,6 +189,33 @@ class _DeltaDumpDict(TypedDict):
tool_calls: NotRequired[Sequence[Mapping[str, object]]]
class _UpstreamFunctionCallLike(Protocol):
arguments: str | None
class _UpstreamToolCallLike(Protocol):
function: _UpstreamFunctionCallLike
class _UpstreamDeltaLike(Protocol):
function_call: _UpstreamFunctionCallLike | None
tool_calls: Sequence[_UpstreamToolCallLike] | None
def __iter__(self) -> Iterator[tuple[str, object]]: ...
def model_dump(self) -> _DeltaDumpDict: ...
class _UpstreamChoiceLike(Protocol):
delta: _UpstreamDeltaLike | None
class _UpstreamChunkLike(Protocol):
id: str
choices: Sequence[_UpstreamChoiceLike]
system_fingerprint: str | None
class _TextCompletionChoiceLike(Protocol):
text: str
finish_reason: str | None
@ -920,7 +947,7 @@ class CustomStreamWrapper:
def copy_model_response_level_provider_specific_fields(
self,
original_chunk: ModelResponseStream | OpenAIChatCompletionChunk,
original_chunk: object,
model_response: ModelResponseStream,
) -> ModelResponseStream:
"""
@ -949,8 +976,8 @@ class CustomStreamWrapper:
or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
or (
"tool_calls" in model_response.choices[0].delta
and model_response.choices[0].delta["tool_calls"] is not None
and len(model_response.choices[0].delta["tool_calls"]) > 0
and model_response.choices[0].delta.tool_calls is not None
and len(model_response.choices[0].delta.tool_calls) > 0
)
or (
"function_call" in model_response.choices[0].delta
@ -1115,7 +1142,8 @@ class CustomStreamWrapper:
## check if openai/azure chunk
original_chunk: Final = response_obj.get("original_chunk", None)
if original_chunk:
if len(original_chunk.choices) > 0:
upstream_chunk: Final[_UpstreamChunkLike] = original_chunk
if len(upstream_chunk.choices) > 0:
original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access
object,
original_chunk,
@ -1151,7 +1179,7 @@ class CustomStreamWrapper:
)
else:
choices: Final = []
for choice in original_chunk.choices:
for choice in upstream_chunk.choices:
try:
if isinstance(choice, BaseModel):
choice_json = choice.model_dump()
@ -1164,11 +1192,11 @@ class CustomStreamWrapper:
setattr(model_response, "choices", choices)
else:
return
model_response.system_fingerprint = original_chunk.system_fingerprint
model_response.system_fingerprint = upstream_chunk.system_fingerprint
setattr(
model_response,
"citations",
stored_attribute(original_chunk, "citations"),
stored_attribute(upstream_chunk, "citations"),
)
preserve_upstream_non_openai_attributes(
model_response=model_response,
@ -1611,7 +1639,9 @@ class CustomStreamWrapper:
model_response.model = self.model
## FUNCTION CALL PARSING
original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None
original_chunk: Final[_UpstreamChunkLike | None] = (
response_obj.get("original_chunk") if response_obj is not None else None
)
if (
original_chunk is not None
): # function / tool calling branch - only set for openai/azure compatible endpoints
@ -1623,31 +1653,19 @@ class CustomStreamWrapper:
original_chunk, model_response
)
if original_chunk.choices and len(original_chunk.choices) > 0:
delta = original_chunk.choices[0].delta
delta: Final = original_chunk.choices[0].delta
if delta is not None and (delta.function_call is not None or delta.tool_calls is not None):
try:
model_response.system_fingerprint = original_chunk.system_fingerprint
## AZURE - check if arguments is not None
if original_chunk.choices[0].delta.function_call is not None:
if (
getattr(
original_chunk.choices[0].delta.function_call,
"arguments",
)
is None
):
original_chunk.choices[0].delta.function_call.arguments = ""
elif original_chunk.choices[0].delta.tool_calls is not None:
if isinstance(original_chunk.choices[0].delta.tool_calls, list):
for t in original_chunk.choices[0].delta.tool_calls:
if hasattr(t, "functions") and hasattr(t.functions, "arguments"):
if (
getattr(
t.function,
"arguments",
)
is None
):
if delta.function_call is not None:
if delta.function_call.arguments is None:
delta.function_call.arguments = ""
elif delta.tool_calls is not None:
if isinstance(delta.tool_calls, list):
for t in delta.tool_calls:
if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"):
if t.function.arguments is None:
t.function.arguments = ""
_json_delta: Final[_DeltaDumpDict] = delta.model_dump()
if "role" not in _json_delta or _json_delta["role"] is None:
@ -1886,7 +1904,7 @@ class CustomStreamWrapper:
else:
asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit))
## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler
litellm_params: Final = self.logging_obj.model_call_details.get("litellm_params", {})
litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {})
if self.logging_obj.is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)

4
litellm/litellm_core_utils/token_counter.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
@ -247,7 +247,7 @@ def get_image_dimensions(
client: Final = get_httpx_client()
response: Final[httpx.Response] = safe_get(client, data)
max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
content_length: Final[str | None] = response.headers.get("Content-Length")
content_length: Final[str | None] = header_value(response.headers, "Content-Length")
if content_length is not None and int(content_length) > max_bytes:
pass # skip download; img_data stays None
else:

59
litellm/litellm_core_utils/url_utils.py
View file

@ -22,11 +22,11 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
import asyncio
import socket
from ipaddress import ip_address, ip_network
from typing import Any, Final, Protocol
from typing import Final, Protocol
from urllib.parse import quote, urlparse, urlsplit, urlunparse
import httpx
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import ReadOnly, TypedDict, Unpack
import litellm
@ -50,6 +50,7 @@ class _UrlFetcher(Protocol):
*,
headers: dict[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
) -> httpx.Response: ...
@ -62,6 +63,8 @@ class _AsyncUrlFetcher(Protocol):
*,
headers: dict[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> httpx.Response: ...
@ -73,8 +76,14 @@ class _AsyncFetcherView(TypedDict):
fetcher: ReadOnly[_AsyncUrlFetcher]
class _CallerHeadersView(TypedDict):
class _FetchOptions(TypedDict, total=False):
headers: ReadOnly[dict[str, str]]
follow_redirects: ReadOnly[bool]
timeout: ReadOnly[float | httpx.Timeout | None]
class _AsyncFetchOptions(_FetchOptions, total=False):
max_response_bytes: ReadOnly[int | None]
# Cloud-internal IPs that ``ip.is_global`` can report as public. Everything
@ -440,7 +449,7 @@ def _extract_redirect_url(response: httpx.Response, request_url: str) -> str:
return str(httpx.URL(request_url).join(location))
def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
def safe_get(client: _UrlFetcher, url: str, **kwargs: Unpack[_FetchOptions]) -> httpx.Response:
"""
Fetch a user-supplied URL with SSRF protection on every redirect hop.
@ -460,21 +469,23 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
The final httpx.Response.
"""
if not getattr(litellm, "user_url_validation", True):
kwargs.setdefault("follow_redirects", True)
unvalidated: Final[_ResponseView] = {"response": client.get(url, **kwargs)}
unvalidated_options: Final[_FetchOptions] = {
**kwargs,
"follow_redirects": kwargs.get("follow_redirects", True),
}
unvalidated: Final[_ResponseView] = {"response": client.get(url, **unvalidated_options)}
return unvalidated["response"]
fetcher_view: Final[_FetcherView] = {"fetcher": client}
fetcher: Final = fetcher_view["fetcher"]
kwargs.pop("follow_redirects", None)
headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})}
caller_headers: Final = kwargs.get("headers", {})
for _ in range(_MAX_REDIRECTS):
validated_url, original_host = validate_url(url)
response = fetcher.get(
validated_url,
headers={**headers_view["headers"], "Host": original_host},
follow_redirects=False,
hop_options: _FetchOptions = {
**kwargs,
)
"headers": {**caller_headers, "Host": original_host},
"follow_redirects": False,
}
response = fetcher.get(validated_url, **hop_options)
if not response.is_redirect:
return response
# Resolve the next hop against the ORIGINAL (pre-rewrite) URL so
@ -483,24 +494,26 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
raise SSRFError("Too many redirects")
async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Any) -> httpx.Response:
async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Unpack[_AsyncFetchOptions]) -> httpx.Response:
"""Async version of safe_get."""
if not getattr(litellm, "user_url_validation", True):
kwargs.setdefault("follow_redirects", True)
unvalidated: Final[_ResponseView] = {"response": await client.get(url, **kwargs)}
unvalidated_options: Final[_AsyncFetchOptions] = {
**kwargs,
"follow_redirects": kwargs.get("follow_redirects", True),
}
unvalidated: Final[_ResponseView] = {"response": await client.get(url, **unvalidated_options)}
return unvalidated["response"]
fetcher_view: Final[_AsyncFetcherView] = {"fetcher": client}
fetcher: Final = fetcher_view["fetcher"]
kwargs.pop("follow_redirects", None)
headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})}
caller_headers: Final = kwargs.get("headers", {})
for _ in range(_MAX_REDIRECTS):
validated_url, original_host = await asyncio.to_thread(validate_url, url)
response = await fetcher.get(
validated_url,
headers={**headers_view["headers"], "Host": original_host},
follow_redirects=False,
hop_options: _AsyncFetchOptions = {
**kwargs,
)
"headers": {**caller_headers, "Host": original_host},
"follow_redirects": False,
}
response = await fetcher.get(validated_url, **hop_options)
if not response.is_redirect:
return response
# Resolve the next hop against the ORIGINAL (pre-rewrite) URL so

3
litellm/llms/__init__.py
View file

@ -3,6 +3,7 @@ import os
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import attribute_or_none
from litellm.types.utils import CallTypes
from . import *
@ -128,7 +129,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans
# Check for guardrail_translation_mappings dictionary
if hasattr(module, "guardrail_translation_mappings"):
mappings = getattr(module, "guardrail_translation_mappings")
mappings = attribute_or_none(module, "guardrail_translation_mappings")
if isinstance(mappings, dict):
discovered_mappings.update(mappings)
verbose_logger.debug(

6
litellm/llms/a2a/common_utils.py
View file

@ -31,6 +31,10 @@ class A2AError(BaseLLMException):
)
def _message_role(message: Mapping[str, object]) -> object:
return message.get("role", "user")
def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str:
"""
Convert OpenAI messages to a single prompt string for A2A agent.
@ -51,7 +55,7 @@ def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str:
# Get role
if isinstance(msg, BaseModel):
role = msg.model_dump().get("role", "user")
role = _message_role(msg.model_dump())
elif isinstance(msg, dict):
role = msg.get("role", "user")
else:

4
litellm/llms/ai21/chat/transformation.py
View file

@ -4,6 +4,8 @@ AI21 Chat Completions API
this is OpenAI compatible - no translation needed / occurs
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from ...openai_like.chat.transformation import OpenAILikeChatConfig
@ -41,7 +43,7 @@ class AI21ChatConfig(OpenAILikeChatConfig):
tool_choice: str | None = None,
user: str | None = None,
) -> None:
locals_: Final = locals().copy()
locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy())
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)

4
litellm/llms/amazon_nova/chat/transformation.py
View file

@ -2,6 +2,8 @@
Translate from OpenAI's `/v1/chat/completions` to Amazon Nova's `/v1/chat/completions`
"""
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
@ -38,7 +40,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
tools: list | None = None,
reasoning_effort: list | None = None,
) -> None:
locals_: Final = locals().copy()
locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy())
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)

2
litellm/llms/anthropic/batches/handler.py
View file

@ -132,7 +132,7 @@ class AnthropicBatchesHandler:
max_retries: int | None,
logging_obj: LiteLLMLoggingObj | None = None,
litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment
) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]:
) -> LiteLLMBatch | Coroutine[object, object, LiteLLMBatch]:
"""
Retrieve a batch from Anthropic.

9
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -1697,14 +1697,13 @@ class AnthropicMessagesHandler(BaseTranslation):
if not line.startswith("data:"):
return line
try:
data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads(
line[len("data:") :].strip()
)
data: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip()))
except json.JSONDecodeError:
return line
if not isinstance(data, dict):
event: Final = as_str_mapping(data)
if not isinstance(event, dict):
return line
rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event)
rewritten: Final = _rewritten_event(event, rewrite_event)
return line if rewritten is data else "data: " + json.dumps(rewritten)
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:

7
litellm/llms/anthropic/chat/transformation.py
View file

@ -831,6 +831,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"else": "conditional (else): {}",
"not": "must not match: {}",
}
schema_fields: Final[Mapping[str, object]] = MappingProxyType(schema)
min_items: Final = schema.get("minItems")
enforced_min_items: Final = (
isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1)
@ -845,7 +846,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
for field, label in constraint_labels.items():
if field not in schema or field not in unsupported_fields:
continue
value = schema[field]
value = schema_fields[field]
# A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no
# real requirement, so don't add a misleading advisory note for it.
if isinstance(value, bool) and not value:
@ -869,7 +870,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
for key, value in schema.items():
for key, value in schema_fields.items():
if key in unsupported_fields:
continue
if key == "type" and drops_conflicting_type:
@ -2385,7 +2386,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params["tools"] = tools
## Load Config
config: Final = litellm.AnthropicConfig.get_config(model=model)
config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model))
for k, v in config.items():
if (
k not in optional_params

3
litellm/llms/anthropic/completion/transformation.py
View file

@ -7,6 +7,7 @@ Litellm provider slug: `anthropic_text/<model_name>`
import json
import time
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
@ -112,7 +113,7 @@ class AnthropicTextConfig(BaseConfig):
) -> dict:
prompt: Final = self._get_anthropic_text_prompt_from_messages(messages=messages, model=model)
## Load Config
config: Final = litellm.AnthropicTextConfig.get_config()
config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicTextConfig.get_config())
for k, v in config.items():
if (
k not in optional_params

5
litellm/llms/anthropic/files/handler.py
View file

@ -5,6 +5,7 @@ from collections.abc import Coroutine, Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -24,6 +25,8 @@ from litellm.types.utils import CallTypes, LlmProviders, ModelResponse
from ..chat.transformation import AnthropicConfig
from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo
_DECODED_JSON: Final = TypeAdapter(object)
class AnthropicFilesHandler:
"""
@ -194,7 +197,7 @@ class AnthropicFilesHandler:
if not line.strip():
continue
anthropic_result: object = json.loads(line)
anthropic_result: object = _DECODED_JSON.validate_python(json.loads(line))
if not isinstance(anthropic_result, dict):
raise TypeError(
f"Anthropic batch result line is not a JSON object: {type(anthropic_result).__name__}"

8
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -25,7 +25,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
@ -1367,9 +1367,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
# open a block: the emitted {"type": "thinking", "thinking": ""} gets
# replayed as history and Anthropic rejects it (LIT-6357). A signed
# entry opens the block so the client receives the replay signature.
thinking_blocks: Final = getattr(delta, "thinking_blocks", None)
if thinking_blocks and any(
isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks
if any(
isinstance(b, dict) and not is_empty_unsigned_thinking_block(b)
for b in attribute_items(delta, "thinking_blocks")
):
return False
return True

15
litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py
View file

@ -15,7 +15,7 @@ from dataclasses import dataclass
from functools import reduce
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Any, Final, Literal, TypeVar, cast
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeVar, cast
from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -32,6 +32,9 @@ from ..constants import (
from ..errors import AnthropicContextManagementError
from ..placeholders import build_cleared_tool_result_content
if TYPE_CHECKING:
from litellm.types.llms.openai import ChatCompletionToolParam
T = TypeVar("T")
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
_TINY_RESULT_BYTES: Final = 46
@ -216,7 +219,7 @@ def _trigger_met(
current_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast(Any, tools),
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
@ -366,9 +369,13 @@ def apply_clear_tool_uses_20250919(
tokens_before: Final = (
counted_tokens
if counted_tokens is not None
else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools))
else litellm.token_counter(
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
tokens_after: Final = litellm.token_counter(
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools))
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
return final_messages, {

4
litellm/llms/anthropic/pass_through/context_management/placeholders.py
View file

@ -1,12 +1,10 @@
"""Placeholder content for cleared ``tool_result`` blocks (string or block list)."""
from typing import Any
from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER
def build_cleared_tool_result_content(
original_content: Any,
original_content: object,
) -> str | list[dict]:
"""Return a string or single text block list, matching ``original_content`` shape."""
if isinstance(original_content, list):

Some files were not shown because too many files have changed in this diff Show more