diff --git a/enterprise/enterprise_hooks/aporia_ai.py b/enterprise/enterprise_hooks/aporia_ai.py index 28b49bfce21..1e840f8dcc5 100644 --- a/enterprise/enterprise_hooks/aporia_ai.py +++ b/enterprise/enterprise_hooks/aporia_ai.py @@ -15,9 +15,10 @@ sys.path.insert( ) # Adds the parent directory to the system path import json import sys -from typing import Any, List, Literal, Optional +from typing import Final, List, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata from litellm.types.guardrails import GuardrailEventHooks GUARDRAIL_NAME = "aporia" +_DECODED_JSON: Final = TypeAdapter(object) class AporiaGuardrail(CustomGuardrail): @@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail): async def prepare_aporia_request( self, new_messages: List[dict], response_string: Optional[str] = None ) -> dict: - data: dict[str, Any] = {} + data: dict[str, object] = {} if new_messages is not None: data["messages"] = new_messages if response_string is not None: @@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Aporia AI response: %s", response.text) if response.status_code == 200: # check if the response was flagged - _json_response = response.json() - action: str = _json_response.get( + _json_response: Final = decoded_json_object( + _DECODED_JSON.validate_python(response.json()) + ) + action: Final = _json_response.get( "action" ) # possible values are modify, passthrough, block, rephrase if action == "block": diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index f6998d58cd5..7e4d2514880 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -7,9 +7,10 @@ ## This accepts a list of user id's for whom calls will be rejected -from typing import Literal, Optional +from typing import Final, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._internal_context import with_service_target @@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import EndUserRepository + +_REQUEST_BODY_VALUE: Final = TypeAdapter(object) class ENTERPRISE_BlockedUserList(CustomLogger): @@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger): ) if end_user_cache_obj is None and self.prisma_client is not None: # check db - end_user_obj = ( - await self.prisma_client.db.litellm_endusertable.find_unique( - where={"user_id": user} - ) - ) + end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user) + end_user_obj = await EndUserRepository( + self.prisma_client + ).table.find_unique(where={"user_id": end_user_id}) if end_user_obj is None: # user not in db - assume not blocked end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False) cache.set_cache(key=cache_key, value=end_user_obj, ttl=60) - if end_user_obj is not None and end_user_obj.blocked is True: + if end_user_obj.blocked is True: raise HTTPException( status_code=400, detail={ diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index b3ef5a3d496..7dbbc9ed6ce 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -6,7 +6,10 @@ # +-----------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan +from typing import Final + from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails._content_utils import iter_message_text from litellm.types.utils import CallTypesLiteral +_THRESHOLD_VALUE: Final = TypeAdapter(object) + class ENTERPRISE_GoogleTextModeration(CustomLogger): user_api_key_cache = None @@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger): f"{category}_confidence_threshold", default_confidence_threshold, ) - set_confidence_value = getattr( - self, - f"{category}_confidence_threshold", + set_confidence_value = _THRESHOLD_VALUE.validate_python( + getattr( + self, + f"{category}_confidence_threshold", + ) ) verbose_proxy_logger.info( f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}" diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py index 14d34f5d1e8..e0190ea8da0 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/example_logging_api.py @@ -1,15 +1,20 @@ # this is an example endpoint to receive data from litellm +from typing import Final + from fastapi import FastAPI, HTTPException, Request +from pydantic import TypeAdapter app = FastAPI() +_LOGGED_EVENT: Final = TypeAdapter(object) + @app.post("/log-event") async def log_event(request: Request): try: print("Received /log-event request") # noqa # Assuming the incoming request has JSON data - data = await request.json() + data: Final = _LOGGED_EVENT.validate_python(await request.json()) print("Received request data:") # noqa print(data) # noqa diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py index 3422e8969b0..e12539eac53 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py @@ -8,25 +8,30 @@ ## This provides an LLM Guard Integration for content moderation on the proxy import asyncio -from typing import Final, Optional +from collections.abc import Mapping +from typing import Final import aiohttp from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.openai.common_utils import decoded_json_object from litellm.proxy._types import UserAPIKeyAuth from litellm.secret_managers.main import get_secret_str from litellm.types.utils import CallTypesLiteral +_DECODED_JSON: Final = TypeAdapter(object) + class _ENTERPRISE_LLMGuard(CustomLogger): # Class variables or attributes def __init__( self, mock_testing: bool = False, - mock_redacted_text: Optional[dict] = None, + mock_redacted_text: Mapping[str, object] | None = None, ): self.mock_redacted_text = mock_redacted_text self.llm_guard_mode = litellm.llm_guard_mode @@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger): async with session.post( analyze_url, json={"prompt": text} ) as response: - redacted_text = await response.json() + redacted_text = _DECODED_JSON.validate_python( + await response.json() + ) verbose_proxy_logger.debug( f"LLM Guard: Received response - {redacted_text}" ) @@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger): "error": f"Invalid content moderation response: {redacted_text}" }, ) - if redacted_text.get("is_valid", None) is False: + redacted_response: Final = decoded_json_object(redacted_text) + if redacted_response.get("is_valid", None) is False: raise HTTPException( status_code=400, detail={"error": "Violated content safety policy"}, ) - sanitized_prompt = redacted_text.get("sanitized_prompt") + sanitized_prompt: Final = redacted_response.get("sanitized_prompt") return sanitized_prompt if isinstance(sanitized_prompt, str) else text except Exception as e: verbose_proxy_logger.exception( diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index df1f263a947..f3b1240f733 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -1040,8 +1040,8 @@ class CheckBatchCost: from litellm.proxy.proxy_server import proxy_config from litellm.secret_managers.main import str_to_bool - api_key: Final = getattr(job, "api_key", None) - team_id: Final = getattr(job, "team_id", None) + api_key: Final = job.api_key if hasattr(job, "api_key") else None + team_id: Final = job.team_id if hasattr(job, "team_id") else None has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key) try: key_row: Final = ( diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index bc35c4a0aba..48cbc0ab8ff 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -44,6 +44,7 @@ from litellm.constants import ( ) from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.internal_call_metadata import requests_background from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -133,7 +134,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache from litellm.proxy.utils import PrismaClient as _PrismaClient - Span = Union[_Span, Any] + Span = _Span InternalUsageCache = _InternalUsageCache PrismaClient = _PrismaClient else: @@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE _PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0) +class _SupportsKeyedGet(Protocol): + def get(self, key: str, default: object, /) -> object: ... + + +def _keyed_get(source: object, key: str, default: object = None) -> object: + return require_attributes(source, _SupportsKeyedGet, "get").get(key, default) + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return holders if holders or "input_file_id" in request else healthy_deployments async def async_pre_call_deployment_hook( - self, kwargs: Dict[str, Any], call_type: Optional[CallTypes] + self, kwargs: dict[str, object], call_type: Optional[CallTypes] ) -> Optional[dict]: """ Allow modifying the request just before it's sent to the deployment. @@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")) # model_info may be at top-level or nested under litellm_metadata # (batch/file operations use litellm_metadata) - model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None)) + model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id")) if model_id is None: model_id = cast( Optional[str], - kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None), + _keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"), ) for accessor_key in accessor_keys: input_file_id = cast(Optional[str], kwargs.get(accessor_key)) @@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ), ) - async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict: + async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict: """ Get model-specific file IDs for a list of proxy file IDs. Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id @@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Optional[Router], _create_file_request: CreateFileRequest, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, ) -> List[OpenAIFileObject]: if llm_router is None: raise Exception("LLM Router not initialized. Ensure models added to proxy.") @@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): create_file_request: CreateFileRequest, llm_router: Router, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: responses = await self.create_file_for_each_model( @@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_objects: List[OpenAIFileObject], create_file_request: CreateFileRequest, internal_usage_cache: InternalUsageCache, - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, target_model_names_list: List[str], ) -> OpenAIFileObject: ## GET THE FILE TYPE FROM THE CREATE FILE REQUEST @@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Continue with other files even if one fails continue - def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str: + def _get_content_type_from_file_object(self, file_object: object) -> str: """ Determine content type from file object. diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py index 0e9657aba96..fcaf58099b9 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache from litellm.proxy.utils import PrismaClient as _PrismaClient - Span = Union[_Span, Any] + Span = _Span InternalUsageCache = _InternalUsageCache PrismaClient = _PrismaClient else: @@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores( llm_router: Router, model: str, request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams, - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, ) -> VectorStoreCreateResponse: """ Create a vector store for a specific model. @@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores( create_request: VectorStoreCreateOptionalRequestParams, llm_router: Router, target_model_names_list: List[str], - litellm_parent_otel_span: Span, + litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, ) -> VectorStoreCreateResponse: """ diff --git a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py index db419fde1e6..b91b4a970e8 100644 --- a/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py +++ b/enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py @@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.project_repository import ProjectRepository +from litellm.repositories.table_repositories import PrismaTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository @@ -73,6 +74,16 @@ def _verification_token_table( return VerificationTokenRepository(prisma_client).table +class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]): + table_name = "litellm_verificationtoken" + + +def _writer_verification_token_table( + prisma_client: PrismaClient, +) -> TableActions["prisma_models.LiteLLM_VerificationToken"]: + return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table + + def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: return BudgetRepository(prisma_client).table @@ -849,7 +860,7 @@ async def update_project( else None ) if current_project is not None and data.team_id != current_project.team_id: - mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count( + mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count( where={ "project_id": data.project_id, "OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}], diff --git a/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py index 9b87f4f1a7f..19d4988452b 100644 --- a/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py +++ b/enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py @@ -14,6 +14,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final, List, Optional, Protocol from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -37,6 +38,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient router = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) class ManagedVectorStoreRow(Protocol): @@ -348,11 +350,11 @@ async def update_vector_store( raise HTTPException(status_code=500, detail="Database not connected") try: - update_data = data.model_dump(exclude_unset=True) - vector_store_id: Final[str] = update_data.pop("vector_store_id") + update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"}) + vector_store_id: Final = data.vector_store_id if update_data.get("vector_store_metadata") is not None: update_data["vector_store_metadata"] = safe_dumps( - update_data["vector_store_metadata"] + _DECODED_JSON.validate_python(update_data["vector_store_metadata"]) ) updated = await managed_vector_store_table(prisma_client).update( diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..be57c8f3f16 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload -argilla_transformation_object: Optional[Dict[str, Any]] = None +argilla_transformation_object: Mapping[str, str] | None = None _async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. @@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False # Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy -proxy_auth: Optional[Any] = None +proxy_auth: _ProxyAuthHandlerType | None = None ### DEFAULT AZURE API VERSION ### AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest ### DEFAULT WATSONX API VERSION ### @@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) -public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} +public_model_groups_links: dict[str, str | dict[str, object]] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None # priority_reservation_settings is lazy-loaded via __getattr__ @@ -2289,6 +2289,10 @@ if TYPE_CHECKING: LiteLLM_UpperboundKeyGenerateParams, ) from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator + from collections.abc import Awaitable + from litellm.types.utils import ModelResponse as _ModelResponseType + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType + from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -2312,7 +2316,7 @@ if TYPE_CHECKING: supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] - acreate: Callable[..., Any] + acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]] get_max_tokens: Callable[..., int] get_model_info: Callable[..., _ModelInfoType] register_prompt_template: Callable[..., None] @@ -2610,8 +2614,10 @@ def __getattr__(name: str) -> Any: cancel_run, delete_run, ) + from types import MappingProxyType - return locals()[name] + evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals()) + return evals_exports[name] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 46126c444a6..1a7dc343745 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -19,7 +19,7 @@ import importlib import sys from collections.abc import Callable, Mapping from types import ModuleType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final from typing_extensions import ReadOnly, TypedDict @@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object: params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"} # Create the client instance - provider_id: Final = cast(Any, "litellm_module_level_client") + provider_id: Final = "litellm_module_level_client" async_client: Final = get_async_httpx_client( llm_provider=provider_id, params=params, diff --git a/litellm/a2a_protocol/cost_calculator.py b/litellm/a2a_protocol/cost_calculator.py index 31ca81c44dd..9a1431d6457 100644 --- a/litellm/a2a_protocol/cost_calculator.py +++ b/litellm/a2a_protocol/cost_calculator.py @@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners to define custom costs per agent query or per token. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex + +from typing_extensions import Buffer if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -15,6 +18,15 @@ else: LitellmLoggingObject = Any +def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None: + response_cost: Final = model_call_details.get("response_cost", None) + if response_cost is None: + return None + if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)): + return float(response_cost) + raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}") + + class A2ACostCalculator: @staticmethod def calculate_a2a_cost( @@ -45,9 +57,9 @@ class A2ACostCalculator: model_call_details: Final = litellm_logging_obj.model_call_details # Check if user set a custom response cost (backward compatibility) - response_cost: Final = model_call_details.get("response_cost", None) - if response_cost is not None: - return float(response_cost) + custom_response_cost: Final = _custom_response_cost(model_call_details) + if custom_response_cost is not None: + return custom_response_cost # Get litellm_params for cost parameters litellm_params: Final = model_call_details.get("litellm_params", {}) or {} diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py index 404bcca712f..1cea0194715 100644 --- a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -84,14 +84,23 @@ def _filter_reserved_headers( return filtered or None +def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None: + principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM) + if principal is None or isinstance(principal, str): + return principal + if not principal: + return None + raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal) + + def _request_scoped_runtime_session_id( params: Mapping[str, object], - litellm_params: Mapping[str, Any], + litellm_params: Mapping[str, object], ) -> str | None: context_id: Final = get_session_id_from_a2a_params(params) if not isinstance(context_id, str) or not context_id: return None - return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)) + return scope_session_to_principal(context_id, _calling_key_hash(litellm_params)) def _validate_runtime_session_id(session_id: str, model: str) -> str: @@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str: ) +def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str: + model: Final = litellm_params.get("model", "") + if isinstance(model, str): + return model + raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model) + + class BedrockAgentCoreA2ATransformation: """ Request/response transformation for Bedrock AgentCore A2A agents. @@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation: """ # Extract model and strip the "bedrock/" prefix # "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..." - model: Final = litellm_params.get("model", "") - if model.startswith("bedrock/"): - agentcore_model = model[len("bedrock/") :] - else: - agentcore_model = model + model: Final = _agentcore_model_name(litellm_params) + agentcore_model: Final = model.removeprefix("bedrock/") # Build optional_params from litellm_params (everything except model and custom_llm_provider) optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")} diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py index 57c4a4677d0..c2d581321cd 100644 --- a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py @@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model: """ import asyncio -from collections.abc import AsyncIterator, Mapping -from typing import Any, Final +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence +from itertools import chain +from typing import Final from uuid import uuid4 from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items +from litellm.llms.openai.common_utils import decoded_json_object + + +def _a2a_text_part(part: object) -> Sequence[object]: + part_fields: Final = as_str_mapping(part) + if not isinstance(part_fields, dict): + return () + if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"): + return (part_fields["text"],) + return () + + +def _join_with_spaces(texts: Sequence[object]) -> str: + strings: Final = [text for text in texts if isinstance(text, str)] + if len(strings) == len(texts): + return " ".join(strings) + first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str)) + raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found") class WatsonxOrchestrateTransformation: @@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation: return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}" @staticmethod - def extract_text_from_a2a_params(params: dict[str, Any]) -> str: + def extract_text_from_a2a_params(params: Mapping[str, object]) -> str: """ Extract user message text from A2A MessageSendParams. A2A format: params.message.parts[*] where part.kind == "text" """ - message: Final = params.get("message", {}) - parts: Final = message.get("parts", []) - texts: Final = [] - for part in parts: - if not isinstance(part, dict): - continue - kind = part.get("kind") - if kind in (None, "", "text") and part.get("text"): - texts.append(part["text"]) - return " ".join(texts) or "" + message: Final = decoded_json_object(params.get("message", {})) + parts: Final[Iterable[object]] = iter_items(message.get("parts", [])) + texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts))) + return _join_with_spaces(texts) or "" @staticmethod def build_wxo_run_body( diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index 7400844bf28..17d1b86a609 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -4,15 +4,20 @@ Utility functions for A2A protocol. import hashlib from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, Protocol import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import has_attributes if TYPE_CHECKING: from a2a.types import SendMessageRequest, SendStreamingMessageRequest +class _SupportsModelDump(Protocol): + def model_dump(self, *, mode: str) -> object: ... + + class A2ARequestUtils: """Utility class for A2A request/response processing.""" @@ -74,7 +79,7 @@ class A2ARequestUtils: @staticmethod def get_input_message_from_request( request: "SendMessageRequest | SendStreamingMessageRequest", - ) -> Any: + ) -> object: """ Extract the input message from an A2A request. @@ -84,7 +89,7 @@ class A2ARequestUtils: Returns: The message object/dict or None """ - params: Final = getattr(request, "params", None) + params: Final[object] = getattr(request, "params", None) if params is None: return None return getattr(params, "message", None) @@ -127,10 +132,12 @@ class A2ARequestUtils: # the dict branch — request-side parts are a2a-sdk Part RootModels whose # kind/text live on part.root, which the object branch cannot read. This # mirrors how the response side already works (it operates on model_dump). - input_message = A2ARequestUtils.get_input_message_from_request(request) - if input_message is not None and hasattr(input_message, "model_dump"): - input_message = input_message.model_dump(mode="json") - input_text: Final = A2ARequestUtils.extract_text_from_message(input_message) + input_message: Final = A2ARequestUtils.get_input_message_from_request(request) + input_text: Final = A2ARequestUtils.extract_text_from_message( + input_message.model_dump(mode="json") + if has_attributes(input_message, _SupportsModelDump, "model_dump") + else input_message + ) prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text) # Count output tokens diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 61e36bd1e96..fec1d8e4465 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -45,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]] _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) +_DECODED_JSON: Final = TypeAdapter(object) class GetAnthropicBetaHeadersConfig: @@ -76,7 +77,7 @@ class GetAnthropicBetaHeadersConfig: } @staticmethod - def _check_is_valid_dict(fetched_config: dict) -> bool: + def _check_is_valid_dict(fetched_config: object) -> bool: """Check if fetched config is a non-empty dict with expected structure.""" if not isinstance(fetched_config, dict): verbose_logger.warning( @@ -110,7 +111,7 @@ class GetAnthropicBetaHeadersConfig: return True @classmethod - def validate_beta_headers_config(cls, fetched_config: dict) -> bool: + def validate_beta_headers_config(cls, fetched_config: object) -> bool: """ Validate the integrity of a fetched beta headers config. @@ -119,7 +120,7 @@ class GetAnthropicBetaHeadersConfig: return cls._check_is_valid_dict(fetched_config) @staticmethod - def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict: + def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object: """ Fetch the beta headers config from a remote URL. @@ -128,7 +129,7 @@ class GetAnthropicBetaHeadersConfig: """ response: Final = httpx.get(url, timeout=timeout) response.raise_for_status() - return response.json() + return _DECODED_JSON.validate_python(response.json()) def get_beta_headers_config(url: str) -> dict: @@ -161,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict: return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() # Validate the fetched config - if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content): + if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance( + content, dict + ): verbose_logger.warning( "LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s", url, diff --git a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py index eb3ec8aaee2..d8198e41239 100644 --- a/litellm/anthropic_interface/exceptions/exception_mapping_utils.py +++ b/litellm/anthropic_interface/exceptions/exception_mapping_utils.py @@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop import json from typing import Final +from pydantic import TypeAdapter + from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType +_PARSED_ERROR_BODY: Final = TypeAdapter(object) + # HTTP status code -> Anthropic error type # Source: https://docs.anthropic.com/en/api/errors ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = { @@ -79,7 +83,7 @@ class AnthropicExceptionMapping: - Generic: {"message": "..."} - Plain strings """ - parsed: Final = safe_json_loads(raw_message) + parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) if isinstance(parsed, dict): # Bedrock format if "detail" in parsed and isinstance(parsed["detail"], dict): @@ -145,9 +149,8 @@ class AnthropicExceptionMapping: AnthropicErrorResponse dict """ # Try to parse as JSON once - parsed: dict | None = safe_json_loads(raw_message) - if not isinstance(parsed, dict): - parsed = None + decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message)) + parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None # If parsed and already in Anthropic format - passthrough if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed): diff --git a/litellm/assistants/main.py b/litellm/assistants/main.py index a3428261636..a5721c6f047 100644 --- a/litellm/assistants/main.py +++ b/litellm/assistants/main.py @@ -77,7 +77,7 @@ def get_assistants( api_base: str | None = None, api_version: str | None = None, **kwargs, -) -> SyncCursorPage[Assistant]: +) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]: aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None) if aget_assistants is not None and not isinstance(aget_assistants, bool): raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed") @@ -100,7 +100,7 @@ def get_assistants( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[Assistant] | None = None + response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -564,7 +564,7 @@ def create_thread( tool_resources: OpenAICreateThreadParamsToolResources | None = None, client: OpenAI | None = None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """ - get the llm provider - if openai - route it there @@ -610,7 +610,7 @@ def create_thread( api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -735,7 +735,7 @@ def get_thread( thread_id: str, client=None, **kwargs, -) -> Thread: +) -> Thread | Coroutine[None, None, Thread]: """Get the thread object, given a thread_id""" aget_thread: Final = kwargs.pop("aget_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -757,7 +757,7 @@ def get_thread( timeout = 600.0 api_base: str | None = None api_key: str | None = None - response: Thread | None = None + response: Thread | Coroutine[None, None, Thread] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -903,7 +903,7 @@ def add_message( metadata: dict | None = None, client=None, **kwargs, -) -> OpenAIMessage: +) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]: ### COMMON OBJECTS ### a_add_message: Final = kwargs.pop("a_add_message", None) _message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata) @@ -935,7 +935,7 @@ def add_message( timeout = 600.0 api_key: str | None = None api_base: str | None = None - response: OpenAIMessage | None = None + response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -1065,7 +1065,7 @@ def get_messages( thread_id: str, client: Any | None = None, **kwargs, -) -> SyncCursorPage[OpenAIMessage]: +) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]: aget_messages: Final = kwargs.pop("aget_messages", None) optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) @@ -1086,7 +1086,7 @@ def get_messages( elif timeout is None: timeout = 600.0 - response: SyncCursorPage[OpenAIMessage] | None = None + response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None api_key: str | None = None api_base: str | None = None if custom_llm_provider == "openai": @@ -1256,7 +1256,7 @@ def run_thread( client: object | None = None, event_handler: AssistantEventHandler | None = None, # for stream=True calls **kwargs, -) -> Run: +) -> Run | Coroutine[None, None, Run]: """Run a given thread + assistant.""" arun_thread: Final = kwargs.pop("arun_thread", None) optional_params: Final = GenericLiteLLMParams(**kwargs) @@ -1278,7 +1278,7 @@ def run_thread( elif timeout is None: timeout = 600.0 - response: Run | None = None + response: Run | Coroutine[None, None, Run] | None = None if custom_llm_provider == "openai": api_base = ( optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there diff --git a/litellm/caching/_embedding_router.py b/litellm/caching/_embedding_router.py index d82e58ecc91..0e7589d88ab 100644 --- a/litellm/caching/_embedding_router.py +++ b/litellm/caching/_embedding_router.py @@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import litellm from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS @@ -25,12 +25,12 @@ if TYPE_CHECKING: def resolve_embedding_router( embedding_model: str, llm_router: Router | None, - llm_model_list: list[dict[str, Any]] | None, + llm_model_list: Sequence[Mapping[str, object]] | None, ) -> Router | None: """Return ``llm_router`` iff it serves ``embedding_model`` as a deployment.""" if llm_router is None: return None - router_model_names: Final[list[str]] = ( + router_model_names: Final[Sequence[object]] = ( [m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else [] ) if embedding_model in router_model_names: diff --git a/litellm/caching/azure_blob_cache.py b/litellm/caching/azure_blob_cache.py index bf0e15421bc..5b9071dc920 100644 --- a/litellm/caching/azure_blob_cache.py +++ b/litellm/caching/azure_blob_cache.py @@ -13,10 +13,14 @@ import json from contextlib import suppress from typing import Final +from pydantic import TypeAdapter + from litellm._logging import print_verbose, verbose_logger from .base_cache import BaseCache +_DECODED_BLOB: Final = TypeAdapter(object) + class AzureBlobCache(BaseCache): def __init__(self, account_url, container) -> None: @@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache): print_verbose(f"Get Azure Blob Cache: key: {key}") as_bytes: Final = self.container_client.download_blob(key).readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", @@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache): blob: Final = await self.async_container_client.download_blob(key) as_bytes: Final = await blob.readall() as_str: Final = as_bytes.decode("utf-8") - cached_response: Final[object] = json.loads(as_str) + cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str)) verbose_logger.debug( "Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s", key, diff --git a/litellm/caching/base_cache.py b/litellm/caching/base_cache.py index 51c169ba796..fb91d13a8c9 100644 --- a/litellm/caching/base_cache.py +++ b/litellm/caching/base_cache.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py index 50939ad51ca..9470f23570a 100644 --- a/litellm/caching/disk_cache.py +++ b/litellm/caching/disk_cache.py @@ -1,15 +1,19 @@ import json from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from .base_cache import BaseCache if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any +_DECODED_DISK_VALUE: Final = TypeAdapter(object) + class DiskCache(BaseCache): def __init__(self, disk_cache_dir: str | None = None): @@ -44,7 +48,7 @@ class DiskCache(BaseCache): original_cached_response: Final = self.disk_cache.get(key) if original_cached_response: try: - cached_response = json.loads(original_cached_response) + cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response)) except Exception: cached_response = original_cached_response return cached_response diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98f6f1409f2..a0da0f6ddca 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1200,12 +1200,16 @@ def get_usage_object( elif isinstance(usage_obj, dict): return Usage(**usage_obj) elif isinstance(usage_obj, BaseModel): - return Usage(**usage_obj.model_dump()) + return _usage_from_model(usage_obj) else: verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj) return None +def _usage_from_model(usage_model: BaseModel) -> Usage: + return Usage(**usage_model.model_dump()) + + def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper: match usage: case DecisionsUsage(): @@ -1686,7 +1690,7 @@ def completion_cost( setattr( completion_response, "usage", - litellm.Usage(**_usage_for_dump.model_dump()), + _usage_from_model(_usage_for_dump), ) if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): _usage = _decisions_usage(usage_obj).model_dump() @@ -3035,8 +3039,11 @@ def _batch_prompt_cost( ) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) + + def _attribute_value(obj: object, name: str) -> object: - return getattr(obj, name) + return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name)) def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]: @@ -3671,7 +3678,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | return None -def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None: +_TranscriptionRateKey: TypeAlias = Literal[ + "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" +] + + +def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None: + match key: + case "input_cost_per_second": + return info.get("input_cost_per_second") + case "input_cost_per_audio_token": + return info.get("input_cost_per_audio_token") + case "input_cost_per_token": + return info.get("input_cost_per_token") + case "output_cost_per_token": + return info.get("output_cost_per_token") + case _: + assert_never(key) + + +def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: @@ -3680,12 +3706,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) if declared is None: return None return next( - (float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None), + ( + float(value) + for key in keys + if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None + ), None, ) -def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float: +def _transcription_rate( + keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None +) -> float: rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base)) return next((rate for rate in rates if rate is not None), 0.0) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 733f1aa72d5..b8e933bf046 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError): self.message = _saved_message self.args = (_saved_message,) - original_headers: Final = getattr(original_exception, "litellm_response_headers", None) + original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None) if original_headers is not None: self.litellm_response_headers = original_headers diff --git a/litellm/experimental_mcp_client/tools.py b/litellm/experimental_mcp_client/tools.py index 8015b065899..4f204b3629e 100644 --- a/litellm/experimental_mcp_client/tools.py +++ b/litellm/experimental_mcp_client/tools.py @@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool from openai.types.chat import ChatCompletionToolParam from openai.types.responses.function_tool_param import FunctionToolParam from openai.types.shared_params.function_definition import FunctionDefinition +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import ( @@ -20,6 +21,8 @@ from litellm.constants import ( from litellm.types.llms.anthropic import AnthropicMessagesTool from litellm.types.utils import ChatCompletionMessageToolCall +_DECODED_JSON: Final = TypeAdapter(object) + ######################################################## # List MCP Tool functions @@ -200,15 +203,18 @@ async def call_mcp_tool( return tool_result +def _decode_json_arguments(arguments: str) -> object: + try: + return _DECODED_JSON.validate_python(json.loads(arguments)) + except json.JSONDecodeError: + return {} + + def _get_function_arguments(function: FunctionDefinition) -> dict: """Helper to safely get and parse function arguments.""" - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError: - arguments = {} - return arguments if isinstance(arguments, dict) else {} + arguments: Final = function.get("arguments", {}) + decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments + return decoded if isinstance(decoded, dict) else {} def transform_openai_tool_call_request_to_mcp_tool_call_request( diff --git a/litellm/files/main.py b/litellm/files/main.py index 16ff5b02f9c..c5172bc01ae 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -9,7 +9,7 @@ import asyncio import contextvars import time import uuid as uuid_module -from collections.abc import Coroutine +from collections.abc import Awaitable, Coroutine from functools import partial from typing import Final, Literal, cast @@ -293,6 +293,14 @@ def create_file( raise e +async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject: + return await pending + + +async def _await_deleted_file(pending: Awaitable[object]) -> object: + return await pending + + @client async def afile_retrieve( file_id: str, @@ -325,7 +333,7 @@ async def afile_retrieve( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_retrieved_file(init_response) else: response = init_response @@ -505,7 +513,7 @@ async def afile_delete( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) if asyncio.iscoroutine(init_response): - response = await init_response + response = await _await_deleted_file(init_response) else: response = init_response diff --git a/litellm/files/streaming.py b/litellm/files/streaming.py index 460050872f4..e9a030e0e32 100644 --- a/litellm/files/streaming.py +++ b/litellm/files/streaming.py @@ -1,7 +1,7 @@ import datetime import traceback from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final, Optional, cast +from typing import TYPE_CHECKING, Final, Optional, cast import anyio @@ -163,7 +163,7 @@ class FileContentStreamingResponse: merged_hidden_params: Final = cast( "StandardLoggingHiddenParams", { - **cast(dict[str, Any], payload.get("hidden_params") or {}), + **(payload.get("hidden_params") or {}), **self._hidden_params, }, ) diff --git a/litellm/harness/handlers/base.py b/litellm/harness/handlers/base.py index 18b7391e988..4721e9d6ff2 100644 --- a/litellm/harness/handlers/base.py +++ b/litellm/harness/handlers/base.py @@ -4,7 +4,6 @@ from __future__ import annotations from abc import ABC, abstractmethod from collections.abc import AsyncIterator -from typing import Any from litellm.harness.context import SessionContext from litellm.harness.errors import CapabilityUnsupported @@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC): async def resume(self, ctx: SessionContext, native_session_id: str) -> None: """Continue the runtime's own session on the next turn.""" - async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape + async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history") diff --git a/litellm/harness/handlers/cli_handler.py b/litellm/harness/handlers/cli_handler.py index 6320a877e6b..b01f449a67d 100644 --- a/litellm/harness/handlers/cli_handler.py +++ b/litellm/harness/handlers/cli_handler.py @@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None: async def private_dir_for(sandbox: Sandbox) -> str: - tempdir = getattr(sandbox, "tempdir", None) + tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None if tempdir is None: raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir") - path: str = await tempdir() + path: Final = await tempdir() return path diff --git a/litellm/harness/handlers/deepagents_handler.py b/litellm/harness/handlers/deepagents_handler.py index 4a36b1a08d1..f6785323292 100644 --- a/litellm/harness/handlers/deepagents_handler.py +++ b/litellm/harness/handlers/deepagents_handler.py @@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler): async def history( self, ctx: SessionContext - ) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts + ) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts agent, deps = self._require_agent() snapshot = await agent.aget_state(self._run_config(ctx, None)) messages = (snapshot.values or MappingProxyType({})).get("messages") or () diff --git a/litellm/harness/runtime.py b/litellm/harness/runtime.py index dc24a1f0ef7..1d946521bfa 100644 --- a/litellm/harness/runtime.py +++ b/litellm/harness/runtime.py @@ -12,11 +12,10 @@ import inspect import logging import os import uuid -from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType from typing import ( - Any, Final, get_args, ) @@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None: queue.put_nowait(end) +async def _await_decision(pending: Awaitable[object]) -> object: + return await pending + + async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None: """Run on_approval (sync in a worker thread, or async) and resolve approval.""" try: if inspect.iscoroutinefunction(handler): - decision: object = await handler(approval) + decision: object = await _await_decision(handler(approval)) else: decision = await asyncio.to_thread(handler, approval) if inspect.isawaitable(decision): @@ -749,7 +752,7 @@ class AsyncSession: async def history( self, - ) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler + ) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler if not self.harness_config.capabilities.history: raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history") await self._ensure_ready() diff --git a/litellm/images/main.py b/litellm/images/main.py index 7f94835d59b..29bac3ffe05 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -3,7 +3,7 @@ import contextvars import importlib from collections.abc import Coroutine from functools import partial -from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, overload from pydantic import TypeAdapter, ValidationError @@ -70,16 +70,17 @@ from litellm.utils import ( _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) # Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls -_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None +_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None -def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils": +def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]: """Get ImageEditRequestUtils, loading it lazily if needed.""" global _ImageEditRequestUtils_cache if _ImageEditRequestUtils_cache is None: # Access via module to trigger __getattr__ if not cached - module: Final = importlib.import_module(__name__) - _ImageEditRequestUtils_cache = module.ImageEditRequestUtils + from litellm.images import main as images_main + + _ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker return _ImageEditRequestUtils_cache diff --git a/litellm/integrations/SlackAlerting/hanging_request_check.py b/litellm/integrations/SlackAlerting/hanging_request_check.py index 6f986144d4c..87baee91780 100644 --- a/litellm/integrations/SlackAlerting/hanging_request_check.py +++ b/litellm/integrations/SlackAlerting/hanging_request_check.py @@ -115,14 +115,16 @@ class AlertingHangingRequestCheck: if hanging_request_data.alerted: continue - request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache( - key=f"request_status:{hanging_request_data.request_id}", - litellm_parent_otel_span=None, - local_only=True, - ) # this means the request status was either success or fail # and is not hanging - if request_status is not None: + if ( + await proxy_logging_obj.internal_usage_cache.async_get_cache( + key=f"request_status:{hanging_request_data.request_id}", + litellm_parent_otel_span=None, + local_only=True, + ) + is not None + ): # clear this request from hanging request cache since the request was either success or failed self.hanging_request_cache._remove_key( key=request_id, diff --git a/litellm/integrations/SlackAlerting/user_spend_alerts.py b/litellm/integrations/SlackAlerting/user_spend_alerts.py index 38794735c1b..08799a2f59f 100644 --- a/litellm/integrations/SlackAlerting/user_spend_alerts.py +++ b/litellm/integrations/SlackAlerting/user_spend_alerts.py @@ -56,8 +56,9 @@ async def fetch_user_spend_rows( today_str: Final = today.strftime("%Y-%m-%d") month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d") baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d") - raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str) - return USER_SPEND_ROWS_ADAPTER.validate_python(raw) + return USER_SPEND_ROWS_ADAPTER.validate_python( + await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str) + ) def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None: diff --git a/litellm/integrations/agentops/agentops.py b/litellm/integrations/agentops/agentops.py index 9043ab1853a..3364520a8ae 100644 --- a/litellm/integrations/agentops/agentops.py +++ b/litellm/integrations/agentops/agentops.py @@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls import os from dataclasses import dataclass -from typing import Any, Final +from typing import Final + +from pydantic import TypeAdapter from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import get_httpx_client +_DECODED_JSON: Final = TypeAdapter(object) + @dataclass class AgentOpsConfig: @@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry): project_id = None if config.api_key: try: - response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint) - jwt_token = response.get("token") - project_id = response.get("project_id") + response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint)) + if response is not None: + jwt_token = response.get("token") + project_id = response.get("project_id") except Exception: pass @@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry): "telemetry.sdk.name": "agentops", } - if project_id: - resource_attrs["project.id"] = project_id + self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs - self.resource_attributes = resource_attrs - - def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]: + def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object: """ Fetch JWT authentication token from AgentOps API @@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry): if response.status_code != 200: raise Exception(f"Failed to fetch auth token: {response.text}") - return response.json() + return _DECODED_JSON.validate_python(response.json()) finally: client.close() diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 24279b443aa..5a03b91d470 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry): primary ``otel`` callback which handles proxy-level parent spans. """ - def set_attributes(self, span: Span, kwargs, response_obj: Any | None): + def set_attributes(self, span: Span, kwargs, response_obj: object): ArizeLogger.set_arize_attributes(span, kwargs, response_obj) @staticmethod diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 501f5749ea4..f72b29a6ad8 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -3,9 +3,10 @@ import os from datetime import datetime -from typing import Final +from typing import Final, Protocol import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm import verbose_logger @@ -24,6 +25,18 @@ from litellm.utils import print_verbose API_BASE: Final = "https://api.braintrustdata.com/v1" +class _BraintrustProjectBody(TypedDict): + id: ReadOnly[str] + + +class _BraintrustProjectResponse(Protocol): + def json(self) -> _BraintrustProjectBody: ... + + +def _project_id(response: _BraintrustProjectResponse) -> str: + return response.json()["id"] + + def get_utc_datetime(): import datetime as dt @@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger): headers=self.headers, json={"name": project_name}, ) - project_dict: Final = response.json() - project_id: Final = project_dict["id"] + project_id: Final = _project_id(response) self._project_id_cache[project_name] = project_id return project_id except httpx.HTTPStatusError as e: @@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger): headers=self.headers, json={"name": project_name}, ) - project_dict: Final = response.json() - project_id: Final = project_dict["id"] + project_id: Final = _project_id(response) self._project_id_cache[project_name] = project_id return project_id except httpx.HTTPStatusError as e: @@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger): f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} ) - project_dict: Final = project.json() - - self.default_project_id = project_dict["id"] + self.default_project_id = _project_id(project) def create_sync_default_project_and_experiment(self): project: Final = self.global_braintrust_sync_http_handler.post( f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"} ) - project_dict: Final = project.json() - - self.default_project_id = project_dict["id"] + self.default_project_id = _project_id(project) def log_success_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug("REACHES BRAINTRUST SUCCESS") diff --git a/litellm/integrations/cloudzero/cz_stream_api.py b/litellm/integrations/cloudzero/cz_stream_api.py index 1e0413e8576..d0b25ea83e8 100644 --- a/litellm/integrations/cloudzero/cz_stream_api.py +++ b/litellm/integrations/cloudzero/cz_stream_api.py @@ -19,8 +19,9 @@ """Output modules for writing CBF data to various destinations.""" import zoneinfo +from collections.abc import Mapping from datetime import datetime, timezone -from typing import Any, Final +from typing import Final import httpx import polars as pl @@ -188,7 +189,7 @@ class CloudZeroStreamer: ) raise - def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]: + def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]: """Prepare batch payload according to CloudZero AnyCost API format.""" # Convert batch_date to month for the API (YYYY-MM format) try: @@ -205,7 +206,7 @@ class CloudZeroStreamer: if record: data_records.append(record) - payload: Final = {"month": month_str, "operation": operation, "data": data_records} + payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records} return payload diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 1c12ac395c7..9e2dc902d55 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger): self, violation_message: str, request_data: dict[str, Any], - detection_info: dict[str, Any] | None = None, + detection_info: Mapping[str, object] | None = None, original_response: object = None, ) -> None: """ @@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger): if event_type is not None: guardrail_mode = event_type elif isinstance(self.event_hook, Mode): - guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) + guardrail_mode = GuardrailMode( + tags=copy.deepcopy(self.event_hook.tags), + default=copy.deepcopy(self.event_hook.default), + ) else: guardrail_mode = self.event_hook diff --git a/litellm/integrations/custom_secret_manager.py b/litellm/integrations/custom_secret_manager.py index e59842d409a..8fc5bc551c8 100644 --- a/litellm/integrations/custom_secret_manager.py +++ b/litellm/integrations/custom_secret_manager.py @@ -37,7 +37,6 @@ Usage: """ from abc import abstractmethod -from typing import Any import httpx @@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Asynchronously write a secret to your custom secret manager. diff --git a/litellm/integrations/dotprompt/__init__.py b/litellm/integrations/dotprompt/__init__.py index 4cf6c0de2df..0a2921db6c7 100644 --- a/litellm/integrations/dotprompt/__init__.py +++ b/litellm/integrations/dotprompt/__init__.py @@ -61,7 +61,9 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom prompt_file: Final = getattr(litellm_params, "prompt_file", None) # Handle dotprompt_content from database - dotprompt_content: Final = getattr(litellm_params, "dotprompt_content", None) + dotprompt_content: Final = ( + litellm_params.dotprompt_content if hasattr(litellm_params, "dotprompt_content") else None + ) if dotprompt_content and not prompt_data and not prompt_file: prompt_data = get_prompt_data_from_dotprompt_content(dotprompt_content) diff --git a/litellm/integrations/dotprompt/dotprompt_manager.py b/litellm/integrations/dotprompt/dotprompt_manager.py index c646dbf4e2e..77d70dd761c 100644 --- a/litellm/integrations/dotprompt/dotprompt_manager.py +++ b/litellm/integrations/dotprompt/dotprompt_manager.py @@ -7,6 +7,8 @@ import json from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final +from typing_extensions import ReadOnly, TypedDict + from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.types.llms.openai import AllMessageValues @@ -18,7 +20,12 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -from .prompt_manager import PromptManager, PromptTemplate +from .prompt_manager import PromptJsonEntry, PromptManager, PromptTemplate + + +class _PromptJsonData(TypedDict, total=False): + content: ReadOnly[str] + metadata: ReadOnly[dict[str, object]] class DotpromptManager(CustomPromptManagement): @@ -342,17 +349,17 @@ class DotpromptManager(CustomPromptManagement): if self._prompt_manager: self._prompt_manager.reload_prompts() - def add_prompt_from_json(self, prompt_id: str, json_data: dict[str, Any]) -> None: + def add_prompt_from_json(self, prompt_id: str, json_data: _PromptJsonData) -> None: """Add a prompt from JSON data.""" content: Final = json_data.get("content", "") metadata: Final = json_data.get("metadata", {}) self.prompt_manager.add_prompt(prompt_id, content, metadata) - def load_prompts_from_json(self, prompts_data: dict[str, dict[str, object]]) -> None: + def load_prompts_from_json(self, prompts_data: Mapping[str, object]) -> None: """Load multiple prompts from JSON data.""" self.prompt_manager.load_prompts_from_json_data(prompts_data) - def get_prompts_as_json(self) -> dict[str, dict[str, object]]: + def get_prompts_as_json(self) -> dict[str, PromptJsonEntry]: """Get all prompts in JSON format.""" return self.prompt_manager.get_all_prompts_as_json() diff --git a/litellm/integrations/dotprompt/prompt_manager.py b/litellm/integrations/dotprompt/prompt_manager.py index 0eda266740c..0e52ca0b62f 100644 --- a/litellm/integrations/dotprompt/prompt_manager.py +++ b/litellm/integrations/dotprompt/prompt_manager.py @@ -12,6 +12,8 @@ from jinja2 import DictLoader, select_autoescape from jinja2.sandbox import ImmutableSandboxedEnvironment from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.litellm_core_utils.core_helpers import as_str_mapping + class _PromptFileJson(TypedDict): """JSON form of a .prompt file: rendered template text plus its frontmatter.""" @@ -20,6 +22,11 @@ class _PromptFileJson(TypedDict): metadata: ReadOnly[NotRequired[dict[str, object]]] +class PromptJsonEntry(TypedDict): + content: ReadOnly[str] + metadata: ReadOnly[dict[str, object]] + + def strip_version_suffix(prompt_id: str) -> str | None: base, separator, version = prompt_id.rpartition(".v") if separator and base and version.isdigit(): @@ -358,20 +365,26 @@ class PromptManager: return f"---\n{frontmatter_yaml}---\n{content}" - def get_all_prompts_as_json(self) -> dict[str, dict[str, Any]]: + def get_all_prompts_as_json(self) -> dict[str, PromptJsonEntry]: """Get all loaded prompts in JSON format. Returns: Dictionary mapping prompt_id to prompt data """ - result: Final = {} - for prompt_id, template in self.prompts.items(): - result[prompt_id] = { + return { + prompt_id: { "content": template.content, "metadata": template.metadata, } - return result + for prompt_id, template in self.prompts.items() + } - def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None: - """Load additional prompts from JSON data (merges with existing prompts).""" - self._load_prompts_from_json(prompt_data) + def load_prompts_from_json_data(self, prompt_data: Mapping[str, object]) -> None: + """Load additional prompts from JSON data (merges with existing prompts). Entries that are not + mappings are skipped, the same as a malformed entry inside the loader.""" + entries: Final = { + prompt_id: dict(entry) + for prompt_id, raw in prompt_data.items() + if (entry := as_str_mapping(raw)) is not None + } + self._load_prompts_from_json(entries) diff --git a/litellm/integrations/focus/destinations/factory.py b/litellm/integrations/focus/destinations/factory.py index 0ecf04326b3..cf871013265 100644 --- a/litellm/integrations/focus/destinations/factory.py +++ b/litellm/integrations/focus/destinations/factory.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import Mapping from typing import Any, Final from .base import FocusDestination @@ -20,7 +21,7 @@ class FocusDestinationFactory: *, provider: str, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, object] | None = None, ) -> FocusDestination: """Return a destination implementation for the requested provider.""" provider_lower: Final = provider.lower() @@ -39,7 +40,7 @@ class FocusDestinationFactory: def _resolve_config( *, provider: str, - overrides: dict[str, object], + overrides: Mapping[str, object], ) -> dict[str, Any]: if provider == "s3": resolved = { diff --git a/litellm/integrations/focus/export_engine.py b/litellm/integrations/focus/export_engine.py index 615aba72954..5c345630984 100644 --- a/litellm/integrations/focus/export_engine.py +++ b/litellm/integrations/focus/export_engine.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Callable -from typing import Any, Final, SupportsFloat, SupportsIndex, SupportsInt, cast +from collections.abc import Callable, Mapping +from typing import Final, SupportsFloat, SupportsIndex, SupportsInt, cast import polars as pl @@ -24,7 +24,7 @@ class FocusExportEngine: provider: str, export_format: str, prefix: str, - destination_config: dict[str, Any] | None = None, + destination_config: Mapping[str, object] | None = None, ) -> None: self.provider = provider self.export_format = export_format @@ -77,7 +77,7 @@ class FocusExportEngine: return FocusParquetSerializer() raise NotImplementedError(f"Export format '{self.export_format}' not supported. Use 'parquet' or 'csv'.") - async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, Any]: + async def dry_run_export_usage_data(self, limit: int | None) -> dict[str, object]: data: Final = await self.database.get_usage_data(limit=limit) normalized: Final = self.transformer.transform(data) diff --git a/litellm/integrations/focus/focus_logger.py b/litellm/integrations/focus/focus_logger.py index e961b53a240..71b61296cfc 100644 --- a/litellm/integrations/focus/focus_logger.py +++ b/litellm/integrations/focus/focus_logger.py @@ -6,6 +6,8 @@ import os from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, ClassVar, Final, cast +from typing_extensions import TypedDict, Unpack + import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger @@ -25,6 +27,10 @@ FOCUS_USAGE_DATA_JOB_NAME: Final = "focus_export_usage_data" DEFAULT_DRY_RUN_LIMIT: Final = 500 +class _CustomLoggerOptions(TypedDict, total=False, extra_items=object): + pass + + class FocusLogger(CustomLogger): """Coordinates Focus export jobs across transformer/serializer/destination layers.""" @@ -40,7 +46,7 @@ class FocusLogger(CustomLogger): interval_seconds: int | None = None, prefix: str | None = None, destination_config: dict[str, Any] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomLoggerOptions], ) -> None: super().__init__(**kwargs) self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower() diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 5bc7247e02a..99cd8b31b63 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -3,6 +3,7 @@ import hashlib import json import os import time +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any, Final @@ -15,6 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase from litellm.litellm_core_utils.cloud_storage_security import ( sanitize_cloud_object_component, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.gcs_bucket import * from litellm.types.utils import StandardBatchLineItemsPayload, StandardLoggingPayload @@ -134,7 +136,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): """ return f"{date_str}/batch-{batch_id}.ndjson" - def _get_config_key(self, kwargs: dict[str, Any]) -> str: + def _get_config_key(self, kwargs: Mapping[str, object]) -> str: """ Extract a synchronous grouping key from kwargs to group items by GCS config. This allows us to batch items with the same bucket/credentials together. @@ -143,7 +145,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key() for logging purposes. """ - standard_callback_dynamic_params: Final = kwargs.get("standard_callback_dynamic_params", None) or {} + standard_callback_dynamic_params: Final = ( + as_str_mapping(kwargs.get("standard_callback_dynamic_params", None)) or {} + ) bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default" path_service_account: Final = ( diff --git a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py index 9dfd75e5559..3f8cf93e11c 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py @@ -9,6 +9,7 @@ Usage: """ import asyncio +import os from collections.abc import AsyncIterable, Iterable from typing import Final @@ -40,7 +41,7 @@ _mocks_initialized = False # Default mock latency in seconds (simulates network round-trip) # Typical GCS API calls take 100-300ms for uploads, 50-150ms for GET/DELETE -_MOCK_LATENCY_SECONDS: Final = float(__import__("os").getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 +_MOCK_LATENCY_SECONDS: Final = float(os.getenv("GCS_MOCK_LATENCY_MS", "150")) / 1000.0 async def _mock_async_handler_get(self, url, params=None, headers=None, follow_redirects=None): diff --git a/litellm/integrations/gcs_pubsub/pub_sub.py b/litellm/integrations/gcs_pubsub/pub_sub.py index c799245c215..cbf6898d978 100644 --- a/litellm/integrations/gcs_pubsub/pub_sub.py +++ b/litellm/integrations/gcs_pubsub/pub_sub.py @@ -12,6 +12,8 @@ import os import traceback from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: @@ -27,6 +29,8 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) +_DECODED_JSON: Final = TypeAdapter(object) + class GcsPubSubLogger(CustomBatchLogger): def __init__( @@ -152,7 +156,7 @@ class GcsPubSubLogger(CustomBatchLogger): finally: self.log_queue.clear() - async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> dict[str, Any] | None: + async def publish_message(self, message: SpendLogsPayload | StandardLoggingPayload) -> object: """ Publish message to Google Cloud Pub/Sub using REST API @@ -188,7 +192,7 @@ class GcsPubSubLogger(CustomBatchLogger): raise Exception(f"Failed to publish message: {response.text}") verbose_logger.debug("Pub/Sub response: %s", response.text) - return response.json() + return _DECODED_JSON.validate_python(response.json()) except Exception as e: verbose_logger.error("Pub/Sub publish error: %s", str(e)) diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 86f80047e52..b8c6e91d0d7 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -20,6 +20,7 @@ from litellm.integrations.langfuse.langfuse_mock_client import ( should_use_langfuse_mock, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, filter_exceptions_from_params, reconstruct_model_name, safe_deep_copy, @@ -1268,11 +1269,11 @@ def log_provider_specific_information_as_span( *, tracing: "LangfuseTracing", parent: "LangfuseObservation", - enrichments: Mapping[str, Any], + enrichments: Mapping[str, object], ): """Logs provider-specific information as spans under the generation.""" - _hidden_params: Final[Mapping[str, object] | None] = enrichments.get("hidden_params", None) + _hidden_params: Final = as_str_mapping(enrichments.get("hidden_params", None)) if _hidden_params is None: return diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index f195cd97e3c..9364a3d4216 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import ( serialize_langfuse_datetime, ) from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.types.integrations.langfuse_otel import ( LangfuseSpanAttributes, @@ -222,7 +223,7 @@ class LangfuseOtelLogger(OpenTelemetry): ) @staticmethod - def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj): + def _set_langfuse_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj): """ Sets Langfuse specific metadata attributes onto the OTEL span. @@ -234,7 +235,7 @@ class LangfuseOtelLogger(OpenTelemetry): from litellm.integrations.arize._utils import safe_set_attribute from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - dynamic_params: Final = kwargs.get("standard_callback_dynamic_params") + dynamic_params: Final = as_str_mapping(kwargs.get("standard_callback_dynamic_params")) langfuse_environment: Final = ( dynamic_params.get("langfuse_environment") if dynamic_params else None ) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 3786087ba91..50ff7002068 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -2,6 +2,7 @@ Call Hook for LiteLLM Proxy which allows Langfuse prompt management. """ +from collections.abc import Mapping from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast @@ -110,6 +111,10 @@ def _remember_trace_id(litellm_call_id: object, logged: LangfuseLoggedEvent) -> in_memory_trace_id_cache.set_cache(litellm_call_id=litellm_call_id, service_name="langfuse", trace_id=trace_id) +def _params_without_model(config: Mapping[str, object]) -> Mapping[str, object]: + return {k: v for k, v in config.items() if k != "model"} + + class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogger): def __init__( self, @@ -182,12 +187,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return compiled_prompt def _get_optional_params_from_langfuse(self, langfuse_prompt_client: PROMPT_CLIENT) -> dict: - config: Final = langfuse_prompt_client.config - optional_params: Final = {} - for k, v in config.items(): - if k != "model": - optional_params[k] = v - return optional_params + return dict(_params_without_model(langfuse_prompt_client.config)) async def async_get_chat_completion_prompt( self, diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 01812149498..ff1f3568685 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -7,10 +7,11 @@ import random import traceback import types from datetime import datetime, timezone -from typing import Any, Final +from typing import Final import httpx from pydantic import BaseModel, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -31,6 +32,10 @@ from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPa _RUN_RESPONSE: Final = TypeAdapter(object) +class _DynamicParamsKwargs(TypedDict, total=False): + standard_callback_dynamic_params: ReadOnly[StandardCallbackDynamicParams | None] + + def is_serializable(value): non_serializable_types: Final = ( types.CoroutineType, @@ -474,7 +479,7 @@ class LangsmithLogger(CustomBatchLogger): return log_queue_by_credentials - def _get_sampling_rate_to_use_for_request(self, kwargs: dict[str, Any]) -> float: + def _get_sampling_rate_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> float: standard_callback_dynamic_params: Final[StandardCallbackDynamicParams | None] = kwargs.get( "standard_callback_dynamic_params", None ) @@ -485,7 +490,7 @@ class LangsmithLogger(CustomBatchLogger): sampling_rate = float(_sampling_rate) return sampling_rate - def _get_credentials_to_use_for_request(self, kwargs: dict[str, Any]) -> LangsmithCredentialsObject: + def _get_credentials_to_use_for_request(self, kwargs: _DynamicParamsKwargs) -> LangsmithCredentialsObject: """ Handles key/team based logging diff --git a/litellm/integrations/langtrace.py b/litellm/integrations/langtrace.py index eeff76e7063..ea411baee7e 100644 --- a/litellm/integrations/langtrace.py +++ b/litellm/integrations/langtrace.py @@ -6,7 +6,7 @@ from litellm.types.integrations.otel_span_attributes import SpanAttributes if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/integrations/levo/levo.py b/litellm/integrations/levo/levo.py index 4d2b4edf3cd..6273f39e09c 100644 --- a/litellm/integrations/levo/levo.py +++ b/litellm/integrations/levo/levo.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: Protocol = _Protocol OpenTelemetryConfig = _OpenTelemetryConfig - Span = _Span | Any + Span = _Span else: Protocol = Any OpenTelemetryConfig = Any diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py index 848369a1cbb..862893fa883 100644 --- a/litellm/integrations/logfire_logger.py +++ b/litellm/integrations/logfire_logger.py @@ -3,8 +3,9 @@ import os import traceback +from collections.abc import Mapping from enum import Enum -from typing import Any, Final, NamedTuple +from typing import Final, NamedTuple from typing_extensions import LiteralString @@ -15,7 +16,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info class SpanConfig(NamedTuple): message_template: LiteralString - span_data: dict[str, Any] + span_data: Mapping[str, object] class LogfireLevel(str, Enum): diff --git a/litellm/integrations/newrelic/newrelic.py b/litellm/integrations/newrelic/newrelic.py index 9829ef4e18f..3623335f599 100644 --- a/litellm/integrations/newrelic/newrelic.py +++ b/litellm/integrations/newrelic/newrelic.py @@ -49,11 +49,12 @@ import time import uuid from collections.abc import Mapping, Sequence from datetime import datetime -from typing import Any, Final +from typing import Final, Protocol import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus from litellm.types.integrations.newrelic import NewRelicInitParams @@ -65,6 +66,10 @@ except ImportError: _newrelic_agent = None +class _SupportsTimestamp(Protocol): + def timestamp(self) -> float: ... + + class NewRelicLogger(CustomLogger): """ New Relic logger for LiteLLM to send AI monitoring events. @@ -126,13 +131,13 @@ class NewRelicLogger(CustomLogger): verbose_logger.error("Failed to initialize New Relic agent: %s. Integration will be disabled.", e) self.enabled = False - def _get_newrelic_params(self) -> dict: + def _get_newrelic_params(self) -> Mapping[str, object]: """ Get the newrelic_params from litellm.newrelic_params These are params specific to initializing the NewRelicLogger e.g. turn_off_message_logging """ - dict_newrelic_params: dict = {} + dict_newrelic_params: Mapping[str, object] = {} if litellm.newrelic_params is not None: if isinstance(litellm.newrelic_params, NewRelicInitParams): dict_newrelic_params = litellm.newrelic_params.model_dump() @@ -401,9 +406,9 @@ class NewRelicLogger(CustomLogger): return choices[0].get("finish_reason") or "unknown" return "unknown" - def _to_epoch_ms(self, t: Any) -> float: + def _to_epoch_ms(self, t: datetime | float) -> float: """Convert a datetime or float timestamp to epoch milliseconds.""" - if hasattr(t, "timestamp"): + if has_attributes(t, _SupportsTimestamp, "timestamp"): return t.timestamp() * 1000.0 return float(t) * 1000.0 @@ -518,13 +523,13 @@ class NewRelicLogger(CustomLogger): sequence = 0 # Extract timestamps, preferring StandardLoggingPayload - start_time = None + start_time: datetime | float | None = None if standard_logging_object: start_time = standard_logging_object.get("startTime") if not start_time: start_time = kwargs.get("start_time") - end_time = None + end_time: datetime | float | None = None if standard_logging_object: end_time = standard_logging_object.get("endTime") if not end_time: diff --git a/litellm/integrations/newrelic/newrelic_metrics.py b/litellm/integrations/newrelic/newrelic_metrics.py index a2cedeba0f5..641d1d86188 100644 --- a/litellm/integrations/newrelic/newrelic_metrics.py +++ b/litellm/integrations/newrelic/newrelic_metrics.py @@ -254,7 +254,7 @@ class NewRelicMetricsLogger(CustomBatchLogger): if not self.log_queue: return if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1: - await asyncio.sleep(2**_pass) + await asyncio.sleep(1 << _pass) async with self.flush_lock: tried_ids: Final = frozenset(id(record) for record in attempted) survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids) diff --git a/litellm/integrations/opik/opik_payload_builder/payload_builders.py b/litellm/integrations/opik/opik_payload_builder/payload_builders.py index 3aaf5bfc162..2909ab9d63c 100644 --- a/litellm/integrations/opik/opik_payload_builder/payload_builders.py +++ b/litellm/integrations/opik/opik_payload_builder/payload_builders.py @@ -57,9 +57,7 @@ def build_span_payload( span_id: Final = utils.create_uuid7() model: Final[str] = response_obj.get("model", "unknown-model") - obj_type: Final[str] = response_obj.get("object", "unknown-object") - created: Final[int] = response_obj.get("created", 0) - span_name: Final = f"{model}_{obj_type}_{created}" + span_name: Final = f"{model}_{response_obj.get('object', 'unknown-object')}_{response_obj.get('created', 0)}" _logging.verbose_logger.debug("OpikLogger creating span with id %s for trace %s", span_id, trace_id) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index cca12b885c1..dd150082252 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -130,7 +130,8 @@ class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource): return self._env_settings.get_field_value(field, field_name) def __call__(self) -> dict[str, object]: - return {key: value for key, value in self._env_settings().items() if key != "excluded_services"} + settings: Final = self._env_settings() + return {key: settings[key] for key in settings if key != "excluded_services"} class OpenTelemetryV2Config(BaseSettings): diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 5d37894bb8f..a69c334a1fb 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -332,7 +332,7 @@ def caller_session_id(kwargs: Mapping[str, object], trace: TraceControls) -> str return (None if echoes_trace_id else explicit) or trace.session_id or None -def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: +def time_to_first_chunk_seconds(kwargs: Mapping[str, object]) -> float | None: """Seconds from the upstream request being issued (``api_call_start_time``) to the first streamed chunk (``completion_start_time``); ``None`` for non-streaming calls, where ``completion_start_time`` is backfilled with the @@ -340,8 +340,8 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None - api_call_start: Final = to_seconds(kwargs.get("api_call_start_time")) - completion_start: Final = to_seconds(kwargs.get("completion_start_time")) + api_call_start: Final = _epoch_seconds(kwargs.get("api_call_start_time")) + completion_start: Final = _epoch_seconds(kwargs.get("completion_start_time")) if api_call_start is None or completion_start is None: return None return completion_start - api_call_start diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d11505ce026..9c62c45e489 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, cast from urllib.parse import urlsplit from typing_extensions import ReadOnly, TypedDict @@ -716,6 +716,15 @@ def _json_or_none(value: object) -> str | None: return None +class _EnumValue(Protocol): + @property + def value(self) -> object: ... + + +def _enum_value(item: _EnumValue) -> object: + return item.value + + def _guardrail_mode_str(value: object) -> str | None: """Normalize ``guardrail_mode`` to a stable string. @@ -732,12 +741,12 @@ def _guardrail_mode_str(value: object) -> str | None: for item in value: if item is None: continue - part = as_str(item.value) if isinstance(item, Enum) else as_str(item) + part = as_str(_enum_value(item)) if isinstance(item, Enum) else as_str(item) if part: parts.append(part) return ",".join(parts) or None if isinstance(value, Enum): - return as_str(value.value) + return as_str(_enum_value(value)) return as_str(value) diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 7ec3711ee65..72d51471ba2 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -173,6 +173,7 @@ class _MetricKwargs(TypedDict, total=False): response_cost: ReadOnly[float | None] completion_start_time: ReadOnly[datetime | float | str | None] api_call_start_time: ReadOnly[datetime | float | str | None] + end_time: ReadOnly[datetime | float | str | None] def resolve_error_type(kwargs: Mapping[str, Any]) -> str: @@ -414,15 +415,12 @@ class GenAIMetricRecorder: if generation_time > 0: self._metrics.time_per_output_token.record(generation_time / completion_tokens, attributes=common_attrs) - def _record_response_duration(self, kwargs: Mapping[str, Any], end_time: datetime, common_attrs: dict) -> None: + def _record_response_duration(self, kwargs: _MetricKwargs, end_time: datetime, common_attrs: dict) -> None: api_call_start_time: Final = kwargs.get("api_call_start_time") if api_call_start_time is None: return - _end_time = kwargs.get("end_time") or end_time - if _end_time is None: - _end_time = datetime.now() api_call_start: Final = to_seconds(api_call_start_time) - end_ts: Final = to_seconds(_end_time) + end_ts: Final = to_seconds(kwargs.get("end_time") or end_time) if api_call_start is None or end_ts is None: return duration: Final = end_ts - api_call_start diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index d787375ca3c..83f85232182 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -13,6 +13,8 @@ import re import traceback from typing import Final +from pydantic import TypeAdapter + import litellm from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( @@ -32,6 +34,7 @@ from litellm.types.utils import StandardLoggingPayload from .custom_batch_logger import CustomBatchLogger +_DECODED_JSON: Final = TypeAdapter(object) _BASE64_INLINE_PATTERN: Final = re.compile( r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+", re.MULTILINE, @@ -270,7 +273,7 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): if self.sqs_queue_url is None: raise ValueError("sqs_queue_url not set") - json_data: Final = json.loads(safe_dumps(payload)) + json_data: Final = _DECODED_JSON.validate_python(json.loads(safe_dumps(payload))) if self.app_crypto: aad_bytes: Final = self.sqs_app_encryption_aad.encode("utf-8") if self.sqs_app_encryption_aad else None encrypted: Final = self.app_crypto.encrypt_json(json_data, aad=aad_bytes) diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py index 8fee0fcd1b5..d84f0b5ffbd 100644 --- a/litellm/interactions/litellm_responses_transformation/handler.py +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -116,10 +116,7 @@ class LiteLLMResponsesInteractionsHandler: """Async handler for interactions API requests.""" # Call litellm.aresponses() # Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] - aresponses_fn: Final[ - Callable[..., Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]] - ] = vars(litellm)["aresponses"] - responses_response: Final = await aresponses_fn( + responses_response: Final = await litellm.aresponses( **responses_request, ) diff --git a/litellm/litellm_core_utils/app_crypto.py b/litellm/litellm_core_utils/app_crypto.py index 4ef3f34f3b8..da844620dce 100644 --- a/litellm/litellm_core_utils/app_crypto.py +++ b/litellm/litellm_core_utils/app_crypto.py @@ -4,6 +4,9 @@ import os from typing import Final from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from pydantic import TypeAdapter + +_DECODED_JSON: Final = TypeAdapter(object) class AppCrypto: @@ -12,7 +15,7 @@ class AppCrypto: raise ValueError("Master key must be 32 bytes for AES-256-GCM") self.key = master_key - def encrypt_json(self, data: dict, aad: bytes | None = None) -> dict: + def encrypt_json(self, data: object, aad: bytes | None = None) -> dict: aes: Final = AESGCM(self.key) nonce: Final = os.urandom(12) plaintext: Final = json.dumps(data).encode("utf-8") @@ -24,10 +27,10 @@ class AppCrypto: "tag": base64.b64encode(tag).decode(), } - def decrypt_json(self, enc: dict, aad: bytes | None = None) -> dict: + def decrypt_json(self, enc: dict, aad: bytes | None = None) -> object: aes: Final = AESGCM(self.key) nonce: Final = base64.b64decode(enc["nonce"]) ct: Final = base64.b64decode(enc["ciphertext"]) tag: Final = base64.b64decode(enc["tag"]) data: Final = aes.decrypt(nonce, ct + tag, aad) - return json.loads(data.decode()) + return _DECODED_JSON.validate_python(json.loads(data.decode())) diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index e9689bd7d8a..8e949694881 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -3,10 +3,11 @@ import re from collections.abc import Mapping, Sequence from itertools import accumulate, repeat from types import MappingProxyType -from typing import Any, Final, cast +from typing import Final from urllib.parse import quote, unquote from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/" BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-" @@ -141,7 +142,8 @@ def should_allow_legacy_cloud_file_ids( if isinstance(litellm_params, Mapping): trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE): - value = cast(Mapping[str, Any], trusted_model_credentials).get("allow_legacy_cloud_file_ids") + trusted_mapping: Final = as_str_mapping(trusted_model_credentials) + value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None if isinstance(value, bool): return value diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 1d61ef8519a..4be5cabac04 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -5,10 +5,11 @@ import logging import re from collections.abc import Collection, Iterable, Iterator, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads import litellm from litellm._logging import verbose_logger @@ -58,6 +59,13 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None: return value if isinstance(value, Mapping) else None +def decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + @runtime_checkable class _SupportsIter(Protocol): def __iter__(self) -> Iterator[object]: ... @@ -72,6 +80,24 @@ def attribute_or_none(source: object, name: str) -> object: return getattr(source, name, None) +_Shape: Final = TypeVar("_Shape") + + +def has_attributes( + value: object, + shape: type[_Shape], + *names: str, +) -> TypeIs[_Shape]: # guard-ok: hasattr sees what __getattr__ serves, which Protocol isinstance skips on 3.12+ + return all(hasattr(value, name) for name in names) + + +def require_attributes(value: object, shape: type[_Shape], *names: str) -> _Shape: + if has_attributes(value, shape, *names): + return value + missing: Final = next(name for name in names if not hasattr(value, name)) + raise AttributeError(f"'{type(value).__name__}' object has no attribute '{missing}'", name=missing, obj=value) + + def iter_items(value: object) -> Iterable[object]: if isinstance(value, _SupportsIter): return value @@ -634,11 +660,14 @@ def preserve_upstream_non_openai_attributes( for key in candidate_keys ) if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): - for key, value in candidate_values: - setattr(model_response, key, value) + _assign_attributes(model_response, candidate_values) return - for key, value in original_chunk.model_dump(include=frozenset(candidate_keys)).items(): - setattr(model_response, key, value) + _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + + +def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: + for key, value in values: + setattr(target, key, value) def safe_deep_copy(data): @@ -764,6 +793,10 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: Filtered data structure with Exception and callable objects removed, or None if the entire input was an Exception or callable """ + return _filter_exceptions_from_params(data, max_depth) + + +def _filter_exceptions_from_params(data: object, max_depth: int) -> object: if max_depth <= 0: return data @@ -785,7 +818,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)): continue try: - filtered = filter_exceptions_from_params(v, max_depth - 1) + filtered = _filter_exceptions_from_params(v, max_depth - 1) if filtered is not None: result[k] = filtered except Exception: @@ -799,7 +832,7 @@ def filter_exceptions_from_params(data: object, max_depth: int = 20) -> Any: if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)): continue try: - filtered = filter_exceptions_from_params(item, max_depth - 1) + filtered = _filter_exceptions_from_params(item, max_depth - 1) if filtered is not None: result_list.append(filtered) except Exception: diff --git a/litellm/litellm_core_utils/cost_discount.py b/litellm/litellm_core_utils/cost_discount.py index a64ad0b20f7..bfcee7928fb 100644 --- a/litellm/litellm_core_utils/cost_discount.py +++ b/litellm/litellm_core_utils/cost_discount.py @@ -23,7 +23,7 @@ def parse_cost_discount_key(key: str) -> CostDiscountKey: def _literal_length(pattern: str) -> int: - return sum(1 for token in _GLOB_TOKEN.findall(pattern) if len(token) == 1 and token not in "*?") + return sum(1 for match in _GLOB_TOKEN.finditer(pattern) if len(match.group()) == 1 and match.group() not in "*?") def _model_names(model: str, provider_prefix: str) -> tuple[str, ...]: diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index c3b6a008411..970dd512a40 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -48,5 +48,5 @@ for attempt in range(_max_retries): # Last attempt, re-raise the exception raise # Exponential backoff with jitter to reduce collision probability - delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1) + delay = _retry_delay * (2.0**attempt) + random.uniform(0, 0.1) time.sleep(delay) diff --git a/litellm/litellm_core_utils/dot_notation_indexing.py b/litellm/litellm_core_utils/dot_notation_indexing.py index 1dac67ecbf6..09720fa413c 100644 --- a/litellm/litellm_core_utils/dot_notation_indexing.py +++ b/litellm/litellm_core_utils/dot_notation_indexing.py @@ -181,7 +181,7 @@ def _delete_nested_value_custom( def delete_nested_value( - data: dict[str, Any], + data: dict[str, object], path: str, depth: int = 0, max_depth: int = 20, diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index e8da59cd33a..1c816b22ef1 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -16,7 +16,7 @@ from litellm.litellm_core_utils.bug_report import ( build_bug_report, should_report_bug, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string from litellm.types.utils import LlmProviders @@ -169,7 +169,7 @@ def get_error_message(error_obj) -> str | None: return None if hasattr(error_obj, "body"): - _error_obj_body: Final = getattr(error_obj, "body") + _error_obj_body: Final = attribute_or_none(error_obj, "body") if isinstance(_error_obj_body, dict): # OpenAI-style: {"message": "...", "type": "...", ...} if _error_obj_body.get("message"): diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 1e2a511d439..7fc6eab6c60 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) +_DECODED_JSON: Final = TypeAdapter(object) _BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -314,7 +315,7 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) - def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome: try: - parsed: Final[object] = json.loads(content) + parsed: Final = _DECODED_JSON.validate_python(json.loads(content)) except ValueError as e: return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): @@ -383,7 +384,8 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp ) if response.is_error: return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") - return _classify_fetch_content(response.content, url, response.headers.get("etag")) + etag_values: Final = response.headers.get_list("etag") + return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None) def _next_retry_wait( diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index f8d44d44cdc..a74337d2549 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -1,4 +1,5 @@ from collections.abc import Iterator, Mapping, MutableMapping +from types import MappingProxyType from typing import Final, Protocol, cast from pydantic import BaseModel @@ -52,7 +53,7 @@ class HiddenParamsModelView(MutableMapping[str, object]): def stored_attribute(obj: object, name: str) -> object: if not isinstance(obj, BaseModel): return getattr(obj, name, None) - values: Final[Mapping[str, object]] = vars(obj) + values: Final[Mapping[str, object]] = MappingProxyType[str, object](vars(obj)) if name in values: return values[name] extras: Final[Mapping[str, object]] = obj.__pydantic_extra__ or {} diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 5a533054f0b..c3769686e45 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -28,7 +28,7 @@ def inherit_message_logging_privacy(disabled: bool) -> Generator[None]: def iter_client_callback_metadata_dicts( - kwargs: dict[str, Any], + kwargs: Mapping[str, object], ) -> Iterator[tuple[str, dict[str, Any]]]: litellm_params: Final = kwargs.get("litellm_params") if isinstance(litellm_params, dict): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 41f13817d75..fc8d1ecc8c1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -556,6 +556,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_REDACTED_RESULT: Final = TypeAdapter(object) def _provider_response_id(source: object) -> str | None: @@ -1171,7 +1172,7 @@ class Logging(LiteLLMLoggingBaseClass): automatically extracts metadata/litellm_metadata from kwargs, so callers don't need to manually plumb them into litellm_params. """ - base_litellm_params: Final[dict[str, Any]] = {} + base_litellm_params: Final[dict[str, object]] = {} if isinstance(kwargs.get("metadata"), dict): base_litellm_params["metadata"] = kwargs["metadata"].copy() @@ -1188,10 +1189,11 @@ class Logging(LiteLLMLoggingBaseClass): lp_metadata: Final = litellm_params.pop("metadata", None) base_litellm_params.update(litellm_params) if lp_metadata and isinstance(lp_metadata, dict): - base_litellm_params.setdefault("metadata", {}) - for k, v in lp_metadata.items(): - if k not in base_litellm_params["metadata"]: - base_litellm_params["metadata"][k] = v + merged_metadata: Final = base_litellm_params.setdefault("metadata", {}) + if isinstance(merged_metadata, dict): + for k, v in lp_metadata.items(): + if k not in merged_metadata: + merged_metadata[k] = v self.update_environment_variables( litellm_params=base_litellm_params, @@ -1793,7 +1795,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) - original_response = redact_message_input_output_from_logging( + redact_message_input_output_from_logging( model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), result=original_response, ) @@ -2023,10 +2025,9 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details.pop("response_cost_failure_debug_information", None) - if cache_hit is None: - cache_hit = self.model_call_details.get("cache_hit", False) + resolved_cache_hit: Final = self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit - if cache_hit is True: + if resolved_cache_hit is True: return 0.0 if is_unbilled_non_inference_call( @@ -2078,7 +2079,7 @@ class Logging(LiteLLMLoggingBaseClass): model_value: Final = litellm_model_name or self.model cost_cache_hit_value: Final = ( - self.model_call_details.get("cache_hit", False) if cache_hit is None else cache_hit + self.model_call_details.get("cache_hit", False) if resolved_cache_hit is None else resolved_cache_hit ) cost_cache_hit: Final[bool | None] = cast( # cast-ok: callback metadata is caller-provided bool | None, cost_cache_hit_value @@ -2512,8 +2513,10 @@ class Logging(LiteLLMLoggingBaseClass): def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" - usage: Final = self.model_call_details.pop("combined_usage_object", None) - response_cost: Final = self.model_call_details.pop("response_cost", None) + usage: Final = self._model_call_details_view.get("combined_usage_object") + response_cost: Final = self._model_call_details_view.get("response_cost") + self.model_call_details.pop("combined_usage_object", None) + self.model_call_details.pop("response_cost", None) if isinstance(usage, Usage): self.model_call_details["current_attempt_api_call_start_time"] = None self.model_call_details["failed_attempts_usage"] = ( @@ -3302,7 +3305,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback_logging_payload is not None or shared_logging_payload is None else shared_logging_payload.copy() ) - call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict + call_details: dict[str, object] # mutable-ok: legacy loggers take a plain dict call_details = ( # rebind-ok: logging hooks may return a new dict self.model_call_details if not for_async_request @@ -3323,9 +3326,11 @@ class Logging(LiteLLMLoggingBaseClass): ) ## REDACT MESSAGES ## - result = redact_message_input_output_from_logging( - model_call_details=call_details, - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=call_details, + result=result, + ) ) ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) @@ -3591,7 +3596,7 @@ class Logging(LiteLLMLoggingBaseClass): print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response") s3Logger.log_event( kwargs=call_details, - response_obj=call_details["complete_streaming_response"], + response_obj=call_details_view["complete_streaming_response"], start_time=start_time, end_time=end_time, print_verbose=print_verbose, @@ -3914,9 +3919,11 @@ class Logging(LiteLLMLoggingBaseClass): self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks ) - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) ## LOGGING HOOK ## @@ -4261,9 +4268,11 @@ class Logging(LiteLLMLoggingBaseClass): result: object = None # result sent to all loggers, init this to None incase it's not created - result = redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging( + model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + result=result, + ) ) self.mark_logging_complete(event_type="sync_failure") for callback in callbacks: @@ -4952,7 +4961,7 @@ class Logging(LiteLLMLoggingBaseClass): """ import httpx - httpx_response: Final = self.model_call_details.get("httpx_response", None) + httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None) if httpx_response is None: raise ValueError("Google GenAI Generate Content: httpx_response is None") dict_result: Final = httpx_response.json() @@ -4982,7 +4991,7 @@ class Logging(LiteLLMLoggingBaseClass): The response object with usage added if available """ # Get usage from model_call_details (set by asend_message) - usage: Final = self.model_call_details.get("usage") + usage: Final[Usage | None] = self.model_call_details.get("usage") if usage is None: return result @@ -5815,7 +5824,9 @@ def build_custom_logger_compatible_class( return callback # Get global BitBucket config - bitbucket_config: Final = getattr(litellm, "global_bitbucket_config", None) + bitbucket_config: Final = ( + litellm.global_bitbucket_config if hasattr(litellm, "global_bitbucket_config") else None + ) if bitbucket_config is None: raise ValueError("BitBucket configuration not found. Please set litellm.global_bitbucket_config first.") diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 04510c20db0..045869e4547 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -450,7 +450,7 @@ class StandardBuiltInToolCostTracking: def _get_code_interpreter_cost( model_info: ModelInfo | None, custom_llm_provider: str | None, - standard_built_in_tools_params: StandardBuiltInToolsParams, + standard_built_in_tools_params: Mapping[str, object], ) -> float: """Calculate code interpreter cost.""" code_interpreter_sessions: Final = standard_built_in_tools_params.get("code_interpreter_sessions", None) diff --git a/litellm/litellm_core_utils/llm_judge.py b/litellm/litellm_core_utils/llm_judge.py index 0c2d7767455..307e000119b 100644 --- a/litellm/litellm_core_utils/llm_judge.py +++ b/litellm/litellm_core_utils/llm_judge.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def default_router_provider() -> Router | None: @@ -40,13 +41,13 @@ def parse_json_verdict(raw: str) -> dict[str, object]: # mutable-ok: plain pars text = fenced.group(1).strip() # rebind-ok: progressively narrowed to the JSON payload parsed: object try: - parsed = json.loads(text) + parsed = _DECODED_JSON.validate_python(json.loads(text)) except json.JSONDecodeError: start: Final = text.find("{") end: Final = text.rfind("}") if start == -1 or end <= start: raise - parsed = json.loads(text[start : end + 1]) + parsed = _DECODED_JSON.validate_python(json.loads(text[start : end + 1])) if not isinstance(parsed, dict): raise ValueError("judge response is not a JSON object") return {str(k): v for k, v in parsed.items()} diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 307353d081b..56c6436eae6 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -1,5 +1,5 @@ import threading -from collections.abc import Callable, Iterable, Iterator +from collections.abc import Callable, Iterable, Iterator, Mapping from types import MethodType from typing import TYPE_CHECKING, Final @@ -20,6 +20,10 @@ _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {} _callback_lists_lock: Final = threading.RLock() +def _instance_attributes(target: object) -> Mapping[str, object]: + return vars(target) + + class LoggingCallbackManager: """ A centralized class that allows easy add / remove callbacks for litellm. @@ -341,7 +345,7 @@ class LoggingCallbackManager: key_parts: Final = [type(custom_logger).__name__] # Add only fundamental type instance variables to the key - for attr_name, attr_value in vars(custom_logger).items(): + for attr_name, attr_value in _instance_attributes(custom_logger).items(): if not attr_name.startswith("_"): # Skip private attributes if isinstance(attr_value, (str, bool, int)): key_parts.append(f"{attr_name}={attr_value}") diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 60be4d44706..59882dcbf2f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -20,6 +20,7 @@ from pydantic import TypeAdapter import litellm from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, @@ -1527,7 +1528,7 @@ def _grammar_branches(node: Mapping[str, object]) -> int: if properties is None: return 0 optional: Final = len(_optional_property_names(node)) - return 2**optional * (len(properties) - optional + 1) + return (1 << optional) * (len(properties) - optional + 1) def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]: @@ -2839,7 +2840,7 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: break try: - obj, end_idx = decoder.raw_decode(raw, idx) + decoded = decoder.raw_decode(raw, idx) except json.JSONDecodeError as e: verbose_logger.warning( "split_concatenated_json_objects: discarding unparseable tool-call " @@ -2849,13 +2850,14 @@ def split_concatenated_json_objects(raw: str) -> list[dict[str, object]]: e, ) break - if isinstance(obj, dict): - results.append(obj) + obj = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if obj is not None: + results.append(dict(obj)) else: # Non-dict JSON value – wrap in empty dict (Bedrock requires # toolUse.input to be an object). results.append({}) - idx = end_idx + idx = decoded[1] return results @@ -2887,14 +2889,16 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], .. if idx >= length: break try: - obj, end_idx = decoder.raw_decode(stripped, idx) + decoded = decoder.raw_decode(stripped, idx) except json.JSONDecodeError: return () - if not isinstance(obj, dict): + decoded_object = as_str_mapping(_DECODED_JSON.validate_python(decoded[0])) + if decoded_object is None: return () - idx = end_idx + idx = decoded[1] if exceeded: continue + obj = dict(decoded_object) count += 1 if not kept: kept = (obj,) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..cfb6e3ab45f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -21,6 +21,7 @@ import litellm.types.llms from litellm import verbose_logger from litellm._uuid import uuid from litellm.constants import REDACTED_BY_LITELLM +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( anthropic_system_messages, find_anthropic_server_tool_result, @@ -1021,11 +1022,12 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str: tool_function = get_attribute_or_key(tool, "function") tool_name = get_attribute_or_key(tool_function, "name") tool_arguments = get_attribute_or_key(tool_function, "arguments") - parsed_args = parse_tool_call_arguments( - tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke" + parsed_args = _DECODED_JSON.validate_python( + parse_tool_call_arguments(tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke") ) - if isinstance(parsed_args, dict): - parameters = "".join(f"<{param}>{val}\n" for param, val in parsed_args.items()) + parsed_mapping = as_str_mapping(parsed_args) + if parsed_mapping is not None: + parameters = "".join(f"<{param}>{val}\n" for param, val in parsed_mapping.items()) else: parameters = f"{parsed_args}\n" invokes += f"\n{tool_name}\n\n{parameters}\n\n" @@ -2342,7 +2344,7 @@ def _add_missing_tool_results( else: tool_name = getattr(function, "name", "unknown_tool") else: - function = getattr(tool_call, "function", None) + function = attribute_or_none(tool_call, "function") if function: tool_name = getattr(function, "name", "unknown_tool") break @@ -3184,10 +3186,14 @@ def anthropic_messages_pt( def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: ext_list = re.findall(f"<{tag}>(.+?)", string, re.DOTALL) if strip: - ext_list = [e.strip() for e in ext_list] + ext_list = list(_stripped(ext_list)) return ext_list +def _stripped(values: Iterable[str]) -> Iterator[str]: + return (value.strip() for value in values) + + def contains_tag(tag: str, string: str) -> bool: return bool(re.search(f"<{tag}>(.+?)", string, re.DOTALL)) @@ -4061,13 +4067,15 @@ def _convert_to_bedrock_tool_call_invoke( arguments_dict = {} else: try: - arguments_dict = json.loads(arguments) + decoded_arguments = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments))) # Ensure arguments_dict is always a dict # (Bedrock requires toolUse.input to be an object). # Some providers return arguments: '""' which # json.loads decodes to a bare string. - if not isinstance(arguments_dict, dict): + if decoded_arguments is None: arguments_dict = {} + else: + arguments_dict = dict(decoded_arguments) except json.JSONDecodeError: # The model may return multiple JSON objects # concatenated in a single arguments string, e.g. @@ -6028,7 +6036,11 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) return ({},) normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw try: - parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) + parsed: Final = as_str_mapping( + _DECODED_JSON.validate_python( + parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context) + ) + ) except ValueError as e: salvaged: Final = salvage_concatenated_tool_arguments(normalized_raw) if salvaged: @@ -6041,7 +6053,7 @@ def _parse_tool_call_arguments(raw: object, tool_name: str | None, context: str) return salvaged verbose_logger.warning("Failed to parse tool call arguments: %s", e) return ({},) - return (parsed,) if isinstance(parsed, dict) else ({},) + return (dict(parsed),) if parsed is not None else ({},) def _choice_tool_calls(choice: object) -> tuple[object, ...]: diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index ecf419695d0..eb3400aee8b 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -482,7 +482,7 @@ def _runs_by_anchor( ) -> Mapping[int, tuple[tuple[int, ...], ...]]: anchored: Final = tuple(sorted(((anchor, run) for run, anchor in anchors if anchor is not None), key=itemgetter(0))) return MappingProxyType( - {anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=itemgetter(0))} + {anchor: tuple(run for _, run in group) for anchor, group in groupby(anchored, key=lambda entry: entry[0])} ) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 52758e617d8..8c402fe9b44 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, Sequence +from collections.abc import ItemsView, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field from typing import Any, Final @@ -30,6 +30,10 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( ) +def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]: + return data.items() + + class SensitiveDataMasker: def __init__( self, @@ -127,7 +131,7 @@ class SensitiveDataMasker: return data masked_data: Final[dict[str, object]] = {} - for k, v in data.items(): + for k, v in _object_items(data): try: key_is_sensitive = self.is_sensitive_key(k, excluded_keys) if isinstance(v, Mapping): diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 39f675099d7..50789406055 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -425,7 +425,7 @@ class ChunkProcessor: if hasattr(chunk["choices"][0], "finish_reason"): chunk_finish_reason = chunk["choices"][0].finish_reason elif "finish_reason" in chunk["choices"][0]: - chunk_finish_reason = chunk["choices"][0]["finish_reason"] + chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) if chunk_finish_reason is not None: finish_reason = chunk_finish_reason @@ -1226,6 +1226,7 @@ def concatenate_base64_list(base64_strings: Sequence[str]) -> str: _CHOICE_DICTS: Final = TypeAdapter(list[dict[str, object]]) +_CHOICE_FIELD_VALUE: Final = TypeAdapter(object) def _text_completion_chunk_from_mapping(chunk: Mapping[str, object]) -> TextCompletionResponse: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f994baa52b1..fdcdca151be 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -37,7 +37,6 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( CacheCreationTokenDetails, @@ -111,14 +110,15 @@ def _prepend_upstream_choice_extras( choice.__pydantic_fields_set__.update(upstream_extras) -def _holds_only_scalars(model: BaseModel) -> bool: - fields: Final[Iterator[tuple[str, object]]] = iter(model) +def _holds_only_scalars(fields: Iterable[tuple[str, object]]) -> bool: return all(value is None or isinstance(value, (str, int, float, bool)) for _, value in fields) -def _delta_from_upstream(upstream_delta: BaseModel | None) -> Delta | None: +def _delta_from_upstream(upstream_delta: "BaseModel | _UpstreamDeltaLike | None") -> Delta | None: if upstream_delta is None: return Delta() + if not isinstance(upstream_delta, BaseModel): + return None try: return Delta(**dict(iter(upstream_delta))) except Exception: @@ -189,6 +189,33 @@ class _DeltaDumpDict(TypedDict): tool_calls: NotRequired[Sequence[Mapping[str, object]]] +class _UpstreamFunctionCallLike(Protocol): + arguments: str | None + + +class _UpstreamToolCallLike(Protocol): + function: _UpstreamFunctionCallLike + + +class _UpstreamDeltaLike(Protocol): + function_call: _UpstreamFunctionCallLike | None + tool_calls: Sequence[_UpstreamToolCallLike] | None + + def __iter__(self) -> Iterator[tuple[str, object]]: ... + + def model_dump(self) -> _DeltaDumpDict: ... + + +class _UpstreamChoiceLike(Protocol): + delta: _UpstreamDeltaLike | None + + +class _UpstreamChunkLike(Protocol): + id: str + choices: Sequence[_UpstreamChoiceLike] + system_fingerprint: str | None + + class _TextCompletionChoiceLike(Protocol): text: str finish_reason: str | None @@ -920,7 +947,7 @@ class CustomStreamWrapper: def copy_model_response_level_provider_specific_fields( self, - original_chunk: ModelResponseStream | OpenAIChatCompletionChunk, + original_chunk: object, model_response: ModelResponseStream, ) -> ModelResponseStream: """ @@ -949,8 +976,8 @@ class CustomStreamWrapper: or ("function_call" in completion_obj and completion_obj["function_call"] is not None) or ( "tool_calls" in model_response.choices[0].delta - and model_response.choices[0].delta["tool_calls"] is not None - and len(model_response.choices[0].delta["tool_calls"]) > 0 + and model_response.choices[0].delta.tool_calls is not None + and len(model_response.choices[0].delta.tool_calls) > 0 ) or ( "function_call" in model_response.choices[0].delta @@ -1115,7 +1142,8 @@ class CustomStreamWrapper: ## check if openai/azure chunk original_chunk: Final = response_obj.get("original_chunk", None) if original_chunk: - if len(original_chunk.choices) > 0: + upstream_chunk: Final[_UpstreamChunkLike] = original_chunk + if len(upstream_chunk.choices) > 0: original_chunk_for_reuse: Final[object] = cast( # cast-ok: runtime type-check before access object, original_chunk, @@ -1151,7 +1179,7 @@ class CustomStreamWrapper: ) else: choices: Final = [] - for choice in original_chunk.choices: + for choice in upstream_chunk.choices: try: if isinstance(choice, BaseModel): choice_json = choice.model_dump() @@ -1164,11 +1192,11 @@ class CustomStreamWrapper: setattr(model_response, "choices", choices) else: return - model_response.system_fingerprint = original_chunk.system_fingerprint + model_response.system_fingerprint = upstream_chunk.system_fingerprint setattr( model_response, "citations", - stored_attribute(original_chunk, "citations"), + stored_attribute(upstream_chunk, "citations"), ) preserve_upstream_non_openai_attributes( model_response=model_response, @@ -1611,7 +1639,9 @@ class CustomStreamWrapper: model_response.model = self.model ## FUNCTION CALL PARSING - original_chunk: Final = response_obj.get("original_chunk") if response_obj is not None else None + original_chunk: Final[_UpstreamChunkLike | None] = ( + response_obj.get("original_chunk") if response_obj is not None else None + ) if ( original_chunk is not None ): # function / tool calling branch - only set for openai/azure compatible endpoints @@ -1623,31 +1653,19 @@ class CustomStreamWrapper: original_chunk, model_response ) if original_chunk.choices and len(original_chunk.choices) > 0: - delta = original_chunk.choices[0].delta + delta: Final = original_chunk.choices[0].delta if delta is not None and (delta.function_call is not None or delta.tool_calls is not None): try: model_response.system_fingerprint = original_chunk.system_fingerprint ## AZURE - check if arguments is not None - if original_chunk.choices[0].delta.function_call is not None: - if ( - getattr( - original_chunk.choices[0].delta.function_call, - "arguments", - ) - is None - ): - original_chunk.choices[0].delta.function_call.arguments = "" - elif original_chunk.choices[0].delta.tool_calls is not None: - if isinstance(original_chunk.choices[0].delta.tool_calls, list): - for t in original_chunk.choices[0].delta.tool_calls: - if hasattr(t, "functions") and hasattr(t.functions, "arguments"): - if ( - getattr( - t.function, - "arguments", - ) - is None - ): + if delta.function_call is not None: + if delta.function_call.arguments is None: + delta.function_call.arguments = "" + elif delta.tool_calls is not None: + if isinstance(delta.tool_calls, list): + for t in delta.tool_calls: + if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"): + if t.function.arguments is None: t.function.arguments = "" _json_delta: Final[_DeltaDumpDict] = delta.model_dump() if "role" not in _json_delta or _json_delta["role"] is None: @@ -1886,7 +1904,7 @@ class CustomStreamWrapper: else: asyncio.run(self.logging_obj.async_success_handler(processed_chunk, None, None, cache_hit)) ## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler - litellm_params: Final = self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: Final[Mapping[str, object]] = self.logging_obj.model_call_details.get("litellm_params", {}) if self.logging_obj.is_sync_litellm_request(litellm_params): self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8aded69aed2..90e79d4d853 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -44,7 +44,7 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace from litellm.litellm_core_utils.url_utils import safe_get -from litellm.llms.custom_httpx.http_handler import get_httpx_client +from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, @@ -247,7 +247,7 @@ def get_image_dimensions( client: Final = get_httpx_client() response: Final[httpx.Response] = safe_get(client, data) max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024) - content_length: Final[str | None] = response.headers.get("Content-Length") + content_length: Final[str | None] = header_value(response.headers, "Content-Length") if content_length is not None and int(content_length) > max_bytes: pass # skip download; img_data stays None else: diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 4a1eaa87d3a..0db6a41b503 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -22,11 +22,11 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config): import asyncio import socket from ipaddress import ip_address, ip_network -from typing import Any, Final, Protocol +from typing import Final, Protocol from urllib.parse import quote, urlparse, urlsplit, urlunparse import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack import litellm @@ -50,6 +50,7 @@ class _UrlFetcher(Protocol): *, headers: dict[str, str] | None = None, follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, ) -> httpx.Response: ... @@ -62,6 +63,8 @@ class _AsyncUrlFetcher(Protocol): *, headers: dict[str, str] | None = None, follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, ) -> httpx.Response: ... @@ -73,8 +76,14 @@ class _AsyncFetcherView(TypedDict): fetcher: ReadOnly[_AsyncUrlFetcher] -class _CallerHeadersView(TypedDict): +class _FetchOptions(TypedDict, total=False): headers: ReadOnly[dict[str, str]] + follow_redirects: ReadOnly[bool] + timeout: ReadOnly[float | httpx.Timeout | None] + + +class _AsyncFetchOptions(_FetchOptions, total=False): + max_response_bytes: ReadOnly[int | None] # Cloud-internal IPs that ``ip.is_global`` can report as public. Everything @@ -440,7 +449,7 @@ def _extract_redirect_url(response: httpx.Response, request_url: str) -> str: return str(httpx.URL(request_url).join(location)) -def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: +def safe_get(client: _UrlFetcher, url: str, **kwargs: Unpack[_FetchOptions]) -> httpx.Response: """ Fetch a user-supplied URL with SSRF protection on every redirect hop. @@ -460,21 +469,23 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: The final httpx.Response. """ if not getattr(litellm, "user_url_validation", True): - kwargs.setdefault("follow_redirects", True) - unvalidated: Final[_ResponseView] = {"response": client.get(url, **kwargs)} + unvalidated_options: Final[_FetchOptions] = { + **kwargs, + "follow_redirects": kwargs.get("follow_redirects", True), + } + unvalidated: Final[_ResponseView] = {"response": client.get(url, **unvalidated_options)} return unvalidated["response"] fetcher_view: Final[_FetcherView] = {"fetcher": client} fetcher: Final = fetcher_view["fetcher"] - kwargs.pop("follow_redirects", None) - headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})} + caller_headers: Final = kwargs.get("headers", {}) for _ in range(_MAX_REDIRECTS): validated_url, original_host = validate_url(url) - response = fetcher.get( - validated_url, - headers={**headers_view["headers"], "Host": original_host}, - follow_redirects=False, + hop_options: _FetchOptions = { **kwargs, - ) + "headers": {**caller_headers, "Host": original_host}, + "follow_redirects": False, + } + response = fetcher.get(validated_url, **hop_options) if not response.is_redirect: return response # Resolve the next hop against the ORIGINAL (pre-rewrite) URL so @@ -483,24 +494,26 @@ def safe_get(client: _UrlFetcher, url: str, **kwargs: Any) -> httpx.Response: raise SSRFError("Too many redirects") -async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Any) -> httpx.Response: +async def async_safe_get(client: _AsyncUrlFetcher, url: str, **kwargs: Unpack[_AsyncFetchOptions]) -> httpx.Response: """Async version of safe_get.""" if not getattr(litellm, "user_url_validation", True): - kwargs.setdefault("follow_redirects", True) - unvalidated: Final[_ResponseView] = {"response": await client.get(url, **kwargs)} + unvalidated_options: Final[_AsyncFetchOptions] = { + **kwargs, + "follow_redirects": kwargs.get("follow_redirects", True), + } + unvalidated: Final[_ResponseView] = {"response": await client.get(url, **unvalidated_options)} return unvalidated["response"] fetcher_view: Final[_AsyncFetcherView] = {"fetcher": client} fetcher: Final = fetcher_view["fetcher"] - kwargs.pop("follow_redirects", None) - headers_view: Final[_CallerHeadersView] = {"headers": kwargs.pop("headers", {})} + caller_headers: Final = kwargs.get("headers", {}) for _ in range(_MAX_REDIRECTS): validated_url, original_host = await asyncio.to_thread(validate_url, url) - response = await fetcher.get( - validated_url, - headers={**headers_view["headers"], "Host": original_host}, - follow_redirects=False, + hop_options: _AsyncFetchOptions = { **kwargs, - ) + "headers": {**caller_headers, "Host": original_host}, + "follow_redirects": False, + } + response = await fetcher.get(validated_url, **hop_options) if not response.is_redirect: return response # Resolve the next hop against the ORIGINAL (pre-rewrite) URL so diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 7452a4a20ef..91f33635658 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -3,6 +3,7 @@ import os from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.utils import CallTypes from . import * @@ -128,7 +129,7 @@ def discover_guardrail_translation_mappings() -> dict[CallTypes, type["BaseTrans # Check for guardrail_translation_mappings dictionary if hasattr(module, "guardrail_translation_mappings"): - mappings = getattr(module, "guardrail_translation_mappings") + mappings = attribute_or_none(module, "guardrail_translation_mappings") if isinstance(mappings, dict): discovered_mappings.update(mappings) verbose_logger.debug( diff --git a/litellm/llms/a2a/common_utils.py b/litellm/llms/a2a/common_utils.py index 1818b406823..633ea73bb16 100644 --- a/litellm/llms/a2a/common_utils.py +++ b/litellm/llms/a2a/common_utils.py @@ -31,6 +31,10 @@ class A2AError(BaseLLMException): ) +def _message_role(message: Mapping[str, object]) -> object: + return message.get("role", "user") + + def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str: """ Convert OpenAI messages to a single prompt string for A2A agent. @@ -51,7 +55,7 @@ def convert_messages_to_prompt(messages: list[AllMessageValues]) -> str: # Get role if isinstance(msg, BaseModel): - role = msg.model_dump().get("role", "user") + role = _message_role(msg.model_dump()) elif isinstance(msg, dict): role = msg.get("role", "user") else: diff --git a/litellm/llms/ai21/chat/transformation.py b/litellm/llms/ai21/chat/transformation.py index 2b8c36d0085..e78cc77c0fc 100644 --- a/litellm/llms/ai21/chat/transformation.py +++ b/litellm/llms/ai21/chat/transformation.py @@ -4,6 +4,8 @@ AI21 Chat Completions API this is OpenAI compatible - no translation needed / occurs """ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final from ...openai_like.chat.transformation import OpenAILikeChatConfig @@ -41,7 +43,7 @@ class AI21ChatConfig(OpenAILikeChatConfig): tool_choice: str | None = None, user: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index a93fbf1e933..4ef729bc76d 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -2,6 +2,8 @@ Translate from OpenAI's `/v1/chat/completions` to Amazon Nova's `/v1/chat/completions` """ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -38,7 +40,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): tools: list | None = None, reasoning_effort: list | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/anthropic/batches/handler.py b/litellm/llms/anthropic/batches/handler.py index 3f3716e0de6..a9d9f930b9e 100644 --- a/litellm/llms/anthropic/batches/handler.py +++ b/litellm/llms/anthropic/batches/handler.py @@ -132,7 +132,7 @@ class AnthropicBatchesHandler: max_retries: int | None, logging_obj: LiteLLMLoggingObj | None = None, litellm_params: dict | None = None, # mutable-ok: handed straight to validate_environment - ) -> LiteLLMBatch | Coroutine[Any, Any, LiteLLMBatch]: + ) -> LiteLLMBatch | Coroutine[object, object, LiteLLMBatch]: """ Retrieve a batch from Anthropic. diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 4c6bded68ea..3ab21099d3a 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -1697,14 +1697,13 @@ class AnthropicMessagesHandler(BaseTranslation): if not line.startswith("data:"): return line try: - data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads( - line[len("data:") :].strip() - ) + data: Final = _DECODED_SSE_DATA.validate_python(json.loads(line[len("data:") :].strip())) except json.JSONDecodeError: return line - if not isinstance(data, dict): + event: Final = as_str_mapping(data) + if not isinstance(event, dict): return line - rewritten: Final = _rewritten_event(_as_str_mapping(data), rewrite_event) + rewritten: Final = _rewritten_event(event, rewrite_event) return line if rewritten is data else "data: " + json.dumps(rewritten) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9a63afd5cc0..92e7c84031c 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -831,6 +831,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "else": "conditional (else): {}", "not": "must not match: {}", } + schema_fields: Final[Mapping[str, object]] = MappingProxyType(schema) min_items: Final = schema.get("minItems") enforced_min_items: Final = ( isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1) @@ -845,7 +846,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): for field, label in constraint_labels.items(): if field not in schema or field not in unsupported_fields: continue - value = schema[field] + value = schema_fields[field] # A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no # real requirement, so don't add a misleading advisory note for it. if isinstance(value, bool) and not value: @@ -869,7 +870,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema) - for key, value in schema.items(): + for key, value in schema_fields.items(): if key in unsupported_fields: continue if key == "type" and drops_conflicting_type: @@ -2385,7 +2386,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): optional_params["tools"] = tools ## Load Config - config: Final = litellm.AnthropicConfig.get_config(model=model) + config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model)) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index 73e94e85f04..5cc4e955d00 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -7,6 +7,7 @@ Litellm provider slug: `anthropic_text/` import json import time from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -112,7 +113,7 @@ class AnthropicTextConfig(BaseConfig): ) -> dict: prompt: Final = self._get_anthropic_text_prompt_from_messages(messages=messages, model=model) ## Load Config - config: Final = litellm.AnthropicTextConfig.get_config() + config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicTextConfig.get_config()) for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index 1a793e32317..279fb92cf6c 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -5,6 +5,7 @@ from collections.abc import Coroutine, Mapping from typing import Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -24,6 +25,8 @@ from litellm.types.utils import CallTypes, LlmProviders, ModelResponse from ..chat.transformation import AnthropicConfig from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo +_DECODED_JSON: Final = TypeAdapter(object) + class AnthropicFilesHandler: """ @@ -194,7 +197,7 @@ class AnthropicFilesHandler: if not line.strip(): continue - anthropic_result: object = json.loads(line) + anthropic_result: object = _DECODED_JSON.validate_python(json.loads(line)) if not isinstance(anthropic_result, dict): raise TypeError( f"Anthropic batch result line is not a JSON object: {type(anthropic_result).__name__}" diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index e9fd77ef441..dadd4d8558d 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -25,7 +25,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame @@ -1367,9 +1367,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # open a block: the emitted {"type": "thinking", "thinking": ""} gets # replayed as history and Anthropic rejects it (LIT-6357). A signed # entry opens the block so the client receives the replay signature. - thinking_blocks: Final = getattr(delta, "thinking_blocks", None) - if thinking_blocks and any( - isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks + if any( + isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) + for b in attribute_items(delta, "thinking_blocks") ): return False return True diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py index 935ce77ef87..d4430ca5ab9 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from functools import reduce from itertools import chain from types import MappingProxyType -from typing import Annotated, Any, Final, Literal, TypeVar, cast +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field, StrictInt, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -32,6 +32,9 @@ from ..constants import ( from ..errors import AnthropicContextManagementError from ..placeholders import build_cleared_tool_result_content +if TYPE_CHECKING: + from litellm.types.llms.openai import ChatCompletionToolParam + T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 @@ -216,7 +219,7 @@ def _trigger_met( current_tokens: Final = litellm.token_counter( model=model, messages=messages, - tools=cast(Any, tools), + tools=cast("list[ChatCompletionToolParam] | None", tools), ) verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens) verbose_logger.debug("context_management polyfill: threshold: %s", threshold) @@ -366,9 +369,13 @@ def apply_clear_tool_uses_20250919( tokens_before: Final = ( counted_tokens if counted_tokens is not None - else litellm.token_counter(model=model, messages=messages, tools=cast(Any, tools)) + else litellm.token_counter( + model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools) + ) + ) + tokens_after: Final = litellm.token_counter( + model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools) ) - tokens_after: Final = litellm.token_counter(model=model, messages=final_messages, tools=cast(Any, tools)) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/pass_through/context_management/placeholders.py b/litellm/llms/anthropic/pass_through/context_management/placeholders.py index b3d3529f105..8a0eb6d48e1 100644 --- a/litellm/llms/anthropic/pass_through/context_management/placeholders.py +++ b/litellm/llms/anthropic/pass_through/context_management/placeholders.py @@ -1,12 +1,10 @@ """Placeholder content for cleared ``tool_result`` blocks (string or block list).""" -from typing import Any - from .constants import CLEARED_TOOL_RESULT_PLACEHOLDER def build_cleared_tool_result_content( - original_content: Any, + original_content: object, ) -> str | list[dict]: """Return a string or single text block list, matching ``original_content`` shape.""" if isinstance(original_content, list): diff --git a/litellm/llms/anthropic/pass_through/context_management/result.py b/litellm/llms/anthropic/pass_through/context_management/result.py index 5b88fe30871..4713b8d79c7 100644 --- a/litellm/llms/anthropic/pass_through/context_management/result.py +++ b/litellm/llms/anthropic/pass_through/context_management/result.py @@ -6,7 +6,7 @@ attach ``iterations`` to ``usage``. """ from dataclasses import dataclass, field -from typing import Any, Final +from typing import Final from litellm.types.llms.anthropic import ( AppliedEdit, @@ -19,8 +19,8 @@ from .constants import COMPACT_EDIT_TYPE @dataclass class PolyfillResult: - messages: list[dict[str, Any]] - system: str | list[dict[str, Any]] | None + messages: list[dict[str, object]] + system: str | list[dict[str, object]] | None applied_edits: list[AppliedEdit] = field(default_factory=list) compaction_block: CompactionBlock | None = None iterations_usage: list[UsageIteration] | None = None diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/pass_through/convertible_blocks.py index fb17910fb97..9d651f8d721 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/pass_through/convertible_blocks.py @@ -3,7 +3,6 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial from itertools import chain, groupby -from operator import itemgetter from types import MappingProxyType from typing import Final, TypeAlias @@ -279,6 +278,14 @@ def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _K return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]} +def _entry_tool_use_id(entry: tuple[str, _KeptParts]) -> str: + return entry[0] + + +def _entry_kept_parts(entry: tuple[str, _KeptParts]) -> _KeptParts: + return entry[1] + + def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object: match as_str_mapping(block): case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if ( @@ -293,13 +300,15 @@ def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]: blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) - by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0))) + by_id: Final = tuple( + sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=_entry_tool_use_id) + ) if not any(parts for _, parts in by_id): return rewritten kept_in_order: Final = MappingProxyType( { - tool_use_id: iter(tuple(map(itemgetter(1), group))) - for tool_use_id, group in groupby(by_id, key=itemgetter(0)) + tool_use_id: iter(tuple(map(_entry_kept_parts, group))) + for tool_use_id, group in groupby(by_id, key=_entry_tool_use_id) } ) return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py index 8d1b358c8f1..2b6161d98f6 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py @@ -9,8 +9,8 @@ the LLM doesn't make a tool call, and we need to return a stream to the user. """ import json -from collections.abc import Mapping -from typing import Any, Final, cast +from collections.abc import Mapping, Sequence +from typing import Final, cast from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -136,8 +136,7 @@ class FakeAnthropicMessagesStreamIterator: """Convert the non-streaming response to streaming chunks""" chunks: Final = [] - # Cast response to dict for easier access - response_dict: Final = cast(dict[str, Any], self.response) + response_dict: Final[Mapping[str, object]] = self.response # 1. message_start event usage: Final = self.response.get("usage") @@ -160,9 +159,8 @@ class FakeAnthropicMessagesStreamIterator: chunks.append(f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode()) # 2-4. For each content block, send start/delta/stop events - content_blocks: Final = response_dict.get("content", []) - for index, block in enumerate(content_blocks): - block_dict = cast(dict[str, Any], block) + content_blocks: Final = cast(Sequence[Mapping[str, object]], response_dict.get("content", [])) + for index, block_dict in enumerate(content_blocks): chunks.extend(self._create_content_block_chunks(block_dict, index)) # 5. message_delta event (with final usage and stop_reason) diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index dfe0788e50a..35db54147c5 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -722,7 +722,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): logging_obj: LiteLLMLoggingObj, ) -> AnthropicMessagesResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[AnthropicMessagesResponse] = raw_response.json() except Exception: raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code) response: Final = AnthropicMessagesResponse(**raw_response_json) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index bd0f57c3ed9..48fc987a95a 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -242,7 +242,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _summary_part_text(part: object) -> str: if isinstance(part, Mapping): - mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json + mapping: Final = cast(Mapping[str, object], part) # cast-ok: summary parts are untyped provider json return str(mapping.get("text") or "") return str(getattr(part, "text", None) or "") @@ -673,7 +673,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config = anthropic_request.get("output_config") reasoning: Final = self.translate_thinking_to_reasoning( thinking, - output_config=cast(dict[str, Any] | None, output_config), + output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, drop_params=drop_params, ) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 99ebd2875a7..df8a0614665 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from openai import AsyncAzureOpenAI, AzureOpenAI from pydantic import BaseModel @@ -38,7 +38,7 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: + ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} if atranscription is True: diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 67b25bf7d12..02f1ba3472a 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -13,7 +13,7 @@ from openai import ( BadRequestError, OpenAI, ) -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES @@ -59,6 +59,8 @@ from .image_generation import ( ) from .image_generation.http_utils import azure_deployment_image_generation_json_body +_DECODED_JSON: Final = TypeAdapter(object) + class AzureOpenAIAssistantsAPIConfig: """ @@ -961,7 +963,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, @@ -1078,7 +1080,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): body=error_data, ) - result: Final = response.json()["result"] + result: Final = _DECODED_JSON.validate_python(response.json()["result"]) return httpx.Response( status_code=200, headers=response.headers, @@ -1358,7 +1360,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except AzureOpenAIError as e: raise e except Exception as e: - error_code: Final = getattr(e, "status_code", None) + error_code: Final[object] = getattr(e, "status_code", None) if error_code is not None: raise AzureOpenAIError(status_code=error_code, message=str(e)) else: diff --git a/litellm/llms/azure/exception_mapping.py b/litellm/llms/azure/exception_mapping.py index dcc3601cd8e..f7146a36384 100644 --- a/litellm/llms/azure/exception_mapping.py +++ b/litellm/llms/azure/exception_mapping.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.exceptions import ContentPolicyViolationError @@ -54,7 +54,7 @@ class AzureOpenAIExceptionMapping: @staticmethod def _extract_azure_error( original_exception: Exception, - ) -> tuple[dict[str, object], dict | None]: + ) -> tuple[dict[str, object], object]: """Extract Azure OpenAI error payload and inner error details. Azure error formats can vary by endpoint/version. Common shapes: @@ -67,13 +67,13 @@ class AzureOpenAIExceptionMapping: return {}, None # Some SDKs place the payload under "error". - azure_error: dict[str, Any] + azure_error: dict[str, object] if isinstance(body_dict.get("error"), dict): azure_error = body_dict.get("error", {}) else: azure_error = body_dict - inner_error: Final = ( + inner_error: Final[object] = ( azure_error.get("inner_error") or azure_error.get("innererror") or body_dict.get("innererror") diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 8cae4bf284b..6d904503df8 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -95,7 +95,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def get_stripped_model_name(self, model: str) -> str: return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "") - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]: """ Handle reasoning items to filter out the status field. Issue: https://github.com/BerriAI/litellm/issues/13484 diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 83d00a621b4..3b541310d28 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -397,7 +397,7 @@ class AzureAIStudioConfig(OpenAIConfig): return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: - fields_to_drop: Final = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) + fields_to_drop: Final[Collection[str]] = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) tools: Final = request_data.get("tools") if fields_to_drop and isinstance(tools, list): for tool in tools: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index b2eb4cde5ce..91d0d52a629 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -94,9 +94,10 @@ class BaseConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("_") and not isinstance( v, @@ -513,7 +514,7 @@ class BaseConfig(ABC): def apply_assembled_streaming_response_metadata( self, response: "ModelResponse", - chunks: list[Any], + chunks: list[object], ) -> None: """Hook for providers to merge chunk metadata into assembled streaming responses.""" return diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 5fe81a7f1b7..6399ded9555 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -121,7 +121,7 @@ class BaseTranslation(ABC): @staticmethod def merge_user_api_key_metadata_into_request( - request_data: dict[str, Any], # mutable-ok: proxy hooks share and mutate the request payload dict in place + request_data: dict[str, object], # mutable-ok: proxy hooks share and mutate the request payload dict in place user_api_key_dict: Optional["UserAPIKeyAuth"], ) -> None: """ diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 55a4b3da0b3..725146fe894 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -9,6 +9,8 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage +_DECODED_JSON: Final = TypeAdapter(object) + def _anthropic_stream_chunk_events(item: object) -> list[dict]: if isinstance(item, dict): @@ -30,7 +32,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]: if not payload or payload == "[DONE]": continue try: - parsed = json.loads(payload) + parsed = _DECODED_JSON.validate_python(json.loads(payload)) except json.JSONDecodeError: continue if isinstance(parsed, dict): diff --git a/litellm/llms/base_llm/image_edit/transformation.py b/litellm/llms/base_llm/image_edit/transformation.py index 4faf0aaaf30..35b4bb90025 100644 --- a/litellm/llms/base_llm/image_edit/transformation.py +++ b/litellm/llms/base_llm/image_edit/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import RequestFiles @@ -32,9 +32,10 @@ class BaseImageEditConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/interactions/transformation.py b/litellm/llms/base_llm/interactions/transformation.py index b7bbf7bde73..39ca1d8f7d7 100644 --- a/litellm/llms/base_llm/interactions/transformation.py +++ b/litellm/llms/base_llm/interactions/transformation.py @@ -11,7 +11,8 @@ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final import httpx @@ -60,9 +61,10 @@ class BaseInteractionsAPIConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index d3b05716452..2663a3c42e7 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -141,7 +141,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): self, llm_router: Router, model: str, - request_data: dict[str, Any], + request_data: dict[str, object], litellm_parent_otel_span: Span, ) -> ResourceObjectType: """ @@ -399,7 +399,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]): async def create_resource_for_each_model( self, llm_router: Router, - request_data: dict[str, Any], + request_data: dict[str, object], target_model_names_list: list[str], litellm_parent_otel_span: Span, ) -> list[ResourceObjectType]: diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index ac5032c08f1..7e58cc6e71b 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -28,6 +28,7 @@ if TYPE_CHECKING: RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class PassthroughMetadata(LiteLLMBaseModel): @@ -71,7 +72,7 @@ def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None def relayed_body(httpx_response: Response) -> str | dict: try: - body: Final[object] = httpx_response.json() + body: Final = _DECODED_JSON.validate_python(httpx_response.json()) except ValueError: return httpx_response.text return body if isinstance(body, dict) else httpx_response.text diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 5d2f92b5e82..7f246bf33da 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -80,7 +80,7 @@ class BaseRerankConfig(ABC): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index fb85cdb7687..be49098b62b 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -1,6 +1,7 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, TypedDict +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, TypedDict import httpx @@ -40,9 +41,10 @@ class BaseTextToSpeechConfig(ABC): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 28f6348e4d9..138700b3212 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -152,7 +152,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: pass @@ -164,7 +164,7 @@ class BaseVectorStoreConfig: api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Optional async version of transform_search_vector_store_request. diff --git a/litellm/llms/baseten/chat.py b/litellm/llms/baseten/chat.py index 381d487c36d..c165831b90a 100644 --- a/litellm/llms/baseten/chat.py +++ b/litellm/llms/baseten/chat.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import litellm @@ -42,7 +43,7 @@ class BasetenConfig(OpenAIGPTConfig): frequency_penalty: int | None = None, stream_options: dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/bedrock/batches/handler.py b/litellm/llms/bedrock/batches/handler.py index 48e79d39b8e..f37abacc883 100644 --- a/litellm/llms/bedrock/batches/handler.py +++ b/litellm/llms/bedrock/batches/handler.py @@ -174,12 +174,13 @@ class BedrockBatchesHandler: ) def job_status() -> "LiteLLMBatch": + auth_kwargs: Final[Mapping[str, object]] = dict(auth_params.model_dump()) return BedrockBatchesHandler.handle_model_invocation_job_status( batch_id=batch_id, aws_region_name=region, logging_obj=logging_obj, aws_bedrock_control_plane_endpoint=control_plane_endpoint, - **auth_params.model_dump(), + **auth_kwargs, ) try: diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index bb5bb246a93..18212b7390f 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, Union from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid @@ -51,6 +52,8 @@ else: HTTPHandler = Any AsyncHTTPHandler = Any +_DECODED_JSON: Final = TypeAdapter(object) + class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): def __init__(self, **kwargs): @@ -314,7 +317,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): return None try: - data: Final[object] = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) # Skip non-dict data (some lines contain JSON strings) return data if isinstance(data, dict) else None except json.JSONDecodeError: @@ -482,7 +485,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): if "application/json" in content_type: # Direct JSON response verbose_logger.debug("Parsing JSON response") - response_json: Final[object] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) verbose_logger.debug("Response JSON: %s", response_json) return self._parse_json_response(response_json) else: @@ -566,7 +569,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -706,7 +709,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = response.read() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, @@ -788,7 +791,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): continue try: - data_obj = json.loads(json_str) + data_obj = _DECODED_JSON.validate_python(json.loads(json_str)) if not isinstance(data_obj, dict): continue @@ -928,7 +931,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) try: body: Final = await response.aread() - response_json: Final[object] = json.loads(body) + response_json: Final = _DECODED_JSON.validate_python(json.loads(body)) except (json.JSONDecodeError, Exception) as e: raise BedrockError( status_code=response.status_code, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..4d7ec663d09 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -12,7 +12,8 @@ from itertools import chain from typing import TYPE_CHECKING, Final, Literal, cast, overload import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -21,6 +22,7 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, filter_exceptions_from_params, filter_internal_params, map_finish_reason, @@ -160,6 +162,14 @@ _TOOLS_AS_SENT: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _JSON_VALUE: Final = TypeAdapter(object) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) _TOOLS_AS_RECEIVED: Final = TypeAdapter(list[object]) + + +@with_config(ConfigDict(extra="allow", strict=True)) +class _ToolCallInContent(TypedDict): + name: ReadOnly[str] + + +_TOOL_CALL_IN_CONTENT: Final = TypeAdapter(_ToolCallInContent) CLAUDE_ON_BEDROCK_INVOKE_REMEDY: Final = " This model runs tool search on Bedrock through `bedrock/invoke/`." _DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) @@ -2453,9 +2463,17 @@ class AmazonConverseConfig(BaseConfig): if message.content is not None: try: tool_call_names: Final = self.get_tool_call_names(tools) - json_content: Final = json.loads(message.content) - if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: - tool_calls: Final = [ChatCompletionMessageToolCall(function=Function(**json_content))] + json_content: Final = as_str_mapping(_JSON_VALUE.validate_python(json.loads(message.content))) + if ( + json_content is not None + and json_content.get("type") == "function" + and json_content.get("name") in tool_call_names + ): + tool_calls: Final = [ + ChatCompletionMessageToolCall( + function=Function(**_TOOL_CALL_IN_CONTENT.validate_python(json_content)) + ) + ] message.tool_calls = tool_calls message.content = None diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py index 3005bc82299..777237b4f18 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_ai21_transformation.py @@ -56,9 +56,10 @@ class AmazonAI21Config(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py index 00c2eaf0f8f..2118d183b22 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py @@ -37,9 +37,10 @@ class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py index d2eb6eef9ef..cb867a09f8f 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_llama_transformation.py @@ -37,9 +37,10 @@ class AmazonLlamaConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py index 44511eeaa3e..4420bb3a418 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_mistral_transformation.py @@ -47,9 +47,10 @@ class AmazonMistralConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py index a18e97c63a6..736b3ea3cd2 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_titan_transformation.py @@ -43,9 +43,10 @@ class AmazonTitanConfig(AmazonInvokeConfig, BaseConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not k.startswith("_abc") and not isinstance( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py index b29ddf01d1e..2655c13be44 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py @@ -44,9 +44,10 @@ class AmazonAnthropicConfig(AmazonInvokeConfig): @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index e48196946cd..d61541aab8a 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -21,6 +21,7 @@ from litellm.types.utils import ModelResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.anthropic.chat.handler import ModelResponseIterator LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -134,7 +135,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "ModelResponseIterator": return self.get_model_response_iterator_for_request( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, optional_params={} ) @@ -145,7 +146,7 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig): sync_stream: bool, json_mode: bool | None, optional_params: Mapping[str, object], - ) -> object: + ) -> "ModelResponseIterator": from litellm.llms.anthropic.chat.handler import ModelResponseIterator return ModelResponseIterator( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4954e646f58..2b946d9664c 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -2021,8 +2021,13 @@ def get_bedrock_response_stream_shape(): _BEDROCK_STREAM_OUTPUT_SHAPES: Final = ("ConverseStreamOutput", "ResponseStream") +class _ModeledErrorMetadata(TypedDict, total=False): + httpStatusCode: ReadOnly[int] + + def _modeled_error_status(member: Shape) -> int | None: - status: Final = (member.metadata or {}).get("error", {}).get("httpStatusCode") + error: Final[_ModeledErrorMetadata] = (member.metadata or {}).get("error", {}) + status: Final = error.get("httpStatusCode") return None if status is None else int(status) diff --git a/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py b/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py index 581b89369f7..fba2673b5aa 100644 --- a/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py @@ -35,9 +35,10 @@ class AmazonTitanG1Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py b/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py index 3b594662a9b..7d319ca6b06 100644 --- a/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py @@ -39,9 +39,10 @@ class AmazonTitanV2Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 3d23b69f846..37f95ff7b9b 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final import httpx @@ -139,7 +139,7 @@ class BedrockFilesHandler(BaseAWSLLM): optional_params: dict, timeout: float | httpx.Timeout, max_retries: int | None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from S3 bucket for Bedrock files. Supports both sync and async operations. diff --git a/litellm/llms/bedrock/image_edit/handler.py b/litellm/llms/bedrock/image_edit/handler.py index 466993de9d1..bdfe6f5a8d2 100644 --- a/litellm/llms/bedrock/image_edit/handler.py +++ b/litellm/llms/bedrock/image_edit/handler.py @@ -10,6 +10,7 @@ import json from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -39,6 +40,8 @@ if TYPE_CHECKING: else: AWSPreparedRequest = Any +_DECODED_JSON: Final = TypeAdapter(object) + class BedrockImageEditPreparedRequest(LiteLLMBaseModel): """ @@ -317,7 +320,7 @@ class BedrockImageEdit(BaseAWSLLM): additional_args={"complete_input_dict": data}, ) verbose_logger.debug("raw model_response: %s", response.text) - response_dict: Final = response.json() + response_dict: Final = _DECODED_JSON.validate_python(response.json()) if response_dict is None: raise ValueError("Error in response object format, got None") diff --git a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py index acc847e0e3c..e5c5edba750 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -20,9 +21,10 @@ class AmazonStability3Config: @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py index 145e6a0781f..5731541da56 100644 --- a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py @@ -3,6 +3,7 @@ Transformation logic for Amazon Titan Image Generation. """ import types +from collections.abc import Mapping from typing import Final from openai.types.image import Image @@ -35,16 +36,17 @@ class AmazonTitanImageGenerationConfig: width: int | None = None, height: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 12421aa94fd..805652630ed 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -145,7 +145,7 @@ class AmazonAnthropicClaudeMessagesConfig( self, headers: dict, model: str, - messages: list[object], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index c71721d800e..d4a2d2eabe9 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -7,7 +7,7 @@ stripping, and the anthropic-version / anthropic-beta placement (headers, never the body) that are specific to the bedrock-mantle endpoint. """ -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -90,7 +90,7 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/bedrock/model_listing.py b/litellm/llms/bedrock/model_listing.py index 99ff48f9400..409bf2c7b97 100644 --- a/litellm/llms/bedrock/model_listing.py +++ b/litellm/llms/bedrock/model_listing.py @@ -4,7 +4,7 @@ from typing import Final, TypeAlias from urllib.parse import quote, urlencode import httpx -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token @@ -15,6 +15,7 @@ from litellm.types.llms.bedrock import AwsAuthParams LIST_MODELS_TIMEOUT: Final = 10.0 INFERENCE_PROFILES_PAGE_SIZE: Final = 1000 INFERENCE_PROFILES_PAGE_CAP: Final = 20 +_DECODED_JSON: Final = TypeAdapter(object) QueryPairs: TypeAlias = tuple[tuple[str, str], ...] @@ -103,7 +104,7 @@ class BedrockModelLister(BaseAWSLLM): f"Status code: {response.status_code}, Response: {response.text}" ), ) - return response.json() + return _DECODED_JSON.validate_python(response.json()) def _auth_headers(self, url: str) -> Mapping[str, str]: if self._bearer_token is not None: diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index a90b4829634..1b01b0ec6d0 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -544,7 +544,7 @@ class _StreamFrame(TypedDict): def _unpack_uint32(buffer: bytes) -> int: - return struct.unpack("!I", buffer)[0] + return int.from_bytes(buffer, "big") def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]: @@ -1255,7 +1255,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): headers_bytes = frame_raw[12 : 12 + orig_hdrs_len] try: - payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + payload_dict = _BODY_BY_REFERENCE.validate_python( + json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4]) + ) for local_idx, (_, container, key) in enumerate( _collect_stream_delta_text_holders(payload_dict.get("delta")) ): diff --git a/litellm/llms/black_forest_labs/image_edit/transformation.py b/litellm/llms/black_forest_labs/image_edit/transformation.py index 7cfb0daf334..38bcbb0d905 100644 --- a/litellm/llms/black_forest_labs/image_edit/transformation.py +++ b/litellm/llms/black_forest_labs/image_edit/transformation.py @@ -51,6 +51,7 @@ class _BflPolledResult(TypedDict): _BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult) +_DECODED_JSON: Final = TypeAdapter(object) _BFL_REQUEST_PARAMS: Final = ( @@ -316,7 +317,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - raw_response_data: Final[object] = raw_response.json() + raw_response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 928af314506..52f2c9a7c50 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -40,6 +40,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): @@ -272,7 +273,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): The response contains: {"status": "Ready", "result": {"sample": "https://..."}} """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise BlackForestLabsError( status_code=raw_response.status_code, diff --git a/litellm/llms/cerebras/chat.py b/litellm/llms/cerebras/chat.py index c3aa26ade35..9fc99500fc5 100644 --- a/litellm/llms/cerebras/chat.py +++ b/litellm/llms/cerebras/chat.py @@ -4,6 +4,7 @@ Cerebras Chat Completions API this is OpenAI compatible - no translation needed / occurs """ +from collections.abc import Mapping from typing import Final from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -41,7 +42,7 @@ class CerebrasConfig(OpenAIGPTConfig): user: str | None = None, reasoning_effort: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/chatgpt/chat/streaming_utils.py b/litellm/llms/chatgpt/chat/streaming_utils.py index ee3120bacea..49be583f6d1 100644 --- a/litellm/llms/chatgpt/chat/streaming_utils.py +++ b/litellm/llms/chatgpt/chat/streaming_utils.py @@ -7,6 +7,8 @@ Normalizes non-spec-compliant tool_call chunks from the ChatGPT backend API. from collections.abc import Awaitable from typing import Final, Protocol +from pydantic import TypeAdapter + from litellm.types.utils import ( ChatCompletionDeltaCustomToolCall, ChatCompletionDeltaToolCall, @@ -23,6 +25,9 @@ class ChatGPTChunkStream(Protocol): def __anext__(self) -> Awaitable[ModelResponseStream]: ... +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) + + def _first_choice_delta(chunk: ModelResponseStream) -> Delta | None: return chunk.choices[0].delta @@ -47,7 +52,7 @@ class ChatGPTToolCallNormalizer: self._last_id: str | None = None # tracks which tool call the next delta belongs to def __getattr__(self, name: str) -> object: - return getattr(self._stream, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._stream, name)) def __iter__(self) -> "ChatGPTToolCallNormalizer": return self diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 8c2fd9e0aff..d07ae5e4215 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -23,6 +23,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class ClarifaiConfig(OpenAIGPTConfig): @@ -114,7 +115,7 @@ class ClarifaiConfig(OpenAIGPTConfig): ) ## Reponse try: - completion_response: Final = raw_response.json() + completion_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenAIError( status_code=raw_response.status_code, diff --git a/litellm/llms/claude_code/harness/transformation.py b/litellm/llms/claude_code/harness/transformation.py index f75d86c7e63..8252d2dd461 100644 --- a/litellm/llms/claude_code/harness/transformation.py +++ b/litellm/llms/claude_code/harness/transformation.py @@ -30,6 +30,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionSetup, @@ -231,12 +232,12 @@ def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> return event_list() -def _record_result(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]: +def _record_result(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]: state.result_seen = True state.is_error = bool(event.get("is_error", False)) result = event.get("result") state.result_text = result if isinstance(result, str) else None - state.errors = [str(e) for e in event.get("errors") or ()] # mutable-ok: mirrors the JSON errors array + state.errors = [str(e) for e in iter_items(event.get("errors") or ())] # mutable-ok: mirrors the JSON errors array state.structured_output = event.get("structured_output") if event.get("session_id"): state.session_id = str(event["session_id"]) diff --git a/litellm/llms/codestral/completion/handler.py b/litellm/llms/codestral/completion/handler.py index 08799c3a0a5..6bebfe2d444 100644 --- a/litellm/llms/codestral/completion/handler.py +++ b/litellm/llms/codestral/completion/handler.py @@ -2,7 +2,7 @@ ## handler file for TextCompletionCodestral Integration - https://codestral.com/ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping from functools import partial from typing import Final, Protocol @@ -268,7 +268,7 @@ class CodestralTextCompletion: prompt = prompt_factory(model=model, messages=messages) ## Load Config - config: Final = litellm.CodestralTextCompletionConfig.get_config() + config: Final[Mapping[str, object]] = litellm.CodestralTextCompletionConfig.get_config() for k, v in config.items(): if ( k not in optional_params diff --git a/litellm/llms/codex/harness/transformation.py b/litellm/llms/codex/harness/transformation.py index 4512e39189e..d7388bd23fc 100644 --- a/litellm/llms/codex/harness/transformation.py +++ b/litellm/llms/codex/harness/transformation.py @@ -15,7 +15,7 @@ import re from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from pydantic import ConfigDict, TypeAdapter @@ -31,6 +31,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionSetup, @@ -77,13 +78,13 @@ class CodexStreamState: started: set[str] = field(default_factory=set) # mutable-ok: parser records announced tool items -def _tool_input(item: Mapping[str, Any]) -> tuple[str, str, Mapping[str, object], bool]: +def _tool_input(item: Mapping[str, object]) -> tuple[str, str, Mapping[str, object], bool]: """(normalized name, native name, input, builtin) for a tool-like item.""" item_type = item.get("type") if item_type == "command_execution": return "bash", "command_execution", MappingProxyType({"command": item.get("command", "")}), True if item_type == "file_change": - changes: Final = list(item.get("changes") or ()) # mutable-ok: JSON array, as codex reports it + changes: Final = list(iter_items(item.get("changes") or ())) # mutable-ok: JSON array, as codex reports it return "edit", "apply_patch", MappingProxyType({"changes": changes}), True if item_type == "web_search": return "web_search", "web_search", MappingProxyType({"query": item.get("query", "")}), True diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fdf44d5333e..e7fadd7f375 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -1,6 +1,6 @@ import json import time -from collections.abc import AsyncIterator, Iterator, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx @@ -221,7 +221,8 @@ class CohereChatConfig(BaseConfig): headers: dict, ) -> dict: ## Load Config - for k, v in litellm.CohereChatConfig.get_config().items(): + config: Final[Mapping[str, object]] = litellm.CohereChatConfig.get_config() + for k, v in config.items(): if ( k not in optional_params ): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index 8b641c24e9b..7d9c8de7a07 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -15,6 +15,7 @@ from litellm.types.rerank import OptionalRerankParams, RerankRequest, RerankResp from ..common_utils import CohereError _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class CohereRerankConfig(BaseRerankConfig): @@ -144,7 +145,7 @@ class CohereRerankConfig(BaseRerankConfig): No transformation required, litellm follows cohere API response format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise CohereError(message=raw_response.text, status_code=raw_response.status_code) diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index 009a220534c..83596f90d1f 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.cohere.rerank.transformation import CohereRerankConfig @@ -39,7 +39,7 @@ class CohereRerankV2Config(CohereRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/cometapi/chat/transformation.py b/litellm/llms/cometapi/chat/transformation.py index 2bf119a0a4a..ae04889bc97 100644 --- a/litellm/llms/cometapi/chat/transformation.py +++ b/litellm/llms/cometapi/chat/transformation.py @@ -6,7 +6,7 @@ Documentation: [CometAPI Documentation Link] """ from collections.abc import AsyncIterator, Iterator -from typing import Any, Final +from typing import Final import httpx @@ -137,7 +137,7 @@ class CometAPIConfig(OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "CometAPIChatCompletionStreamingHandler": """ Get model response iterator for streaming responses """ diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index a02db2338d8..87fc61985e6 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -3,10 +3,10 @@ CompactifAI chat completion transformation """ from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.common_utils import OpenAIError @@ -25,11 +25,32 @@ else: LiteLLMLoggingObj = Any +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + +class _CompactifAIToolCall(TypedDict): + function: ReadOnly[Mapping[str, str]] + + +class _CompactifAIMessage(TypedDict, total=False): + content: str | None # writable-ok: json mode moves a lone tool call's arguments into the content in place + tool_calls: Sequence[_CompactifAIToolCall] | None # writable-ok: json mode clears the tool call it moved + + +class _CompactifAIChoice(TypedDict, total=False): + message: ReadOnly[_CompactifAIMessage] + + class CompactifAIResponseFields(TypedDict, total=False): """The chat completion fields of a CompactifAI response body.""" id: ReadOnly[str] - choices: ReadOnly[Sequence[Mapping[str, object]]] + choices: ReadOnly[Required[Sequence[_CompactifAIChoice]]] created: ReadOnly[int] model: ReadOnly[str] system_fingerprint: ReadOnly[str | None] @@ -82,22 +103,19 @@ class CompactifAIChatConfig(OpenAIGPTConfig): ) ## RESPONSE OBJECT - response_json: Final = raw_response.json() + response_json: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + CompactifAIResponseFields, _json_body(raw_response) + ) # Handle JSON mode if needed if json_mode: for choice in response_json["choices"]: message = choice.get("message") - if message and message.get("tool_calls"): - # Convert tool calls to content for JSON mode - tool_calls = message.get("tool_calls", []) - if len(tool_calls) == 1: - message["content"] = tool_calls[0]["function"].get("arguments", "") - message["tool_calls"] = None + if message and (tool_calls := message.get("tool_calls")) and len(tool_calls) == 1: + message["content"] = tool_calls[0]["function"].get("arguments", "") + message["tool_calls"] = None - response_fields: Final[CompactifAIResponseFields] = response_json - - returned_response: Final = ModelResponse(**response_fields) + returned_response: Final = ModelResponse(**response_json) # Set model name with provider prefix returned_response.model = f"compactifai/{model}" diff --git a/litellm/llms/custom_httpx/async_client_cleanup.py b/litellm/llms/custom_httpx/async_client_cleanup.py index 6ef61b6f58f..3c0cc90f30f 100644 --- a/litellm/llms/custom_httpx/async_client_cleanup.py +++ b/litellm/llms/custom_httpx/async_client_cleanup.py @@ -20,7 +20,7 @@ async def close_litellm_async_clients(): cache_dict: Final = getattr(litellm.in_memory_llm_clients_cache, "cache_dict", {}) - for key, handler in cache_dict.items(): + for handler in cache_dict.values(): # Handle BaseLLMAIOHTTPHandler instances (aiohttp_openai provider) if isinstance(handler, BaseLLMAIOHTTPHandler) and hasattr(handler, "close"): try: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a692f3f964e..55b2c21718d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -49,7 +49,7 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -412,6 +412,27 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool: return False +def _run_responses_deployment_hook( + *, + model: str, + request_input: str | ResponseInputParam, + custom_llm_provider: str, + optional_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, +) -> Mapping[str, object]: + return run_async_function( + async_pre_call_deployment_hook, + { + **dict(litellm_params), + **optional_params, + "model": model, + "input": request_input, + "custom_llm_provider": custom_llm_provider, + }, + CallTypes.responses.value, + ) + + def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict: """A pre-signed request carries its auth inside its own ``headers`` key, which logging treats as request body (only the top-level headers channel gets masked), @@ -477,6 +498,7 @@ _DECODED_BODY_STALE_HEADERS: Final[frozenset[str]] = frozenset({"content-encodin _OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) _JSON_OBJECTS: Final = TypeAdapter(list[dict[str, object]]) +_DECODED_JSON: Final = TypeAdapter(object) def _decoded_body_headers(response: httpx.Response) -> httpx.Headers: @@ -2765,16 +2787,12 @@ class BaseLLMHTTPHandler: litellm_params, ) - modified_kwargs: Final = run_async_function( - async_pre_call_deployment_hook, - { - **dict(litellm_params), - **response_api_optional_request_params, - "model": model, - "input": input, - "custom_llm_provider": custom_llm_provider, - }, - CallTypes.responses.value, + modified_kwargs: Final = _run_responses_deployment_hook( + model=model, + request_input=input, + custom_llm_provider=custom_llm_provider, + optional_params=response_api_optional_request_params, + litellm_params=litellm_params, ) if modified_kwargs is None: return ( @@ -3753,7 +3771,7 @@ class BaseLLMHTTPHandler: response: httpx.Response, upload_url_location: str, upload_url_key: str = "upload_url", - ) -> tuple[str | None, dict | None]: + ) -> tuple[object, Mapping[str, object] | None]: """ Extract upload URL from initial file creation response. @@ -3773,8 +3791,10 @@ class BaseLLMHTTPHandler: else: # Response body style (e.g., Manus, S3 presigned URLs) try: - response_data: Final = response.json() - upload_url = response_data.get(upload_url_key) + response_data: Final = as_str_mapping(_DECODED_JSON.validate_python(response.json())) + if response_data is None: + return None, None + upload_url: Final = response_data.get(upload_url_key) return upload_url, response_data if upload_url else None except Exception: return None, None @@ -3940,7 +3960,7 @@ class BaseLLMHTTPHandler: elif isinstance(transformed_request, dict) and "file" in transformed_request: # Handle multipart form-data uploads (e.g., Anthropic Files API) # The dict contains tuples suitable for httpx's `files` parameter - file_request: Final = cast(dict[str, Any], transformed_request) + file_request: Final = cast(dict[str, object], transformed_request) upload_response = sync_httpx_client.post( url=api_base, headers=headers, diff --git a/litellm/llms/custom_httpx/mock_transport.py b/litellm/llms/custom_httpx/mock_transport.py index 0322cceec0f..6acadb8e98a 100644 --- a/litellm/llms/custom_httpx/mock_transport.py +++ b/litellm/llms/custom_httpx/mock_transport.py @@ -12,6 +12,9 @@ import uuid from typing import Final import httpx +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import decoded_json_object # --------------------------------------------------------------------------- # Pre-built response templates @@ -22,7 +25,7 @@ def _mock_id() -> str: return f"chatcmpl-mock-{uuid.uuid4().hex[:8]}" -def _chat_completion_json(model: str) -> dict: +def _chat_completion_json(model: object) -> dict: """Return a minimal valid ChatCompletion object.""" return { "id": _mock_id(), @@ -55,6 +58,8 @@ _JSON_HEADERS: Final = { "content-type": "application/json", } +_DECODED_JSON: Final = TypeAdapter(object) + class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @@ -64,13 +69,13 @@ class MockOpenAITransport(httpx.AsyncBaseTransport, httpx.BaseTransport): """ @staticmethod - def _parse_request(request: httpx.Request) -> tuple[str, bool]: + def _parse_request(request: httpx.Request) -> tuple[object, bool]: """Extract model from the request body.""" try: - body: Final = json.loads(request.content) + body: Final = _DECODED_JSON.validate_python(json.loads(request.content)) except (json.JSONDecodeError, ValueError): return ("mock-model", False) - model: Final = body.get("model", "mock-model") + model: Final = decoded_json_object(body).get("model", "mock-model") return (model, False) async def handle_async_request(self, request: httpx.Request) -> httpx.Response: diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 3f388637a83..55d6fb1a0e2 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -55,10 +55,9 @@ def _extract_token_breakdown(usage: Usage) -> TokenBreakdown: ) -def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) -> float: - value: Final = model_info.get(cost_key) +def _flat_rate(value: float | None, fallback_value: float | None) -> float: if value is None: - return float(model_info.get(fallback_cost_key) or 0.0) + return float(fallback_value or 0.0) return float(value) @@ -66,8 +65,12 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates: reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token") return TokenRates( input_rate=float(model_info.get("input_cost_per_token") or 0.0), - cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"), + cache_read_rate=_flat_rate( + model_info.get("cache_read_input_token_cost"), model_info.get("input_cost_per_token") + ), + cache_creation_rate=_flat_rate( + model_info.get("cache_creation_input_token_cost"), model_info.get("input_cost_per_token") + ), output_rate=float(model_info.get("output_cost_per_token") or 0.0), reasoning_rate=None if reasoning_rate is None else float(reasoning_rate), ) diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 6a172e1e074..d1faa6c6cd0 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -49,6 +49,7 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" @@ -187,7 +188,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): ) try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse DashScope image generation response: {e}", diff --git a/litellm/llms/databricks/embed/transformation.py b/litellm/llms/databricks/embed/transformation.py index ddaf2bbb621..0feb2aea5a0 100644 --- a/litellm/llms/databricks/embed/transformation.py +++ b/litellm/llms/databricks/embed/transformation.py @@ -3,6 +3,7 @@ Translates from OpenAI's `/v1/embeddings` to Databricks' `/embeddings` """ import types +from collections.abc import Mapping from typing import Final @@ -16,16 +17,17 @@ class DatabricksEmbeddingConfig: ) def __init__(self, instruction: str | None = None) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index cb8d2b652a2..f5ccbd92fb4 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -3,6 +3,7 @@ from collections.abc import AsyncIterator, Mapping from typing import Final import httpx +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items @@ -25,6 +26,7 @@ DROP_CONTAINER_WARNING: Final = ( "and the chat completions bridge it replaces never forwarded it." ) BEDROCK_INVOCATION_METRICS_FIELD: Final = "amazon-bedrock-invocationMetrics" +_DECODED_JSON: Final = TypeAdapter(object) def _is_unsupported_server_tool(tool: object) -> bool: @@ -57,7 +59,7 @@ def _without_unsupported_server_tools(params: Mapping[str, object]) -> dict[str, def _without_invocation_metrics(sse_line: bytes) -> bytes: if not sse_line.startswith(b"data:") or BEDROCK_INVOCATION_METRICS_FIELD.encode() not in sse_line: return sse_line - event: Final = as_str_mapping(json.loads(sse_line.removeprefix(b"data:"))) + event: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(sse_line.removeprefix(b"data:")))) if event is None: return sse_line kept_fields: Final = {key: value for key, value in event.items() if key != BEDROCK_INVOCATION_METRICS_FIELD} diff --git a/litellm/llms/deepagents/harness/sandbox_backend.py b/litellm/llms/deepagents/harness/sandbox_backend.py index d39690ccdd1..8ad90faa2b4 100644 --- a/litellm/llms/deepagents/harness/sandbox_backend.py +++ b/litellm/llms/deepagents/harness/sandbox_backend.py @@ -22,7 +22,7 @@ import shlex import uuid from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import Any, Final, TypeVar +from typing import Final, TypeVar from deepagents.backends.protocol import ( DeleteResult, @@ -171,7 +171,7 @@ class SandboxBackend(SandboxBackendProtocol): # pyright: ignore[reportUntypedBa # -- sync bridge (deepagents only calls these outside the event loop) --- - def _sync(self, coro: Coroutine[Any, Any, T]) -> T: + def _sync(self, coro: Coroutine[object, object, T]) -> T: try: running = asyncio.get_running_loop() except RuntimeError: diff --git a/litellm/llms/deepagents/harness/transformation.py b/litellm/llms/deepagents/harness/transformation.py index cb80e3e12f7..54a6a5573ac 100644 --- a/litellm/llms/deepagents/harness/transformation.py +++ b/litellm/llms/deepagents/harness/transformation.py @@ -11,11 +11,13 @@ from __future__ import annotations import itertools import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol +from pydantic import TypeAdapter + from litellm.harness.options import DeepAgentsOptions from litellm.harness.types import ( Capabilities, @@ -41,6 +43,7 @@ DEEPAGENTS_DEFAULT_RECURSION_LIMIT: Final = 1000 _MODEL_NODE: Final = "model" # Only these graph nodes produce new messages; middleware hooks may re-emit history. _EVENT_NODES: Final = frozenset({"model", "tools"}) +_TOOL_CALLS: Final = TypeAdapter(Iterable[Mapping[str, object]]) NORMALIZED_TO_NATIVE: Final[Mapping[str, str]] = MappingProxyType( { @@ -208,7 +211,7 @@ def update_events( def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event, ...]: kind = getattr(message, "type", None) if kind == "ai": - calls = getattr(message, "tool_calls", None) or () + calls: Final = _TOOL_CALLS.validate_python(getattr(message, "tool_calls", None) or ()) return tuple(tool_call_event(call) for call in calls if call.get("name") not in skip_tools) if kind == "tool" and getattr(message, "name", None) not in skip_tools: status: Final[object] = getattr(message, "status", None) diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index a1c3021c9f5..1c8a056116f 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -2,12 +2,13 @@ Translates from OpenAI's `/v1/audio/transcriptions` to Deepgram's `/v1/listen` """ -from collections.abc import Iterable, Mapping -from typing import Final +from collections.abc import Iterable, Mapping, Sequence +from typing import Final, Protocol, cast from urllib.parse import urlencode from httpx import Headers, Response -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, InstanceOf, TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.litellm_core_utils.url_utils import join_api_base @@ -27,6 +28,31 @@ from ..common_utils import DeepgramException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_RESPONSE_BODY: Final = TypeAdapter(InstanceOf[dict[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_WORD_LIST: Final = TypeAdapter(InstanceOf[list[object]], config=ConfigDict(hide_input_in_errors=True)) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + +class _DeepgramAlternative(TypedDict): + transcript: ReadOnly[str] + words: ReadOnly[Sequence[Mapping[str, object]]] + paragraphs: ReadOnly[NotRequired[Mapping[str, str]]] + + +class _DeepgramChannel(TypedDict): + alternatives: ReadOnly[Sequence[_DeepgramAlternative]] + + +class _DeepgramResponse(TypedDict): + results: ReadOnly[Mapping[str, Sequence[_DeepgramChannel]]] + metadata: ReadOnly[Mapping[str, float]] class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -82,7 +108,10 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from Deepgram to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + body: Final = _json_body(raw_response) + response_json: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + _DeepgramResponse, body + ) # Get the first alternative from the first channel first_channel: Final = response_json["results"]["channels"][0] @@ -102,7 +131,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): text = first_alternative["paragraphs"]["transcript"] else: # Diarization without paragraphs: reconstruct from words - text = self._reconstruct_diarized_transcript(first_alternative["words"]) + text = self._reconstruct_diarized_transcript(_WORD_LIST.validate_python(first_alternative["words"])) # Create TranscriptionResponse object response: Final = TranscriptionResponse(text=text) @@ -124,7 +153,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ] # Store full response in hidden params - response.hidden_params = response_json + response.hidden_params = _RESPONSE_BODY.validate_python(body) return response diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 5196c44d504..62a780396a1 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -6,6 +6,7 @@ from collections.abc import Mapping, Sequence from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._uuid import uuid @@ -24,6 +25,8 @@ from litellm.types.rerank import ( RerankTokens, ) +_DECODED_JSON: Final = TypeAdapter(object) + class _DeepinfraInferenceStatus(TypedDict, total=False): """The ``inference_status`` block of a DeepInfra rerank response.""" @@ -251,7 +254,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): try: error_data = error_message if isinstance(error_message, str): - error_data = json.loads(error_message) + error_data = _DECODED_JSON.validate_python(json.loads(error_message)) if isinstance(error_data, dict): # Check for {"detail": {"error": "..."}} detail: Final = error_data.get("detail") diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 6ddee99aea3..3b2dd5af260 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -2,8 +2,10 @@ DeepSeek Anthropic-compatible messages transformation config. """ -from collections.abc import Sequence -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -13,6 +15,24 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams +_TOOL_LIST: Final = TypeAdapter(list[object]) +_TOOL_FIELDS: Final = TypeAdapter(dict[object, object]) + + +def _custom_tool_without_type(tool: object) -> Mapping[object, object] | None: + if not isinstance(tool, dict): + return None + fields: Final = _TOOL_FIELDS.validate_python(tool) + if fields.get("type") == "custom": + fields.pop("type", None) + return fields + return None + + +def _sanitized_tool(tool: object) -> object: + stripped: Final = _custom_tool_without_type(tool) + return tool if stripped is None else stripped + class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): """ @@ -96,19 +116,11 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): return join_api_base(base_url, "v1/messages") @staticmethod - def _sanitize_tools_for_deepseek(tools: Any) -> object: + def _sanitize_tools_for_deepseek(tools: object) -> object: if not isinstance(tools, list): return tools - sanitized_tools: Final = [] - for tool in tools: - if isinstance(tool, dict) and tool.get("type") == "custom": - sanitized_tool = dict(tool) - sanitized_tool.pop("type", None) - sanitized_tools.append(sanitized_tool) - else: - sanitized_tools.append(tool) - return sanitized_tools + return [_sanitized_tool(tool) for tool in _TOOL_LIST.validate_python(tools)] def transform_anthropic_messages_request( self, diff --git a/litellm/llms/deprecated_providers/palm.py b/litellm/llms/deprecated_providers/palm.py index 785cffa48ea..b4a32630c42 100644 --- a/litellm/llms/deprecated_providers/palm.py +++ b/litellm/llms/deprecated_providers/palm.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final @@ -41,16 +42,17 @@ class PalmConfig: top_p: float | None = None, max_output_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/edenai/embedding/transformation.py b/litellm/llms/edenai/embedding/transformation.py index c79a6839434..13fa7142916 100644 --- a/litellm/llms/edenai/embedding/transformation.py +++ b/litellm/llms/edenai/embedding/transformation.py @@ -5,7 +5,7 @@ per-request cost at the top level of the body. Docs: https://www.edenai.co/docs/v3/llms/embeddings """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol, cast import httpx @@ -24,6 +24,14 @@ if TYPE_CHECKING: _SUPPORTED_PARAMS: Final = ("dimensions", "encoding_format", "user") +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIEmbeddingConfig(BaseEmbeddingConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited contract return list(_SUPPORTED_PARAMS) @@ -80,10 +88,14 @@ class EdenAIEmbeddingConfig(BaseEmbeddingConfig): optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract ) -> EmbeddingResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) response: Final[EmbeddingResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="embedding" + response_object=cast( # cast-ok: the body passes through unvalidated, as before it was typed + dict[str, object], body + ), + model_response_object=model_response, + response_type="embedding", ) set_response_cost_in_hidden_params(response, reported_cost(body)) return response diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index e2a0b8684f6..a0c4e1b0458 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -5,7 +5,7 @@ for every image model in the catalog with the real per-request cost at the top l Docs: https://www.edenai.co/docs/v3/llms/image-generation """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol, cast import httpx @@ -36,6 +36,14 @@ _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( ) +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class EdenAIImageGenerationConfig(BaseImageGenerationConfig): def get_supported_openai_params( self, model: str @@ -97,10 +105,14 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: - body: Final = raw_response.json() + body: Final = _json_body(raw_response) logging_obj.post_call(original_response=body) response: Final[ImageResponse] = convert_to_model_response_object( - response_object=body, model_response_object=model_response, response_type="image_generation" + response_object=cast( # cast-ok: the body passes through unvalidated, as before it was typed + dict[str, object], body + ), + model_response_object=model_response, + response_type="image_generation", ) set_response_cost_in_hidden_params(response, reported_cost(body)) return response diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 5a673b5ac41..3dc42f35d3c 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/audio/transcriptions` to ElevenLabs's `/v1/speech- """ from collections.abc import Iterable, Mapping -from typing import Final +from typing import Final, Protocol from httpx import Headers, Response from pydantic import ConfigDict, TypeAdapter @@ -27,6 +27,15 @@ from ..common_utils import ElevenLabsException _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object]) + + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -120,7 +129,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): Transforms the raw response from ElevenLabs to the TranscriptionResponse format """ try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) response_object: Final = _JSON_OBJECT.validate_python(response_json) # Extract the main transcript text @@ -135,20 +144,18 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): # Map ElevenLabs words to OpenAI format if "words" in response_object: - response["words"] = [] - for word_data in _JSON_OBJECTS.validate_python(response_object["words"]): - # Only include actual words, skip spacing and audio events - if word_data.get("type") == "word": - response["words"].append( - { - "word": word_data.get("text", ""), - "start": word_data.get("start", 0), - "end": word_data.get("end", 0), - } - ) + response["words"] = [ + { + "word": word_data.get("text", ""), + "start": word_data.get("start", 0), + "end": word_data.get("end", 0), + } + for word_data in _JSON_OBJECTS.validate_python(response_object["words"]) + if word_data.get("type") == "word" + ] # Store full response in hidden params - response.hidden_params = response_json + response.hidden_params = _HIDDEN_PARAMS.validate_python(response_json) return response diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 0fdd89aec91..fc215e3d94e 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -19,6 +19,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIFluxProV11UltraConfig(FalAIBaseConfig): @@ -222,7 +223,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 038a06c5749..26b257f5c89 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -18,6 +18,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIRecraftV3Config(FalAIBaseConfig): @@ -195,7 +196,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index 30d51cc8084..fdd552efd73 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -19,6 +19,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class FalAIStableDiffusionConfig(FalAIBaseConfig): @@ -236,7 +237,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): } """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/featherless_ai/chat/transformation.py b/litellm/llms/featherless_ai/chat/transformation.py index a68261c58e9..4bf0486f723 100644 --- a/litellm/llms/featherless_ai/chat/transformation.py +++ b/litellm/llms/featherless_ai/chat/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final import litellm @@ -42,7 +44,7 @@ class FeatherlessAIConfig(OpenAIGPTConfig): tool_choice: str | None = None, tools: list | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 964f85c821e..84c93974cd7 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -44,6 +44,7 @@ class _FireworksAIResultFields(LiteLLMBaseModel): _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): @@ -200,7 +201,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): Transform Fireworks AI rerank response to LiteLLM RerankResponse format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse response: {e}", diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index cd9ffe9e03e..680db52b283 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -10,13 +10,17 @@ from collections.abc import Callable from typing import Final, Protocol from urllib.parse import urlsplit +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, Unpack import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues +_DECODED_JSON: Final = TypeAdapter(object) + class _OpenAIGPTConfigOptions(TypedDict, total=False): """The sampling defaults ``OpenAIGPTConfig.__init__`` accepts and stashes on the class.""" @@ -203,10 +207,10 @@ class GDCGeminiConfig(OpenAILikeChatConfig): import google.auth try: - json_obj: Final = json.loads(api_key) + json_obj: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(api_key))) except json.JSONDecodeError: return None, False - if not isinstance(json_obj, dict) or json_obj.get("type") != self._GDCH_CREDENTIAL_TYPE: + if json_obj is None or json_obj.get("type") != self._GDCH_CREDENTIAL_TYPE: raise ValueError( "GDC only accepts a GDCH service account credential as a JSON api_key " '(expected "type": "gdch_service_account"). Other Google credential types are ' diff --git a/litellm/llms/gemini/agents/transformation.py b/litellm/llms/gemini/agents/transformation.py index c19bd9ce87c..f7352da5fb6 100644 --- a/litellm/llms/gemini/agents/transformation.py +++ b/litellm/llms/gemini/agents/transformation.py @@ -10,11 +10,12 @@ Proxies the Gemini v1beta Agents API: """ from collections.abc import Mapping -from typing import Any, Final +from typing import Any, Final, Protocol import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo @@ -29,6 +30,15 @@ from litellm.types.agents import ( # create-agent body verbatim. _GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment") + +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + # LiteLLM-internal keys that must never be forwarded to Gemini. _LITELLM_INTERNAL_KEYS: Final = frozenset( { @@ -156,18 +166,19 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data: dict[str, Any] = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: verbose_logger.warning( "GeminiAgentsConfig: non-JSON create response (status=%d).", raw_response.status_code, ) - data = {"id": name} + decoded = {"id": name} + data: Final = dict(decoded_json_object(decoded)) # Gemini uses "id" as the identifier; normalise to both fields. data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig create response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # LIST # @@ -192,13 +203,16 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): ) -> AgentListResponse: self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = decoded_json_object(decoded) verbose_logger.debug("GeminiAgentsConfig list response: %s", data) - return AgentListResponse( - agents=data.get("agents", []), - next_page_token=data.get("nextPageToken"), + return AgentListResponse.model_validate( + { + "agents": data.get("agents", []), + "next_page_token": data.get("nextPageToken"), + } ) # ------------------------------------------------------------------ # @@ -222,13 +236,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """Same shape as create response — Gemini returns "id" as identifier.""" self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {"id": name} + decoded = {"id": name} + data: Final = dict(decoded_json_object(decoded)) data.setdefault("id", name) data.setdefault("name", data["id"]) verbose_logger.debug("GeminiAgentsConfig get response: %s", data) - return AgentCreateResponse(**data) + return AgentCreateResponse.model_validate(data) # ------------------------------------------------------------------ # # DELETE # @@ -285,11 +300,14 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): """ self._raise_for_status(raw_response) try: - data = raw_response.json() + decoded: object = _json_body(raw_response) except Exception: - data = {} + decoded = {} + data: Final = decoded_json_object(decoded) verbose_logger.debug("GeminiAgentsConfig list_versions response for '%s': %s", name, data) - return AgentVersionsResponse( - agent_versions=data.get("agentVersions", []), - next_page_token=data.get("nextPageToken"), + return AgentVersionsResponse.model_validate( + { + "agent_versions": data.get("agentVersions", []), + "next_page_token": data.get("nextPageToken"), + } ) diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index ea116a7e9de..3457b22faa4 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from typing import Final from httpx import Headers, Response +from pydantic import TypeAdapter from litellm.litellm_core_utils.audio_utils.subtitle_utils import SUBTITLE_RESPONSE_FORMATS from litellm.litellm_core_utils.audio_utils.utils import ( @@ -36,6 +37,7 @@ from litellm.types.utils import ( INTERACTIONS_API_REVISION: Final = "2026-05-20" WORD_INFO_ANNOTATION_TYPE: Final = "word_info" +_DECODED_JSON: Final = TypeAdapter(object) class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @@ -132,7 +134,7 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): raw_response: Response, ) -> TranscriptionResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError: raise GeminiError( status_code=raw_response.status_code, diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 13df3921d13..9634c580858 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -2,9 +2,9 @@ import base64 import datetime import json import math -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, cast import httpx from pydantic import JsonValue, TypeAdapter @@ -389,7 +389,7 @@ class GeminiModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model.replace("gemini/", "") - def process_model_name(self, models: list[dict[str, str]]) -> list[str]: + def process_model_name(self, models: Iterable[Mapping[str, str]]) -> list[str]: litellm_model_names: Final = [] for model in models: stripped_model_name = model["name"].replace("models/", "") @@ -416,9 +416,11 @@ class GeminiModelInfo(BaseLLMModelInfo): f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}" ) - models: Final[list[dict[str, str]]] = response.json()["models"] + listing: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + Mapping[str, Iterable[Mapping[str, str]]], _JSON_VALUE.validate_python(response.json()) + ) - litellm_model_names: Final = self.process_model_name(models) + litellm_model_names: Final = self.process_model_name(listing["models"]) return litellm_model_names def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: diff --git a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py index 44cdac11325..34007bc8fd4 100644 --- a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py +++ b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py @@ -15,7 +15,10 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamTransformSink, @@ -28,6 +31,7 @@ if TYPE_CHECKING: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _EMPTY_REQUEST_DATA: Final[Mapping[str, object]] = MappingProxyType({}) +_DECODED_JSON: Final = TypeAdapter(object) def _field(container: object, name: str) -> object | None: @@ -108,12 +112,10 @@ def _write_back_texts(text_parts: Sequence[object], guardrailed_texts: Sequence[ def _parse_json_dict_or_none(payload: str) -> Mapping[str, object] | None: try: - parsed: Final = json.loads(payload) + parsed: Final = _DECODED_JSON.validate_python(json.loads(payload)) except json.JSONDecodeError: return None - if isinstance(parsed, dict): - return parsed - return None + return as_str_mapping(parsed) def _sse_payload_texts(sse_text: str) -> tuple[str, ...]: diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index bc3180a2129..0e29df36e80 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -3,7 +3,7 @@ Transformation for Calling Google models in their native format. """ from copy import deepcopy -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast import httpx @@ -36,6 +36,14 @@ else: from ..common_utils import get_api_key_from_env +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): """ Configuration for calling Google models in their native format. @@ -382,7 +390,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): from litellm.types.google_genai.main import GenerateContentResponse try: - response = raw_response.json() + decoded: Final = _json_body(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming generate content response: {e}", @@ -391,7 +399,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): ) logging_obj.model_call_details["httpx_response"] = raw_response - response = self.convert_citation_sources_to_citations(response) + response: Final = self.convert_citation_sources_to_citations( + cast(dict[str, object], decoded) # cast-ok: the body passes through unvalidated, as before it was typed + ) return GenerateContentResponse(**response) diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index 450c38a2206..d48a66f1f93 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -1,12 +1,13 @@ import base64 from collections.abc import Mapping from io import BufferedReader, BytesIO -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import httpx from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.gemini.common_utils import ( @@ -18,6 +19,7 @@ from litellm.llms.gemini.image_usage_transformation import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.llms.vertex_ai import GenerateContentResponseBody from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( FileTypes, @@ -34,6 +36,14 @@ else: LiteLLMLoggingObj = Any +class _JsonResponse(Protocol): + def json(self) -> object: ... + + +def _json_body(response: _JsonResponse) -> object: + return response.json() + + class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" SUPPORTED_PARAMS: list[str] = ["n", "size", "imageConfig"] @@ -125,7 +135,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final = _json_body(raw_response) except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", @@ -133,7 +143,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): headers=raw_response.headers, ) - candidates: Final = response_json.get("candidates", []) + payload: Final = cast( # cast-ok: the body passes through unvalidated, as before it was typed + GenerateContentResponseBody, response_json + ) + candidates: Final = payload.get("candidates", []) data_list: Final[list[ImageObject]] = [] for candidate in candidates: @@ -141,17 +154,18 @@ class GeminiImageEditConfig(BaseImageEditConfig): parts = content.get("parts", []) for part in parts: inline_data = part.get("inlineData") - if inline_data and inline_data.get("data"): + image_data = inline_data.get("data") if inline_data else None + if image_data: data_list.append( ImageObject( - b64_json=inline_data["data"], + b64_json=image_data, url=None, ) ) model_response.data = cast(list[OpenAIImage], data_list) - if "usageMetadata" in response_json: - model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"]) + if "usageMetadata" in payload: + model_response.usage = transform_gemini_image_usage(dict(decoded_json_object(payload["usageMetadata"]))) return model_response def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 1aa3d074b1a..1f986b11af2 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -37,6 +37,7 @@ else: _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter( Mapping[str, object] | str | Sequence[object], config=ConfigDict(hide_input_in_errors=True) ) @@ -188,7 +189,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): Transform Google AI Imagen response to litellm ImageResponse format """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", @@ -226,9 +227,9 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): ) # Extract usage metadata for Gemini models - if "usageMetadata" in response_data: + if "usageMetadata" in response_object: model_response.usage = transform_gemini_image_usage( - _JSON_DICT.validate_python(response_data["usageMetadata"]) + _JSON_DICT.validate_python(response_object["usageMetadata"]) ) web_search_requests: Final = get_gemini_image_web_search_requests(response_object) if web_search_requests and model_response.usage is not None: diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 9af7edeac86..6f778ab2761 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -5,13 +5,14 @@ This file contains the transformation logic for the Gemini realtime API. import json from collections import OrderedDict from collections.abc import Mapping, Sequence -from typing import Any, Final, cast +from typing import Final, cast from typing_extensions import ReadOnly, Required, TypedDict import litellm from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig @@ -167,16 +168,17 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return True @staticmethod - def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]: - if not isinstance(details, dict): + def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]: + detail_mapping: Final = as_str_mapping(details) + if detail_mapping is None or not isinstance(details, dict): return dict(defaults) return { **defaults, - **{key: value for key, value in details.items() if value is not None}, + **{key: value for key, value in detail_mapping.items() if value is not None}, } @staticmethod - def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, Any]) -> dict[str, object]: + def _add_pipecat_usage_detail_aliases(usage_dict: dict[str, object]) -> dict[str, object]: usage_dict.setdefault( "input_token_details", GeminiRealtimeConfig._usage_detail_alias( diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 0656095927c..3f6b900fd01 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from httpx._types import RequestFiles +from pydantic import TypeAdapter import litellm from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS @@ -37,6 +38,8 @@ else: LiteLLMLoggingObj = Any BaseLLMException = Any +_DECODED_JSON: Final = TypeAdapter(object) + def _convert_image_to_gemini_format(image_file) -> dict[str, str]: """ @@ -60,7 +63,7 @@ def _convert_image_to_gemini_format(image_file) -> dict[str, str]: def _json_payload(raw_response: httpx.Response) -> object: """Read an HTTP response body as an opaque JSON payload.""" - return raw_response.json() + return _DECODED_JSON.validate_python(raw_response.json()) def _usage_video_resolution_from_parameters( diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 936e6339865..65921b628fb 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -273,7 +273,7 @@ class GigaChatConfig(BaseConfig): # Default to None (don't set function_call) return None - def _upload_image(self, image_url: str) -> str | None: + def _upload_image(self, image_url: str) -> object: """ Upload image to GigaChat and return file_id. @@ -293,7 +293,7 @@ class GigaChatConfig(BaseConfig): verbose_logger.error("Failed to upload image: %s", e) return None - def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[str]]: + def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]: """ Extract text and image attachments from a multimodal message content list. @@ -304,7 +304,7 @@ class GigaChatConfig(BaseConfig): Tuple of (combined text, list of attachment file ids) """ texts: Final[list[str]] = [] # mutable-ok: accumulator - attachments: Final[list[str]] = [] # mutable-ok: accumulator + attachments: Final[list[object]] = [] # mutable-ok: accumulator for part in content: if isinstance(part, dict): if part.get("type") == "text": diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index aff0465d1ec..d53ae3af334 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -12,6 +12,8 @@ import uuid from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( @@ -19,12 +21,14 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, ) from litellm.llms.gigachat.utils import get_api_base +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.utils import LlmProviders from .authenticator import get_access_token, get_access_token_async # Simple in-memory cache for file IDs -_file_cache: Final[dict[str, str]] = {} +_file_cache: Final[dict[str, object]] = {} +_DECODED_JSON: Final = TypeAdapter(object) def _get_url_hash(url: str) -> str: @@ -91,7 +95,7 @@ def upload_file_sync( credentials: str | None = None, api_base: str | None = None, litellm_params: Mapping[str, object] | None = None, -) -> str | None: +) -> object: """ Upload file to GigaChat and return file_id (sync). @@ -137,9 +141,9 @@ def upload_file_sync( timeout=60, ) response.raise_for_status() - result: Final = response.json() + result: Final = _DECODED_JSON.validate_python(response.json()) - file_id: Final = result.get("id") + file_id: Final = decoded_json_object(result).get("id") if file_id: _file_cache[url_hash] = file_id verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) @@ -156,7 +160,7 @@ async def upload_file_async( credentials: str | None = None, api_base: str | None = None, litellm_params: Mapping[str, object] | None = None, -) -> str | None: +) -> object: """ Upload file to GigaChat and return file_id (async). @@ -205,9 +209,9 @@ async def upload_file_async( timeout=60, ) response.raise_for_status() - result: Final = response.json() + result: Final = _DECODED_JSON.validate_python(response.json()) - file_id: Final = result.get("id") + file_id: Final = decoded_json_object(result).get("id") if file_id: _file_cache[url_hash] = file_id verbose_logger.debug("File uploaded successfully, file_id: %s", file_id) diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index 872eee6dbf9..27f75eece50 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -5,11 +5,13 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig from litellm.llms.gigachat.authenticator import get_access_token from litellm.llms.gigachat.chat.streaming import GigaChatModelResponseIterator from litellm.llms.gigachat.utils import GIGACHAT_BASE_URL +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse @@ -20,6 +22,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import CostResponseTypes +_DECODED_JSON: Final = TypeAdapter(object) + class GigaChatPassthroughConfig(BasePassthroughConfig): def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool: @@ -161,7 +165,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): continue chunk = chunk.removeprefix("data: ") try: - message = json.loads(chunk) + message = _DECODED_JSON.validate_python(json.loads(chunk)) except json.JSONDecodeError: continue @@ -169,7 +173,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): streaming_response=None, sync_stream=False, ) - translated_chunk = gigachat_iterator.chunk_parser(chunk=message) + translated_chunk = gigachat_iterator.chunk_parser(chunk=decoded_json_object(message)) if isinstance(translated_chunk, dict) and generic_chunk_has_all_required_fields( # pyright: ignore[reportUnnecessaryIsInstance] # runtime guard for patched chunk_parser dict(translated_chunk) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 9553327db97..83bc8df23a6 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -4,9 +4,11 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # spread dict loses the role literal without a cast import httpx +from pydantic import TypeAdapter from litellm.exceptions import AuthenticationError from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.openai.common_utils import decoded_json_object from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk from litellm.types.utils import ModelResponse @@ -23,6 +25,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding +_DECODED_JSON: Final = TypeAdapter(object) + class GithubCopilotConfig(OpenAIConfig): def __init__( @@ -309,7 +313,7 @@ class GithubCopilotConfig(OpenAIConfig): json_mode: bool | None = None, ) -> "ModelResponse": try: - response_json = raw_response.json() + decoded_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: return super().transform_response( model=model, @@ -325,12 +329,13 @@ class GithubCopilotConfig(OpenAIConfig): json_mode=json_mode, ) + response_json: Final = decoded_json_object(decoded_response) if not response_json.get("choices"): - response_json = self._synthesize_choices_for_anthropic_native(response_json) + synthesized: Final = self._synthesize_choices_for_anthropic_native(dict(response_json)) raw_response = httpx.Response( status_code=raw_response.status_code, headers=raw_response.headers, - content=json.dumps(response_json).encode(), + content=json.dumps(synthesized).encode(), ) return super().transform_response( diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 56c3693a02c..613b7e42100 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary +from collections.abc import Sequence +from typing import Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -54,7 +55,7 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index bf0b8c79068..b055d388dd5 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -9,6 +9,7 @@ https://github.com/caozhiyuan/copilot-api """ import os +from collections.abc import Mapping from typing import ( TYPE_CHECKING, Any, @@ -16,6 +17,8 @@ from typing import ( cast, # noqa: TID251 # narrows untyped request dicts at the session boundary ) +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -47,6 +50,9 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_OBJECT_LIST: Final = TypeAdapter(list[object]) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[object, object]) + def github_copilot_supports_responses_api(model: str) -> bool: """ @@ -387,7 +393,9 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): """ return self._contains_vision_content(input_param) - def _contains_vision_content(self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> bool: + def _contains_vision_content( + self, value: object, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH + ) -> bool: """ Recursively check if a value contains vision content. @@ -404,22 +412,28 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Check arrays if isinstance(value, list): - return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value) + items: Final = _OBJECT_LIST.validate_python(value) + return any(self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in items) # Only check dict/object types if not isinstance(value, dict): return False + fields: Final = _OBJECT_MAPPING.validate_python(value) + # Check if this item is an input_image - item_type: Final = value.get("type") + item_type: Final = fields.get("type") if isinstance(item_type, str) and item_type.lower() == "input_image": return True # Check content field recursively - if "content" in value and isinstance(value["content"], list): - return any( - self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in value["content"] - ) + if "content" in fields: + content: Final = fields["content"] + if isinstance(content, list): + content_items: Final = _OBJECT_LIST.validate_python(content) + return any( + self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth) for item in content_items + ) return False diff --git a/litellm/llms/gradient_ai/chat/transformation.py b/litellm/llms/gradient_ai/chat/transformation.py index 0ba5ca45c2a..2a2368ad8de 100644 --- a/litellm/llms/gradient_ai/chat/transformation.py +++ b/litellm/llms/gradient_ai/chat/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final, Literal from litellm.litellm_core_utils.url_utils import join_api_base @@ -43,7 +45,7 @@ class GradientAIConfig(OpenAILikeChatConfig): include_guardrails_info: bool | None = None, provide_citations: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 8c269cb5dfd..349a4b4ab87 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -40,6 +40,7 @@ class GroqExecutedToolIdentity(LiteLLMBaseModel): _EXECUTED_TOOLS_ADAPTER: Final = TypeAdapter(tuple[GroqExecutedToolIdentity, ...]) +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) class GroqChatConfig(OpenAILikeChatConfig): @@ -324,7 +325,7 @@ class GroqChatConfig(OpenAILikeChatConfig): json_mode=json_mode, ) - original_service_tier: Final[object] = getattr(model_response, "service_tier") + original_service_tier: Final = _ATTRIBUTE_VALUE.validate_python(getattr(model_response, "service_tier")) mapped_service_tier: Final[Literal["auto", "default", "flex"]] = self._map_groq_service_tier( original_service_tier=original_service_tier ) diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py index bcebb347aea..7f75a607410 100644 --- a/litellm/llms/groq/cost_calculator.py +++ b/litellm/llms/groq/cost_calculator.py @@ -19,7 +19,10 @@ if TYPE_CHECKING: def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: search_costs: Final = model_info.get("search_context_cost_per_query") cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 - server_tool_use: Final = getattr(usage, "server_tool_use", None) + try: + server_tool_use: Final = usage.server_tool_use + except AttributeError: + return 0.0 if server_tool_use is None: return 0.0 searches: Final = server_tool_use.web_search_requests or 0 diff --git a/litellm/llms/groq/stt/transformation.py b/litellm/llms/groq/stt/transformation.py index 93d36dd7b0f..33e01ab7c0e 100644 --- a/litellm/llms/groq/stt/transformation.py +++ b/litellm/llms/groq/stt/transformation.py @@ -3,6 +3,7 @@ Translate from OpenAI's `/v1/audio/transcriptions` to Groq's `/v1/audio/transcri """ import types +from collections.abc import Mapping from typing import Final import litellm @@ -39,16 +40,17 @@ class GroqSTTConfig: tools: list | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 4a5729945b4..0b77a04c69a 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import Final import httpx -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from litellm._uuid import uuid from litellm.exceptions import UnsupportedParamsError @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( HostedVLLMRerankTruncationParams, @@ -28,6 +29,8 @@ from litellm.types.rerank import ( RerankTokens, ) +_DECODED_JSON: Final = TypeAdapter(object) + class HostedVLLMRerankError(BaseLLMException): def __init__( @@ -193,11 +196,11 @@ class HostedVLLMRerankConfig(BaseRerankConfig): Process response from Hosted VLLM rerank API """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise ValueError(f"Error parsing response: {raw_response.text}, status_code={raw_response.status_code}") - return self._transform_response(raw_response_json) + return self._transform_response(dict(decoded_json_object(raw_response_json))) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return HostedVLLMRerankError(message=error_message, status_code=status_code, headers=headers) diff --git a/litellm/llms/hosted_vllm/videos/transformation.py b/litellm/llms/hosted_vllm/videos/transformation.py index 30ae6835263..2045d2c8988 100644 --- a/litellm/llms/hosted_vllm/videos/transformation.py +++ b/litellm/llms/hosted_vllm/videos/transformation.py @@ -8,6 +8,7 @@ from typing import Final from urllib.parse import urlparse from httpx._types import FileTypes, RequestFiles +from pydantic import TypeAdapter from litellm.images.utils import ImageEditRequestUtils from litellm.litellm_core_utils.url_utils import SSRFError, join_api_base, validate_url @@ -16,6 +17,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoCreateOptionalRequestParams +_DECODED_JSON: Final = TypeAdapter(object) _EXCLUDED_FORM_KEYS: Final = frozenset( { "model", @@ -82,7 +84,7 @@ def _maybe_json(value: object) -> object: stripped: Final = value.strip() if not stripped or stripped[0] not in "{[": return value - return json.loads(stripped) + return _DECODED_JSON.validate_python(json.loads(stripped)) def _reject_unsafe_media_url(url: str) -> None: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 60917c68221..a1d1a3f6a19 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -3,6 +3,7 @@ import os from typing import Final, Literal, Protocol, get_args import httpx +from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -11,6 +12,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, get_async_httpx_client, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.utils import EmbeddingResponse from ...base import BaseLLM @@ -21,6 +23,8 @@ config: Final = HuggingFaceEmbeddingConfig() HF_HUB_URL: Final = "https://huggingface.co" +_DECODED_JSON: Final = TypeAdapter(object) + hf_tasks_embeddings: Final = ( Literal[ # pipeline tags + hf tei endpoints - https://huggingface.github.io/text-embeddings-inference/#/ "sentence-similarity", "feature-extraction", "rerank", "embed", "similarity" @@ -34,7 +38,7 @@ class _SupportsTokenEncode(Protocol): def encode(self, text: str) -> list[int]: ... -def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: +def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -44,14 +48,14 @@ def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}") - model_info_dict: Final = model_info.json() + model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) - pipeline_tag: Final[str | None] = model_info_dict.get("pipeline_tag", None) + pipeline_tag: Final = model_info_dict.get("pipeline_tag", None) return pipeline_tag -async def async_get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: +async def async_get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -63,9 +67,9 @@ async def async_get_hf_task_embedding_for_model(model: str, task_type: str | Non model_info: Final = await http_client.get(url=f"{api_base}/api/models/{model}") - model_info_dict: Final = model_info.json() + model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) - pipeline_tag: Final[str | None] = model_info_dict.get("pipeline_tag", None) + pipeline_tag: Final = model_info_dict.get("pipeline_tag", None) return pipeline_tag @@ -77,7 +81,7 @@ class HuggingFaceEmbedding(BaseLLM): def __init__(self) -> None: super().__init__() - def _transform_input_on_pipeline_tag(self, input: list, pipeline_tag: str | None) -> dict: + def _transform_input_on_pipeline_tag(self, input: list, pipeline_tag: object) -> dict: if pipeline_tag is None: return {"inputs": input} if pipeline_tag == "sentence-similarity" or pipeline_tag == "similarity": diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 0883a9a8077..1b7d6483739 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -447,14 +447,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig): except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass - output_text: Final = model_response["choices"][0]["message"].get("content", "") + first_message: Final = model_response.choices[0].message + output_text: Final = first_message.content if output_text is not None and len(output_text) > 0: completion_tokens = 0 try: if encoding is not None: - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) ##[TODO] use the llama2 tokenizer here + completion_tokens = len(encoding.encode(output_text)) ##[TODO] use the llama2 tokenizer here except Exception: # this should remain non blocking we should not block a response returning if calculating usage fails pass diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 1217a2540d4..1f130f84283 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -86,7 +86,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 0938cc0d07b..3807be562cf 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -1,9 +1,11 @@ """LangFlow run API: POST {api_base}/api/v1/run/{flow_id}""" +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from urllib.parse import quote import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -11,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -27,6 +30,8 @@ else: AsyncHTTPHandler = Any CustomStreamWrapper = Any +_DECODED_JSON: Final = TypeAdapter(object) + class LangFlowError(BaseLLMException): """Exception class for LangFlow API errors.""" @@ -179,7 +184,7 @@ class LangFlowConfig(BaseConfig): verbose_logger.debug("LangFlow request payload: %s", payload) return payload - def _extract_content_from_response(self, response_json: dict) -> str | None: + def _extract_content_from_response(self, response_json: Mapping[str, object]) -> str | None: """ Extract the assistant text from a LangFlow run response. @@ -236,7 +241,7 @@ class LangFlowConfig(BaseConfig): json_mode: bool | None = None, ) -> ModelResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise LangFlowError( message=f"LangFlow returned a non-JSON response: {e}", @@ -245,7 +250,7 @@ class LangFlowConfig(BaseConfig): verbose_logger.debug("LangFlow response: %s", response_json) - content: Final = self._extract_content_from_response(response_json) + content: Final = self._extract_content_from_response(decoded_json_object(response_json)) if content is None: raise LangFlowError( message=( diff --git a/litellm/llms/langgraph/chat/sse_iterator.py b/litellm/llms/langgraph/chat/sse_iterator.py index 499cbab3b7b..575820bceee 100644 --- a/litellm/llms/langgraph/chat/sse_iterator.py +++ b/litellm/llms/langgraph/chat/sse_iterator.py @@ -9,10 +9,13 @@ import uuid from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices +_DECODED_JSON: Final = TypeAdapter(object) + class LangGraphSSEStreamIterator: """ @@ -60,7 +63,7 @@ class LangGraphSSEStreamIterator: return None try: - data: Final = json.loads(json_str) + data: Final = _DECODED_JSON.validate_python(json.loads(json_str)) return self._process_data(data) except json.JSONDecodeError: verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100]) diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index ce624eecc0e..603d5a07292 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Optional, Union import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.langgraph.chat.sse_iterator import LangGraphSSEStreamIterator +from litellm.llms.openai.common_utils import decoded_json_object from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -37,6 +39,8 @@ else: AsyncHTTPHandler = Any CustomStreamWrapper = Any +_DECODED_JSON: Final = TypeAdapter(object) + class LangGraphError(BaseLLMException): """Exception class for LangGraph API errors.""" @@ -233,7 +237,7 @@ class LangGraphConfig(BaseConfig): verbose_logger.debug("LangGraph request payload: %s", payload) return payload - def _extract_content_from_response(self, response_json: dict) -> str: + def _extract_content_from_response(self, response_json: Mapping[str, object]) -> str: """ Extract content from LangGraph non-streaming response. @@ -434,10 +438,10 @@ class LangGraphConfig(BaseConfig): Transform the LangGraph response to LiteLLM ModelResponse format. """ try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) verbose_logger.debug("LangGraph response: %s", response_json) - content: Final = self._extract_content_from_response(response_json) + content: Final = self._extract_content_from_response(decoded_json_object(response_json)) # Create the message message: Final = Message(content=content, role="assistant") diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 01fdc0b8bf8..42bd63b0160 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -3,7 +3,7 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio """ from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from urllib.parse import quote import httpx @@ -27,6 +27,7 @@ if TYPE_CHECKING: _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str) +_OPAQUE_VALUE: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -123,7 +124,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): return ["lemonade/" + _STR.validate_python(model["id"]) for model in model_list] @staticmethod - def _get_positive_int(value: Any) -> int | None: + def _get_positive_int(value: object) -> int | None: if isinstance(value, bool): return None if isinstance(value, int) and value > 0: @@ -163,7 +164,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): provider_specific_entry.get("context_window"), provider_specific_entry.get("max_context_window"), ): - parsed = self._get_positive_int(value) + parsed = self._get_positive_int(_OPAQUE_VALUE.validate_python(value)) if parsed is not None: return parsed return None diff --git a/litellm/llms/litellm_proxy/skills/prompt_injection.py b/litellm/llms/litellm_proxy/skills/prompt_injection.py index 3377338a03e..e8e1c68a112 100644 --- a/litellm/llms/litellm_proxy/skills/prompt_injection.py +++ b/litellm/llms/litellm_proxy/skills/prompt_injection.py @@ -9,9 +9,10 @@ import posixpath import zipfile from collections.abc import Mapping from io import BytesIO -from typing import Any, Final +from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.models.skills import LiteLLM_SkillsTable @@ -251,27 +252,30 @@ class SkillPromptInjectionHandler: if len(description) > max_desc_length: description = description[: max_desc_length - 3] + "..." - tool: Final[dict[str, Any]] = { + params: Final = ( + as_str_mapping(skill.metadata.get("parameters")) + if skill.metadata and isinstance(skill.metadata, dict) + else None + ) + parameters: Final[Mapping[str, object]] = ( + params + if params and isinstance(params, dict) + else { + "type": "object", + "properties": {}, + "required": [], + } + ) + + return { "type": "function", "function": { "name": func_name, "description": description, - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, + "parameters": parameters, }, } - # If skill has metadata with parameter definitions, use them - if skill.metadata and isinstance(skill.metadata, dict): - params: Final = skill.metadata.get("parameters") - if params and isinstance(params, dict): - tool["function"]["parameters"] = params - - return tool - def convert_skill_to_anthropic_tool(self, skill: LiteLLM_SkillsTable) -> Mapping[str, object]: """ Convert a LiteLLM skill to an Anthropic-style tool (messages API format). diff --git a/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/litellm/llms/litellm_proxy/skills/sandbox_executor.py index 3e38dd81905..8e5c9effa17 100644 --- a/litellm/llms/litellm_proxy/skills/sandbox_executor.py +++ b/litellm/llms/litellm_proxy/skills/sandbox_executor.py @@ -7,7 +7,8 @@ Supports Docker, Podman, and Kubernetes backends. import base64 import os -from typing import Any, Final, Protocol, TypedDict +from collections.abc import Sequence +from typing import Final, Protocol, TypedDict from typing_extensions import ReadOnly @@ -41,6 +42,13 @@ class _GeneratedFile(TypedDict): mime_type: ReadOnly[str] +class _SandboxExecution(TypedDict): + success: ReadOnly[bool] + output: ReadOnly[str] + error: ReadOnly[str] + files: ReadOnly[Sequence[_GeneratedFile]] + + class SkillsSandboxExecutor: """ Executes skill code in llm-sandbox Docker container. @@ -76,7 +84,7 @@ class SkillsSandboxExecutor: code: str, skill_files: dict[str, bytes], requirements: str | None = None, - ) -> dict[str, Any]: + ) -> _SandboxExecution: """ Execute code with skill files in sandbox. diff --git a/litellm/llms/lm_studio/embed/transformation.py b/litellm/llms/lm_studio/embed/transformation.py index 66b2dbb534d..d6264ceb6e0 100644 --- a/litellm/llms/lm_studio/embed/transformation.py +++ b/litellm/llms/lm_studio/embed/transformation.py @@ -7,6 +7,7 @@ Docs - https://lmstudio.ai/docs/basics/server """ import types +from collections.abc import Mapping from typing import Final @@ -18,16 +19,17 @@ class LmStudioEmbeddingConfig: def __init__( self, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/maritalk.py b/litellm/llms/maritalk.py index 094dee4cee1..36bde5847fb 100644 --- a/litellm/llms/maritalk.py +++ b/litellm/llms/maritalk.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from httpx._models import Headers @@ -32,7 +33,7 @@ class MaritalkConfig(OpenAIGPTConfig): tools: list[dict] | None = None, tool_choice: str | dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 67de3a06af8..14470922e57 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -4,12 +4,14 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, VectorStoreEmbeddingExecutor, ) +from litellm.llms.openai.common_utils import decoded_json_object from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -30,6 +32,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + MILVUS_OPTIONAL_PARAMS: Final = { "annsField", "limit", @@ -218,15 +222,17 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): results: Final = response_json.get("data", []) # Try to get text_field from optional_params first, then litellm_params - optional_params: Final = litellm_logging_obj.model_call_details.get("optional_params", {}) + optional_params: Final = decoded_json_object( + _DECODED_JSON.validate_python(litellm_logging_obj.model_call_details.get("optional_params", {})) + ) text_field = optional_params.get("milvus_text_field", "") # Fallback to litellm_params if not in optional_params if not text_field: - text_field = litellm_logging_obj.model_call_details.get("litellm_params", {}).get( - "milvus_text_field", "" - ) + text_field = decoded_json_object( + _DECODED_JSON.validate_python(litellm_logging_obj.model_call_details.get("litellm_params", {})) + ).get("milvus_text_field", "") # Transform results to standard format search_results: Final[list[VectorStoreSearchResult]] = [] diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py index f9f3502237c..f7a263a467c 100644 --- a/litellm/llms/minimax/messages/transformation.py +++ b/litellm/llms/minimax/messages/transformation.py @@ -2,8 +2,6 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API """ -from typing import Any # noqa: TID251 # override below must mirror the legacy base signature - import litellm from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -54,7 +52,7 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig): self, headers: dict, # mutable-ok: mirrors the legacy base override signature model: str, - messages: list[Any], # mutable-ok: mirrors the legacy base override signature + messages: list[object], # mutable-ok: mirrors the legacy base override signature optional_params: dict, # mutable-ok: mirrors the legacy base override signature litellm_params: dict, # mutable-ok: mirrors the legacy base override signature api_key: str | None = None, diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py index 36c8705f256..c308de92d1d 100644 --- a/litellm/llms/modelscope/image_generation/transformation.py +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -33,6 +33,7 @@ else: LiteLLMLoggingObj = object _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) @@ -173,7 +174,7 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): {"created": timestamp, "data": [{"url": "..."}]} """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error parsing ModelScope response: {e}", diff --git a/litellm/llms/nadir/chat/transformation.py b/litellm/llms/nadir/chat/transformation.py index d22b1a51bab..c8bde7d9fad 100644 --- a/litellm/llms/nadir/chat/transformation.py +++ b/litellm/llms/nadir/chat/transformation.py @@ -2,8 +2,9 @@ import math from typing import Final import httpx +from pydantic import TypeAdapter -from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_response_cost_in_hidden_params from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -19,13 +20,17 @@ _SUPPORTED_OPENAI_PARAMS: Final = ( "temperature", "top_p", ) +_DECODED_JSON: Final = TypeAdapter(object) def _reported_cost_usd(raw_response: httpx.Response) -> float | None: try: - cost: Final = raw_response.json()["nadir_metadata"]["cost"]["total_cost_usd"] + body: Final = as_str_mapping(_DECODED_JSON.validate_python(raw_response.json())) except (ValueError, KeyError, TypeError): return None + metadata: Final = as_str_mapping(body.get("nadir_metadata")) if body is not None else None + cost_block: Final = as_str_mapping(metadata.get("cost")) if metadata is not None else None + cost: Final = cost_block.get("total_cost_usd") if cost_block is not None else None if isinstance(cost, bool) or not isinstance(cost, (int, float)): return None if not math.isfinite(cost) or cost < 0: diff --git a/litellm/llms/nvidia_nim/embed.py b/litellm/llms/nvidia_nim/embed.py index 5a0105a3776..d2ef446299e 100644 --- a/litellm/llms/nvidia_nim/embed.py +++ b/litellm/llms/nvidia_nim/embed.py @@ -9,6 +9,7 @@ API calling is done using the OpenAI SDK with an api_base """ import types +from collections.abc import Mapping from typing import Final @@ -32,16 +33,17 @@ class NvidiaNimEmbeddingConfig: input_type: str | None = None, truncate: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index 748203b22d9..cafeece0a4e 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -28,7 +28,7 @@ without the optional STT extras installed. import asyncio import inspect -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol @@ -48,6 +48,8 @@ from litellm.llms.nvidia_riva.audio_transcription.transformation import ( RIVA_TARGET_NUM_CHANNELS, RIVA_TARGET_SAMPLE_RATE_HZ, NvidiaRivaAudioTranscriptionConfig, + RivaFinalResult, + RivaWordTiming, ) from litellm.llms.nvidia_riva.common_utils import ( NvidiaRivaException, @@ -535,14 +537,14 @@ class NvidiaRivaAudioTranscription: yield chunk @staticmethod - def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> list[dict[str, object]]: + def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]: """ Walk the gRPC responses, ignore empty / non-final chunks, and return a list of normalized final-result dicts. Matching the user's note: the ``id`` blocks with no ``results`` are streaming heartbeats and must be skipped. """ - final_results: Final[list[dict[str, object]]] = [] + final_results: Final[list[RivaFinalResult]] = [] for response in stream: results = getattr(response, "results", None) or [] for result in results: @@ -554,16 +556,19 @@ class NvidiaRivaAudioTranscription: top = alternatives[0] transcript = getattr(top, "transcript", "") or "" words_proto = getattr(top, "words", None) or [] - words = [] - for word in words_proto: - words.append( - { - "word": getattr(word, "word", ""), - "start_time_ms": int(getattr(word, "start_time", 0) or 0), - "end_time_ms": int(getattr(word, "end_time", 0) or 0), - } - ) - final_results.append({"transcript": transcript, "words": words}) + final_results.append( + { + "transcript": transcript, + "words": [ + RivaWordTiming( + word=getattr(word, "word", ""), + start_time_ms=int(getattr(word, "start_time", 0) or 0), + end_time_ms=int(getattr(word, "end_time", 0) or 0), + ) + for word in words_proto + ], + } + ) return final_results diff --git a/litellm/llms/nvidia_riva/audio_transcription/transformation.py b/litellm/llms/nvidia_riva/audio_transcription/transformation.py index 262dfb7bd78..5862a5d10b1 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/transformation.py +++ b/litellm/llms/nvidia_riva/audio_transcription/transformation.py @@ -11,9 +11,11 @@ dict at call time. Reference: https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html """ -from typing import Any, Final +from collections.abc import Sequence +from typing import Final, TypedDict from httpx import Headers, Response +from typing_extensions import ReadOnly from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import ( @@ -34,6 +36,17 @@ RIVA_TARGET_NUM_CHANNELS: Final = 1 RIVA_TARGET_ENCODING: Final = "LINEAR_PCM" +class RivaWordTiming(TypedDict): + word: ReadOnly[str] + start_time_ms: ReadOnly[int] + end_time_ms: ReadOnly[int] + + +class RivaFinalResult(TypedDict): + transcript: ReadOnly[str] + words: ReadOnly[Sequence[RivaWordTiming]] + + class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig): """ Config for NVIDIA Riva ASR (gRPC). @@ -219,7 +232,7 @@ class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig): @staticmethod def build_transcription_response( - final_results: list[dict[str, Any]], + final_results: Sequence[RivaFinalResult], response_format: str, duration_seconds: float | None, timestamp_granularities: list[str] | None, diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e1502b2610..e8d37f857e6 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -189,6 +189,7 @@ def _normalize_tool_choice(selected_params: dict) -> None: _TOOL_CHOICE_MODES: Final = frozenset({"auto", "none", "required", "any"}) _TOOLS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) _JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) def _as_json_object(value: object) -> Mapping[str, object] | None: @@ -701,7 +702,7 @@ class OCIChatConfig(BaseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) if not isinstance(response_json, dict): raise OCIError( diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py index ffcb156bb16..6dee64bf19a 100644 --- a/litellm/llms/oci/embed/transformation.py +++ b/litellm/llms/oci/embed/transformation.py @@ -56,6 +56,7 @@ else: LiteLLMLoggingObj = Any _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) # OCI sends up to 96 texts per embedText request (Cohere limit). OCI_EMBED_BATCH_LIMIT: Final = 96 @@ -272,7 +273,7 @@ class OCIEmbedConfig(BaseEmbeddingConfig): ) try: - json_response: Final = raw_response.json() + json_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OCIError( status_code=raw_response.status_code, diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 7894904c870..f313c62f026 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -13,6 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) class OllamaResponseIdNormalizer: @@ -152,7 +153,7 @@ class OllamaModelInfo(BaseLLMModelInfo): try: resp: Final = httpx.get(join_api_base(base, "api/tags"), headers=headers) resp.raise_for_status() - data: Final = resp.json() + data: Final = _DECODED_JSON.validate_python(resp.json()) # Expecting a dict with a 'models' list models_list = [] if isinstance(data, dict) and "models" in data and isinstance(data["models"], list): diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 30de22b17ff..239b4e32645 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -547,7 +547,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): held: Final = self.held_content self.held_content = "" try: - parsed: Final[object] = json.loads(_strip_json_code_fence(held)) + parsed: Final = _JSON_VALUE.validate_python(json.loads(_strip_json_code_fence(held))) except json.JSONDecodeError: return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage) if isinstance(parsed, dict) and "name" in parsed and "arguments" in parsed: diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index e472661efb9..12ead582975 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -23,13 +23,14 @@ from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.completion_usage import CompletionUsage -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter if TYPE_CHECKING: from aiohttp import ClientSession import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -134,6 +135,9 @@ class OpenAIError(BaseLLMException): ) +_DECODED_JSON: Final = TypeAdapter(object) + + ####### Error Handling Utils for OpenAI API ####################### ################################################################### def drop_params_from_unprocessable_entity_error( @@ -152,19 +156,20 @@ def drop_params_from_unprocessable_entity_error( """ invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): - error_json: Final = e.response.json() + error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) error_message: Final = error_json.get("error", {}) error_body = error_message else: error_body = e.body - if error_body is not None and isinstance(error_body, dict) and error_body.get("message"): - message = error_body.get("message", {}) + error_mapping: Final = as_str_mapping(error_body) + if isinstance(error_mapping, dict) and error_mapping.get("message"): + message = error_mapping.get("message", {}) if isinstance(message, str): try: - message = json.loads(message) + message = _DECODED_JSON.validate_python(json.loads(message)) except json.JSONDecodeError: message = {"detail": message} - detail: Final = message.get("detail") + detail: Final = decoded_json_object(message).get("detail") if isinstance(detail, list) and len(detail) > 0 and isinstance(detail[0], dict): for error_dict in detail: diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 581e5e8ad87..73d211e1613 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -2,10 +2,10 @@ OpenAI Evals API configuration and transformations """ -from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base @@ -32,9 +32,11 @@ from litellm.types.llms.openai_evals import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +_DECODED_JSON: Final = TypeAdapter(object) -def _parsed_response_json(raw_response: httpx.Response) -> Mapping[str, object]: - return raw_response.json() + +def _parsed_response_json(raw_response: httpx.Response) -> object: + return _DECODED_JSON.validate_python(raw_response.json()) class OpenAIEvalsConfig(BaseEvalsAPIConfig): diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 437f19a4dc6..deee502c9f2 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -20,7 +20,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter -from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials +from litellm.llms.openai.common_utils import OpenAIError, decoded_json_object, get_openai_credentials from litellm.llms.openai.responses.count_tokens.handler import ( OpenAICountTokensHandler, ) @@ -244,7 +244,7 @@ class OpenAITokenCounter(BaseTokenCounter): if result is not None: return TokenCountResponse.model_validate( { - "total_tokens": result.get("input_tokens", 0), + "total_tokens": decoded_json_object(result).get("input_tokens", 0), "request_model": request_model, "model_used": model_to_use, "tokenizer_type": "openai_api", diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index c1ee42444c5..0f885146e36 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -6,12 +6,13 @@ This module handles the transformation of requests to OpenAI's /v1/responses/inp import copy from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, Protocol, TypeAlias from openai.types.responses.input_token_count_params import InputTokenCountParams from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error @@ -61,6 +62,26 @@ class _AnthropicCountMessage(TypedDict): _ANTHROPIC_COUNT_MESSAGES: Final = TypeAdapter(tuple[_AnthropicCountMessage, ...]) + + +class _FieldReader(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +class _FunctionFields(_FieldReader, Protocol): + def __contains__(self, key: object, /) -> bool: ... + + def __getitem__(self, key: str, /) -> object: ... + + +def _field_reader(value: object) -> _FieldReader: + return require_attributes(value, _FieldReader, "get") + + +def _function_fields(value: object) -> _FunctionFields: + return require_attributes(value, _FunctionFields, "get", "__contains__", "__getitem__") + + _ANTHROPIC_OUTPUT_SCHEMA: Final = TypeAdapter(AnthropicOutputSchema) @@ -196,7 +217,7 @@ class OpenAICountTokensConfig: @staticmethod def _transform_tools_for_responses_api( - tools: list[dict[str, Any]], + tools: list[dict[str, object]], ) -> list[dict[str, object]]: """ Transform OpenAI chat and Anthropic tools to Responses API tools format. @@ -208,7 +229,7 @@ class OpenAICountTokensConfig: transformed: Final[list[dict[str, object]]] = [] for tool in tools: if tool.get("type") == "function" and "function" in tool: - func = tool["function"] + func = _function_fields(tool["function"]) item: dict[str, object] = { "type": "function", "name": func.get("name", ""), @@ -235,7 +256,7 @@ class OpenAICountTokensConfig: @staticmethod def messages_to_responses_input( - messages: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], ) -> tuple[list[dict[str, object]], str | None]: """ Convert standard chat messages format to OpenAI Responses API input format. @@ -276,14 +297,17 @@ class OpenAICountTokensConfig: if content: input_items.append({"role": "assistant", "content": content}) if tool_calls: - for tc in tool_calls: - func = tc.get("function", {}) + for tc in iter_items(tool_calls): + tool_call = _field_reader(tc) + function = tool_call.get("function", {}) + call_id = tool_call.get("id", "") + function_fields = _field_reader(function) input_items.append( { "type": "function_call", - "call_id": tc.get("id", ""), - "name": func.get("name", ""), - "arguments": func.get("arguments", ""), + "call_id": call_id, + "name": function_fields.get("name", ""), + "arguments": function_fields.get("arguments", ""), } ) elif not content: diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 3ea126c5a2a..d37a12a91d6 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -36,7 +36,7 @@ from collections.abc import Mapping, MutableSequence, Sequence from dataclasses import dataclass from itertools import accumulate, chain, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast +from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict @@ -47,6 +47,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i OpenAiResponsesToChatCompletionStreamIterator, tool_call_dict_from_output_item, ) +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -237,6 +238,10 @@ class ResponsesStreamChunk(TypedDict, total=False): content_index: ReadOnly[int] +class _StreamEventFields(Protocol): + def get(self, key: str, /) -> object: ... + + _TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset( { ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, @@ -1264,7 +1269,7 @@ class OpenAIResponsesHandler(BaseTranslation): def _sync_stream_events_with_rewrites( self, - stream_events: Sequence[Any], + stream_events: Sequence[object], rewrites_by_position: Mapping[tuple[int, int], str], ) -> None: """Sync pre-completion stream events with the rewritten completed @@ -1278,7 +1283,7 @@ class OpenAIResponsesHandler(BaseTranslation): {position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()} ) for event in stream_events: - if not (isinstance(event, dict) or hasattr(event, "get")): + if not has_attributes(event, _StreamEventFields, "get"): continue event_type = event.get("type") output_index = event.get("output_index") diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 12c8b1da166..0ab55d653e0 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -41,7 +41,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, @@ -151,7 +151,7 @@ class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): self, headers: dict[str, str], model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/opencode/harness/transformation.py b/litellm/llms/opencode/harness/transformation.py index 46e8b750ca5..e801b66c448 100644 --- a/litellm/llms/opencode/harness/transformation.py +++ b/litellm/llms/opencode/harness/transformation.py @@ -26,10 +26,10 @@ import itertools import json import secrets import string -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError @@ -45,6 +45,7 @@ from litellm.harness.types import ( ToolCall, ToolResult, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.base_llm.harness.transformation import ( BaseCLIHarnessConfig, HarnessSessionCopy, @@ -269,10 +270,17 @@ def session_copy(exported: str, path: str, workdir: str) -> HarnessSessionCopy | ) -def _as_dict(value: object) -> Mapping[str, Any]: +def _as_dict(value: object) -> Mapping[str, object]: return value if isinstance(value, dict) else MappingProxyType({}) +def _spread_items(value: object) -> Iterable[object]: + try: + return iter_items(value) + except TypeError: + raise TypeError(f"Value after * must be an iterable, not {type(value).__name__}") from None + + def _tool_events(part: Mapping[str, object]) -> Sequence[Event]: native = str(part.get("tool") or "") call_id = str(part.get("callID") or part.get("id") or "") @@ -346,7 +354,7 @@ def build_opencode_config( token_path: str, permissions: PermissionMode, disable_tools: Sequence[str] = (), - user_config: Mapping[str, Any] | None = None, + user_config: Mapping[str, object] | None = None, instructions_path: str | None = None, skills_path: str | None = None, ) -> Mapping[str, object]: @@ -355,10 +363,16 @@ def build_opencode_config( validate_user_config(user) qualified = f"{OPENCODE_PROVIDER_ID}/{model}" extra_instructions: Final = (instructions_path,) if instructions_path else () - instructions: Final = [*(user.get("instructions") or ()), *extra_instructions] # mutable-ok: opencode config JSON + instructions: Final = [ + *_spread_items(user.get("instructions") or ()), + *extra_instructions, + ] # mutable-ok: opencode config JSON user_skills = _as_dict(user.get("skills")) extra_skills: Final = (skills_path,) if skills_path else () - skill_paths: Final = [*(user_skills.get("paths") or ()), *extra_skills] # mutable-ok: opencode config JSON + skill_paths: Final = [ + *_spread_items(user_skills.get("paths") or ()), + *extra_skills, + ] # mutable-ok: opencode config JSON options: Final = {"baseURL": base_url, "apiKey": "{file:" + token_path + "}"} # mutable-ok: opencode config JSON models: Final[dict[str, Mapping[str, object]]] = {model: {}} # mutable-ok: opencode config JSON provider: Final = { # mutable-ok: opencode config JSON diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index 9644ee3f280..c2d8398aead 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -47,7 +47,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles -from pydantic import ConfigDict, RootModel, with_config +from pydantic import ConfigDict, RootModel, TypeAdapter, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -74,6 +74,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) + @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) class _ImageUrl(TypedDict): @@ -248,7 +250,7 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): logging_obj: LiteLLMLoggingObj, ) -> ImageResponse: try: - response_json: Final[object] = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenRouterException( message=f"Error parsing OpenRouter response: {e}", diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index ce6e62aaa88..a9e07555d6a 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -59,6 +59,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) @@ -359,7 +360,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): ImageResponse: Populated image response """ try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise OpenRouterException( message=f"Error parsing OpenRouter response: {e}", diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py index 574fbfa9e5b..d30a9ff534e 100644 --- a/litellm/llms/opensandbox/sandbox/transformation.py +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -20,6 +20,7 @@ from litellm.constants import ( OPEN_SANDBOX_POLL_INTERVAL, OPEN_SANDBOX_READY_TIMEOUT, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.sandbox.transformation import ( @@ -44,6 +45,13 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_VALUE: Final = TypeAdapter(object) +def _decoded_json_object(value: object) -> Mapping[str, object]: + mapping: Final = as_str_mapping(value) + if mapping is not None: + return mapping + raise AttributeError(f"{type(value).__name__!r} object has no attribute 'get'", name="get", obj=value) + + class OpenSandboxSandboxConfig(BaseSandboxConfig): def _http(self, client: AsyncHTTPHandler | None) -> AsyncHTTPHandler: if client is not None: @@ -321,7 +329,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): headers=headers, params={"use_server_proxy": use_server_proxy}, ) - data: Final = response.json() + data: Final = _decoded_json_object(_JSON_VALUE.validate_python(response.json())) endpoint: Final = data.get("endpoint") if not endpoint: raise ValueError(f"OpenSandbox did not return an execd endpoint for {sandbox_id}") @@ -492,7 +500,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): if not data: return None try: - parsed: Final = json.loads(data) + parsed: Final = _JSON_VALUE.validate_python(json.loads(data)) except json.JSONDecodeError: return None if not isinstance(parsed, dict): diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 885fc98ffea..3ce8d7ddad5 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -6,11 +6,14 @@ It uses the field targeting configuration from litellm_logging_obj to extract specific fields for guardrail processing. """ -from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping, Sequence +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional, Protocol, runtime_checkable +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import without_server_streaming_classification +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, StreamingScanKey, @@ -215,6 +218,19 @@ class _EventStreamEndpointChecker(Protocol): def event_stream_endpoint_is_de_anonymizable(self, endpoint: str, model: str, body: object) -> bool: ... +class _EventStreamDeAnonymizer(Protocol): + async def de_anonymize_event_stream( + self, + body_bytes: bytes, + proxy_logging_obj: "ProxyLogging", + user_api_key_dict: "UserAPIKeyAuth", + data: Mapping[str, object], + ) -> bytes: ... + + +_TRANSLATION_RESULT: Final = TypeAdapter(object) + + _PROVIDER_HANDLERS: dict[str, type[BaseTranslation]] = {} @@ -260,10 +276,12 @@ class LlmPassthroughRouteHandler(BaseTranslation): provider, ) return data - return await handler_cls().process_input_messages( - data=data, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, + return _TRANSLATION_RESULT.validate_python( + await handler_cls().process_input_messages( + data=data, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + ) ) async def process_output_response( @@ -282,12 +300,14 @@ class LlmPassthroughRouteHandler(BaseTranslation): provider, ) return response - return await handler_cls().process_output_response( - response=response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=request_data, + return _TRANSLATION_RESULT.validate_python( + await handler_cls().process_output_response( + response=response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=request_data, + ) ) @staticmethod @@ -399,9 +419,11 @@ class LlmPassthroughRouteHandler(BaseTranslation): return handler_cls.event_stream_media_type() @staticmethod - def _resolve_event_stream_de_anonymizer(provider: str | None): + def _resolve_event_stream_de_anonymizer(provider: str | None) -> Callable[..., Awaitable[bytes]] | None: handler_cls: Final = _get_provider_handlers().get(provider or "") - return getattr(handler_cls, "de_anonymize_event_stream", None) + if not has_attributes(handler_cls, _EventStreamDeAnonymizer, "de_anonymize_event_stream"): + return None + return handler_cls.de_anonymize_event_stream @staticmethod def supports_event_stream_relay(provider: object, endpoint: object, model: object, body: object) -> bool: diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index a2b7eceea40..77817a5e5bb 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -12,6 +12,7 @@ Ref: https://docs.perplexity.ai/api-reference/responses-post from typing import Any, Final import httpx +from pydantic import TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base @@ -22,6 +23,8 @@ from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +_DECODED_JSON: Final = TypeAdapter(object) + class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def get_supported_openai_params(self, model: str) -> list: @@ -104,7 +107,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): ) -> ResponsesAPIResponse: """Check for Perplexity's status:'failed' on HTTP 200 before delegating to base.""" try: - raw_response_json = raw_response.json() + raw_response_json = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raw_response_json = None diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index ee20c2b12d5..a875992448a 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final from httpx import Headers, Response @@ -57,7 +59,7 @@ class PetalsConfig(BaseConfig): top_p: float | None = None, repetition_penalty: float | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index eeecd02653d..89851f60d76 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -76,7 +76,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): api_base: str, litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/search") diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 7b1a7714ec6..49a2938e1c0 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -26,6 +26,7 @@ else: _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class RecraftImageGenerationConfig(BaseImageGenerationConfig): @@ -134,7 +135,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): https://www.recraft.ai/docs#generate-image """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index e721ac678a1..23c8c86e7ca 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -1,9 +1,10 @@ import asyncio import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger @@ -11,6 +12,7 @@ from litellm.constants import ( RUNWAYML_DEFAULT_API_VERSION, RUNWAYML_POLLING_TIMEOUT, ) +from litellm.litellm_core_utils.core_helpers import decoded_json_object from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -36,9 +38,25 @@ class _RunwayMLTask(TypedDict, total=False): id: ReadOnly[str] status: ReadOnly[str] - output: ReadOnly[Sequence[str | Mapping[str, str]]] - failure: ReadOnly[str] - failureCode: ReadOnly[str] + output: ReadOnly[Sequence[str | Mapping[str, str | None]] | None] + failure: ReadOnly[str | None] + failureCode: ReadOnly[str | None] + + +_JSON_VALUE: Final = TypeAdapter(object) + + +def _decoded_json(response: httpx.Response) -> object: + return _JSON_VALUE.validate_python(response.json()) + + +def _runway_task(decoded: object) -> _RunwayMLTask: + task: Final = decoded_json_object(decoded) + return cast(_RunwayMLTask, task) # cast-ok: the payload passes through unvalidated, as before it was typed + + +def _parse_runway_task(response: httpx.Response) -> _RunwayMLTask: + return _runway_task(_decoded_json(response)) class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): @@ -234,7 +252,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -282,7 +300,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayMLTask = response.json() + response_data: _RunwayMLTask = _parse_runway_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -328,13 +346,14 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): } """ try: - response_data: _RunwayMLTask = raw_response.json() + decoded: Final = _decoded_json(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) + response_data: _RunwayMLTask = _runway_task(decoded) verbose_logger.debug("RunwayML starting polling...") @@ -358,7 +377,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete, transforming to OpenAI format") @@ -388,13 +407,14 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): We need to poll the task until it completes (status SUCCEEDED) using async polling. """ try: - response_data: _RunwayMLTask = raw_response.json() + decoded: Final = _decoded_json(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, ) + response_data: _RunwayMLTask = _runway_task(decoded) verbose_logger.debug("RunwayML starting polling (async)...") @@ -418,7 +438,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): ) # Update response_data with polled result - response_data = raw_response.json() + response_data = _parse_runway_task(raw_response) verbose_logger.debug("RunwayML polling complete (async), transforming to OpenAI format") diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index 1f052648f13..ce1ec1714f1 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -41,6 +41,11 @@ class _RunwayTtsTaskResponse(TypedDict, total=False): failureCode: ReadOnly[str] +def _parse_runway_tts_task(response: httpx.Response) -> _RunwayTtsTaskResponse: + task: Final[_RunwayTtsTaskResponse] = response.json() + return task + + class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ Configuration for RunwayML Text-to-Speech @@ -321,7 +326,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -368,7 +373,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): response = await client.get(url=task_url, headers=headers) response.raise_for_status() - response_data: _RunwayTtsTaskResponse = response.json() + response_data: _RunwayTtsTaskResponse = _parse_runway_tts_task(response) # Check task status status = self._check_task_status(response_data=response_data) @@ -459,7 +464,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -489,7 +494,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete, downloading audio") @@ -528,7 +533,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): from litellm.types.llms.openai import HttpxBinaryResponseContent try: - response_data: Final[_RunwayTtsTaskResponse] = raw_response.json() + response_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(raw_response) except Exception as e: raise self.get_error_class( error_message=f"Error parsing RunwayML TTS response: {e}", @@ -558,7 +563,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): ) # Get the completed task data - task_data: Final[_RunwayTtsTaskResponse] = polled_response.json() + task_data: Final[_RunwayTtsTaskResponse] = _parse_runway_tts_task(polled_response) verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio") diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 106567f6fba..ad8a24c1535 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -119,7 +119,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): @staticmethod def _parse_task_response(raw_response: httpx.Response) -> _RunwayTaskResponse: - return raw_response.json() + return _parse_runway_task_response(raw_response) def get_supported_openai_params(self, model: str) -> list: """ diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index 14f0e3ead62..eaa805e7769 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -1,3 +1,4 @@ +import builtins import codecs import functools import json @@ -34,6 +35,14 @@ class _SageMakerMessagesChunkData(TypedDict, total=False): choices: ReadOnly[Required[list[Mapping[str, object]]]] +class _SageMakerPayloadPart(TypedDict, total=False): + bytes: ReadOnly[builtins.bytes] + + +class _SageMakerStreamEvent(TypedDict, total=False): + chunk: ReadOnly[_SageMakerPayloadPart] + + class _SageMakerChunkDataRoot(RootModel[_SageMakerChunkData]): model_config = ConfigDict(frozen=True, hide_input_in_errors=True) @@ -345,7 +354,7 @@ class AWSEventStreamDecoder: ), ) response_dict: Final = event.to_response_dict() - parsed_response: Final = self.parser.parse(response_dict, response_stream_shape) + parsed_response: Final[_SageMakerStreamEvent] = self.parser.parse(response_dict, response_stream_shape) body: Final = response_dict.get("body") if response_dict["status_code"] != 200: diff --git a/litellm/llms/sagemaker/embedding/cohere_transformation.py b/litellm/llms/sagemaker/embedding/cohere_transformation.py index 35352f65dda..8c0f4437edf 100644 --- a/litellm/llms/sagemaker/embedding/cohere_transformation.py +++ b/litellm/llms/sagemaker/embedding/cohere_transformation.py @@ -10,7 +10,8 @@ be of type Object`. Reference: https://docs.cohere.com/v2/reference/embed """ -from typing import TYPE_CHECKING, Any, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -25,6 +26,7 @@ from litellm.llms.bedrock.embed.cohere_transformation import ( BedrockCohereEmbeddingConfig, ) from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse from ..common_utils import SagemakerError @@ -123,7 +125,7 @@ class SagemakerCohereEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sagemaker/embedding/openai_transformation.py b/litellm/llms/sagemaker/embedding/openai_transformation.py index a3f0ebfbf59..1222c697832 100644 --- a/litellm/llms/sagemaker/embedding/openai_transformation.py +++ b/litellm/llms/sagemaker/embedding/openai_transformation.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from litellm.types.llms.openai import AllEmbeddingInputValues from httpx._models import Headers, Response -from pydantic import BaseModel, ConfigDict, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig @@ -25,6 +25,7 @@ from litellm.types.utils import EmbeddingResponse, Usage from ..common_utils import STR_KEYED_MAPPING, SagemakerError SUPPORTED_OPENAI_PARAMS: Final = ("dimensions", "encoding_format", "user") +_DECODED_JSON: Final = TypeAdapter(object) class OpenAIEmbeddingUsage(BaseModel): @@ -94,7 +95,7 @@ class SagemakerOpenAIEmbeddingConfig(BaseEmbeddingConfig): litellm_params: Mapping[str, object] | None = None, ) -> "EmbeddingResponse": try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError as e: raise SagemakerError( message=f"Failed to parse response: {e}", diff --git a/litellm/llms/sagemaker/embedding/transformation.py b/litellm/llms/sagemaker/embedding/transformation.py index 1395f9b98ec..2cd02683167 100644 --- a/litellm/llms/sagemaker/embedding/transformation.py +++ b/litellm/llms/sagemaker/embedding/transformation.py @@ -4,7 +4,8 @@ Translate from OpenAI's `/v1/embeddings` to Sagemaker's `/invoke` In the Huggingface TGI format. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Sequence +from typing import TYPE_CHECKING, Final if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -15,6 +16,7 @@ from httpx._models import Headers, Response from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage from ..common_utils import SagemakerError @@ -158,7 +160,7 @@ class SagemakerEmbeddingConfig(BaseEmbeddingConfig): self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 8a7baf18e8f..622c28f3c90 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator from .handler import ( AsyncSAPStreamIterator, GenAIHubOrchestrationError, @@ -186,11 +186,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): # Keep a short, tight client lifecycle here to avoid fd leaks client: Final = litellm.module_level_client # with httpx.Client(timeout=30) as client: - deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + join_api_base(self.base_url, "lm/deployments"), headers=self.headers + ).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): if dep.get("scenarioId") == "orchestration": - cfg = client.get( + cfg: AICoreConfiguration = client.get( join_api_base(self.base_url, f"lm/configurations/{dep['configurationId']}"), headers=self.headers, ).json() diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index 2622d0fa9b6..74d065956cc 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path from threading import Lock -from typing import Any, Final, Protocol +from typing import Final, Protocol import httpx from pydantic import TypeAdapter @@ -34,6 +34,21 @@ DEFAULT_HOME_PATH: Final = os.path.join(os.path.expanduser("~"), ".aicore") _DECODED_JSON: Final = TypeAdapter(object) +class AICoreDeployment(TypedDict): + configurationId: ReadOnly[str] + scenarioId: ReadOnly[str] + deploymentUrl: ReadOnly[str] + createdAt: ReadOnly[str] + + +class AICoreDeploymentList(TypedDict, total=False): + resources: ReadOnly[Sequence[AICoreDeployment]] + + +class AICoreConfiguration(TypedDict): + executableId: ReadOnly[str] + + def _get_home() -> str: return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH) @@ -59,7 +74,7 @@ def _get_nested(d: object, path: Sequence[str]) -> object: return cur -def _load_json_env(var_name: str) -> dict[str, object] | None: +def _load_json_env(var_name: str) -> Mapping[str, Sequence[Mapping[str, object]]] | None: raw: Final = os.environ.get(var_name) if not raw: return None @@ -76,7 +91,7 @@ def _str_or_none(value: object) -> str | None: return None -def _load_vcap() -> dict[str, Any]: +def _load_vcap() -> Mapping[str, Sequence[Mapping[str, object]]]: return _load_json_env(VCAP_SERVICES_ENV_VAR) or {} @@ -196,7 +211,7 @@ def resolve_resource_group(sources: list[Source]) -> str | None: def _parse_service_key_once( service_key: str | dict | None, -) -> dict[str, object] | None: +) -> object: """ Pre-parse service_key if it's a string to avoid repeated JSON parsing. @@ -208,7 +223,7 @@ def _parse_service_key_once( return service_key if isinstance(service_key, str): try: - return json.loads(service_key) + return _DECODED_JSON.validate_python(json.loads(service_key)) except json.JSONDecodeError: verbose_logger.warning("SAP service key is a string but not valid JSON. Skipping this source.") return None @@ -218,7 +233,7 @@ def _parse_service_key_once( return None -def _resolve_credential_from_service_key(service_key: str | dict | None, cv: CredentialsValue) -> str | None: +def _resolve_credential_from_service_key(service_key: object, cv: CredentialsValue) -> str | None: if service_key is None: return None val: Final = _str_or_none(_get_nested(service_key, (("credentials",) + cv.vcap_key) if cv.vcap_key else (cv.name,))) @@ -253,14 +268,16 @@ def fetch_credentials( """ config: Final = init_conf(profile) - service_key = _parse_service_key_once(service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR)) + parsed_service_key: Final = _parse_service_key_once( + service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR) + ) vcap_service: Final = _get_vcap_service(VCAP_AICORE_SERVICE_NAME) sources: Final = [ Source("kwargs", lambda cv: _str_or_none(kwargs.get(cv.name))), Source( "service key", - lambda cv: _resolve_credential_from_service_key(service_key, cv), + lambda cv: _resolve_credential_from_service_key(parsed_service_key, cv), ), Source( "environment variables", diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index 107fef0b355..a59b7e1e00d 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllEmbeddingInputValues from litellm.types.utils import EmbeddingResponse from ..chat.handler import GenAIHubOrchestrationError -from ..credentials import get_token_creator +from ..credentials import AICoreConfiguration, AICoreDeploymentList, get_token_creator class Usage(LiteLLMBaseModel): @@ -102,10 +102,12 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): def deployment_url(self) -> str: with httpx.Client(timeout=30) as client: valid_deployments: Final = [] - deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() + deployments: Final[AICoreDeploymentList] = client.get( + join_api_base(self.base_url, "lm/deployments"), headers=self.headers + ).json() for deployment in deployments.get("resources", []): if deployment["scenarioId"] == "orchestration": - config_details = client.get( + config_details: AICoreConfiguration = client.get( join_api_base(self.base_url, f"lm/configurations/{deployment['configurationId']}"), headers=self.headers, ).json() diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 5d5fad6128e..974996c5e48 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -6,10 +6,12 @@ Handles transformation between OpenAI-compatible format and Stability AI API for API Reference: https://platform.stability.ai/docs/api-reference """ +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles +from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -30,6 +32,13 @@ else: LiteLLMLoggingObj = Any +class _StabilityImageEditResponse(TypedDict, total=False): + image: ReadOnly[str] + finish_reason: ReadOnly[str] + seed: ReadOnly[int] + errors: ReadOnly[Sequence[str]] + + class StabilityImageEditConfig(BaseImageEditConfig): """ Configuration for Stability AI image edit. @@ -259,7 +268,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp} """ try: - response_data: Final = raw_response.json() + response_data: Final[_StabilityImageEditResponse] = raw_response.json() except Exception as e: raise self.get_error_class( error_message=f"Error parsing Stability AI response: {e}", diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py index 1e52bc37d47..30c3ecaf3a9 100644 --- a/litellm/llms/tencent/messages/transformation.py +++ b/litellm/llms/tencent/messages/transformation.py @@ -5,8 +5,6 @@ Tencent TokenHub exposes an Anthropic-compatible Messages API endpoint alongside its standard OpenAI-compatible chat completions endpoint. """ -from typing import Any - import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -48,7 +46,7 @@ class TencentAnthropicMessagesConfig(AnthropicMessagesConfig): self, headers: dict, model: str, - messages: list[Any], + messages: list[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index b37bbd78f2b..fe0f96cb805 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Final, Literal from httpx import Headers, Response +from typing_extensions import ReadOnly, TypedDict from litellm.constants import DEFAULT_MAX_TOKENS_FOR_TRITON from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory @@ -32,6 +33,10 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +class _TritonGenerateResponse(TypedDict): + text_output: ReadOnly[str] + + class TritonConfig(BaseConfig): """ Base class for Triton configurations. @@ -220,7 +225,7 @@ class TritonGenerateConfig(TritonConfig): json_mode: bool | None = None, ) -> ModelResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonGenerateResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) model_response.choices = [Choices(index=0, message=Message(content=raw_response_json["text_output"]))] diff --git a/litellm/llms/triton/embedding/transformation.py b/litellm/llms/triton/embedding/transformation.py index 8c6e0b832fc..3a735f741fd 100644 --- a/litellm/llms/triton/embedding/transformation.py +++ b/litellm/llms/triton/embedding/transformation.py @@ -1,6 +1,8 @@ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.base_llm.chat.transformation import AllMessageValues, BaseLLMException from litellm.llms.base_llm.embedding.transformation import ( @@ -14,6 +16,16 @@ from litellm.utils import token_counter from ..common_utils import TritonError +class _TritonEmbeddingOutput(TypedDict): + shape: ReadOnly[Sequence[int]] + data: ReadOnly[Sequence[float]] + + +class _TritonEmbeddingResponse(TypedDict): + outputs: ReadOnly[Sequence[_TritonEmbeddingOutput]] + model_name: NotRequired[ReadOnly[str]] + + class TritonEmbeddingConfig(BaseEmbeddingConfig): """ Transformations for triton /embeddings endpoint (This is a trtllm model) @@ -79,7 +91,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict = {}, ) -> EmbeddingResponse: try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_TritonEmbeddingResponse] = raw_response.json() except Exception: raise TritonError(message=raw_response.text, status_code=raw_response.status_code) @@ -134,7 +146,7 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig): return TritonError(message=error_message, status_code=status_code, headers=headers) @staticmethod - def split_embedding_by_shape(data: list[float], shape: list[int]) -> list[list[float]]: + def split_embedding_by_shape(data: Sequence[float], shape: Sequence[int]) -> list[Sequence[float]]: if len(shape) != 2: raise ValueError("Shape must be of length 2.") embedding_size: Final = shape[1] diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index 3fc74b6acc1..cf633358a41 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -6,9 +6,11 @@ Calls done in OpenAI/openai.py as Vercel AI Gateway is openai-compatible. Docs: https://vercel.com/docs/ai-gateway """ +from collections.abc import Sequence from typing import Final import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.url_utils import join_api_base @@ -20,6 +22,14 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import VercelAIGatewayException +class _VercelAIGatewayModel(TypedDict): + id: ReadOnly[str] + + +class _VercelAIGatewayModelList(TypedDict): + data: ReadOnly[Sequence[_VercelAIGatewayModel]] + + class VercelAIGatewayConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> str | None: @@ -99,5 +109,5 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): if response.status_code != 200: raise Exception(f"Failed to get models: {response.text}") - models: Final = response.json()["data"] - return [model["id"] for model in models] + models: Final[_VercelAIGatewayModelList] = response.json() + return [model["id"] for model in models["data"]] diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 39d2e9b4c49..26de755d5ca 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from typing import Final from httpx import Headers, Response +from pydantic import TypeAdapter import litellm from litellm.exceptions import UnsupportedParamsError @@ -45,6 +46,7 @@ from litellm.types.utils import ( DEFAULT_GEMINI_TRANSCRIBE_LOCATION: Final = "global" AUDIO_MODALITY: Final = "AUDIO" +_DECODED_JSON: Final = TypeAdapter(object) class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase): @@ -177,7 +179,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB raw_response: Response, ) -> TranscriptionResponse: try: - response_json: Final = raw_response.json() + response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except ValueError: raise VertexAIError( status_code=raw_response.status_code, diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index a8d02045740..99ef750b5fc 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -202,7 +202,7 @@ class VertexAIBatchTransformation: @classmethod def transform_vertex_ai_batch_list_response_to_openai_list_response( cls, response: dict[str, Any] - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Transforms Vertex AI batch list response into OpenAI-compatible list response. """ diff --git a/litellm/llms/vertex_ai/files/handler.py b/litellm/llms/vertex_ai/files/handler.py index f2da04a7db7..54c0fa97550 100644 --- a/litellm/llms/vertex_ai/files/handler.py +++ b/litellm/llms/vertex_ai/files/handler.py @@ -3,7 +3,7 @@ import json import os import time from collections.abc import Coroutine, Mapping -from typing import Any, Final +from typing import Final from urllib.parse import unquote import httpx @@ -197,7 +197,7 @@ class VertexAIFilesHandler(GCSBucketBase): timeout: float | httpx.Timeout, max_retries: int | None, litellm_params: dict | None = None, - ) -> HttpxBinaryResponseContent | Coroutine[Any, Any, HttpxBinaryResponseContent]: + ) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]: """ Download file content from GCS bucket for VertexAI files. Supports both sync and async operations. diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index e36aba00293..410f29ef88a 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -5,7 +5,7 @@ import json import os import re import time -from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping +from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping, Sequence from contextlib import aclosing from dataclasses import dataclass from functools import partial @@ -16,6 +16,7 @@ from urllib.parse import quote, unquote import httpx from httpx import Headers, Response from openai.types.file_deleted import FileDeleted +from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required import litellm @@ -31,7 +32,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, @@ -96,6 +97,7 @@ _JSONL_NEWLINE: Final = b"\n" _BATCH_OUTPUT_FIRST_ROW_PEEK_LIMIT_BYTES: Final = 32 * 1024 * 1024 _PASSTHROUGH_MANAGED_GCS_PREFIX: Final = f"{VERTEX_AI_MANAGED_GCS_PREFIX}passthrough/" _RAW_UPLOAD_CHUNK_BYTES: Final = 1024 * 1024 +_DECODED_JSON: Final = TypeAdapter(object) class _GcsObjectMetadataJson(TypedDict, total=False): @@ -827,7 +829,7 @@ def _iter_openai_jsonl_entries( def _load_openai_jsonl_row(lineno: int, line: str) -> object: try: - return json.loads(line) + return _DECODED_JSON.validate_python(json.loads(line)) except json.JSONDecodeError as e: raise VertexAIError( status_code=400, @@ -967,7 +969,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def _get_gcs_object_name_from_batch_jsonl( self, - openai_jsonl_content: list[dict[str, Any]], + openai_jsonl_content: Sequence[Mapping[str, object]], deployment_model: str | None = None, ) -> str: """ @@ -985,8 +987,9 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): `projects/../locations/../endpoints/` batch job model instead of a nonexistent publisher model. """ - first_entry: Final = openai_jsonl_content[0] - body_model: Final = first_entry["body"].get("model") if isinstance(first_entry.get("body"), dict) else None + body: Final = openai_jsonl_content[0].get("body") + body_fields: Final = as_str_mapping(body) + body_model: Final = body_fields.get("model") if body_fields is not None and isinstance(body, dict) else None jsonl_model: Final = body_model if isinstance(body_model, str) else "" raw_model: Final = deployment_model.removeprefix("vertex_ai/") if deployment_model else jsonl_model return _managed_batch_object_name(raw_model, passthrough=False) diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 40564d536a7..7cb52281a6a 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -27,6 +27,11 @@ from litellm.types.utils import LiteLLMFineTuningJob _JSON_VALUE: Final = TypeAdapter(object) +def _response_tuning_job(response: httpx.Response) -> ResponseTuningJob: + tuning_job: Final[ResponseTuningJob] = response.json() + return ResponseTuningJob(**tuning_job) + + class VertexFineTuningAPI(VertexLLM): """ Vertex methods to support for batches @@ -187,9 +192,7 @@ class VertexFineTuningAPI(VertexLLM): "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) @@ -272,9 +275,7 @@ class VertexFineTuningAPI(VertexLLM): verbose_logger.debug( "got response from creating fine tuning job: %s", _JSON_VALUE.validate_python(response.json()) ) - vertex_response: Final = ResponseTuningJob( - **response.json(), - ) + vertex_response: Final = _response_tuning_job(response) verbose_logger.debug("vertex_response %s", vertex_response) open_ai_response: Final = self.convert_vertex_response_to_open_ai_response(vertex_response) diff --git a/litellm/llms/vertex_ai/gemini/long_decode.py b/litellm/llms/vertex_ai/gemini/long_decode.py index 571fa94d4a5..25aae70a853 100644 --- a/litellm/llms/vertex_ai/gemini/long_decode.py +++ b/litellm/llms/vertex_ai/gemini/long_decode.py @@ -332,7 +332,8 @@ def with_initial_prompt_tokens(usage: Usage, initial_prompt_tokens: int | None) def parse_body(response: httpx.Response) -> GenerateContentResponseBody: try: - return GenerateContentResponseBody(**response.json()) + body: Final[GenerateContentResponseBody] = response.json() + return GenerateContentResponseBody(**body) except Exception as e: raise VertexAIError( message=( diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index 0a800497de1..f15c6393cb2 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -6,6 +6,7 @@ import json from typing import TYPE_CHECKING, Final, Literal import httpx +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.llms.custom_httpx.http_handler import ( @@ -35,6 +36,11 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +class _GeminiFileResource(TypedDict, total=False): + mimeType: ReadOnly[str] + uri: ReadOnly[str] + + class GoogleBatchEmbeddings(VertexLLM): @staticmethod def _flatten_and_detect_file_refs( @@ -74,7 +80,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), @@ -111,7 +117,7 @@ class GoogleBatchEmbeddings(VertexLLM): if response.status_code != 200: raise Exception(f"Error fetching file {element}: {response.status_code} {response.text}") - file_data = response.json() + file_data: _GeminiFileResource = response.json() resolved_files[element] = { "mime_type": file_data.get("mimeType", ""), "uri": file_data.get("uri", element), diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 725a7f39917..6d071363823 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -190,7 +190,7 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): payload: Final = json.dumps(request_body) empty_files: Final = cast(RequestFiles, []) - return cast(tuple[dict[str, Any], RequestFiles | None], (payload, empty_files)) + return cast(tuple[dict[str, object], RequestFiles | None], (payload, empty_files)) def transform_image_edit_response( self, diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index 085ff715c30..deaa32f7308 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx from httpx._types import RequestFiles from pydantic import ConfigDict, TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -34,6 +35,10 @@ _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object] ) +class _VertexImagenPredictResponse(TypedDict, total=False): + predictions: ReadOnly[Sequence[object]] + + class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): """ Vertex AI Imagen Image Edit Configuration @@ -207,7 +212,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): ) -> ImageResponse: model_response: Final = ImageResponse() try: - response_json: Final = raw_response.json() + response_json: Final[_VertexImagenPredictResponse] = raw_response.json() except Exception as exc: raise self.get_error_class( error_message=f"Error transforming image edit response: {exc}", diff --git a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py index 7ba7b060991..ec915e10fcd 100644 --- a/litellm/llms/vertex_ai/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai/image_generation/image_generation_handler.py @@ -168,7 +168,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) async def aimage_generation( @@ -272,7 +272,7 @@ class VertexImageGeneration(VertexLLM): if response.status_code != 200: raise Exception(f"Error: {response.status_code} {response.text}") - json_response: Final = response.json() + json_response: Final[Mapping[str, object]] = response.json() return self.process_image_generation_response(json_response, model_response, model) def is_image_generation_response(self, json_response: dict[str, Any]) -> bool: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 916e25a027e..f6df772dd56 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_DECODED_JSON: Final = TypeAdapter(object) _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter( @@ -299,7 +300,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): Transform Gemini image generation response to litellm ImageResponse format """ try: - response_data: Final = raw_response.json() + response_data: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Error transforming image generation response: {e}", diff --git a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py index da77fa338db..c282ae22ea6 100644 --- a/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/multimodal_embeddings/transformation.py @@ -208,7 +208,7 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig): if raw_response.status_code != 200: raise Exception(f"Error: {raw_response.status_code} {raw_response.text}") - _json_response: Final = raw_response.json() + _json_response: Final[MultimodalPredictions] = raw_response.json() if "predictions" not in _json_response: raise InternalServerError( message=f"embedding response does not contain 'predictions', got {_json_response}", diff --git a/litellm/llms/vertex_ai/rag_engine/transformation.py b/litellm/llms/vertex_ai/rag_engine/transformation.py index a34ef519578..61cb083fa4c 100644 --- a/litellm/llms/vertex_ai/rag_engine/transformation.py +++ b/litellm/llms/vertex_ai/rag_engine/transformation.py @@ -4,7 +4,10 @@ Transformation utilities for Vertex AI RAG Engine. Handles transforming LiteLLM's unified formats to Vertex AI RAG Engine API format. """ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final + +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE @@ -13,6 +16,15 @@ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.rag import RAGChunkingStrategy +class VertexRAGChunkingConfig(TypedDict): + chunk_size: ReadOnly[int] + chunk_overlap: ReadOnly[int] + + +class VertexRAGTransformationConfig(TypedDict): + chunking_config: ReadOnly[VertexRAGChunkingConfig] + + class VertexAIRAGTransformation(VertexBase): """ Transformation class for Vertex AI RAG Engine API. @@ -55,7 +67,7 @@ class VertexAIRAGTransformation(VertexBase): def transform_chunking_strategy_to_vertex_format( self, chunking_strategy: RAGChunkingStrategy | None, - ) -> dict[str, Any]: + ) -> VertexRAGTransformationConfig: """ Transform LiteLLM's unified chunking_strategy to Vertex AI RAG format. @@ -105,7 +117,7 @@ class VertexAIRAGTransformation(VertexBase): self, gcs_uri: str, chunking_strategy: RAGChunkingStrategy | None = None, - ) -> dict[str, Any]: + ) -> Mapping[str, object]: """ Build the request payload for importing RAG files. diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index dce4d2f2a87..398471a7c9b 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -6,10 +6,11 @@ Why separate file? Make it easy to see how transformation works import math import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final import httpx +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -27,6 +28,17 @@ from litellm.types.rerank import ( ) +class _VertexRankedRecord(TypedDict): + id: ReadOnly[str] + score: NotRequired[ReadOnly[float]] + title: NotRequired[ReadOnly[str]] + content: NotRequired[ReadOnly[str]] + + +class _VertexRankResponse(TypedDict, total=False): + records: ReadOnly[Sequence[_VertexRankedRecord]] + + class VertexAIRerankConfig(BaseRerankConfig, VertexBase): """ Configuration for Vertex AI Discovery Engine Rerank API @@ -172,7 +184,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): Transform Vertex AI Discovery Engine response to Cohere format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final[_VertexRankResponse] = raw_response.json() except Exception as e: raise ValueError(f"Failed to parse response: {e}") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py index 45cc5320401..3873a4be0b5 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final import litellm @@ -18,16 +19,17 @@ class VertexAIAi21Config(OpenAIGPTConfig): self, max_tokens: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 5d748ad6e74..cfd5b5d3980 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -1,4 +1,5 @@ -from typing import Any, Final +from collections.abc import Sequence +from typing import Final from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.pass_through.messages.transformation import ( @@ -29,7 +30,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert self, headers: dict, model: str, - messages: list[Any], + messages: Sequence[object], optional_params: dict, litellm_params: dict, api_key: str | None = None, diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index ddc83bc53a5..96fccf007a5 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -1,4 +1,5 @@ import types +from collections.abc import Mapping from typing import Final, Literal from pydantic import TypeAdapter @@ -70,16 +71,17 @@ class VertexAITextEmbeddingConfig(LiteLLMBaseModel): | None = None, title: str | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = types.MappingProxyType(locals().copy()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod def get_config(cls): + class_attributes: Final[Mapping[str, object]] = types.MappingProxyType(cls.__dict__) return { k: v - for k, v in cls.__dict__.items() + for k, v in class_attributes.items() if not k.startswith("__") and not isinstance( v, diff --git a/litellm/llms/volcengine/chat/transformation.py b/litellm/llms/volcengine/chat/transformation.py index 52b3242c186..4ccab38e761 100644 --- a/litellm/llms/volcengine/chat/transformation.py +++ b/litellm/llms/volcengine/chat/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -34,7 +35,7 @@ class VolcEngineChatConfig(OpenAILikeChatConfig): top_p: int | None = None, response_format: dict | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[Mapping[str, object]] = dict(locals()) for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 74f6182bd89..747ea21e4fb 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -45,6 +45,11 @@ class _EventModelClass(Protocol): def model_validate(self, obj: Mapping[str, object]) -> ResponsesAPIStreamingResponse: ... +class _FieldDefaultCarrier(Protocol): + @property + def default(self) -> object: ... + + class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): _SUPPORTED_OPTIONAL_PARAMS: list[str] = [ # Doc-listed knobs @@ -430,6 +435,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): annotation: Final[object] = field.annotation return annotation + @staticmethod + def _field_default(field: _FieldDefaultCarrier) -> object: + return field.default + @staticmethod def _fill_missing_fields(chunk: Mapping[str, object], event_model: object | None) -> Mapping[str, object]: """ @@ -451,7 +460,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): continue # Explicit default or factory - field_default: object = field.default + field_default: object = VolcEngineResponsesAPIConfig._field_default(field) if field_default is not pyd_fields.PydanticUndefined and field_default is not None: patched[name] = field_default continue diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 7fe643870d5..21c6e6a82b9 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -29,6 +29,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _OPTIONAL_INT: Final[TypeAdapter[int | None]] = TypeAdapter(int | None) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class VoyageRerankConfig(BaseRerankConfig): @@ -112,7 +113,7 @@ class VoyageRerankConfig(BaseRerankConfig): logging_obj.post_call(original_response=raw_response.text) try: - _json_response: Final = raw_response.json() + _json_response: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception: raise VoyageError( message=f"Failed to parse response: {raw_response.text}", diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index e5da7dd412f..35fda05c028 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -29,6 +29,7 @@ from ...openai.transcriptions.whisper_transformation import ( from ..common_utils import IBMWatsonXMixin _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTranscriptionConfig): @@ -172,7 +173,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran removed before creating the TranscriptionResponse object. """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 591ddda3cbd..87f30483196 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -30,6 +30,7 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) _OPTIONAL_INT: Final[TypeAdapter[int | None]] = TypeAdapter(int | None) _STR: Final = TypeAdapter(str) +_DECODED_JSON: Final = TypeAdapter(object) class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): @@ -170,7 +171,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): Transform IBM watsonx.ai rerank response to LiteLLM RerankResponse format """ try: - raw_response_json: Final = raw_response.json() + raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: raise self.get_error_class( error_message=f"Failed to parse response: {e}", diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 984b62eeb59..2b943fa4107 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -32,6 +32,7 @@ from ...openai.chat.gpt_transformation import ( ) _RESPONSE_BODY: Final = TypeAdapter(dict[object, object], config=ConfigDict(hide_input_in_errors=True)) +_FILTERED_TOOL: Final = TypeAdapter(object) def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None: @@ -204,11 +205,13 @@ class XAIChatConfig(OpenAIGPTConfig): if param == "max_completion_tokens": optional_params["max_tokens"] = value elif param == "tools" and value is not None: - tools = [] - for tool in value: - tool = filter_value_from_dict(tool, "strict") - if tool is not None: - tools.append(tool) + tools = [ + filtered_tool + for filtered_tool in ( + _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value + ) + if filtered_tool is not None + ] if len(tools) > 0: optional_params["tools"] = tools elif param in supported_openai_params: diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index de2cbf5f9e3..aec535c2086 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -120,12 +120,11 @@ def _web_search_cost_per_call_from_model_info(model_info: "ModelInfo") -> float: search_costs: Final = model_info.get("search_context_cost_per_query") if not isinstance(search_costs, Mapping): return _DEFAULT_WEB_SEARCH_COST_PER_CALL - for key in ( - "search_context_size_medium", - "search_context_size_low", - "search_context_size_high", + for value in ( + search_costs.get("search_context_size_medium"), + search_costs.get("search_context_size_low"), + search_costs.get("search_context_size_high"), ): - value = search_costs.get(key) if value is None: continue try: diff --git a/litellm/main.py b/litellm/main.py index 327a7a1c4b5..c22cd8eb11c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -802,6 +802,11 @@ def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, return TranscriptionResponse(**cached_response_dict) +def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse: + fields: Final = {key: value for key, value in cached_response_dict.items() if key != "stream"} + return TextCompletionResponse(stream=bool(cached_response_dict.get("stream", False)), **fields) + + async def _async_streaming(response, model, custom_llm_provider, args): try: print_verbose(f"received response in _async_streaming: {response}") @@ -3560,7 +3565,7 @@ def _complete_nadir(ctx: CompletionDispatchContext) -> _CompletionDispatchResult encoding=_get_encoding(), api_key=api_key, logging_obj=ctx.logging, - client=ctx.client, + client=_dispatch_client_http(ctx), ) ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) @@ -4197,6 +4202,7 @@ _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints) _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint) +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _AWS_BEDROCK_CLIENT_CREDENTIAL_KEYS: Final = ( @@ -4741,7 +4747,7 @@ def _complete_petals(ctx: CompletionDispatchContext) -> _CompletionDispatchResul ) if stream is True: ## [BETA] # Fake streaming for petals - resp_string: Final = model_response["choices"][0]["message"]["content"] + resp_string: Final = _OPAQUE_VALUE_ADAPTER.validate_python(model_response["choices"][0]["message"]["content"]) return CustomStreamWrapper( resp_string, model, @@ -7719,7 +7725,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp init_response: Final = await loop.run_in_executor(None, func_with_context) if isinstance(init_response, dict) or isinstance(init_response, TextCompletionResponse): ## CACHING SCENARIO if isinstance(init_response, dict): - response = TextCompletionResponse(**init_response) + response = _text_completion_response_from_cached_dict(init_response) else: response = init_response elif asyncio.iscoroutine(init_response): @@ -7899,7 +7905,6 @@ def text_completion( tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): - responses: Final = [None for x in prompt] # init responses def process_prompt(i, individual_prompt): decoded_prompt: Final = tokenizer.decode(individual_prompt) @@ -7916,16 +7921,16 @@ def text_completion( text_completion_response["object"] = "text_completion" text_completion_response["created"] = response.get("created", None) text_completion_response["model"] = response.get("model", None) - return response["choices"][0] + return response.choices[0] with concurrent.futures.ThreadPoolExecutor() as executor: completed_futures: Final = [ executor.submit(process_prompt, i, individual_prompt) for i, individual_prompt in enumerate(prompt) ] - for i, future in enumerate(concurrent.futures.as_completed(completed_futures)): - responses[i] = future.result() - text_completion_response.choices = responses + text_completion_response.choices = [ + future.result() for future in concurrent.futures.as_completed(completed_futures) + ] return text_completion_response # else: diff --git a/litellm/models/organization_membership.py b/litellm/models/organization_membership.py index 6a0d22cb63b..f3abbc2e93d 100644 --- a/litellm/models/organization_membership.py +++ b/litellm/models/organization_membership.py @@ -6,6 +6,7 @@ Canonical definition for ``litellm_organizationmembership``. Re-exported from """ from datetime import datetime +from typing import Final from pydantic import ConfigDict, model_validator @@ -31,9 +32,10 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): @model_validator(mode="after") def populate_user_email(self) -> "LiteLLM_OrganizationMembershipTable": - if self.user_email is None and self.user is not None: - if isinstance(self.user, dict): - self.user_email = self.user.get("user_email") + user: Final[object] = self.user + if self.user_email is None and user is not None: + if isinstance(user, dict): + self.user_email = user.get("user_email") else: - self.user_email = getattr(self.user, "user_email", None) + self.user_email = getattr(user, "user_email", None) return self diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 7f0d392ab1b..824633bdf5a 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -99,6 +99,7 @@ if TYPE_CHECKING: _EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({}) OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +_CACHED_VALUE: Final = TypeAdapter(object) def _normalize_caller_admission_credential(value: str) -> str: @@ -4258,7 +4259,7 @@ class MCPRequestHandler: cache_key: Final = f"agent_object_permission_id:{agent_id}" try: - cached: Final[object] = await user_api_key_cache.async_get_cache(key=cache_key) + cached: Final = _CACHED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key)) if cached == MCPRequestHandler._AGENT_NO_PERMISSION_SENTINEL: return None if isinstance(cached, str) and cached: diff --git a/litellm/proxy/_experimental/mcp_server/catalog.py b/litellm/proxy/_experimental/mcp_server/catalog.py index 71eb2112bd5..2db548f5863 100644 --- a/litellm/proxy/_experimental/mcp_server/catalog.py +++ b/litellm/proxy/_experimental/mcp_server/catalog.py @@ -1173,12 +1173,9 @@ async def list_gateway_catalog( try: page: Final = await fetch_optional_catalog_page(context, request, server, allowed, cursor) + page_meta: Final[Mapping[str, object]] = page.meta or {} return page.model_copy( - update={ - "meta": { - key: value for key, value in (page.meta or {}).items() if key != SERVER_OUTCOMES_META_KEY - } - } + update={"meta": {key: value for key, value in page_meta.items() if key != SERVER_OUTCOMES_META_KEY}} ) except Exception as error: if cursor is not None: diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 7302b74f15a..0666cc7e6c0 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -165,6 +165,7 @@ _CLIENT_FORWARDED_AUTH_TYPES: Final["frozenset[str]"] = frozenset({"true_passthr _MINTED_TOKEN_CREDENTIAL_FIELDS: Final["frozenset[str]"] = frozenset({"access_token", "refresh_token", "expires_in"}) _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_DECODED_JSON: Final = TypeAdapter(object) def _bind_submitted_oauth_client( @@ -339,7 +340,7 @@ def _decrypt_env_vars_on_returned_row(row: object) -> None: return if isinstance(env_vars, str): try: - env_vars = json.loads(env_vars) + env_vars = _DECODED_JSON.validate_python(json.loads(env_vars)) except (json.JSONDecodeError, TypeError): return if not isinstance(env_vars, list): @@ -476,8 +477,9 @@ def _prepare_mcp_server_data( # Serialize JSON fields from ``data_dict`` (not ``data``) so the # exclude_unset filter is respected. Reading back from ``data`` would # reintroduce defaults (e.g. ``env={}``) for fields the caller never set. - if data_dict.get("static_headers") is not None: - data_dict["static_headers"] = encrypt_secret_map(data_dict["static_headers"]) + static_headers: Final[Mapping[str, str] | None] = data_dict.get("static_headers") + if static_headers is not None: + data_dict["static_headers"] = encrypt_secret_map(static_headers) # env_vars is read from ``data_dict`` (not ``data``) like every other JSON # column so the exclude_unset filter is respected: a partial update that @@ -489,11 +491,13 @@ def _prepare_mcp_server_data( _encrypt_global_env_var_values(serialized_env_vars) data_dict["env_vars"] = safe_dumps(serialized_env_vars) - if data_dict.get("mcp_info") is not None: - data_dict["mcp_info"] = safe_dumps(data_dict["mcp_info"]) + mcp_info: Final[object] = data_dict.get("mcp_info") + if mcp_info is not None: + data_dict["mcp_info"] = safe_dumps(mcp_info) - if data_dict.get("env") is not None: - data_dict["env"] = encrypt_secret_map(data_dict["env"]) + env: Final[Mapping[str, str] | None] = data_dict.get("env") + if env is not None: + data_dict["env"] = encrypt_secret_map(env) if "tool_name_to_display_name" in data_dict: data_dict["tool_name_to_display_name"] = safe_dumps(data_dict["tool_name_to_display_name"] or {}) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 3b2c2e09df5..f5c54451006 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -701,7 +701,7 @@ async def _store_per_user_token_server_side( expires_in = None refresh_token: Final[str | None] = token_response.get("refresh_token") or None - raw_scope: Final[object] = token_response.get("scope", requested_scope) + raw_scope: Final[object] = _DECODED_JSON.validate_python(token_response.get("scope", requested_scope)) scopes: Final[list[str] | None] = raw_scope.split() if isinstance(raw_scope, str) else None try: diff --git a/litellm/proxy/_experimental/mcp_server/faults/classify.py b/litellm/proxy/_experimental/mcp_server/faults/classify.py index bb41436b495..7826fc34eac 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/classify.py +++ b/litellm/proxy/_experimental/mcp_server/faults/classify.py @@ -10,6 +10,7 @@ from __future__ import annotations from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.faults.types import ( @@ -25,6 +26,8 @@ from litellm.proxy._experimental.mcp_server.faults.types import ( UpstreamReportedFault, ) +_DECODED_JSON: Final = TypeAdapter(object) + def _safe_text(response: httpx.Response) -> str: try: @@ -35,7 +38,7 @@ def _safe_text(response: httpx.Response) -> str: def _safe_json(response: httpx.Response) -> object: try: - return response.json() + return _DECODED_JSON.validate_python(response.json()) except Exception: return None diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py index 84673c8c738..a3e33929f6d 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py @@ -226,7 +226,7 @@ def rate_limit_retry_delay( ) -> float: if retry_after_seconds is not None: return min(retry_after_seconds, max_delay) - return min(max_delay, base_delay * 2**attempt * (1 + jitter)) + return min(max_delay, base_delay * 2.0**attempt * (1 + jitter)) T: Final = TypeVar("T") diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7eddf592262..db70c7d77de 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -488,6 +488,10 @@ class _AuthorizationServerMetadataPayload(TypedDict, total=False): token_endpoint_auth_methods_supported: Sequence[str] +def _mapping_value(source: Mapping[str, object], key: str, default: object) -> object: + return source.get(key, default) + + def _blank_to_none(value: str | None) -> str | None: """Collapse an absent, empty, or whitespace-only string to ``None``. @@ -2757,7 +2761,9 @@ class MCPServerManager: server_id=server_id, name=name_for_prefix, protocol_version=TypeAdapter(MCPUpstreamProtocol).validate_python( - server_config.get("protocol_version", mcp_info.get("protocol_version", "auto")) + _mapping_value( + server_config, "protocol_version", _mapping_value(mcp_info, "protocol_version", "auto") + ) ), alias=alias, server_name=server_name, diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 6d4fefcdb6c..cc92b8cce9b 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -11,6 +11,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Final import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -41,6 +42,8 @@ from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer +_DECODED_JSON: Final = TypeAdapter(object) + class MCPOAuth2TokenCache(InMemoryCache): """ @@ -160,7 +163,7 @@ class MCPOAuth2TokenCache(InMemoryCache): f"failed with status {exc.response.status_code}" ) from exc - body: Final = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( diff --git a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py index c31560a9f63..b3aa040c507 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_identity_binding.py @@ -18,6 +18,7 @@ from typing import Final, Literal, Protocol, TypeAlias import jwt from fastapi import HTTPException from jwt.types import Options +from pydantic import TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -38,6 +39,7 @@ _ALLOWED_ID_TOKEN_ALGORITHMS: Final = ( "PS512", ) _JWKS_CACHE_TTL_SECONDS: Final = 3600 +_DECODED_JSON: Final = TypeAdapter(object) _jwks_cache: Final = InMemoryCache(default_ttl=_JWKS_CACHE_TTL_SECONDS) JwksFetcher: TypeAlias = Callable[ @@ -103,7 +105,7 @@ async def _fetch_issuer_jwks(binding: MCPOAuthIdentityBinding) -> Sequence[Mappi client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(jwks_url) response.raise_for_status() - document: Final = response.json() + document: Final = _DECODED_JSON.validate_python(response.json()) keys: Final = document.get("keys") if isinstance(document, dict) else None if not isinstance(keys, list): raise TypeError(f"JWKS document at {jwks_url} has no 'keys' array") @@ -116,7 +118,7 @@ async def _discover_jwks_url(issuer: str) -> str: client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response: Final = await client.get(discovery_url) response.raise_for_status() - metadata: Final = response.json() + metadata: Final = _DECODED_JSON.validate_python(response.json()) jwks_uri: Final = metadata.get("jwks_uri") if isinstance(metadata, dict) else None if not isinstance(jwks_uri, str) or not jwks_uri: raise ValueError(f"OIDC discovery at {discovery_url} returned no jwks_uri") diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py index 2501f751d8e..d84a65fee02 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/token_exchange_provider.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import Final import httpx +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import ( @@ -34,6 +35,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchanger _GATEWAY_FAULT_OAUTH_ERRORS: Final = frozenset( {"invalid_client", "unauthorized_client", "unsupported_grant_type", "invalid_target", "invalid_scope"} ) +_DECODED_JSON: Final = TypeAdapter(object) def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | None]: @@ -46,7 +48,7 @@ def _oauth_error_fields(response: httpx.Response) -> tuple[str | None, str | Non it can carry IdP internals and must never reach the caller. """ try: - body: Final[object] = response.json() + body: Final = _DECODED_JSON.validate_python(response.json()) except Exception: # noqa: BLE001 return None, None if not isinstance(body, dict): diff --git a/litellm/proxy/_experimental/mcp_server/result_conversion.py b/litellm/proxy/_experimental/mcp_server/result_conversion.py index 61435f94c94..fdb9a01fea6 100644 --- a/litellm/proxy/_experimental/mcp_server/result_conversion.py +++ b/litellm/proxy/_experimental/mcp_server/result_conversion.py @@ -13,7 +13,7 @@ from __future__ import annotations import json import math from collections.abc import Sequence -from typing import Final, TypeAlias, TypeVar +from typing import Final, Protocol, TypeAlias, TypeVar from mcp.types import CacheableResult, CallToolResult, ContentBlock, InputRequiredResult, TextContent, Tool from typing_extensions import ReadOnly, TypedDict, assert_never @@ -105,8 +105,17 @@ def complete_call_tool_result(outcome: ToolOutcome, compat: WireCompat) -> CallT return converted +class _StructuredContentCarrier(Protocol): + @property + def structured_content(self) -> object: ... + + +def _structured_content(result: _StructuredContentCarrier) -> object: + return result.structured_content + + def _downgrade_structured_content(result: CallToolResult) -> CallToolResult: - structured: Final = result.structured_content + structured: Final = _structured_content(result) if structured is None or isinstance(structured, dict): return result fallback: Final = TextContent(type="text", text=json.dumps(structured)) diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index 2f88d8c361f..a7748b9ca88 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -6,7 +6,7 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. import asyncio from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, TypeVar from litellm._logging import verbose_logger from litellm.exceptions import ContextWindowExceededError @@ -20,6 +20,8 @@ if TYPE_CHECKING: from litellm.router import Router +_Tool: Final = TypeVar("_Tool") + class SemanticToolFilterContextWindowError(Exception): """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" @@ -265,9 +267,9 @@ class SemanticMCPToolFilter: async def filter_tools( self, query: str, - available_tools: list[Any], + available_tools: list[_Tool], top_k: int | None = None, - ) -> list[Any]: + ) -> list[_Tool]: """ Filter tools semantically based on query. @@ -390,7 +392,7 @@ class SemanticMCPToolFilter: separator: Final = client_name[-len(canonical) - 1] return separator in ("_", "-") - def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[object]) -> list[object]: + def _get_tools_by_names(self, tool_names: Sequence[str], available_tools: Sequence[_Tool]) -> list[_Tool]: """ Get tools from available_tools by their names, preserving the semantic router's ordering. @@ -406,13 +408,13 @@ class SemanticMCPToolFilter: # Exact matches win over suffix matches when both are present, and # each incoming tool is returned at most once even if two canonical # names happen to be tail-compatible with the same incoming name. - available_by_name: Final[dict[str, object]] = {} + available_by_name: Final[dict[str, _Tool]] = {} for tool in available_tools: client_name, _ = self.extract_tool_info(tool) if client_name and client_name not in available_by_name: available_by_name[client_name] = tool - matched: Final[list[object]] = [] + matched: Final[list[_Tool]] = [] used_ids: Final[set[int]] = set() for canonical in tool_names: tool = available_by_name.get(canonical) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index bb7c81f9d60..7207c4150ad 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -14,7 +14,7 @@ import time import types from collections import Counter from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence, Set -from typing import TYPE_CHECKING, Final, NoReturn, Protocol +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol import httpx from fastapi import FastAPI, HTTPException @@ -150,6 +150,16 @@ def _scope_headers(scope: Scope) -> Sequence[tuple[bytes, bytes]]: return asgi_scope_fields(scope).get("headers", []) +class _RawHeaderPairsSource(Protocol): + def get( + self, key: Literal["headers"], default: Sequence[tuple[bytes | str, bytes | str]], / + ) -> Sequence[tuple[bytes | str, bytes | str]]: ... + + +def _raw_header_pairs(source: _RawHeaderPairsSource) -> Sequence[tuple[bytes | str, bytes | str]]: + return source.get("headers", []) + + def unsupported_protocol_version(scope: Scope) -> str | None: """Admit configured HTTP revisions while keeping SSE on the legacy protocol.""" from litellm.proxy._experimental.mcp_server.capabilities import configured_versions @@ -1319,7 +1329,7 @@ if MCP_AVAILABLE: Extract mcp-session-id from ASGI scope headers. Returns None if not present. """ - scope_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + scope_headers: Final = _raw_header_pairs(scope) for header_name, header_value in scope_headers: name = header_name if isinstance(header_name, bytes) else header_name.encode() if name.lower() == b"mcp-session-id": @@ -1669,7 +1679,7 @@ if MCP_AVAILABLE: Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header: Final = b"mcp-session-id" - _headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = scope.get("headers", []) + _headers: Final = _raw_header_pairs(scope) def _normalize_header_name(header_name: object) -> bytes | None: if isinstance(header_name, bytes): @@ -2869,7 +2879,7 @@ if MCP_AVAILABLE: mcp_servers, _sse_client_ip, scoped_server_endpoint=scoped_server_endpoint, - is_initialize=scope.get("method") == "GET", + is_initialize=asgi_scope_fields(scope).get("method", "") == "GET", ): async with ( sse.connect_sse(transport_scope, receive, send) as (read_stream, write_stream), @@ -3090,7 +3100,7 @@ if MCP_AVAILABLE: ) -> Send: async def wrapped_send(message: Message) -> None: if message.get("type") == "http.response.start": - response_headers: Final[Sequence[tuple[bytes | str, bytes | str]]] = message.get("headers", []) + response_headers: Final = _raw_header_pairs(message) for key, value in response_headers: header_name = key if isinstance(key, bytes) else str(key).encode() if header_name.lower() == b"mcp-session-id": diff --git a/litellm/proxy/_experimental/mcp_server/toolset_db.py b/litellm/proxy/_experimental/mcp_server/toolset_db.py index f3a78d206d7..35906c793b4 100644 --- a/litellm/proxy/_experimental/mcp_server/toolset_db.py +++ b/litellm/proxy/_experimental/mcp_server/toolset_db.py @@ -131,6 +131,10 @@ async def get_mcp_toolset_by_name( return _toolset_from_row(row) +def _set_fields(data: UpdateMCPToolsetRequest) -> Mapping[str, object]: + return data.model_dump(exclude_unset=True) + + async def update_mcp_toolset( prisma_client: PrismaClient, data: UpdateMCPToolsetRequest, @@ -143,7 +147,7 @@ async def update_mcp_toolset( data_dict: Final = dict( ( (field, json.dumps(value) if field == "tools" else value) - for field, value in data.model_dump(exclude_unset=True).items() + for field, value in _set_fields(data).items() if field != "toolset_id" and (field not in ("toolset_name", "tools") or value is not None) ), updated_by=touched_by, diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 1684b9c5c30..ee8f3ea3b30 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -13,6 +13,8 @@ from collections.abc import Set as AbstractSet from typing import Any, Final, Literal, Protocol, overload from urllib.parse import quote +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -21,6 +23,7 @@ if typing.TYPE_CHECKING: MCP_SERVERS_TARGET: Final = "mcp_servers" MCP_OAUTH_TOKENS_TARGET: Final = "mcp_oauth_tokens" +_DECODED_JSON: Final = TypeAdapter(object) class ASGIScopeFields(Protocol): @@ -243,7 +246,7 @@ def _parse_mcp_info_dict(mcp_info: object) -> Mapping[str, object] | None: return mcp_info if isinstance(mcp_info, str): try: - parsed: Final[object] = json.loads(mcp_info) + parsed: Final = _DECODED_JSON.validate_python(json.loads(mcp_info)) except (ValueError, TypeError): return None return parsed if isinstance(parsed, dict) else None diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 28c701e0fc2..aa67d36af4f 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -356,7 +356,7 @@ class LazyFeatureMiddleware: async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # Short-circuit once every feature has loaded. if scope["type"] in ("http", "websocket") and len(self._loaded) < len(self._features): - path: str = scope.get("path", "") + path: str = str(_scope_value(scope, "path")) # Strip the request's root_path so prefix matching works under a # server root path. Without this, requests like /api/v1/policies/... # never match the registered prefixes (/policies/...) and lazy @@ -584,6 +584,7 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY from fastapi import APIRouter, Depends, HTTPException from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router: Final = APIRouter() @@ -603,17 +604,22 @@ def _make_warmup_router(app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY await _force_load(app, feat, features) feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] - full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) + full: Final[Mapping[str, object]] = MappingProxyType( + get_openapi(title=app.title, version=app.version, routes=feat_routes) + ) + paths: Final = as_str_mapping(full.get("paths", {})) or {} + components: Final = as_str_mapping(full.get("components", {})) or {} # Force all operations under one tag so they group under a single Swagger # section — many lazy modules tag routes inconsistently. - for path_ops in full.get("paths", {}).values(): - for op in path_ops.values(): - if isinstance(op, dict): - op["tags"] = [feat.name] + for path_ops in paths.values(): + for op in (as_str_mapping(path_ops) or {}).values(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation["tags"] = [feat.name] return { "stub_path": feat.path_prefixes[0], "paths": full.get("paths", {}), - "components": {"schemas": full.get("components", {}).get("schemas", {})}, + "components": {"schemas": components.get("schemas", {})}, } return router diff --git a/litellm/proxy/_lazy_openapi_snapshot.py b/litellm/proxy/_lazy_openapi_snapshot.py index 00a05307a89..c5a63b0f990 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.py +++ b/litellm/proxy/_lazy_openapi_snapshot.py @@ -16,6 +16,7 @@ import sys from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import TYPE_CHECKING, Final from typing_extensions import ReadOnly, TypedDict @@ -61,7 +62,7 @@ def load_snapshot() -> dict[str, dict] | None: return None -def _normalize_operation_ids(paths: dict[str, dict]) -> None: +def _normalize_operation_ids(paths: Mapping[str, object]) -> None: """Make FastAPI-generated operation IDs stable for multi-method routes. FastAPI derives the default operation ID suffix from the first item in the @@ -69,26 +70,30 @@ def _normalize_operation_ids(paths: dict[str, dict]) -> None: set iteration order can vary between processes, which makes the snapshot drift even when no routes changed. """ + from litellm.litellm_core_utils.core_helpers import as_str_mapping + for path_ops in paths.values(): - if not isinstance(path_ops, dict): + operations = as_str_mapping(path_ops) + if not isinstance(operations, dict): continue - methods = {method for method in path_ops if method in HTTP_METHOD_SUFFIXES} + methods = {method for method in operations if method in HTTP_METHOD_SUFFIXES} if not methods: continue - for method, operation in path_ops.items(): - if method not in HTTP_METHOD_SUFFIXES or not isinstance(operation, dict): + for method, operation in operations.items(): + entry = as_str_mapping(operation) + if method not in HTTP_METHOD_SUFFIXES or not isinstance(entry, dict): continue - operation_id = operation.get("operationId") + operation_id = entry.get("operationId") if not isinstance(operation_id, str): continue for suffix in methods: suffix_token = f"_{suffix}" if operation_id.endswith(suffix_token): - operation["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" + entry["operationId"] = operation_id[: -len(suffix_token)] + f"_{method}" break @@ -117,6 +122,7 @@ def _register_feature(app: "FastAPI", feat: "LazyFeature") -> str | None: def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: set[str]) -> SnapshotFragment | None: from fastapi.openapi.utils import get_openapi + from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.proxy_server import ensure_unique_openapi_operation_ids feat_routes: Final = [r for r in app.routes if feat.matches(getattr(r, "path", ""))] @@ -124,22 +130,29 @@ def _feature_fragment(app: "FastAPI", feat: "LazyFeature", used_operation_ids: s return None _stabilize_multi_method_route_ids(feat_routes) full: Final = get_openapi(title=app.title, version=app.version, routes=feat_routes) - paths: Final = full.get("paths", {}) + openapi: Final[Mapping[str, object]] = MappingProxyType(full) + paths: Final = as_str_mapping(openapi.get("paths", {})) or {} _normalize_operation_ids(paths) for path_ops in paths.values(): - for method, op in path_ops.items(): - if isinstance(op, dict): - operation_id = op.get("operationId") + for method, op in (as_str_mapping(path_ops) or {}).items(): + operation = as_str_mapping(op) + if isinstance(operation, dict): + operation_id = operation.get("operationId") if isinstance(operation_id, str): for suffix in HTTP_METHOD_SUFFIXES: if operation_id.endswith(f"_{suffix}"): - op["operationId"] = operation_id[: -len(suffix)] + method + operation["operationId"] = operation_id[: -len(suffix)] + method break - op["tags"] = [feat.name] - unique: Final = ensure_unique_openapi_operation_ids(full, used_operation_ids) + operation["tags"] = [feat.name] + unique: Final[Mapping[str, object]] = MappingProxyType( + ensure_unique_openapi_operation_ids(full, used_operation_ids) + ) + components: Final = as_str_mapping(unique.get("components", {})) or {} return { - "paths": paths, - "components": {"schemas": unique.get("components", {}).get("schemas", {})}, + "paths": { + path: operations for path, entry in paths.items() if (operations := as_str_mapping(entry)) is not None + }, + "components": {"schemas": as_str_mapping(components.get("schemas", {})) or {}}, } diff --git a/litellm/proxy/a2a/discovery.py b/litellm/proxy/a2a/discovery.py index ff58c9c85ec..03d1be7d185 100644 --- a/litellm/proxy/a2a/discovery.py +++ b/litellm/proxy/a2a/discovery.py @@ -16,10 +16,13 @@ fetcher dispatches by ``discovery_mode``: from collections.abc import Mapping from enum import Enum -from typing import Any, Final +from typing import Final from urllib.parse import urlencode +from pydantic import TypeAdapter + from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.custom_http import httpxSpecialProvider @@ -46,6 +49,8 @@ AGENT_CARD_WELL_KNOWN_PATHS: Final[tuple[str, ...]] = ( DEFAULT_DISCOVERY_TIMEOUT_SECONDS: Final = 10.0 +_DECODED_JSON: Final = TypeAdapter(object) + class AgentCardDiscoveryError(Exception): """Raised when none of the well-known paths returned a usable agent card.""" @@ -87,7 +92,7 @@ async def fetch_well_known_card( params: Mapping[str, object] | None = None, timeout: float = DEFAULT_DISCOVERY_TIMEOUT_SECONDS, headers: dict[str, str] | None = None, -) -> dict[str, Any]: +) -> dict[str, object]: """ Fetch an agent card from ``base_url`` using the strategy chosen by ``discovery_mode``. Returns the parsed JSON from the first path that @@ -137,13 +142,14 @@ async def fetch_well_known_card( continue try: - card = response.json() + decoded = _DECODED_JSON.validate_python(response.json()) except Exception as exc: last_error = f"{url}: invalid JSON ({exc})" continue + card = as_str_mapping(decoded) if not isinstance(card, dict): - last_error = f"{url}: expected JSON object, got {type(card).__name__}" + last_error = f"{url}: expected JSON object, got {type(decoded).__name__}" continue verbose_proxy_logger.debug("A2A discovery succeeded at %s", url) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index c11d296837e..4691c7b0bc5 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -23,6 +23,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header from litellm.proxy._types import UserAPIKeyAuth @@ -281,7 +282,7 @@ async def _a2a_sse_event_source( request_id: str | int | None = None, extra_headers: Mapping[str, str] | None = None, served_version: A2AVersion = "0.3", -) -> AsyncGenerator[Mapping[str, object], None]: +) -> AsyncGenerator[object, None]: """Stream an upstream A2A SSE response as parsed JSON-RPC event dicts. Upstream HTTP/JSON-RPC errors are surfaced as a single JSON-RPC error event @@ -327,12 +328,14 @@ async def _a2a_sse_event_source( if not payload: continue try: - event = json.loads(payload) + event = _DECODED_JSON.validate_python(json.loads(payload)) except Exception: continue - if isinstance(event, dict): - event = normalize_stream_event(event, served_version, request_id=request_id) - yield event + event_fields = as_str_mapping(event) + if isinstance(event_fields, dict): + yield normalize_stream_event(event_fields, served_version, request_id=request_id) + else: + yield event finally: await resp.aclose() @@ -741,7 +744,7 @@ async def invoke_agent_a2a( # Get backend URL and agent name agent_card_params: Final = agent.agent_card_params or {} agent_url: Final = agent_card_params.get("url") - agent_name: Final = agent_card_params.get("name", agent_id) + agent_name: Final = (as_str_mapping(agent_card_params) or {}).get("name", agent_id) # Get litellm_params (may include custom_llm_provider for completion bridge) litellm_params: dict[str, object] = agent.litellm_params or {} diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 9a6f674db2c..fb727367caf 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -18,6 +18,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.handler import ( bridge_model_name, ) from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints.databricks_oauth import ( resolve_databricks_backend_auth, @@ -173,13 +174,14 @@ async def route_a2a_agent_request( return _call_route(route_type, _bridge_agent_request(data, agent, bridge_auth.litellm_params)) # Get API base URL from agent config - if not agent.agent_card_params or "url" not in agent.agent_card_params: + card_params: Final = as_str_mapping(agent.agent_card_params) + if not card_params or "url" not in card_params: verbose_proxy_logger.error("[A2A] Agent '%s' has no URL configured", agent_name) route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) raise ProxyModelNotFoundError(route=route_name, model_name=model_name, retryable_with_model_read_through=False) # Inject API base and route to litellm - data["api_base"] = agent.agent_card_params["url"] + data["api_base"] = card_params["url"] verbose_proxy_logger.debug("[A2A] Routing %s to %s", model_name, data["api_base"]) return _call_route(route_type, data) diff --git a/litellm/proxy/agent_endpoints/databricks_oauth.py b/litellm/proxy/agent_endpoints/databricks_oauth.py index 7425c240d10..bebba009d82 100644 --- a/litellm/proxy/agent_endpoints/databricks_oauth.py +++ b/litellm/proxy/agent_endpoints/databricks_oauth.py @@ -48,6 +48,7 @@ _DEFAULT_SCOPE: Final = "all-apis" _TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60 _DEFAULT_TTL_SECONDS: Final = 3600 _OAUTH_BLOCK: Final = TypeAdapter(Mapping[str, object]) +_TOKEN_RESPONSE_BODY: Final = TypeAdapter(object) def _resolve_secret(value: object) -> str | None: @@ -275,7 +276,7 @@ class DatabricksAppOAuthTokenCache(InMemoryCache): except httpx.HTTPError as exc: raise ValueError(f"Databricks App OAuth token request failed: {exc}") from exc - body: Final[object] = response.json() + body: Final = _TOKEN_RESPONSE_BODY.validate_python(response.json()) if not isinstance(body, dict): raise ValueError( f"Databricks App OAuth token response returned non-object JSON (got {type(body).__name__})" diff --git a/litellm/proxy/agent_endpoints/managed_identity.py b/litellm/proxy/agent_endpoints/managed_identity.py index abab21901ee..fab7ebbc61d 100644 --- a/litellm/proxy/agent_endpoints/managed_identity.py +++ b/litellm/proxy/agent_endpoints/managed_identity.py @@ -119,6 +119,10 @@ def managed_write_fields( return AgentIdentityFailure(message=f"Invalid agent identity configuration: {exc}") +def _identity_fields(identity: EntraIdentityConfig) -> Mapping[str, object]: + return identity.model_dump() + + def _identity_write(identity: EntraIdentityConfig | None, existing: AgentResponse | None) -> ManagedWriteFields: if identity is None: unbind: Final[ManagedWriteFields] = { @@ -134,7 +138,7 @@ def _identity_write(identity: EntraIdentityConfig | None, existing: AgentRespons existing and existing.identity and existing.identity.active - and all(getattr(existing.identity, name) == value for name, value in identity.model_dump().items()) + and all(getattr(existing.identity, name) == value for name, value in _identity_fields(identity).items()) ): unchanged: Final[ManagedWriteFields] = {} return unchanged diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 8e348423837..b0088d93112 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -21,6 +21,7 @@ import json import re from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Annotated, Final, Protocol, TypedDict, get_args from urllib.parse import urlsplit @@ -299,7 +300,7 @@ def _validate_installation_preference(spec: PluginSpec) -> None: def _build_plugin_manifest(name: str, spec: PluginSpec) -> Mapping[str, object]: """Build the stored manifest dict shared by plugin create and update.""" - dumped: Final[Mapping[str, object]] = spec.model_dump(exclude_none=True) + dumped: Final[Mapping[str, object]] = MappingProxyType(spec.model_dump(exclude_none=True)) return {"name": name, **{key: value for key, value in dumped.items() if value and key != "name"}} diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..2c6768b8423 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -393,7 +393,7 @@ def _member_allowed_models(membership: _TeamMembershipModelScope) -> Sequence[st class _RawCacheRead(Protocol): - async def async_get_cache(self, *, key: str) -> object: ... + async def async_get_cache(self, *, key: str, local_only: bool = ...) -> object: ... def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead: @@ -2712,7 +2712,7 @@ async def get_team_membership( use_writer=True, ) - l1_cached: Final[object] = await user_api_key_cache.async_get_cache(key=_key, local_only=True) + l1_cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=_key, local_only=True) l1_membership: Final = _membership_from_cached_payload(l1_cached) if not isinstance(l1_membership, _TeamMembershipCacheMiss): return l1_membership diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a186d84aa5..25f900a5f9f 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -924,8 +924,9 @@ def get_request_route(request: Request) -> str: scope: Final = request.scope if not isinstance(scope, dict): return str(request.url.path) - raw_path: Final[str] = str(scope.get("path", request.url.path)) - root_path: Final[str] = str(scope.get("app_root_path", scope.get("root_path", ""))).rstrip("/") + scope_fields: Final[Mapping[str, object]] = MappingProxyType(scope) + raw_path: Final[str] = str(scope_fields.get("path", request.url.path)) + root_path: Final[str] = str(scope_fields.get("app_root_path", scope_fields.get("root_path", ""))).rstrip("/") if not isinstance(raw_path, str): return str(request.url.path) # Strip root_path only when it matches whole path segments — guarding @@ -1520,8 +1521,15 @@ def get_model_rate_limit_from_metadata( metadata_accessor_key: Literal["team_metadata", "organization_metadata", "project_metadata"], rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit", "model_itpm_limit", "model_otpm_limit"], ) -> dict[str, int] | None: - if getattr(user_api_key_dict, metadata_accessor_key): - return getattr(user_api_key_dict, metadata_accessor_key).get(rate_limit_key) + metadata: Final = ( + user_api_key_dict.team_metadata + if metadata_accessor_key == "team_metadata" + else user_api_key_dict.organization_metadata + if metadata_accessor_key == "organization_metadata" + else user_api_key_dict.project_metadata + ) + if metadata: + return metadata.get(rate_limit_key) return None diff --git a/litellm/proxy/auth/ip_address_utils.py b/litellm/proxy/auth/ip_address_utils.py index f1ca2c70f3b..0b349e6a733 100644 --- a/litellm/proxy/auth/ip_address_utils.py +++ b/litellm/proxy/auth/ip_address_utils.py @@ -283,7 +283,7 @@ class IPAddressUtils: @staticmethod def get_mcp_client_ip( request: Request, - general_settings: dict[str, Any] | None = None, + general_settings: dict[str, object] | None = None, ) -> str | None: """ Extract client IP from a FastAPI request for MCP access control. @@ -371,4 +371,4 @@ class IPAddressUtils: return client_ip case _HopCountUnset(): pass - return get_request_ip_address(request, use_x_forwarded_for=use_xff) + return get_request_ip_address(request, use_x_forwarded_for=use_xff is True) diff --git a/litellm/proxy/auth/rds_iam_token.py b/litellm/proxy/auth/rds_iam_token.py index a9f349d1750..e291c78294c 100644 --- a/litellm/proxy/auth/rds_iam_token.py +++ b/litellm/proxy/auth/rds_iam_token.py @@ -3,7 +3,7 @@ import re from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Final, Protocol import httpx @@ -223,11 +223,15 @@ def rds_region_from_hostname(db_host: str) -> str | None: return match.group("region").lower() +class _RdsAuthTokenClient(Protocol): + def generate_db_auth_token(self, *, DBHostname: str, Port: str, DBUsername: str, Region: str | None) -> str: ... + + def generate_iam_auth_token( db_host: str, db_port: str, db_user: str, - client: Any | None = None, + client: _RdsAuthTokenClient | None = None, *, region: str | None = None, ) -> str: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 720bfabb95a..3ae671d40ac 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -160,7 +160,7 @@ class RouteChecks: if allowed_route in LiteLLMRoutes._member_names_: if RouteChecks.check_route_access( route=route, - allowed_routes=LiteLLMRoutes._member_map_[allowed_route].value, + allowed_routes=LiteLLMRoutes[allowed_route].value, ): if ( allowed_route in _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 27fa4731baf..b1e8bdbe787 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -13,6 +13,7 @@ import re import secrets from collections.abc import Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import Final, NamedTuple, Protocol, Union, cast import fastapi @@ -177,6 +178,7 @@ from litellm.proxy.utils import ( normalize_route_for_root_path, ) from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository +from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model from litellm.router_utils.fallback_event_handlers import ( @@ -1319,7 +1321,7 @@ async def fetch_global_spend_with_event_coordination( async def _load_global_spend() -> float | None: proxy_budget_row: Final = await bounded_db_lookup( - prisma_client.db.litellm_usertable.find_unique(where={"user_id": LITELLM_PROXY_BUDGET_NAME}), + UserRepository(prisma_client).table.find_unique(where={"user_id": LITELLM_PROXY_BUDGET_NAME}), name="proxy_budget", ) return float(proxy_budget_row.spend) if proxy_budget_row is not None else None @@ -1635,7 +1637,7 @@ async def _auto_register_jwt_mapping( if minted: try: async with db_span("delete_orphaned_jwt_key", "LiteLLM_VerificationToken"): - await prisma_client.db.litellm_verificationtoken.delete(where={"token": token_hash}) + await VerificationTokenRepository(prisma_client).table.delete(where={"token": token_hash}) except Exception as delete_err: # Don't fail the request if cleanup fails — the orphan is # unmapped and inert. Log so an operator can prune it later. @@ -2693,7 +2695,7 @@ async def user_api_key_auth_builder( and valid_token.last_refreshed_at is not None and team_obj.last_refreshed_at > valid_token.last_refreshed_at ): - team_obj_dict: Final = team_obj.__dict__ + team_obj_dict: Final[Mapping[str, object]] = MappingProxyType(team_obj.__dict__) for k, v in team_obj_dict.items(): field_name = f"team_{k}" diff --git a/litellm/proxy/caching_routes.py b/litellm/proxy/caching_routes.py index 597080be3b3..775d00ed53e 100644 --- a/litellm/proxy/caching_routes.py +++ b/litellm/proxy/caching_routes.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request @@ -19,6 +20,10 @@ router: Final = APIRouter( ) +def _copy_cache_params(cache_params: Mapping[str, object]) -> Mapping[str, object]: + return dict(**cache_params) + + def _extract_cache_params() -> dict[str, object]: """ Safely extracts and cleans cache parameters. @@ -41,7 +46,7 @@ def _extract_cache_params() -> dict[str, object]: try: cache_params: Final = vars(litellm.cache.cache) cleaned_params: Final = ( - HealthCheckCacheParams.model_validate(dict(**cache_params)).model_dump() if cache_params else {} + HealthCheckCacheParams.model_validate(_copy_cache_params(cache_params)).model_dump() if cache_params else {} ) return masker.mask_dict(cleaned_params) except (AttributeError, TypeError) as e: diff --git a/litellm/proxy/client/chat.py b/litellm/proxy/client/chat.py index c027bf2fe98..ccdef44f9e8 100644 --- a/litellm/proxy/client/chat.py +++ b/litellm/proxy/client/chat.py @@ -59,7 +59,7 @@ class ChatClient: frequency_penalty: float | None = None, user: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a chat completion. @@ -184,7 +184,8 @@ class ChatClient: response.raise_for_status() # Parse SSE stream - for line in response.iter_lines(): + lines: Final[Iterator[object]] = response.iter_lines() + for line in lines: if line: line = _SSE_LINE.validate_python(line).decode("utf-8") if line.startswith("data: "): diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index ef73b4f0d2d..61c8db617af 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -4,9 +4,14 @@ from typing import Final import click import rich +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping from ...http_client import HTTPClient -from .auth import cli_context_obj +from ._cli_context import cli_context_values + +_DECODED_JSON: Final = TypeAdapter(object) @click.group() @@ -40,23 +45,32 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool): lite encryption migrate --check # attestation scan, no writes lite encryption migrate # perform the migration """ - client: Final = HTTPClient(cli_context_obj(ctx)["base_url"], ctx.obj["api_key"]) + context: Final = cli_context_values(ctx) + client: Final = HTTPClient(context["base_url"], context["api_key"]) - if check_only: - response = client.request("GET", "/credentials/migrate-encryption/check") - else: - response = client.request( + response: Final = _DECODED_JSON.validate_python( + client.request("GET", "/credentials/migrate-encryption/check") + if check_only + else client.request( "POST", "/credentials/migrate-encryption", json={}, params={"dry_run": "true"} if dry_run else None, ) + ) rich.print_json(data=response) - report: Final = response.get("report", {}) if isinstance(response, dict) else {} + report_value: Final = (as_str_mapping(response) or {}).get("report", {}) + report: Final = as_str_mapping(report_value) + if report is None: + raise click.ClickException(f"Unexpected report in the proxy response: {report_value!r}") residual: Final = report.get("residual_legacy") - if residual is not None and residual > 0: + if residual is None: + return + if not isinstance(residual, int | float): + raise click.ClickException(f"Unexpected residual_legacy in the proxy response: {residual!r}") + if residual > 0: rich.print(f"[yellow]Residual legacy values remaining: {residual}[/yellow]") elif residual == 0: rich.print("[green]No legacy values remaining (residual_legacy == 0).[/green]") diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py index 5aeefa0c739..fef3e313ee5 100644 --- a/litellm/proxy/client/cli/commands/statusline_script.py +++ b/litellm/proxy/client/cli/commands/statusline_script.py @@ -89,6 +89,28 @@ class Fetch(Protocol): def __call__(self, credentials: Credentials, session_id: str) -> Fetched: ... +class _ResponseBody(Protocol): + def read(self) -> bytes: ... + + +class _OpenedResponse(Protocol): + def __enter__(self) -> _ResponseBody: ... + + def __exit__(self, *exc_info: object) -> None: ... + + +class _UrlOpener(Protocol): + def __call__(self, url: urllib.request.Request, /, *, timeout: float) -> _OpenedResponse: ... + + +def _url_opener() -> _UrlOpener: + return urllib.request.urlopen + + +def _json_decoder() -> Callable[[bytes | str], object]: + return json.loads + + def as_mapping(value: object) -> Mapping[str, object]: return value if isinstance(value, dict) else EMPTY @@ -106,7 +128,7 @@ def printable(value: object) -> str: def load_json(raw: bytes | str) -> object: try: - return json.loads(raw) + return _json_decoder()(raw) except ValueError: return None @@ -195,7 +217,7 @@ def fetch_session(credentials: Credentials, session_id: str) -> Fetched: }, ) try: - with urllib.request.urlopen(request, timeout=FETCH_TIMEOUT_SECONDS) as response: + with _url_opener()(request, timeout=FETCH_TIMEOUT_SECONDS) as response: raw: Final[bytes] = response.read() except urllib.error.HTTPError as error: return Fetched(session=None, definitive=400 <= error.code < 500) diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py index f7c545ef48e..7c01732119e 100644 --- a/litellm/proxy/client/cli/interface.py +++ b/litellm/proxy/client/cli/interface.py @@ -189,7 +189,7 @@ def execute_command(path: ShellScope, ctx: click.Context, cli: click.Group) -> N try: if path[0].startswith("-"): cli.main( - ["--base-url", ctx.obj["base_url"], *path], + ["--base-url", cli_context_obj(ctx)["base_url"], *path], prog_name="lite", obj={"secret_vault": context_secret_vault(ctx)}, standalone_mode=False, diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 3f1e7f4259c..a1964d8a1bb 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Iterator, Mapping, MutableMapping from typing import Final, Literal, Protocol import click +from pydantic import TypeAdapter from litellm._version import version as litellm_version from litellm.litellm_core_utils.cli_token_utils import load_cli_token @@ -89,10 +90,13 @@ class LazyCommands(MutableMapping[str, click.Command]): return len(self._loaders) +_MODULE_ATTRIBUTE: Final = TypeAdapter(object) + + def _command_in(module: str, attribute: str) -> Callable[[], click.Command]: @functools.cache def load() -> click.Command: - match getattr(importlib.import_module(module, __package__), attribute): + match _MODULE_ATTRIBUTE.validate_python(getattr(importlib.import_module(module, __package__), attribute)): case click.Command() as command: return command case _: diff --git a/litellm/proxy/client/credentials.py b/litellm/proxy/client/credentials.py index ebcd79457a6..649fe529936 100644 --- a/litellm/proxy/client/credentials.py +++ b/litellm/proxy/client/credentials.py @@ -75,7 +75,7 @@ class CredentialsManagementClient: credential_values: Mapping[str, object], return_request: bool = False, display_name: str | None = None, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Create a new credential. @@ -121,7 +121,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Delete a credential by name. @@ -182,7 +182,7 @@ class CredentialsManagementClient: self, credential_name: str, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get a credential by name. diff --git a/litellm/proxy/client/http_client.py b/litellm/proxy/client/http_client.py index aa0b986b1ad..96a170c75cb 100644 --- a/litellm/proxy/client/http_client.py +++ b/litellm/proxy/client/http_client.py @@ -4,6 +4,11 @@ from collections.abc import Mapping from typing import Any, Final import requests +from typing_extensions import TypedDict, Unpack + + +class _RequestOptions(TypedDict, total=False, extra_items=object): + pass class HTTPClient: @@ -29,7 +34,7 @@ class HTTPClient: data: Mapping[str, object] | list | bytes | None = None, json: Mapping[str, object] | list | None = None, headers: dict[str, str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_RequestOptions], ) -> Any: """Make an HTTP request to the LiteLLM proxy server. diff --git a/litellm/proxy/client/model_groups.py b/litellm/proxy/client/model_groups.py index fef307600c4..1555d170212 100644 --- a/litellm/proxy/client/model_groups.py +++ b/litellm/proxy/client/model_groups.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final import requests @@ -31,7 +31,7 @@ class ModelGroupsManagementClient: headers["Authorization"] = f"Bearer {self._api_key}" return headers - def info(self, return_request: bool = False) -> list[dict[str, Any]] | requests.Request: + def info(self, return_request: bool = False) -> list[dict[str, object]] | requests.Request: """ Get detailed information about all model groups from the server. diff --git a/litellm/proxy/client/models.py b/litellm/proxy/client/models.py index 1b50f8c24f3..7342974c0cc 100644 --- a/litellm/proxy/client/models.py +++ b/litellm/proxy/client/models.py @@ -158,7 +158,7 @@ class ModelsManagementClient: model_id: str | None = None, model_name: str | None = None, return_request: bool = False, - ) -> dict[str, Any] | requests.Request: + ) -> dict[str, object] | requests.Request: """ Get information about a specific model by its ID or name. diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index 40cdb117476..b2640d2a765 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -1,11 +1,15 @@ import asyncio import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import asdict, dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy.common_utils.config_sync_pubsub import ( # noqa: F401 # legacy module exports ConfigSyncPubSub, _ConfigSyncPubSub, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -30,6 +34,7 @@ _pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong _in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES) _BACKOFF_INITIAL_SECONDS: Final = 5.0 _BACKOFF_MAX_SECONDS: Final = 60.0 +_DECODED_JSON: Final = TypeAdapter(object) def auth_cache_invalidation_channel(redis_cache: "RedisCache") -> str: @@ -46,7 +51,9 @@ class _CacheInvalidationMessage: def _cache_invalidation_message_json(cache_key: str, new_value: float | None = None, ttl: float | None = None) -> str: - message: Final = asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + message: Final[Mapping[str, object]] = MappingProxyType( + asdict(_CacheInvalidationMessage(cache_key=cache_key, new_value=new_value, ttl=ttl)) + ) return json.dumps({field: value for field, value in message.items() if value is not None}) @@ -62,10 +69,10 @@ def _message_from_data(data: object) -> _CacheInvalidationMessage | None: if not isinstance(data, str): return None try: - parsed: Final = json.loads(data) + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(data))) except json.JSONDecodeError: return None - if not isinstance(parsed, dict): + if parsed is None: return None cache_key: Final = parsed.get("cache_key") if not isinstance(cache_key, str): diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 46434a997b7..f889748c504 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from itertools import accumulate from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -39,6 +40,7 @@ from litellm.types.utils import ( ) _CALLBACK_VAR_MASKER: Final = SensitiveDataMasker() +_LOADED_CALLBACK: Final = TypeAdapter(object) # Compound names that are credential-bearing but don't contain any of the # default sensitive segments (so SensitiveDataMasker won't flag them). _EXTRA_SENSITIVE_CALLBACK_KEYS: Final = {"gcs_path_service_account"} @@ -389,9 +391,11 @@ def initialize_callbacks_on_proxy( imported_list.append( _loaded_callback_or_raise( entry=callback, - loaded=get_instance_fn( - value=callback, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=callback, + config_file_path=config_file_path, + ) ), ) ) @@ -408,9 +412,11 @@ def initialize_callbacks_on_proxy( litellm.callbacks = [ _loaded_callback_or_raise( entry=value, - loaded=get_instance_fn( - value=value, - config_file_path=config_file_path, + loaded=_LOADED_CALLBACK.validate_python( + get_instance_fn( + value=value, + config_file_path=config_file_path, + ) ), ) ] diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 33633150711..ec4baa76514 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -11,12 +11,14 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import TypeAdapter from typing_extensions import ReadOnly from litellm import get_secret_str from litellm._logging import verbose_proxy_logger from litellm.constants import PYTHON_GC_THRESHOLD from litellm.litellm_core_utils.bug_report import EnvironmentReport +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.bug_report_config import build_proxy_environment_report @@ -26,6 +28,7 @@ if TYPE_CHECKING: from litellm.router import Router router: Final = APIRouter() +_DECODED_JSON: Final = TypeAdapter(object) # Configure garbage collection thresholds from environment variables @@ -885,8 +888,8 @@ def init_verbose_loggers(): return if os.path.isfile(worker_config): return - _settings: Final = json.loads(worker_config) - if not isinstance(_settings, dict): + _settings: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(worker_config))) + if _settings is None: return debug: Final = _settings.get("debug", None) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 245a56678eb..e34d03a04c3 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -164,7 +164,7 @@ def coerce_numeric_form_fields( def _parse_binary_body(body: bytes) -> dict: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = orjson.loads(body) + parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) if isinstance(parsed, dict): return parsed except orjson.JSONDecodeError: diff --git a/litellm/proxy/common_utils/load_config_utils.py b/litellm/proxy/common_utils/load_config_utils.py index 4a082eb307b..640a947f9c5 100644 --- a/litellm/proxy/common_utils/load_config_utils.py +++ b/litellm/proxy/common_utils/load_config_utils.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase _BUCKET_CONFIG_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_YAML: Final = TypeAdapter(object) class BucketObjectFetcher(Protocol): @@ -31,7 +32,7 @@ class SyncBucketObjectReader(Protocol): def _parsed_config(object_key: str, file_contents: str) -> object | None: try: - parsed: Final = yaml.safe_load(file_contents) + parsed: Final = _DECODED_YAML.validate_python(yaml.safe_load(file_contents)) except yaml.YAMLError as e: verbose_proxy_logger.error("Config object %s is not valid YAML: %s", object_key, e) return None diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 089d0b7a226..02da6f706d1 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -37,7 +37,7 @@ This module provides a single proxy-side error class that: import json from collections.abc import Mapping -from typing import Any, Final +from typing import Final from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError @@ -171,7 +171,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): def __init__( self, - detail: Any, + detail: object, headers: Mapping[str, object] | None = None, category: str | RateLimitErrorCategory = RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type: str | RateLimitType | None = None, diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py index 35bd69ef0bd..a2ffdd4d59d 100644 --- a/litellm/proxy/container_endpoints/handler_factory.py +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -13,6 +13,7 @@ from typing import Final from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -26,6 +27,8 @@ from litellm.proxy.container_endpoints.ownership import ( get_container_forwarding_params, ) +_UPLOADED_FILE_FIELD: Final = TypeAdapter(object) + def _load_endpoints_config() -> dict: """Load the endpoints configuration from JSON file.""" @@ -304,7 +307,7 @@ async def _process_multipart_upload_request( raise HTTPException(status_code=400, detail="Missing required 'file' field") # convert_upload_files_to_file_data returns list of tuples, extract single file - file_list: Final = data["file"] + file_list: Final = _UPLOADED_FILE_FIELD.validate_python(data["file"]) if isinstance(file_list, list) and len(file_list) > 0: data["file"] = file_list[0] diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py index aeb0bb1e3d0..9e4ab3c3c4e 100644 --- a/litellm/proxy/container_endpoints/ownership.py +++ b/litellm/proxy/container_endpoints/ownership.py @@ -1,13 +1,14 @@ import json from collections.abc import Awaitable, Callable, Mapping, Sequence from collections.abc import Set as AbstractSet -from typing import TYPE_CHECKING, Any, Final, TypeAlias +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from fastapi import HTTPException from pydantic import BaseModel from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.core_helpers import has_attributes from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.resource_ownership import ( get_primary_resource_owner_scope, @@ -111,12 +112,20 @@ def _get_response_id(response: object) -> str | None: return value if isinstance(value, str) else None -def _dump_response(response: Any) -> dict[str, object]: +class _SupportsModelDump(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + +class _SupportsPydanticDict(Protocol): + def dict(self) -> Mapping[str, object]: ... + + +def _dump_response(response: object) -> Mapping[str, object]: if isinstance(response, dict): return dict(response) - if hasattr(response, "model_dump"): + if has_attributes(response, _SupportsModelDump, "model_dump"): return response.model_dump() - if hasattr(response, "dict"): + if has_attributes(response, _SupportsPydanticDict, "dict"): return response.dict() return {"id": _get_response_id(response)} @@ -199,9 +208,11 @@ async def record_container_owner( original_container_id, resolved_provider = decode_container_id_for_ownership(container_id, custom_llm_provider) model_object_id: Final = _container_model_object_id(original_container_id, resolved_provider) - file_object: Final = _dump_response(response) - file_object["custom_llm_provider"] = resolved_provider - file_object["provider_container_id"] = original_container_id + file_object: Final = { + **_dump_response(response), + "custom_llm_provider": resolved_provider, + "provider_container_id": original_container_id, + } # Prisma Python requires Json fields to be serialized as a JSON string. file_object_json: Final[str] = json.dumps(file_object) diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 75882bc3fb1..4ddcf7e600b 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -526,7 +526,7 @@ async def _upsert_turn_with_retry( except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise - await asyncio.sleep(2**attempt + random.uniform(0, 1)) + await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) else: return diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index be5b307b5ae..b290e0f85b4 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -16,6 +16,7 @@ from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mappi from contextlib import asynccontextmanager from contextvars import ContextVar from datetime import datetime, timedelta, timezone +from functools import reduce from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overload from urllib.parse import quote, unquote @@ -104,6 +105,7 @@ else: RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value}) _SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARG_VALUE: Final = TypeAdapter(object) _JsonPayload: TypeAlias = Mapping[str, object] | Sequence[object] | str | int | float | bool | None _JSON_VALUE_ADAPTER: Final[TypeAdapter[_JsonPayload]] = TypeAdapter( _JsonPayload, config=ConfigDict(strict=True, hide_input_in_errors=True) @@ -313,6 +315,20 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +class _SpendRestore(TypedDict, total=False): + db_spend_update_transactions: ReadOnly[DBSpendUpdateTransactions] + daily_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_team_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_org_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_end_user_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + daily_agent_spend_update_transactions: ReadOnly[Mapping[str, BaseDailySpendTransaction]] + window_spend_update_transactions: ReadOnly[Sequence[WindowSpendTransaction]] + + +def _merged_spend_restore(first: _SpendRestore, second: _SpendRestore) -> _SpendRestore: + return {**first, **second} + + def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None: spend_log_error( "Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s", @@ -845,7 +861,7 @@ class DBSpendUpdateWriter: spend=payload["spend"], total_tokens=payload["total_tokens"], completion_response=completion_response, - realtime_tool_calls=(kwargs or {}).get("realtime_tool_calls"), + realtime_tool_calls=_CALL_KWARG_VALUE.validate_python((kwargs or {}).get("realtime_tool_calls")), ) if transaction is None: return @@ -1627,7 +1643,7 @@ class DBSpendUpdateWriter: ): verbose_proxy_logger.debug("acquired lock for spend updates") - uncommitted: dict[str, Any] = {} # mutable-ok: tracks popped categories still needing commit + uncommitted: dict[str, _SpendRestore] = {} # mutable-ok: tracks popped categories still needing commit committed_spend_tables: Final[list[_SpendTableName]] = [] # mutable-ok: filled as each table lands try: @@ -1642,13 +1658,41 @@ class DBSpendUpdateWriter: ) = await self.redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() uncommitted = { - "db_spend_update_transactions": db_spend_update_transactions, - "daily_spend_update_transactions": daily_spend_update_transactions, - "daily_team_spend_update_transactions": daily_team_spend_update_transactions, - "daily_org_spend_update_transactions": daily_org_spend_update_transactions, - "daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions, - "daily_agent_spend_update_transactions": daily_agent_spend_update_transactions, - "window_spend_update_transactions": window_spend_update_transactions, + "db_spend_update_transactions": ( + {} + if db_spend_update_transactions is None + else {"db_spend_update_transactions": db_spend_update_transactions} + ), + "daily_spend_update_transactions": ( + {} + if daily_spend_update_transactions is None + else {"daily_spend_update_transactions": daily_spend_update_transactions} + ), + "daily_team_spend_update_transactions": ( + {} + if daily_team_spend_update_transactions is None + else {"daily_team_spend_update_transactions": daily_team_spend_update_transactions} + ), + "daily_org_spend_update_transactions": ( + {} + if daily_org_spend_update_transactions is None + else {"daily_org_spend_update_transactions": daily_org_spend_update_transactions} + ), + "daily_end_user_spend_update_transactions": ( + {} + if daily_end_user_spend_update_transactions is None + else {"daily_end_user_spend_update_transactions": daily_end_user_spend_update_transactions} + ), + "daily_agent_spend_update_transactions": ( + {} + if daily_agent_spend_update_transactions is None + else {"daily_agent_spend_update_transactions": daily_agent_spend_update_transactions} + ), + "window_spend_update_transactions": ( + {} + if window_spend_update_transactions is None + else {"window_spend_update_transactions": window_spend_update_transactions} + ), } if db_spend_update_transactions is not None: @@ -1677,9 +1721,12 @@ class DBSpendUpdateWriter: on_table_committed=committed_spend_tables.append, ) except Exception as e: - uncommitted["db_spend_update_transactions"] = _spend_tables_left_to_send( + left_to_send: Final = _spend_tables_left_to_send( db_spend_update_transactions, committed_spend_tables, e ) + uncommitted["db_spend_update_transactions"] = ( + {} if left_to_send is None else {"db_spend_update_transactions": left_to_send} + ) raise uncommitted.pop("db_spend_update_transactions", None) @@ -1753,7 +1800,7 @@ class DBSpendUpdateWriter: exc=e, ) finally: - to_restore = {name: txns for name, txns in uncommitted.items() if txns is not None} + to_restore: Final = reduce(_merged_spend_restore, uncommitted.values(), _SpendRestore()) if to_restore: await self.redis_update_buffer.restore_transactions_to_redis(**to_restore) await self.pod_lock_manager.release_lock( diff --git a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py index 627b30e83f7..1116cd604a3 100644 --- a/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py +++ b/litellm/proxy/db/db_transaction_queue/pod_lock_manager.py @@ -1,7 +1,7 @@ import asyncio import json import logging -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, TypeAlias from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -14,7 +14,9 @@ from litellm.types.services import ServiceTypes POD_LOCK_TARGET: Final = "pod_lock" if TYPE_CHECKING: - ProxyLogging = Any + from litellm.proxy.utils import ProxyLogging as _ProxyLogging + + ProxyLogging: TypeAlias = _ProxyLogging else: ProxyLogging = Any diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 54d20bd4507..66833fd688a 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -11,6 +11,7 @@ from datetime import datetime from functools import reduce from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from pydantic import TypeAdapter from redis.exceptions import RedisError from litellm._internal_context import with_service_target @@ -98,6 +99,7 @@ _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = ( ) _ValueT = TypeVar("_ValueT") +_DECODED_JSON: Final = TypeAdapter(object) def _spend_log_json_default(value: object) -> str: @@ -109,7 +111,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str: def _decode_spend_log_row(encoded: str) -> dict[str, object] | None: - decoded: Final = json.loads(encoded) + decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded)) return decoded if isinstance(decoded, dict) else None diff --git a/litellm/proxy/db/prisma_query_span.py b/litellm/proxy/db/prisma_query_span.py index 75625c6c0f2..2747a765f2a 100644 --- a/litellm/proxy/db/prisma_query_span.py +++ b/litellm/proxy/db/prisma_query_span.py @@ -20,6 +20,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from litellm.integrations.otel.model.spans import PG_CATALOG, PRISMA_RELATIONS @@ -34,6 +36,7 @@ class PrismaQuery: UNKNOWN_PRISMA_QUERY: Final = PrismaQuery("prisma_query", None, None) +_DECODED_JSON: Final = TypeAdapter(object) _ROOT_FIELD: Final = re.compile(r"result:\s*(\w+)") _RAW_SQL: Final = re.compile(r'query:\s*"((?:[^"\\]|\\.)*)"') @@ -111,7 +114,7 @@ def sql_operation(sql: str) -> tuple[str | None, str | None]: def _query_text(content: str) -> str: try: - payload: Final[object] = json.loads(content) + payload: Final = _DECODED_JSON.validate_python(json.loads(content)) except ValueError: return content query: Final = payload.get("query") if isinstance(payload, dict) else None diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py index 673b549b645..3214c0974b4 100644 --- a/litellm/proxy/db/proxy_worker_heartbeat.py +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -56,6 +56,7 @@ class _LiveWorkerCountRow(TypedDict): _COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...]) +_RAW_QUERY_RESULT: Final = TypeAdapter(object) class _HeartbeatExecute(Protocol): @@ -99,7 +100,9 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None: db: Final = prisma_client.db primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"): - rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + rows: Final = _RAW_QUERY_RESULT.validate_python( + await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + ) return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"] except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503 verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err) diff --git a/litellm/proxy/db/routing_prisma_wrapper.py b/litellm/proxy/db/routing_prisma_wrapper.py index 48ad57fd06e..52a0be3d49e 100644 --- a/litellm/proxy/db/routing_prisma_wrapper.py +++ b/litellm/proxy/db/routing_prisma_wrapper.py @@ -298,7 +298,7 @@ class RoutingPrismaWrapper: def __getattr__(self, name: str) -> Any: if name in _TOP_LEVEL_READ_METHODS: - return getattr(self.read_target, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self.read_target, name)) writer_attr: Final = _DELEGATED_ATTRIBUTE.validate_python(getattr(self._writer, name)) # Per-model action accessors are non-callable instances that expose # both `find_many` and `create`. Methods like execute_raw / batch_ / diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 7ac414bec00..591ad81a4e5 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -19,7 +19,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import chain, groupby -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol +from typing import TYPE_CHECKING, Final, Literal, Protocol from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import LiteralString @@ -122,7 +122,7 @@ def build_tool_usage_transaction( spend: float, total_tokens: int, completion_response: object, - realtime_tool_calls: Any = None, + realtime_tool_calls: object = None, ) -> ToolUsageTransaction | None: """None when the request invoked no tools. Realtime sessions carry invoked tools in kwargs["realtime_tool_calls"] (OpenAI tool_calls shape) rather than diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py index 1f2fca3992e..94a0c3e2cf1 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_archive.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -16,6 +16,7 @@ from types import MappingProxyType from typing import Final import yaml +from pydantic import TypeAdapter MAX_ARCHIVE_UNPACKED_BYTES: Final = 50 * 1024 * 1024 MAX_ARCHIVE_ENTRIES: Final = 1000 @@ -26,6 +27,7 @@ _ZIP_ENTRY_PERMISSIONS: Final = 0o644 << 16 _FRONTMATTER_PATTERN: Final = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL) _WINDOWS_DRIVE_PATTERN: Final = re.compile(r"^[A-Za-z]:") _EMPTY_FRONTMATTER: Final[Mapping[str, object]] = MappingProxyType({}) +_DECODED_YAML: Final = TypeAdapter(object) @dataclass(frozen=True, slots=True) @@ -100,7 +102,7 @@ def _manifest_frontmatter(manifest: bytes) -> Mapping[str, object]: if match is None: return _EMPTY_FRONTMATTER try: - parsed: Final = yaml.safe_load(match.group(1)) + parsed: Final = _DECODED_YAML.validate_python(yaml.safe_load(match.group(1))) except yaml.YAMLError: return _EMPTY_FRONTMATTER if not isinstance(parsed, dict): diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py index 2050cc40e6d..830f7b0334e 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -10,7 +10,6 @@ import asyncio import re from collections.abc import Sequence from itertools import groupby -from operator import itemgetter from types import MappingProxyType from typing import Final @@ -173,11 +172,15 @@ def _slugify(raw: str | None) -> str | None: return _NON_SLUG_PATTERN.sub("-", raw.lower()).strip("-")[:MAX_SKILL_NAME_LENGTH].rstrip("-") or None +def _entry_name(entry: tuple[int, str]) -> str: + return entry[1] + + def _deduplicated(names: Sequence[str]) -> tuple[str, ...]: ordinals: Final = MappingProxyType( { position: ordinal - for _, duplicates in groupby(sorted(enumerate(names), key=itemgetter(1)), key=itemgetter(1)) + for _, duplicates in groupby(sorted(enumerate(names), key=_entry_name), key=_entry_name) for ordinal, (position, _) in enumerate(duplicates) } ) diff --git a/litellm/proxy/example_config_yaml/custom_auth.py b/litellm/proxy/example_config_yaml/custom_auth.py index b7646ce5e3d..c53074638c6 100644 --- a/litellm/proxy/example_config_yaml/custom_auth.py +++ b/litellm/proxy/example_config_yaml/custom_auth.py @@ -30,7 +30,7 @@ async def generate_key_fn(data: GenerateKeyRequest): data_json: Final = data.json() # Unpacking variables - team_id: Final = data_json.get("team_id") + team_id: Final = data.team_id data_json.get("duration") data_json.get("models") data_json.get("aliases") diff --git a/litellm/proxy/example_config_yaml/custom_callbacks.py b/litellm/proxy/example_config_yaml/custom_callbacks.py index 2c689811dc8..5ae55060930 100644 --- a/litellm/proxy/example_config_yaml/custom_callbacks.py +++ b/litellm/proxy/example_config_yaml/custom_callbacks.py @@ -7,10 +7,14 @@ import traceback sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path import inspect +from typing import Final + +from pydantic import TypeAdapter import litellm from litellm.integrations.custom_logger import CustomLogger -from typing import Final + +_ATTRIBUTE_VALUE: Final = TypeAdapter(object) # This file includes the custom callbacks for LiteLLM Proxy @@ -27,7 +31,11 @@ class MyCustomHandler(CustomLogger): print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger") try: print_verbose("Logger Initialized with following methods:") - methods: Final = [method for method in dir(self) if inspect.ismethod(getattr(self, method))] + methods: Final = [ + method + for method in dir(self) + if inspect.ismethod(_ATTRIBUTE_VALUE.validate_python(getattr(self, method))) + ] # Pretty print_verbose the methods for method in methods: diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 17bea031794..8581dca603b 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -9,8 +9,9 @@ every text fragment. """ from collections.abc import Callable, Iterator, Mapping, Sequence -from typing import Any, Final +from typing import Final +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, input_items, @@ -231,7 +232,7 @@ def is_string_batch_input(data: Mapping[str, object]) -> bool: return isinstance(input_value, list) and bool(input_value) and all(isinstance(item, str) for item in input_value) -def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequence[object]) -> bool: +def apply_redacted_messages_back(data: dict[str, object], redacted_messages: Sequence[object]) -> bool: """Write redacted messages back to whichever field(s) the caller used. Mask/anonymize paths take a synthesised messages list (from @@ -249,9 +250,9 @@ def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequen carry exactly one message per inspected element: a partial rewrite would forward the remaining originals unredacted. Callers must block on False. """ + input_value: Final = data.get("input") if is_string_batch_input(data): - batch: Final = data["input"] - inspected_indices: Final = tuple(idx for idx, item in enumerate(batch) if item) + inspected_indices: Final = tuple(idx for idx, item in enumerate(iter_items(input_value)) if item) if len(redacted_messages) != len(inspected_indices): return False if any(not isinstance(message, Mapping) or message.get("content") is None for message in redacted_messages): @@ -259,12 +260,12 @@ def apply_redacted_messages_back(data: dict[str, Any], redacted_messages: Sequen redacted_texts: Final = tuple( "\n".join(_iter_text_parts_in_content(message["content"])) for message in redacted_messages ) - for idx, text in zip(inspected_indices, redacted_texts): - batch[idx] = text + if isinstance(input_value, list): + for idx, text in zip(inspected_indices, redacted_texts): + input_value[idx] = text return True if "messages" in data: data["messages"] = redacted_messages - input_value: Final = data.get("input") if isinstance(input_value, str): text_parts: Final[list[str]] = [] for msg in redacted_messages: diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 70e7aac12b5..a5822bfb0e3 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -90,6 +90,7 @@ if TYPE_CHECKING: router: Final = APIRouter() GUARDRAIL_REGISTRY: Final = GuardrailRegistry() _DECODED_JSON: Final = TypeAdapter(object) +_DECODED_YAML: Final = TypeAdapter(object) def _as_str_object_mapping(mapping: Mapping[str, object]) -> Mapping[str, object]: @@ -1682,7 +1683,7 @@ async def validate_blocked_words_file(request: dict[str, str]): if not file_content: return {"valid": False, "error": "No file content provided"} - data: Final = yaml.safe_load(file_content) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(file_content)) if not isinstance(data, dict) or "blocked_words" not in data: return { diff --git a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py index ac0a6911184..aabac874ca2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py +++ b/litellm/proxy/guardrails/guardrail_hooks/agent_365/agent_365.py @@ -63,6 +63,7 @@ _INVALID_ASSERTION_AADSTS_PREFIX: Final = "50027" _AADSTS_CODES_ADAPTER: Final = TypeAdapter(tuple[int, ...]) _MCP_CALL_TYPES: Final[tuple[str, ...]] = ("mcp_call", "call_mcp_tool") _TOOL_INPUT_SCHEMA_ADAPTER: Final = TypeAdapter(dict[str, object]) +_DECODED_JSON: Final = TypeAdapter(object) _OBO_CACHE_MAX_ENTRIES: Final = 1000 _DEFAULT_TOKEN_TTL_SECONDS: Final = 3599.0 _TOKEN_EXPIRY_SLACK_SECONDS: Final = 60.0 @@ -343,7 +344,7 @@ class Agent365Guardrail(CustomGuardrail): latency_ms: float, ) -> dict: # mutable-ok: returns the request data dict per hook contract try: - parsed_verdict: Final = response.json() + parsed_verdict: Final = _DECODED_JSON.validate_python(response.json()) except ValueError: return self._handle_unavailable( data=data, @@ -486,7 +487,7 @@ class Agent365Guardrail(CustomGuardrail): response=response, ) try: - parsed_body: Final = response.json() + parsed_body: Final = _DECODED_JSON.validate_python(response.json()) except ValueError as exc: raise Agent365MalformedResponseError("the Entra token endpoint returned a non-JSON body") from exc if not isinstance(parsed_body, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py index 993c1d30949..389706d7417 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py +++ b/litellm/proxy/guardrails/guardrail_hooks/airia/airia.py @@ -2,12 +2,13 @@ import os import uuid from collections.abc import Mapping from typing import ( - Any, # noqa: TID251 # the only type CustomGuardrail.__init__ accepts for its open-ended kwargs Final, Literal, ) import httpx +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -36,6 +37,12 @@ DEFAULT_BLOCKED_MESSAGE: Final = "Blocked by your organization's content policy. SUPPORTED_EVENT_HOOKS: Final = (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call) +_DECODED_JSON: Final = TypeAdapter(object) + + +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + def _stream_cannot_carry_a_rewrite(request_data: Mapping[str, object] | None) -> bool: """Whether this is a streamed response on a route the framework can only block, not rewrite. @@ -97,7 +104,7 @@ class AiriaGuardrail(CustomGuardrail): timeout: float | None = None, supported_event_hooks: list[GuardrailEventHooks] | None = None, # mutable-ok: matches CustomGuardrail.__init__ async_handler: AsyncHTTPHandler | None = None, # dependency-injected in tests; defaults to a real client - **kwargs: Any, # kwargs-ok: passed straight through to CustomGuardrail.__init__ + **kwargs: Unpack[_CustomGuardrailOptions], # kwargs-ok: passed straight through to CustomGuardrail.__init__ ) -> None: resolved_timeout: Final = timeout or float(os.getenv("AIRIA_TIMEOUT", "10")) @@ -161,7 +168,7 @@ class AiriaGuardrail(CustomGuardrail): headers={"Authorization": f"Bearer {self.api_key}"}, ) response.raise_for_status() - raw_body: Final[object] = response.json() + raw_body: Final = _DECODED_JSON.validate_python(response.json()) body: Final = _as_object(raw_body) if body is None: raise TypeError(f"expected a JSON object in the response body, got {type(raw_body).__name__}") diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py index 64770fdf0f7..5bf471265c8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/__init__.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from .block_code_execution import BlockCodeExecutionGuardrail @@ -26,7 +27,7 @@ def _get_param( value: Final = getattr(litellm_params, key, default) if value is not None: return value - raw: Final = guardrail.get("litellm_params") + raw: Final = as_str_mapping(guardrail.get("litellm_params")) if isinstance(raw, dict) and key in raw: return raw[key] return default diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 936f862b10b..19b9bccd8aa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -23,6 +23,7 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_ssl_configuration, @@ -222,7 +223,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return [] @staticmethod - def _iter_schema_string_refs(data: Mapping[str, Any]): + def _iter_schema_string_refs(data: Mapping[str, object]): """Yield ``(container, key)`` for every non-empty schema string the proxy forwards to the model inside tool/function and structured-output schemas: each ``tools[].function`` and legacy ``functions[]`` entry plus the @@ -235,10 +236,10 @@ class CatoNetworksGuardrail(CustomGuardrail): list_keys: Final = ("enum", "examples") stack: Final[list] = [] - for tool in data.get("tools") or []: + for tool in iter_items(data.get("tools") or ()): if isinstance(tool, dict) and isinstance(tool.get("function"), dict): stack.append(tool["function"]) - for function in data.get("functions") or []: + for function in iter_items(data.get("functions") or ()): if isinstance(function, dict): stack.append(function) response_format: Final = data.get("response_format") diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index ab43d32e260..45e785fd53c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -35,6 +35,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -1743,7 +1744,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @staticmethod def _extract_masked_entity_count( - rules: list[dict[str, Any]], + rules: Sequence[Mapping[str, object]], ) -> dict[str, int] | None: """Count entity-type detections per Cisco rule for the logging payload.""" if not rules: @@ -1752,8 +1753,7 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): for rule in rules: if not isinstance(rule, dict): continue - entity_types = rule.get("entity_types") or [] - for entity_type in entity_types: + for entity_type in iter_items(rule.get("entity_types") or ()): if not isinstance(entity_type, str): continue counts[entity_type] = counts.get(entity_type, 0) + 1 diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py index 498f9bf4099..c5ca97ff760 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Final +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.guardrails import ( GuardrailEventHooks, Mode, @@ -26,7 +27,7 @@ def _coerce_event_hook( def _get_optional_value(litellm_params: LitellmParams, optional_params: object | None, attribute_name: str) -> object: if optional_params is not None: - value: Final = getattr(optional_params, attribute_name, None) + value: Final = attribute_or_none(optional_params, attribute_name) if value is not None: return value return getattr(litellm_params, attribute_name, None) diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 029feba89fb..4d910cde603 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -28,6 +28,7 @@ from urllib.parse import urlparse import httpx from fastapi import HTTPException from httpx import Response as HttpxResponse +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -72,6 +73,7 @@ if TYPE_CHECKING: ) BYPASS_HEADER: Final = "x-compresr-bypass" +_AGENTIC_LOOP_MODEL: Final = TypeAdapter(object) COMPRESR_RETRIEVE_TOOL_NAME: Final = "compresr_retrieve" DEFAULT_API_BASE: Final = "https://api.compresr.ai" DEFAULT_COMPRESSION_MODEL: Final = "latte_v2" @@ -1118,7 +1120,7 @@ class CompresrGuardrail(CustomGuardrail): full_model_name = model if logging_obj is not None: agentic_params: Final = getattr(logging_obj, "model_call_details", {}).get("agentic_loop_params", {}) - candidate: Final = agentic_params.get("model", model) + candidate: Final = _AGENTIC_LOOP_MODEL.validate_python(agentic_params.get("model", model)) if isinstance(candidate, str) and candidate: full_model_name = candidate diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 34725803c60..ee3a957f607 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -5,7 +5,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Optional, cast from fastapi import HTTPException -from pydantic import ConfigDict, Field, ValidationError +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import override from litellm._logging import verbose_proxy_logger @@ -37,6 +37,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class CrowdStrikeAIDRGuardrailMissingSecrets(Exception): """Custom exception for missing CrowdStrike AIDR secrets.""" @@ -362,7 +364,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): assert response is not None response.raise_for_status() - response_body: Final[object] = response.json() + response_body: Final = _DECODED_JSON.validate_python(response.json()) raw_result: Final[object] = response_body.get("result") if isinstance(response_body, dict) else None blocked_signal: Final[object] = raw_result.get("blocked") if isinstance(raw_result, dict) else None diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py index 13d7178dfe9..0f9ce15e65b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/__init__.py @@ -39,7 +39,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" raise ValueError("Custom code guardrail requires a guardrail_name") # Get the custom code from litellm_params - custom_code: Final = getattr(litellm_params, "custom_code", None) + custom_code: Final = litellm_params.custom_code if not custom_code: raise ValueError("Custom code guardrail requires 'custom_code' in litellm_params") diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 199da7df30d..ecd23d86540 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -21,6 +21,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, @@ -126,6 +127,13 @@ def _sanitize_inbound_headers( return sanitized or None +def _logged_inbound_headers(details: Mapping[str, object]) -> Mapping[object, object] | None: + litellm_params: Final = as_str_mapping(details.get("litellm_params", {})) + metadata: Final = as_str_mapping(litellm_params.get("metadata", {})) if litellm_params is not None else None + headers: Final = metadata.get("headers", None) if metadata is not None else None + return headers if isinstance(headers, dict) else None + + def _extract_inbound_headers( request_data: dict, logging_obj: Optional["LiteLLMLoggingObj"], @@ -163,7 +171,7 @@ def _extract_inbound_headers( if logging_obj and getattr(logging_obj, "model_call_details", None): try: details: Final = logging_obj.model_call_details or {} - headers = details.get("litellm_params", {}).get("metadata", {}).get("headers", None) + headers = _logged_inbound_headers(details) if headers: return _sanitize_inbound_headers(headers, extra_allowlist=extra_allowlist) except Exception: @@ -283,7 +291,7 @@ class GenericGuardrailAPI(CustomGuardrail): ) self.num_retries: int = num_retries or 0 self.headers = headers or {} - self.extra_headers = extra_headers or [] + self.extra_headers: Sequence[object] = extra_headers or [] # If api_key is provided, add it as x-api-key header if api_key: diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index f32228a6204..7ff5aa2466b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -34,6 +34,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj GRAYSWAN_BLOCK_ERROR_MSG: Final = "Blocked by Gray Swan Guardrail" +_DECODED_JSON: Final = TypeAdapter(object) class _GraySwanMonitorResponse(TypedDict): @@ -705,7 +706,9 @@ class GraySwanGuardrail(CustomGuardrail): {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers ) if cleaned_litellm_metadata: - sanitized: Final[object] = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + sanitized: Final = _DECODED_JSON.validate_python( + safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) + ) if isinstance(sanitized, dict) and sanitized: payload["litellm_metadata"] = sanitized diff --git a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py index d5edcc19a02..a1d0900ebed 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py +++ b/litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Final from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -23,6 +24,8 @@ from litellm.types.utils import CallTypesLiteral, GuardrailStatus if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class JavelinGuardrail(CustomGuardrail): @classmethod @@ -113,7 +116,9 @@ class JavelinGuardrail(CustomGuardrail): json=dict(request), timeout=self.timeout, ) - verbose_proxy_logger.debug("Javelin Guardrail: Javelin guard API response: %s", response.json()) + verbose_proxy_logger.debug( + "Javelin Guardrail: Javelin guard API response: %s", _DECODED_JSON.validate_python(response.json()) + ) response_data: Final = response.json() # Ensure the response has the required assessments field if "assessments" not in response_data: diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py index 8efd1dc79e3..7c0cdb0d36f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -40,6 +41,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import ( ) from litellm.types.utils import CallTypesLiteral, GuardrailStatus, ModelResponse +_DECODED_JSON: Final = TypeAdapter(object) +_RESPONSE_FIELDS: Final = TypeAdapter(Mapping[str, object]) + _DETECTOR_CATEGORY_PHRASES: Final[Mapping[str, str]] = MappingProxyType( { "prompt_injection": "a potential prompt injection attempt", @@ -402,7 +406,9 @@ class LakeraAIGuardrail(CustomGuardrail): json=request, timeout=self.timeout, ) - verbose_proxy_logger.debug("Lakera AI v2 guard response: %s", response.json()) + verbose_proxy_logger.debug( + "Lakera AI v2 guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) lakera_response = LakeraAIResponse(**response.json()) return lakera_response, masked_entity_count except Exception as e: @@ -801,7 +807,7 @@ class LakeraAIGuardrail(CustomGuardrail): choice_idx = choice_indices[idx] response_dict["choices"][choice_idx]["message"]["content"] = msg.get("content", "") add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) - return ModelResponse(**response_dict) + return ModelResponse(**_RESPONSE_FIELDS.validate_python(response_dict)) # inject_system_message has nothing left to inject into once a response # already exists, so it is treated the same as monitor: log and allow. diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index e05db5003fa..97b35769f63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -9,7 +9,7 @@ import json import os import uuid from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict +from typing import TYPE_CHECKING, Final, Literal, TypedDict try: import ulid @@ -39,6 +39,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.integrations.custom_guardrail import dc as global_cache +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -397,13 +398,15 @@ class LassoGuardrail(CustomGuardrail): HTTPException: If blocking violations are detected """ raw_messages: Final[list[dict[str, object]]] = data.get("messages") or [] - messages: list[dict[str, Any]] = self._expand_messages_for_classification(raw_messages) if raw_messages else [] - messages_count: Final = len(messages) - if data.get("input") is not None: - # Responses-API payloads carry text in data["input"]. Inspect it - # alongside any "messages" array — otherwise a caller can attach - # benign messages and stash blocked content in input to bypass. - messages.extend(build_inspection_messages({"input": data["input"]})) + expanded_messages: Final = self._expand_messages_for_classification(raw_messages) if raw_messages else [] + messages_count: Final = len(expanded_messages) + # Responses-API payloads carry text in data["input"]. Inspect it + # alongside any "messages" array — otherwise a caller can attach + # benign messages and stash blocked content in input to bypass. + input_messages: Final = ( + build_inspection_messages({"input": data["input"]}) if data.get("input") is not None else [] + ) + messages: Final[Sequence[Mapping[str, object]]] = [*expanded_messages, *input_messages] if not messages: return data @@ -421,7 +424,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], ) -> dict: """Handle classification without masking.""" try: @@ -439,7 +442,7 @@ class LassoGuardrail(CustomGuardrail): data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"], - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], messages_count: int, ) -> dict: """Handle masking with classifix endpoint. @@ -470,7 +473,7 @@ class LassoGuardrail(CustomGuardrail): text_parts = [msg["content"] for msg in masked_for_input if isinstance(msg.get("content"), str)] if text_parts: data["input"] = "\n".join(text_parts) - self._log_masking_applied(message_type, dict(response)) + self._log_masking_applied(message_type, response) return data except Exception as e: @@ -479,7 +482,7 @@ class LassoGuardrail(CustomGuardrail): def _map_masked_messages_back( self, - original_messages: list[dict[str, Any]], + original_messages: Sequence[Mapping[str, object]], masked_messages: Sequence[Mapping[str, object]], ) -> list[dict[str, object]]: """Map Lasso-format masked messages back onto the original OpenAI-format messages. @@ -492,7 +495,7 @@ class LassoGuardrail(CustomGuardrail): """ # Index masked content by type so we can look up by id without caring about order. masked_tool_use: Final[dict[object, dict[str, object]]] = {} - masked_tool_result: Final[dict[str, str]] = {} + masked_tool_result: Final[dict[object, str]] = {} masked_text: Final[list[str]] = [] for msg in masked_messages: @@ -567,12 +570,12 @@ class LassoGuardrail(CustomGuardrail): def _update_tool_calls_from_masked( self, - tool_calls: list[object], + tool_calls: object, masked_tool_use: Mapping[object, Mapping[str, object]], ) -> list[object]: """Replace tool_call arguments with masked values returned by Lasso.""" updated: Final = [] - for call in tool_calls: + for call in iter_items(tool_calls): call_id = self._get_field(call, "id") if call_id and call_id in masked_tool_use: masked_input = masked_tool_use[call_id].get("input") @@ -627,7 +630,7 @@ class LassoGuardrail(CustomGuardrail): def _log_masking_applied( self, message_type: Literal["PROMPT", "COMPLETION"], - response: dict[str, Any], + response: LassoResponse, ) -> None: """Log masking application with structured context.""" conversation_id: Final = getattr(self, "conversation_id", "unknown") @@ -637,12 +640,12 @@ class LassoGuardrail(CustomGuardrail): "guardrail_name": getattr(self, "guardrail_name", "unknown"), "message_type": message_type, "violations_count": len(response.get("findings", {})), - "masked_fields": len(response.get("messages", [])), + "masked_fields": len(response.get("messages") or []), "conversation_id": conversation_id, }, ) - def _expand_messages_for_classification(self, messages: list[dict[str, Any]]) -> list[dict[str, object]]: + def _expand_messages_for_classification(self, messages: Sequence[Mapping[str, object]]) -> list[dict[str, object]]: """ Convert raw OpenAI-format messages to Lasso API format with content blocks. @@ -701,7 +704,7 @@ class LassoGuardrail(CustomGuardrail): expanded.append({"role": role, "content": content}) if role == "assistant": - for call in msg.get("tool_calls") or []: + for call in iter_items(msg.get("tool_calls") or []): call_id, name, input_data = self._extract_tool_call_fields(call) if not call_id or not name: verbose_proxy_logger.warning( @@ -749,7 +752,7 @@ class LassoGuardrail(CustomGuardrail): def _prepare_payload( self, - messages: list[dict[str, object]], + messages: Sequence[Mapping[str, object]], data: dict, cache: DualCache, message_type: Literal["PROMPT", "COMPLETION"] = "PROMPT", diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py index 5e2b85250d2..76c3a86d701 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/competitor_intent/airline.py @@ -13,6 +13,9 @@ import json from pathlib import Path from typing import Any, Final +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.competitor_intent.base import ( # noqa: F401 # legacy module exports BaseCompetitorIntentChecker, _compile_marker, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -114,6 +117,7 @@ AIRLINE_EXPLICIT_OTHER_MEANING_MARKER: Final = ( ) _MAJOR_AIRLINES_PATH: Final = Path(__file__).resolve().parent / "major_airlines.json" +_DECODED_JSON: Final = TypeAdapter(object) def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: @@ -127,11 +131,11 @@ def _load_competitors_excluding_brand(brand_self: list[str]) -> list[str]: return [] try: with open(_MAJOR_AIRLINES_PATH, encoding="utf-8") as f: - airlines: Final = json.load(f) + airlines: Final = _DECODED_JSON.validate_python(json.load(f)) except (json.JSONDecodeError, OSError): return [] result: Final[list[str]] = [] - for entry in airlines: + for entry in iter_items(airlines): if not isinstance(entry, dict): continue match_str = entry.get("match") or "" diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index 40e44151e65..f58c8705815 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -78,6 +78,7 @@ from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 _DECODED_ARGUMENTS: Final = TypeAdapter(object) +_JSON_ENTRIES: Final = TypeAdapter(list[object]) GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b") SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+") @@ -699,9 +700,8 @@ class ContentFilterGuardrail(CustomGuardrail): Severity mapping: 4,3 -> high; 2 -> medium; 1 -> low. """ with open(file_path, "r") as f: - entries = json.load(f) - if not isinstance(entries, list): - entries = [entries] + decoded: Final = _DECODED_ARGUMENTS.validate_python(json.load(f)) + entries: Final = _JSON_ENTRIES.validate_python(decoded) if isinstance(decoded, list) else [decoded] # Derive category name from filename (e.g. harm_toxic_abuse.json -> harm_toxic_abuse) category_name: Final = os.path.splitext(os.path.basename(file_path))[0] severity_map: Final = {4: "high", 3: "high", 2: "medium", 1: "low"} @@ -806,7 +806,7 @@ class ContentFilterGuardrail(CustomGuardrail): """ try: with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_ARGUMENTS.validate_python(yaml.safe_load(f)) if not isinstance(data, dict) or "blocked_words" not in data: raise ValueError("Invalid format: file must contain 'blocked_words' key with list of words") diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index 7fb87080cd1..d2dafb04fe0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -5,7 +5,7 @@ import uuid from collections.abc import AsyncGenerator, AsyncIterable, Iterator, Mapping, Sequence from typing import ( TYPE_CHECKING, - Any, # noqa: TID251 # the streaming hook and the flush copy pass provider chunks through untyped + Any, # noqa: TID251 # the streaming hook passes provider chunks through untyped ClassVar, Final, Literal, @@ -23,7 +23,7 @@ from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) -from litellm.litellm_core_utils.core_helpers import attribute_items +from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, iter_items, require_attributes from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -99,13 +99,8 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): pass -class _ContentDelta(Protocol): - content: object - - -class _FinishingDelta(Protocol): - content: object - tool_calls: object +class _DeepCopyable(Protocol): + def model_copy(self, *, deep: bool) -> object: ... class LLMShieldProxyGuardrail(CustomGuardrail): @@ -528,7 +523,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): one tool call's accumulating `arguments`. A shared window would splice the text held back for one stream onto another. """ - delta: Final = getattr(choice, "delta", None) + delta: Final = attribute_or_none(choice, "delta") index: Final = choice_index(choice) is_final: Final = bool(getattr(choice, "finish_reason", None)) if isinstance(choice, TextChoices): @@ -566,7 +561,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): async def _restore_content_window( self, - delta: _ContentDelta, + delta: object, key: CarryKey, carries: CarryWindows, session_id: str, @@ -574,19 +569,19 @@ class LLMShieldProxyGuardrail(CustomGuardrail): ) -> None: """Restores one delta's content through its own window.""" carry: Final = carries.get(key, "") - text: Final = getattr(delta, "content", None) + text: Final = attribute_or_none(delta, "content") if not isinstance(text, str) or not text: if is_final and carry: flushed, remaining = await self._stream_step("", carry, True, session_id) carries[key] = remaining # rebind-ok: this stream's window advances. if flushed: - delta.content = flushed + write_field(delta, "content", flushed) return emitted, remaining = await self._stream_step(text, carry, is_final, session_id) carries[key] = remaining # rebind-ok: this stream's window advances. - delta.content = emitted + write_field(delta, "content", emitted) async def _restore_tool_call_window( self, @@ -618,7 +613,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): async def _flush_finished_choice( self, - delta: _FinishingDelta, + delta: object, choice_index: int, carries: CarryWindows, session_id: str, @@ -643,12 +638,12 @@ class LLMShieldProxyGuardrail(CustomGuardrail): if not text: continue if tool_index is None: - delta.content = text + write_field(delta, "content", text) else: continuations.extend(continuation_delta(tool_index, text)) if continuations: - existing: Final = tuple(getattr(delta, "tool_calls", None) or ()) - delta.tool_calls = [*existing, *continuations] + existing: Final = tuple(attribute_items(delta, "tool_calls")) + write_field(delta, "tool_calls", [*existing, *continuations]) async def _flush_trailing( self, last_chunk: object, carries: CarryWindows, session_id: str @@ -676,7 +671,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): chunk = self._chunk_for_choice(last_chunk, choice_index) if chunk is None: continue - choice: object = chunk.choices[0] + choice = read_list(chunk, "choices")[0] delta = read_field(choice, "delta") if isinstance(choice, TextChoices): choice.text = text @@ -688,26 +683,26 @@ class LLMShieldProxyGuardrail(CustomGuardrail): yield chunk @staticmethod - def _chunk_for_choice(last_chunk: Any, index: int) -> Any: + def _chunk_for_choice(last_chunk: object, index: int) -> object | None: """A single-choice copy of the last chunk, carrying only `index`. Emitting one choice per chunk keeps a flush from reading as content on a choice it does not belong to. """ - chunk: Final = last_chunk.model_copy(deep=True) - raw_choices: Final = getattr(chunk, "choices", None) + chunk: Final = require_attributes(last_chunk, _DeepCopyable, "model_copy").model_copy(deep=True) + raw_choices: Final = attribute_or_none(chunk, "choices") if not raw_choices: return None - choices: Final[tuple[object, ...]] = tuple(raw_choices) + choices: Final = tuple(iter_items(raw_choices)) position: Final = next((at for at, choice in enumerate(choices) if choice_index(choice) == index), 0) - kept: Final = raw_choices[position] - if getattr(kept, "delta", None) is None and not isinstance(kept, TextChoices): + kept: Final = choices[position] + if attribute_or_none(kept, "delta") is None and not isinstance(kept, TextChoices): return None - kept.index = index - kept.finish_reason = None - chunk.choices = [kept] + write_field(kept, "index", index) + write_field(kept, "finish_reason", None) + write_field(chunk, "choices", [kept]) if hasattr(chunk, "usage"): - del chunk.usage + delattr(chunk, "usage") return chunk async def _stream_step(self, text: str, carry: str, final: bool, session_id: str) -> tuple[str, str]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py index 4ef8bbdbb3e..ef007155c46 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/stream_restorers.py @@ -7,9 +7,12 @@ from enum import Enum from types import MappingProxyType from typing import ( Final, + Protocol, TypeAlias, ) +from pydantic import TypeAdapter + from .payload import ( JsonBody, MutableRequest, @@ -24,6 +27,8 @@ from .payload import ( write_field, ) +_OPAQUE_VALUE: Final = TypeAdapter(object) + ANTHROPIC_DELTA_FIELDS: Final = MappingProxyType({"text_delta": "text", "input_json_delta": "partial_json"}) SSE_EVENT_BOUNDARY: Final = re.compile(rb"(\r?\n\r?\n)") @@ -80,6 +85,15 @@ def opens_like_sse(head: bytes) -> bool | None: return False +class _ValuedEnum(Protocol): + @property + def value(self) -> object: ... + + +def _enum_value(member: _ValuedEnum) -> object: + return member.value + + def responses_event_type(chunk: object) -> str | None: """The event type of a Responses API stream event, or None for any other chunk. @@ -90,7 +104,7 @@ def responses_event_type(chunk: object) -> str | None: if isinstance(chunk, (bytes, str)): return None kind: Final = read_field(chunk, "type") - value: Final = kind.value if isinstance(kind, Enum) else kind + value: Final = _enum_value(kind) if isinstance(kind, Enum) else kind return value if isinstance(value, str) and value.startswith("response.") else None @@ -173,7 +187,7 @@ class AnthropicSSERestorer: return block line: Final = lines[data_lines[0]] try: - parsed: Final[object] = json.loads(line[len("data:") :]) + parsed: Final = _OPAQUE_VALUE.validate_python(json.loads(line[len("data:") :])) except ValueError: return block event: Final = as_object(parsed) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py index 95db4bd4f77..fa64d306f22 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_end_user_permission/__init__.py @@ -1,4 +1,6 @@ -from typing import TYPE_CHECKING, Any, Final, cast +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Protocol, cast from litellm.types.guardrails import SupportedGuardrailIntegrations @@ -8,13 +10,22 @@ if TYPE_CHECKING: from litellm.types.guardrails import Guardrail, LitellmParams +class _ParamsLookup(Protocol): + def get(self, key: str, /) -> object: ... + + +_NO_PARAMS: Final = MappingProxyType[str, object]({}) + + def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm # Default to always-on. Only disable if the user explicitly sets default_on: false. # We check the raw guardrail dict because LitellmParams normalizes None → False, # making it impossible to distinguish "not set" from "explicitly false" via litellm_params. - _raw_default_on: Final = cast(dict[str, Any], guardrail).get("litellm_params", {}).get("default_on") + _raw_default_on: Final = ( + cast(Mapping[str, _ParamsLookup], guardrail).get("litellm_params", _NO_PARAMS).get("default_on") + ) _default_on: Final = _raw_default_on is not False _callback: Final = MCPEndUserPermissionGuardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py index 221c4b3752b..2e3f7120726 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_jwt_signer/mcp_jwt_signer.py @@ -74,14 +74,14 @@ import os import re import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Final, Optional import httpx import jwt from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -90,7 +90,6 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypesLiteral @@ -98,6 +97,10 @@ if TYPE_CHECKING: from jwt.types import Options +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _OIDCDiscoveryDocument(TypedDict, total=False): jwks_uri: str @@ -279,11 +282,13 @@ class MCPJWTSigner(CustomGuardrail): debug_headers: bool = False, # FR-10: Configurable scopes allowed_scopes: list[str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) # --- Signing key setup --- key_material: Final = os.environ.get(self.SIGNING_KEY_ENV) diff --git a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py index 8482567b6da..3cb20a00e2f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/mcp_security/mcp_security_guardrail.py @@ -6,7 +6,7 @@ on the LiteLLM gateway. Blocks or alerts when unregistered servers are found. """ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal from fastapi import HTTPException @@ -21,6 +21,9 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( ) from litellm.types.guardrails import GuardrailEventHooks +if TYPE_CHECKING: + from litellm.caching.caching import DualCache + class MCPSecurityGuardrail(CustomGuardrail): def __init__( @@ -40,7 +43,7 @@ class MCPSecurityGuardrail(CustomGuardrail): async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, - cache: Any, + cache: "DualCache", data: dict, call_type: str, ) -> Exception | str | dict | None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py index ed955ac829d..06de4ef0e93 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/microsoft_purview/__init__.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Final +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.guardrails import SupportedGuardrailIntegrations from .purview_dlp import MicrosoftPurviewDLPGuardrail @@ -11,8 +12,8 @@ if TYPE_CHECKING: def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm - tenant_id: Final = getattr(litellm_params, "tenant_id", None) - client_id: Final = getattr(litellm_params, "client_id", None) + tenant_id: Final = attribute_or_none(litellm_params, "tenant_id") + client_id: Final = attribute_or_none(litellm_params, "client_id") # client_secret can be passed via the standard api_key field or as # a dedicated client_secret parameter. diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py index 718401b6e68..43eb54d1464 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py @@ -10,9 +10,12 @@ import json import os from collections.abc import Callable, Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional, TypeAlias, cast from urllib.parse import urlparse +from pydantic import TypeAdapter +from typing_extensions import TypedDict, Unpack + from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -25,7 +28,6 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage -from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.guardrails.guardrail_hooks.noma import NomaV2StreamingParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus @@ -39,6 +41,7 @@ _DEFAULT_API_BASE: Final = "https://api.noma.security/" _AIDR_SCAN_ENDPOINT: Final = "/litellm/guardrail" _INTERVENED_INPUT_FIELDS: Final = ("texts", "images", "tools", "tool_calls") _DEFAULT_API_BASE_HOSTNAME: Final = urlparse(_DEFAULT_API_BASE).hostname +_DECODED_JSON: Final = TypeAdapter(object) _GuardrailJsonResponse: TypeAlias = Exception | str | dict[str, object] @@ -50,6 +53,10 @@ _LOGGING_KEYS_DUPLICATING_SCAN_INPUTS: Final = _KEYS_DUPLICATING_SCAN_INPUTS + ( ) +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + pass + + class _Action(str, enum.Enum): BLOCKED = "BLOCKED" NONE = "NONE" @@ -74,7 +81,7 @@ class NomaV2Guardrail(CustomGuardrail): streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, streaming_buffer_release_on_scan: bool | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) @@ -115,10 +122,11 @@ class NomaV2Guardrail(CustomGuardrail): if streaming.streaming_buffer_release_on_scan is not None: self.streaming_buffer_release_on_scan: bool = streaming.streaming_buffer_release_on_scan - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - - base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs - super().__init__(**base_kwargs) + options: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } + super().__init__(**options) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -219,7 +227,7 @@ class NomaV2Guardrail(CustomGuardrail): except (ValueError, TypeError): json_str = safe_dumps(payload) - safe_payload: Final[object] = safe_json_loads(json_str, default={}) + safe_payload: Final = _DECODED_JSON.validate_python(safe_json_loads(json_str, default={})) if safe_payload == {} and payload: verbose_proxy_logger.warning( "Noma v2 guardrail: payload serialization failed, falling back to empty payload" diff --git a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py index 1a6feb47215..7c26c4c4c56 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py +++ b/litellm/proxy/guardrails/guardrail_hooks/onyx/onyx.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Final, Literal, Optional import httpx from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -27,6 +28,8 @@ from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + class OnyxGuardrail(CustomGuardrail): @classmethod @@ -115,7 +118,7 @@ class OnyxGuardrail(CustomGuardrail): try: response: Final = ModelResponse(**request_data) parsed: Final = response.json() - payload = parsed.get("response", {}) + payload = _DECODED_JSON.validate_python(parsed.get("response", {})) except Exception as e: verbose_proxy_logger.error( "Error in converting request_data to ModelResponse: %s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py index fe526b72ac3..4d120a2dd26 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/__init__.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Final import litellm +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) @@ -40,12 +41,12 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" return openai_moderation_guardrail -def _get_config_value(litellm_params, optional_params, attribute_name): +def _get_config_value(litellm_params: object, optional_params: object, attribute_name: str) -> object: if optional_params is not None: - value: Final = getattr(optional_params, attribute_name, None) + value: Final = attribute_or_none(optional_params, attribute_name) if value is not None: return value - return getattr(litellm_params, attribute_name, None) + return attribute_or_none(litellm_params, attribute_name) guardrail_initializer_registry: Final = { diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 7af5eaf1da9..98be35079c3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -40,6 +41,8 @@ if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIModerationResponse from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +_DECODED_JSON: Final = TypeAdapter(object) + def _config_secret(value: object) -> object: return get_secret_str(value) if isinstance(value, str) and value.startswith("os.environ/") else value @@ -151,7 +154,9 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): timeout=self.timeout, ) - verbose_proxy_logger.debug("OpenAI Moderation guard response: %s", response.json()) + verbose_proxy_logger.debug( + "OpenAI Moderation guard response: %s", _DECODED_JSON.validate_python(response.json()) + ) if response.status_code != 200: raise HTTPException( diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 8e03f7db4f5..6426bfb8788 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -18,6 +18,7 @@ from urllib.parse import urlparse import httpx from fastapi import HTTPException from pydantic import ConfigDict, TypeAdapter, ValidationError, field_validator +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -129,6 +130,21 @@ _RESPONSES_INPUT: Final[TypeAdapter[_ResponsesInput]] = TypeAdapter(_ResponsesIn _DECODED_ARGUMENTS: Final = TypeAdapter(object) +class _AirsToolEvent(TypedDict): + metadata: ReadOnly[Mapping[str, object]] + input: NotRequired[ReadOnly[str]] + + +def _mcp_tool_input(mcp_arguments: object) -> str | None: + if mcp_arguments is not None and mcp_arguments != "": + serialized_args: Final = ( + json.dumps(mcp_arguments) if isinstance(mcp_arguments, (dict, list)) else str(mcp_arguments) + ) + if serialized_args.strip(): + return serialized_args + return None + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -339,7 +355,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): is_response: bool = False, metadata: Mapping[str, object] | None = None, call_id: object = None, - tool_event: dict[str, Any] | None = None, + tool_event: _AirsToolEvent | None = None, ) -> dict[str, object]: """Call PANW Prisma AIRS API to scan content or a tool_event.""" @@ -894,7 +910,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): if not line.startswith("data: "): continue try: - data = json.loads(line[6:]) + data = _DECODED_ARGUMENTS.validate_python(json.loads(line[6:])) except (json.JSONDecodeError, ValueError): continue if not isinstance(data, dict): @@ -1954,24 +1970,21 @@ class PanwPrismaAirsHandler(CustomGuardrail): # "mcp_tool_name"/"mcp_arguments". Check canonical first, then fallback. mcp_tool_name: Final = request_data.get("mcp_tool_name") or self._mcp_name_fallback(request_data) if mcp_tool_name and input_type == "request": - mcp_tool_event: Final[dict[str, object]] = { - "metadata": { - "ecosystem": "mcp", - "method": "tools/call", - "server_name": self._get_mcp_server_name(request_data, mcp_tool_name), - "tool_invoked": mcp_tool_name, - }, + mcp_tool_metadata: Final = { + "ecosystem": "mcp", + "method": "tools/call", + "server_name": self._get_mcp_server_name(request_data, mcp_tool_name), + "tool_invoked": mcp_tool_name, } mcp_arguments = request_data.get("mcp_arguments") if mcp_arguments is None: mcp_arguments = request_data.get("arguments") - if mcp_arguments is not None and mcp_arguments != "": - if isinstance(mcp_arguments, (dict, list)): - serialized_args = json.dumps(mcp_arguments) - else: - serialized_args = str(mcp_arguments) - if serialized_args.strip(): - mcp_tool_event["input"] = serialized_args + mcp_tool_input: Final = _mcp_tool_input(mcp_arguments) + mcp_tool_event: Final[_AirsToolEvent] = ( + {"metadata": mcp_tool_metadata} + if mcp_tool_input is None + else {"metadata": mcp_tool_metadata, "input": mcp_tool_input} + ) mcp_scan_result: Final = await self._call_panw_api( tool_event=mcp_tool_event, diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index cadc9ca689b..7e2f6bfec33 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -85,6 +85,7 @@ class _PresidioAnonymizeResponse(TypedDict): _PRESIDIO_ANONYMIZE_ADAPTER: Final[TypeAdapter[_PresidioAnonymizeResponse | None]] = TypeAdapter( _PresidioAnonymizeResponse | None ) +_DECODED_JSON: Final = TypeAdapter(object) class _JsonResponse(Protocol): @@ -1595,10 +1596,11 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if line.startswith("data: ") and line != "data: [DONE]": raw_json = line[6:] try: - event = json.loads(raw_json) + event = _DECODED_JSON.validate_python(json.loads(raw_json)) delta = event.get("delta") if isinstance(event, dict) else None if ( - isinstance(delta, dict) + isinstance(event, dict) + and isinstance(delta, dict) and event.get("type") == "content_block_delta" and delta.get("type") == "text_delta" and isinstance(delta.get("text"), str) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index a92e67046bd..6150fccac0c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -10,6 +10,7 @@ import os from typing import Any, Final, Literal from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -29,6 +30,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs GUARDRAIL_NAME: Final = "qualifire" DEFAULT_QUALIFIRE_API_BASE: Final = "https://proxy.qualifire.ai" +_DECODED_JSON: Final = TypeAdapter(object) class QualifireGuardrail(CustomGuardrail): @@ -188,7 +190,7 @@ class QualifireGuardrail(CustomGuardrail): args = function_info.get("arguments", {}) if isinstance(args, str): try: - args = json.loads(args) + args = _DECODED_JSON.validate_python(json.loads(args)) except json.JSONDecodeError: args = {} api_tool_calls.append( diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index 1a571047977..342e5b64714 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import yaml +from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD @@ -22,6 +23,7 @@ if TYPE_CHECKING: ROUTE_TEMPLATES_DIR: Final = os.path.join(os.path.dirname(__file__), "route_templates") +_DECODED_YAML: Final = TypeAdapter(object) class SemanticGuardRouteLoader: @@ -55,7 +57,7 @@ class SemanticGuardRouteLoader: if not os.path.exists(file_path): raise ValueError(f"SemanticGuard: custom routes file not found: {file_path}") with open(file_path, "r") as f: - data: Final = yaml.safe_load(f) + data: Final = _DECODED_YAML.validate_python(yaml.safe_load(f)) if isinstance(data, list): return data if isinstance(data, dict): diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py index 2fbd50b5863..37acee28b53 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/semantic_guard.py @@ -6,13 +6,14 @@ via embedding similarity. Smarter than regex (understands intent), lighter than an LLM call (~20-50ms per request for embedding). """ -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, ) +from litellm.litellm_core_utils.core_helpers import attribute_items, attribute_or_none, require_attributes from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.route_loader import ( SemanticGuardRouteLoader, ) @@ -188,17 +189,21 @@ def _extract_user_text(messages: list) -> str: return "" -def _extract_response_text(response: Any) -> str: +class _MessageContent(Protocol): + @property + def content(self) -> object: ... + + +def _extract_response_text(response: object) -> str: """Extract text from every LLM response choice.""" - if hasattr(response, "choices") and response.choices: - text_parts: Final[list[str]] = [] - for choice in response.choices: - if hasattr(choice, "message") and choice.message: - text = _content_to_text(choice.message.content) - if text: - text_parts.append(text) - return "\n".join(text_parts) - return "" + text_parts: Final[list[str]] = [] + for choice in attribute_items(response, "choices"): + message = attribute_or_none(choice, "message") + if message: + text = _content_to_text(require_attributes(message, _MessageContent, "content").content) + if text: + text_parts.append(text) + return "\n".join(text_parts) def _content_to_text(content: object) -> str: diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py index de0f8856522..fbf1f2576ff 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py @@ -105,7 +105,7 @@ def _resolve_tool_names_from_messages(messages: list[dict]) -> dict[str, str]: fn = (tc.get("function") or {}).get("name") else: tc_id = getattr(tc, "id", None) - fn_obj = getattr(tc, "function", None) + fn_obj: object = getattr(tc, "function", None) fn = getattr(fn_obj, "name", None) if fn_obj else None if tc_id and fn: mapping[tc_id] = fn diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 3ad9d1a2c42..d713f1078a2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, runtime import anyio from fastapi import HTTPException +from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -50,6 +51,8 @@ A2A_CALL_TYPES: Final = (CallTypes.asend_message, CallTypes.send_message) GUARDRAIL_NAME: Final = "unified_llm_guardrails" +_DECODED_JSON: Final = TypeAdapter(object) + _RequestData: TypeAlias = dict[str, object] @@ -183,7 +186,7 @@ def _get_a2a_request_id(responses_so_far: Sequence[object], request_data: dict) return item.get("id") if isinstance(item, str): try: - obj = json.loads(item.strip()) + obj = _DECODED_JSON.validate_python(json.loads(item.strip())) if isinstance(obj, dict) and "id" in obj: return obj.get("id") except (json.JSONDecodeError, TypeError): @@ -729,7 +732,7 @@ class UnifiedLLMGuardrails(CustomLogger): user_api_key_dict: UserAPIKeyAuth, call_type: str, sampling_rate: int, - end_of_stream_only: bool, + end_of_stream_only: object, mappings: Mapping[CallTypes, type["BaseTranslation"]], ) -> AsyncGenerator[object, None]: """Emit guardrail text transformations as new deltas on the stream. @@ -1135,16 +1138,19 @@ class UnifiedLLMGuardrails(CustomLogger): guardrail_to_apply = request_data.pop("guardrail_to_apply", None) typed_request_data: Final[_RequestData] = request_data - def _streaming_flag(name: str, default: object) -> Any: + def _streaming_setting(name: str, default: object) -> object: return self.resolve_streaming_flag(guardrail_to_apply, name, default) + def _streaming_flag(name: str, default: object) -> Any: + return _streaming_setting(name, default) + sampling_rate: Final[int] = _streaming_flag("streaming_sampling_rate", 5) # Only apply the guardrail at end of stream (not per chunk). - end_of_stream_only: bool = _streaming_flag("streaming_end_of_stream_only", False) + end_of_stream_only: object = _streaming_setting("streaming_end_of_stream_only", False) # "block_only" (default) drops guardrail text rewrites on the streaming # path; "incremental_diff" emits them as synthetic deltas (see # _run_incremental_transform_stream). - streaming_transform_mode: Final[str] = _streaming_flag("streaming_transform_mode", "block_only") + streaming_transform_mode: Final = _streaming_setting("streaming_transform_mode", "block_only") # Withhold every chunk until end-of-stream moderation passes, then # release the original chunks (clean) or only the block message # (blocked) -- moderating the whole response *before* any content @@ -1152,10 +1158,10 @@ class UnifiedLLMGuardrails(CustomLogger): # release the original chunks are replayed as-is, so a # content-rewriting guardrail (e.g. PII masking) would leak # unredacted content. Guarded below via mask_response_content. - buffer_until_moderated: bool = _streaming_flag( + buffer_until_moderated: object = _streaming_setting( "streaming_buffer_until_moderated", buffer_until_moderated_default ) - release_on_scan: Final[bool] = _streaming_flag("streaming_buffer_release_on_scan", False) + release_on_scan: Final = _streaming_setting("streaming_buffer_release_on_scan", False) if ( buffer_until_moderated diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 7521c1e9f45..24b30e65e2b 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -102,6 +102,16 @@ def _as_json_array(value: object) -> list[object] | None: return _JSON_ARRAY.validate_python(value) if isinstance(value, list) else None +def _extra_param_items(litellm_params: LitellmParams) -> Iterable[tuple[str, object]]: + return ( + litellm_params.model_dump(exclude_none=True).items() + if hasattr(litellm_params, "model_dump") + else iter(litellm_params) + if litellm_params + else () + ) + + def contains_encrypted_marker(value: object, depth: int = 0) -> bool: """True if any string in value, at any JSON depth, starts with the encrypted-value prefix.""" if depth > DEFAULT_MAX_RECURSE_DEPTH: @@ -904,13 +914,7 @@ class InMemoryGuardrailHandler: # This matches the behavior of other guardrail initializers (e.g., initialize_lakera) # and aligns with the documented behavior for custom guardrails excluded_extra_param_keys: Final = frozenset(("guardrail", "mode", "default_on", "stream_scope")) - extra_params_items: Final = ( - litellm_params.model_dump(exclude_none=True).items() - if hasattr(litellm_params, "model_dump") - else iter(litellm_params) - if litellm_params - else () - ) + extra_params_items: Final = _extra_param_items(litellm_params) extra_params: Final = MappingProxyType( {key: value for key, value in extra_params_items if key not in excluded_extra_param_keys} ) diff --git a/litellm/proxy/guardrails/tool_name_extraction.py b/litellm/proxy/guardrails/tool_name_extraction.py index 526f58dcd36..c49835905fb 100644 --- a/litellm/proxy/guardrails/tool_name_extraction.py +++ b/litellm/proxy/guardrails/tool_name_extraction.py @@ -6,8 +6,8 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl with standalone extractors for generate_content and MCP. """ -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Callable, Mapping +from typing import Final from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings @@ -15,7 +15,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import list_field from litellm.types.utils import CallTypes # Call types that have no guardrail translation handler; we use standalone extractors -STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {} +STANDALONE_EXTRACTORS: Final[dict[str, Callable[[Mapping[str, object]], list[str]]]] = {} def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]: @@ -28,7 +28,7 @@ def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str return names -def _extract_mcp_tool_names(data: dict) -> list[str]: +def _extract_mcp_tool_names(data: Mapping[str, object]) -> list[str]: """MCP call_tool: name or mcp_tool_name in body""" names: Final[list[str]] = [] name: Final = data.get("name") or data.get("mcp_tool_name") diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 2f466f22ef8..caa66bddf39 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -11,10 +11,11 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, overload from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.usage_tracking import guardrail_status_to_action @@ -838,28 +839,41 @@ def _snippet(text: object, max_len: int = 200) -> str | None: return result -def _input_snippet_for_log(sl: "prisma_models.LiteLLM_SpendLogs") -> str | None: - """Snippet for request input: prefer messages, fall back to proxy_server_request (same as drawer).""" - out = _snippet(sl.messages) +_DECODED_JSON: Final = TypeAdapter(object) + + +def _request_dict(value: object) -> Mapping[str, object] | None: + mapping: Final = as_str_mapping(value) + return mapping if isinstance(mapping, dict) else None + + +def _proxy_request_snippet(psr: object) -> str | None: + request: Final = _request_dict(psr) + if request is None: + return _snippet(psr) + msgs: Final = request.get("messages") + body: Final = _request_dict(request.get("body")) if msgs is None else None + out: Final = _snippet(body.get("messages") if body is not None else msgs) if out: return out - psr = getattr(sl, "proxy_server_request", None) + return _snippet(psr) + + +def _input_snippet_for_log(sl: "prisma_models.LiteLLM_SpendLogs") -> str | None: + """Snippet for request input: prefer messages, fall back to proxy_server_request (same as drawer).""" + out: Final = _snippet(sl.messages) + if out: + return out + psr: Final[object] = getattr(sl, "proxy_server_request", None) if not psr: return None - if isinstance(psr, str): - try: - psr = json.loads(psr) - except Exception: - return _snippet(psr) - if isinstance(psr, dict): - msgs = psr.get("messages") - if msgs is None and isinstance(psr.get("body"), dict): - msgs = psr["body"].get("messages") - out = _snippet(msgs) - if out: - return out + if not isinstance(psr, str): + return _proxy_request_snippet(psr) + try: + decoded: Final = _DECODED_JSON.validate_python(json.loads(psr)) + except Exception: return _snippet(psr) - return _snippet(psr) + return _proxy_request_snippet(decoded) @router.get( diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index 47c3869fb64..8f9aebecf30 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -455,10 +455,10 @@ class KeyManagementEventHooks: try: from litellm.proxy import proxy_server as proxy_server_module + from litellm.proxy.proxy_server import prisma_client except ImportError: return None - prisma_client: Final = getattr(proxy_server_module, "prisma_client", None) user_api_key_cache: Final = getattr(proxy_server_module, "user_api_key_cache", None) if prisma_client is None or user_api_key_cache is None: diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index 47ea50debc9..05c64317a44 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -892,7 +892,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): {**_hidden_params, "additional_headers": _additional_headers}, ) - return await super().async_post_call_success_hook(data, user_api_key_dict, response) + await super().async_post_call_success_hook(data, user_api_key_dict, response) PROXY_MaxParallelRequestsHandler: Final = _PROXY_MaxParallelRequestsHandler diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a2bb691b401..567b8452ae2 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -101,7 +101,7 @@ if TYPE_CHECKING: from litellm.types.agents import AgentResponse from litellm.types.caching import RedisPipelineIncrementOperation - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any diff --git a/litellm/proxy/hooks/rate_limiter_utils.py b/litellm/proxy/hooks/rate_limiter_utils.py index bbe38068638..c4949834896 100644 --- a/litellm/proxy/hooks/rate_limiter_utils.py +++ b/litellm/proxy/hooks/rate_limiter_utils.py @@ -7,6 +7,7 @@ from typing import Final import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.types.router import ModelGroupInfo from litellm.types.utils import PriorityReservationDict @@ -85,10 +86,10 @@ def _resolve_provider_from_router_alias( if llm_router is None: return None try: - model_list: Final = getattr(llm_router, "model_list", None) + model_list: Final[object] = getattr(llm_router, "model_list", None) if not model_list: return None - for deployment in model_list: + for deployment in iter_items(model_list): if not isinstance(deployment, dict): continue if deployment.get("model_name") != model: diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 5f31aa788e2..a87455e5a60 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -9,6 +9,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import iter_items from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._types import ( CommonProxyErrors, @@ -166,6 +167,16 @@ class _AccessGroupTx(Protocol): def litellm_agentstable(self) -> _AgentTable: ... +class _AccessGroupTransaction(Protocol): + async def __aenter__(self) -> _AccessGroupTx: ... + + async def __aexit__(self, *exc_info: object) -> None: ... + + +def _access_group_transaction(prisma_client: PrismaClient) -> _AccessGroupTransaction: + return prisma_client.db.tx() + + def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: raise HTTPException( @@ -191,6 +202,10 @@ def _tool_permissions_json(value: Mapping[str, Sequence[str]] | None) -> "Json": return Json(None) if value is None else Json({server: list(tools) for server, tools in value.items()}) +def _explicitly_set_fields(data: AccessGroupUpdateRequest) -> Mapping[str, object]: + return data.model_dump(exclude_unset=True) + + @dataclass(frozen=True, slots=True) class _ResourceNames: mcp_servers: Mapping[str, str] @@ -539,7 +554,7 @@ async def create_access_group( try: tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: existing: Final = await tx.litellm_accessgrouptable.find_unique( where={"access_group_name": data.access_group_name} ) @@ -647,7 +662,7 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - update_fields: Final = data.model_dump(exclude_unset=True) + update_fields: Final = _explicitly_set_fields(data) update_data: Final[dict] = {"updated_by": user_api_key_dict.user_id} for field, value in update_fields.items(): if field in ( @@ -658,7 +673,7 @@ async def update_access_group( "access_agent_ids", "access_skill_names", ): - value = list(value or ()) + value = list(iter_items(value or ())) update_data[field] = ( _tool_permissions_json(data.mcp_tool_permissions) if field == "mcp_tool_permissions" else value ) @@ -672,7 +687,7 @@ async def update_access_group( try: tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: # Read inside the transaction so delta computation is consistent with the write, # avoiding a TOCTOU race where a concurrent update could make deltas stale. existing: Final = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) @@ -687,10 +702,10 @@ async def update_access_group( old_team_ids: Final[set[str]] = set(attached[access_group_id]) old_key_ids: Final[set[str]] = set(existing.assigned_key_ids or []) new_team_ids: Final[set[str]] = ( - set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + set(data.assigned_team_ids or []) if "assigned_team_ids" in update_fields else old_team_ids ) new_key_ids: Final[set[str]] = ( - set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids + set(data.assigned_key_ids or []) if "assigned_key_ids" in update_fields else old_key_ids ) teams_to_add = list(new_team_ids - old_team_ids) @@ -747,7 +762,7 @@ async def delete_access_group( affected_key_tokens: list[str] = [] tx: _AccessGroupTx - async with prisma_client.db.tx() as tx: + async with _access_group_transaction(prisma_client) as tx: existing: Final = await tx.litellm_accessgrouptable.find_unique(where={"access_group_id": access_group_id}) if existing is None: raise HTTPException( diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d40f64f435f..9c46e711cf6 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -54,6 +54,7 @@ from litellm.proxy.management_helpers.auto_router_permissions import ( from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.daily_activity_sql import build_where_clause +from litellm.repositories.table_repositories import PrismaTableRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository @@ -179,7 +180,7 @@ class _ShadowEvalAttemptRow(Protocol): class _ShadowEvalFunnelTable(Protocol): - async def create_many(self, data: Sequence[Mapping[str, object]], skip_duplicates: bool) -> int: ... + async def create_many(self, data: Sequence[Mapping[str, object]], *, skip_duplicates: bool) -> int: ... class _ShadowEvalAttemptTable(Protocol): @@ -204,16 +205,28 @@ def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable: return UserRepository(prisma_client).table +class _ShadowEvalJobRepository(PrismaTableRepository[_ShadowEvalJobRow]): + table_name = "litellm_shadowevaljob" + + +class _ShadowEvalFunnelRepository(PrismaTableRepository[object]): + table_name = "litellm_shadowevalfunnel" + + +class _ShadowEvalAttemptRepository(PrismaTableRepository[_ShadowEvalAttemptRow]): + table_name = "litellm_shadowevalattempt" + + def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable: - return prisma_client.db.litellm_shadowevaljob + return _ShadowEvalJobRepository(prisma_client).table def _shadow_eval_funnel(prisma_client: "PrismaClient") -> _ShadowEvalFunnelTable: - return prisma_client.db.litellm_shadowevalfunnel # pyright: ignore[reportAttributeAccessIssue] # generated client + return _ShadowEvalFunnelRepository(prisma_client).table def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTable: - return prisma_client.db.litellm_shadowevalattempt + return _ShadowEvalAttemptRepository(prisma_client).table async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]: @@ -221,6 +234,10 @@ async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) - return await prisma_client.db.query_raw(query, *args) +async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int: + return await prisma_client.db.execute_raw(query, *args) + + async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None: from litellm.proxy.management_endpoints.model_management_endpoints import ( ModelManagementAuthChecks, @@ -1788,7 +1805,7 @@ async def start_shadow_eval( # deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id # that happens to equal a key hash never matches the other kind's slot. for target_type, ids in requested_by_type: - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) + await _execute_raw(prisma_client, _SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ "OR": [ @@ -2005,8 +2022,8 @@ async def stop_shadow_eval_job( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) stamp: Final = datetime.now(timezone.utc) operator: Final = user_api_key_dict.user_id or "operator" - claimed: Final = await prisma_client.db.execute_raw( - _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() + claimed: Final = await _execute_raw( + prisma_client, _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() ) legs: Final = _LEG_ROWS.validate_python( await _shadow_eval_jobs(prisma_client).find_many(where={"group_id": job_id}) or () diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index b1f0002e143..14dd9ce6e8a 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -42,6 +42,7 @@ from litellm.proxy.db.db_span import db_span if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import PrismaClient + from litellm.repositories.prisma_protocols import TableActions from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports _ALGO_AES_GCM, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _ENCRYPTION_ALGORITHM_SETTING, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -523,7 +524,7 @@ async def _scan_one_table( scalar_columns: tuple, ) -> LocationReport: report: Final = LocationReport(location=location) - table: Final = getattr(prisma_client.db, db_attr, None) + table: Final[TableActions[object] | None] = getattr(prisma_client.db, db_attr, None) if table is None: return report try: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index eebc537509b..0ad4258fb04 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -211,6 +211,7 @@ if TYPE_CHECKING: from prisma import models as prisma_models _RepositoryModelT = TypeVar("_RepositoryModelT", bound=BaseModel) +_RawRowT = TypeVar("_RawRowT") class _UserRowLike(Protocol): @@ -234,6 +235,12 @@ class _TxTables(Protocol): litellm_proxymodeltable: TableActions[object] +class _ModelReencryptTransaction(Protocol): + async def __aenter__(self) -> _TxTables: ... + + async def __aexit__(self, *exc_info: object) -> None: ... + + class _ModelParamsUpdate(TypedDict): litellm_params: ReadOnly["prisma.Json"] @@ -334,6 +341,14 @@ def _credentials_table(prisma_client: PrismaClient) -> TableActions[CredentialIt ) +async def _query_raw_rows(prisma_client: PrismaClient, query: str, *args: object) -> Sequence[_RawRowT]: + return await prisma_client.db.query_raw(query, *args) + + +def _model_reencrypt_transaction(prisma_client: PrismaClient) -> _ModelReencryptTransaction: + return prisma_client.db.tx(timeout=timedelta(minutes=2)) + + def _config_table(prisma_client: PrismaClient, *, use_writer: bool = False) -> _ConfigTableActions: return cast( # cast-ok: ConfigRepository.table hides the write actions this module needs on that same object "_ConfigTableActions", ConfigRepository(prisma_client, use_writer=use_writer).table @@ -1313,11 +1328,6 @@ async def validate_team_id_used_in_service_account_request( return True -_BUDGET_NUMERIC_KEYS = frozenset( - ["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"] -) - - def _enforce_upperbound_key_params( data: GenerateKeyRequest | UpdateKeyRequest, fill_defaults: bool = True, @@ -1332,13 +1342,20 @@ def _enforce_upperbound_key_params( # (GHSA-2rv4-xv66-fpjg): float('nan') passes every `< 0` check because # nan < 0 is False, and spend >= nan is always False, permanently disabling # budget enforcement for any key that carries it. - for elem in data: - key, value = elem - if key in _BUDGET_NUMERIC_KEYS and value is not None: - if not math.isfinite(value): + budget_numeric_fields: Final = ( + ("max_budget", data.max_budget), + ("max_parallel_requests", data.max_parallel_requests), + ("tpm_limit", data.tpm_limit), + ("rpm_limit", data.rpm_limit), + ("tpd_limit", data.tpd_limit), + ("soft_budget", data.soft_budget), + ) + for field_name, field_value in budget_numeric_fields: + if field_value is not None: + if not math.isfinite(field_value): raise HTTPException( status_code=400, - detail={"error": f"{key} must be a finite number. Received: {value}"}, + detail={"error": f"{field_name} must be a finite number. Received: {field_value}"}, ) if litellm.upperbound_key_generate_params is None: @@ -5746,8 +5763,7 @@ async def rotate_master_key( ] ) verbose_proxy_logger.debug("Re-encrypting litellm_params on %s model rows", len(reencrypted_models)) - async with prisma_client.db.tx(timeout=timedelta(minutes=2)) as tx_ctx: - tx: Final[_TxTables] = tx_ctx + async with _model_reencrypt_transaction(prisma_client) as tx: for reencrypted_model in reencrypted_models: await tx.litellm_proxymodeltable.update_many( data=_ModelParamsUpdate(litellm_params=prisma.Json(reencrypted_model.litellm_params)), @@ -7344,7 +7360,7 @@ async def key_aliases( where_sql: Final = " AND ".join(where_parts) count_sql: Final = f'SELECT COUNT(*) AS count FROM "LiteLLM_VerificationToken" WHERE {where_sql}' - count_rows: Final[Sequence[Mapping[str, int]]] = await prisma_client.db.query_raw(count_sql, *query_params) + count_rows: Final[Sequence[Mapping[str, int]]] = await _query_raw_rows(prisma_client, count_sql, *query_params) total_count: Final = int(count_rows[0]["count"]) if count_rows else 0 aliases_params: Final = query_params + [size, (page - 1) * size] @@ -7357,7 +7373,9 @@ async def key_aliases( f" ORDER BY key_alias ASC" f" LIMIT ${limit_idx} OFFSET ${offset_idx}" ) - alias_rows: Final[Sequence[Mapping[str, str]]] = await prisma_client.db.query_raw(aliases_sql, *aliases_params) + alias_rows: Final[Sequence[Mapping[str, str]]] = await _query_raw_rows( + prisma_client, aliases_sql, *aliases_params + ) aliases: Final[list[str]] = [row["key_alias"] for row in alias_rows if row.get("key_alias")] total_pages: Final = -(-total_count // size) if total_count > 0 else 0 diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index b93c05c555f..da84c2d0013 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,10 +1,12 @@ """`/management/v1/spend_logs` facets.""" +from collections.abc import Sequence from datetime import datetime, timezone from functools import partial -from typing import Annotated, Final, Literal +from typing import Annotated, Final, Literal, Protocol from fastapi import APIRouter, Depends, Query, Request +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth @@ -19,6 +21,7 @@ from litellm.proxy.list_api.common import ( reject_unknown_query_params, ) from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX +from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, PageMeta, @@ -33,6 +36,19 @@ router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX) SPEND_LOGS_FACET_SCAN_CAP: Final = 10000 +class _FacetRow(TypedDict, total=False): + end_user: ReadOnly[str | None] + user: ReadOnly[str | None] + + +class _FacetQuery(Protocol): + async def __call__(self, query: str, *args: object) -> Sequence[_FacetRow]: ... + + +def _facet_query(prisma_client: PrismaClient) -> _FacetQuery: + return prisma_client.db.query_raw + + def _as_utc(value: datetime) -> datetime: return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) @@ -125,8 +141,8 @@ async def _list_spend_log_facet( f" ORDER BY {column_sql} ASC" f" LIMIT ${scan_idx + 1} OFFSET ${scan_idx + 2}" ) - rows: Final = await prisma_client.db.query_raw(facet_sql, *params) - values: Final[list[str]] = [row[column] for row in rows if row.get(column)] + rows: Final = await _facet_query(prisma_client)(facet_sql, *params) + values: Final[list[str]] = [value for row in rows if (value := row.get(column))] has_more: Final = len(values) > page_size return FacetListResponse( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index a014b9cf8a5..776072c69be 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -104,6 +104,14 @@ class _HasServerId(Protocol): server_id: str +class _TemporaryServerCacheWriter(Protocol): + async def async_set_cache(self, *, key: str, value: str, ttl: int) -> object: ... + + +class _TemporaryServerCacheReader(Protocol): + async def async_get_cache(self, *, key: str) -> object: ... + + def _session_cookie_claims(claims: Mapping[str, object]) -> Mapping[str, object]: return claims @@ -527,7 +535,7 @@ if MCP_AVAILABLE: """ if litellm.cache is None or not hasattr(litellm.cache, "cache"): return - cache_backend: Final = getattr(litellm.cache, "cache", None) + cache_backend: Final[_TemporaryServerCacheWriter | None] = getattr(litellm.cache, "cache", None) if cache_backend is None or not hasattr(cache_backend, "async_set_cache"): return @@ -566,7 +574,7 @@ if MCP_AVAILABLE: """ if litellm.cache is None or not hasattr(litellm.cache, "cache"): return None - cache_backend: Final = getattr(litellm.cache, "cache", None) + cache_backend: Final[_TemporaryServerCacheReader | None] = getattr(litellm.cache, "cache", None) if cache_backend is None or not hasattr(cache_backend, "async_get_cache"): return None diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index fc0868a5fc9..11f616a5e18 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3217,12 +3217,13 @@ async def update_useful_links( ) -def _validated_labeled_tiers( - tier_labels: dict[ComplexityTier, str], # mutable-ok: Pydantic materializes JSON object fields as dicts -) -> tuple[tuple[ComplexityTier, str], ...]: +_DECODED_JSON: Final = TypeAdapter(object) + + +def _validated_labeled_tiers(tier_labels: object) -> tuple[tuple[ComplexityTier, str], ...]: """Validate tier labels once for both prompt-preview transports.""" try: - return ComplexityRouterConfig(tier_labels=tier_labels).labeled_tiers() + return ComplexityRouterConfig.model_validate({"tier_labels": tier_labels}).labeled_tiers() except (TypeError, ValidationError) as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", @@ -3237,7 +3238,7 @@ def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[Complexity if not tier_labels: return None try: - parsed: Final = json.loads(tier_labels) + parsed: Final = _DECODED_JSON.validate_python(json.loads(tier_labels)) except JSONDecodeError as e: raise ProxyException( message=f"tier_labels must be a JSON object of tier name to display name: {e}", diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 64566221d4d..1f4ca442d20 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -71,10 +71,11 @@ def _get_routing_strategies_from_router_class() -> list[str]: # Get the routing_strategy parameter routing_strategy_param: Final = sig.parameters.get("routing_strategy") + routing_strategy_annotation: Final[object] = routing_strategy_param.annotation if routing_strategy_param else None - if routing_strategy_param and routing_strategy_param.annotation: + if routing_strategy_param and routing_strategy_annotation: # Extract Literal values using get_args - literal_values: Final = get_args(routing_strategy_param.annotation) + literal_values: Final = get_args(routing_strategy_annotation) if literal_values: return list(literal_values) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c7adae2ce8d..361778ab2ea 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -479,6 +479,14 @@ def _team_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models. return TeamRepository(prisma_client).table +class _ExecuteRaw(Protocol): + async def __call__(self, query: str, *args: object) -> int: ... + + +def _execute_raw(prisma_client: PrismaClient) -> _ExecuteRaw: + return prisma_client.db.execute_raw + + def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]": return cast( # cast-ok: generated actions type Json columns as str; TableActions widens inputs to Mapping "TableActions[prisma_models.LiteLLM_TeamTable]", tx.litellm_teamtable @@ -3092,7 +3100,7 @@ async def _process_team_members( # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models member_allowed_models = data.allowed_models - team_default_member_models: Final = getattr(complete_team_data, "default_team_member_models", None) + team_default_member_models: Final = complete_team_data.default_team_member_models if member_allowed_models is None and team_default_member_models: member_allowed_models = team_default_member_models @@ -6572,7 +6580,7 @@ async def append_team_models( # When the team currently has models=[] (unrestricted access), the # CASE expression inserts the 'all-proxy-models' sentinel first. models_to_add: Final = list(data.models) - await prisma_client.db.execute_raw( + await _execute_raw(prisma_client)( 'UPDATE "LiteLLM_TeamTable" ' "SET models = (" " SELECT ARRAY(SELECT DISTINCT unnest(" diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index fd5eb672811..02494de8c87 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -278,6 +278,12 @@ class _CustomSsoCall(Protocol): async def __call__(self, sso_response: object) -> SSOUserDefinedValues | None: ... +def _configured_custom_sso() -> _CustomSsoCall | None: + from litellm.proxy.proxy_server import user_custom_sso + + return user_custom_sso + + class _ServicePrincipalAssignment(Protocol): def get(self, key: str) -> str: ... @@ -1051,6 +1057,11 @@ class _UserinfoFallbackCapableSSO(Protocol): ) -> CustomOpenID | None: ... +class _GenericSSOClient(_UserinfoFallbackCapableSSO, Protocol): + @property + def refresh_token(self) -> str | None: ... + + async def _verify_generic_sso_or_fall_back_to_id_token( generic_sso: _UserinfoFallbackCapableSSO, request: Request, @@ -1578,7 +1589,7 @@ async def get_generic_sso_response( team_mappings: Final = await _setup_team_mappings() generic_include_token_claims: Final = os.getenv("GENERIC_INCLUDE_TOKEN_CLAIMS", "false").lower() == "true" - def response_convertor(response: Mapping[str, object], httpx_session: object): + def response_convertor(response: Mapping[str, object], httpx_session: object) -> CustomOpenID: nonlocal received_response # return for user debugging response_id_token: Final = response.get("id_token") response_access_token: Final = response.get("access_token") @@ -1613,7 +1624,7 @@ async def get_generic_sso_response( discovery_document=discovery, response_convertor=response_convertor, ) - generic_sso: Final = SSOProvider( + generic_sso: Final[_GenericSSOClient] = SSOProvider( client_id=generic_client_id, client_secret=generic_client_secret, redirect_uri=redirect_url, @@ -1712,9 +1723,7 @@ async def get_generic_sso_response( access_token_str: str | None = combined_response.get("access_token") else: result = await _verify_generic_sso_or_fall_back_to_id_token( - generic_sso=cast( # cast-ok: fastapi-sso builds the provider class dynamically via create_provider - _UserinfoFallbackCapableSSO, generic_sso - ), + generic_sso=generic_sso, request=request, token_exchange_params=token_exchange_params, additional_headers=additional_generic_sso_headers_dict, @@ -2350,9 +2359,7 @@ async def _build_cli_sso_user_defined_values( result: OpenID | dict, parsed_openid_result: ParsedOpenIDResult, ) -> SSOUserDefinedValues | None: - from litellm.proxy.proxy_server import user_custom_sso - - custom_sso_handler: Final[_CustomSsoCall | None] = user_custom_sso + custom_sso_handler: Final = _configured_custom_sso() user_id: Final = parsed_openid_result.get("user_id") if custom_sso_handler is not None: if inspect.iscoroutinefunction(custom_sso_handler): @@ -2438,10 +2445,8 @@ async def _complete_cli_sso_callback_session( ): from fastapi.responses import HTMLResponse - from litellm.proxy.proxy_server import user_custom_sso - user_id: Final = parsed_openid_result.get("user_id") - if user_custom_sso is None: + if _configured_custom_sso() is None: _require_sso_user_id(user_id) user_email: Final = parsed_openid_result.get("user_email") user_info: Final = await get_user_info_from_db( @@ -3807,7 +3812,6 @@ class SSOAuthenticationHandler: proxy_logging_obj, redis_usage_cache, user_api_key_cache, - user_custom_sso, ) from litellm.proxy.utils import get_prisma_client_or_throw from litellm.types.proxy.ui_sso import ReturnedUITokenObject @@ -3832,7 +3836,7 @@ class SSOAuthenticationHandler: # But if it is, we want their models preferences user_defined_values: SSOUserDefinedValues | None = None - custom_sso_handler: Final[_CustomSsoCall | None] = user_custom_sso + custom_sso_handler: Final = _configured_custom_sso() if custom_sso_handler is not None: if inspect.iscoroutinefunction(custom_sso_handler): user_defined_values = await custom_sso_handler(result) diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 3468396c6f6..920970427ba 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -310,7 +310,7 @@ async def _resolve_mcp_server_identifiers_to_ids( identifiers=identifiers, prisma_client=prisma_client, ): - server_id = getattr(server, "server_id", None) + server_id = server.server_id if not server_id: continue for identifier in identifiers: diff --git a/litellm/proxy/memory/memory_endpoints.py b/litellm/proxy/memory/memory_endpoints.py index 34c80bcdce3..4457df460d4 100644 --- a/litellm/proxy/memory/memory_endpoints.py +++ b/litellm/proxy/memory/memory_endpoints.py @@ -181,8 +181,8 @@ async def _assert_write_access( """ if _is_admin(user_api_key_dict): return - row_user_id: Final = getattr(row, "user_id", None) - row_team_id: Final = getattr(row, "team_id", None) + row_user_id: Final = row.user_id + row_team_id: Final = row.team_id # Personal ownership. if row_user_id and row_user_id == user_api_key_dict.user_id: diff --git a/litellm/proxy/moyai_endpoints.py b/litellm/proxy/moyai_endpoints.py index c4d8ad139f7..45f4502ec4e 100644 --- a/litellm/proxy/moyai_endpoints.py +++ b/litellm/proxy/moyai_endpoints.py @@ -20,9 +20,10 @@ from typing import TYPE_CHECKING, Annotated, Final from urllib.parse import urlencode, urlparse from fastapi import APIRouter, Depends, HTTPException, Request, status -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter from litellm._internal_context import with_service_target +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( @@ -114,6 +115,9 @@ def _sign_connect_code(master_key: str, moyai_url: str, user_id: str | None) -> return f"{_b64url(payload)}.{_b64url(signature)}" +_DECODED_CONNECT_PAYLOAD: Final = TypeAdapter(object) + + def _decode_connect_code(master_key: str, code: str) -> _MoyaiConnectCode: try: payload_b64, signature_b64 = code.split(".", 1) @@ -125,18 +129,23 @@ def _decode_connect_code(master_key: str, code: str) -> _MoyaiConnectCode: if not hmac.compare_digest(signature, expected): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") try: - payload: Final = json.loads(payload_raw) + payload: Final = as_str_mapping(_DECODED_CONNECT_PAYLOAD.validate_python(json.loads(payload_raw))) except (ValueError, TypeError): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - if not isinstance(payload, dict) or not isinstance(payload.get("exp"), int) or payload["exp"] < int(time.time()): + if payload is None: raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - if not isinstance(payload.get("moyai_origin"), str) or not isinstance(payload.get("nonce"), str): + exp: Final = payload.get("exp") + if not isinstance(exp, int) or exp < int(time.time()): + raise HTTPException(status_code=400, detail="Invalid Moyai connect code") + moyai_origin: Final = payload.get("moyai_origin") + nonce: Final = payload.get("nonce") + if not isinstance(moyai_origin, str) or not isinstance(nonce, str): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") user_id: Final = payload.get("user_id") return _MoyaiConnectCode( - moyai_origin=payload["moyai_origin"], - nonce=payload["nonce"], - exp=payload["exp"], + moyai_origin=moyai_origin, + nonce=nonce, + exp=exp, user_id=user_id if isinstance(user_id, str) else None, ) diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py index 1aa24eed4cd..5d596aece32 100644 --- a/litellm/proxy/openai_files_endpoints/batch_file_validation.py +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -4,12 +4,15 @@ from dataclasses import dataclass from itertools import chain from typing import BinaryIO, Final, NoReturn +from pydantic import TypeAdapter from typing_extensions import assert_never +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException from litellm.proxy.openai_files_endpoints.file_usage_caps import FileUsageLimit, describe_limit_source _MB: Final = 1024 * 1024 +_DECODED_LINE: Final = TypeAdapter(object) @dataclass(frozen=True, slots=True) @@ -95,10 +98,10 @@ def _iter_lines(file_source: bytes | BinaryIO) -> Iterator[bytes]: def _check_line(line_number: int, raw_line: bytes, line_shape: BatchLineShape) -> BatchFileValidationFailure | None: try: - parsed: Final = json.loads(raw_line) + parsed: Final = as_str_mapping(_DECODED_LINE.validate_python(json.loads(raw_line))) except (ValueError, RecursionError): return BatchFileInvalidJsonLine(line_number=line_number) - if not isinstance(parsed, dict): + if parsed is None: return BatchFileLineNotObject(line_number=line_number) missing: Final = next((key for key in line_shape.required_keys if key not in parsed), None) if missing is None: diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index afc1e1c96fa..1f507982e65 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -8,10 +8,11 @@ storage backends (e.g., Azure Blob Storage) and managing associated metadata. import base64 import time from collections.abc import Sequence -from typing import Any, Final, cast +from typing import Final from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid as uuid_module +from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.llms.base_llm.files.storage_backend import BaseFileStorageBackend from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend @@ -19,6 +20,7 @@ from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose +from litellm.types.passthrough_endpoints.managed_id_rewriter import ManagedFileIdWriter from litellm.types.utils import ExtractedFileData, SpecialEnums @@ -232,11 +234,10 @@ class StorageBackendFileService: proxy_logging_obj: Proxy logging object user_api_key_dict: User API key authentication data """ - managed_files_obj = proxy_logging_obj.get_proxy_hook("managed_files") + managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files") if not managed_files_obj or not isinstance(managed_files_obj, BaseFileEndpoints): verbose_proxy_logger.warning("Managed files hook not available, skipping managed files storage") return - managed_files_obj = cast(Any, managed_files_obj) # Create model mappings using storage URL model_mappings: Final = {model_name: storage_url for model_name in target_model_names} @@ -260,7 +261,10 @@ class StorageBackendFileService: ) # Store in managed files - await managed_files_obj.store_unified_file_id( + managed_files_writer: Final = require_attributes( + managed_files_obj, ManagedFileIdWriter, "store_unified_file_id" + ) + await managed_files_writer.store_unified_file_id( file_id=base64_unified_file_id, file_object=file_object, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 16fbd1de376..9eef4f0ef9d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1305,7 +1305,7 @@ async def handle_bedrock_count_tokens( verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e) from litellm.litellm_core_utils.llm_response_utils.get_headers import get_response_headers - provider_headers: Final = getattr(getattr(e, "response", None), "headers", None) + provider_headers: Final = e.response.headers raise HTTPException( status_code=e.status_code, detail={"error": e.message}, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py index 7527c504c59..3fb4fbd555d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/comprehend_medical_passthrough_logging_handler.py @@ -27,11 +27,15 @@ COMPREHEND_MEDICAL_COST_PER_UNIT_USD: Final[Mapping[str, float]] = MappingProxyT COMPREHEND_MEDICAL_SUPPORTED_OPERATIONS: Final = frozenset(COMPREHEND_MEDICAL_COST_PER_UNIT_USD) +def _operation_from_headers(headers: Mapping[str, str]) -> str: + target: Final = headers.get("x-amz-target", "") + return target.split(".")[-1] + + class ComprehendMedicalPassthroughLoggingHandler: @staticmethod def _operation_from_response(httpx_response: httpx.Response) -> str: - target: Final[str] = httpx_response.request.headers.get("x-amz-target", "") - return target.split(".")[-1] + return _operation_from_headers(httpx_response.request.headers) @staticmethod def get_cost_for_operation(operation: str, text: str) -> float: diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py index e277655f1e1..2afc1d17bc2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -36,6 +36,7 @@ from litellm.types.passthrough_endpoints.tinyfish import ( from litellm.types.utils import StandardPassThroughResponseObject _RUN_ADAPTER: Final = TypeAdapter(TinyfishRun) +_DECODED_JSON: Final = TypeAdapter(object) _EMPTY_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) @@ -323,7 +324,7 @@ class TinyFishPassthroughLoggingHandler: "TinyFish passthrough: GET /v1/runs/%s returned %s", safe_run_id, response.status_code ) return None - payload: Final[object] = response.json() # any-ok: httpx Response.json() -> Any + payload: Final = _DECODED_JSON.validate_python(response.json()) return _parse_run(payload) except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request verbose_proxy_logger.warning("[Non blocking logging error] TinyFish run fetch failed: %s", e) @@ -413,7 +414,7 @@ def _run_id_from_sse_chunks(all_chunks: Sequence[str]) -> str | None: if not line.startswith("data:"): continue try: - event_payload: object = json.loads(line[5:].strip()) # any-ok: json.loads -> Any + event_payload: object = _DECODED_JSON.validate_python(json.loads(line[5:].strip())) except json.JSONDecodeError: continue event = _parse_run(event_payload) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py index 79f6dee90f7..5b45316a44c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/transcribe_passthrough_logging_handler.py @@ -492,6 +492,10 @@ def s3_media_url(media_uri: str, aws_region_name: str) -> str | None: return f"https://{bucket}.s3.{aws_region_name}.{dns_suffix}/{quote(key)}" +def _header_or_default(headers: Mapping[str, str], name: str, default: str) -> str: + return headers.get(name, default) + + def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> bool: try: modified_at: Final = parsedate_to_datetime(headers["last-modified"]).timestamp() @@ -501,7 +505,7 @@ def media_predates_job(headers: Mapping[str, str], job_created_at: float) -> boo async def write_media_within_limit(response: httpx.Response, media_file: IO[bytes], max_bytes: int) -> bool: - if int(response.headers.get("content-length", "0")) > max_bytes: + if int(_header_or_default(response.headers, "content-length", "0")) > max_bytes: return False async for chunk in response.aiter_bytes(): _ = media_file.write(chunk) @@ -591,8 +595,7 @@ class TranscribePassthroughLoggingHandler: @staticmethod def _operation_from_response(httpx_response: httpx.Response) -> str: - headers: Final[Mapping[str, str]] = httpx_response.request.headers - target: Final[str] = headers.get("x-amz-target", "") + target: Final = _header_or_default(httpx_response.request.headers, "x-amz-target", "") return target.split(".")[-1] @staticmethod diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 59b8d40577a..e70d583a027 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -7,6 +7,7 @@ records the reported values without recomputing them. """ import math +from collections.abc import Mapping from dataclasses import dataclass from typing import Final @@ -89,7 +90,7 @@ def _parse_total_tokens(raw_value: str | None) -> int | None: return total_tokens -def parse_upstream_reported_usage(headers: httpx.Headers) -> UpstreamReportedUsage | None: +def parse_upstream_reported_usage(headers: Mapping[str, str]) -> UpstreamReportedUsage | None: """Read the reported totals off an upstream pass-through response. Returns ``None`` when neither header is present, which is the normal case diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py index c751091ff9b..15e7b7e8482 100644 --- a/litellm/proxy/policy_engine/attachment_registry.py +++ b/litellm/proxy/policy_engine/attachment_registry.py @@ -5,10 +5,10 @@ Attachments define WHERE policies apply, separate from the policy definitions. This allows the same policy to be attached to multiple scopes. """ -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict +from typing import TYPE_CHECKING, Final, TypedDict from litellm._logging import verbose_proxy_logger from litellm.repositories.table_repositories import PolicyAttachmentRepository @@ -102,7 +102,7 @@ class AttachmentRegistry: self._initialized = True verbose_proxy_logger.info("Loaded %s policy attachments", len(self._attachments)) - def _parse_attachment(self, attachment_data: dict[str, Any]) -> PolicyAttachment: + def _parse_attachment(self, attachment_data: Mapping[str, object]) -> PolicyAttachment: """ Parse an attachment from raw configuration data. @@ -112,15 +112,17 @@ class AttachmentRegistry: Returns: Parsed PolicyAttachment object """ - return PolicyAttachment( - policy=attachment_data.get("policy", ""), - scope=attachment_data.get("scope"), - teams=attachment_data.get("teams"), - keys=attachment_data.get("keys"), - models=attachment_data.get("models"), - tags=attachment_data.get("tags"), - priority=attachment_data.get("priority"), - default=attachment_data.get("default", False), + return PolicyAttachment.model_validate( + { + "policy": attachment_data.get("policy", ""), + "scope": attachment_data.get("scope"), + "teams": attachment_data.get("teams"), + "keys": attachment_data.get("keys"), + "models": attachment_data.get("models"), + "tags": attachment_data.get("tags"), + "priority": attachment_data.get("priority"), + "default": attachment_data.get("default", False), + } ) def get_attached_policies( diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 1b1b2302236..4ad29bd3478 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -11,7 +11,7 @@ from collections.abc import Callable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -52,6 +52,8 @@ try: except ImportError: HTTPException = None +_LEGACY_HOOK_REPLACEMENT: Final = TypeAdapter(object) + class UndeliverableStreamRewrite(Exception): def __init__(self, guardrail_name: str, reason: str) -> None: @@ -232,10 +234,12 @@ class _LegacyHookStreamAdapter(CustomGuardrail): logging_obj: "LiteLLMLoggingObj | None" = None, ) -> GenericGuardrailAPIInputs: hooked: Final = self.endpoint_translation.post_call_hook_response(request_data.get("response")) - replacement: Final = await self.inner.async_post_call_success_hook( - data=request_data, - user_api_key_dict=self.user_api_key_dict, - response=hooked, + replacement: Final = _LEGACY_HOOK_REPLACEMENT.validate_python( + await self.inner.async_post_call_success_hook( + data=request_data, + user_api_key_dict=self.user_api_key_dict, + response=hooked, + ) ) rewrite: Final = hooked if replacement is None else replacement if rewrite is None: diff --git a/litellm/proxy/policy_engine/policy_endpoints.py b/litellm/proxy/policy_engine/policy_endpoints.py index f4b38bea14e..68ac71c567b 100644 --- a/litellm/proxy/policy_engine/policy_endpoints.py +++ b/litellm/proxy/policy_engine/policy_endpoints.py @@ -623,7 +623,7 @@ async def test_pipeline( ``` """ try: - validated_pipeline: Final = GuardrailPipeline(**request.pipeline) + validated_pipeline: Final = GuardrailPipeline.model_validate(request.pipeline) except Exception as e: raise HTTPException(status_code=400, detail=f"Invalid pipeline: {e}") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ade719ae0ed..44f43808dc8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -24,6 +24,7 @@ from collections.abc import ( Awaitable, Callable, Collection, + Container, Iterable, Mapping, MutableMapping, @@ -360,6 +361,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_parent_otel_span_from_kwargs, iter_items, normalize_drop_params, + require_attributes, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -474,6 +476,7 @@ from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored +from litellm.proxy.common_utils.custom_openapi_spec import JsonObject from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.discoverable_model_filter import ( @@ -1140,7 +1143,7 @@ def _redact_worker_config_for_logging(worker_config: str | dict[str, JsonValue] return None if isinstance(worker_config, dict): return _redact_secret_values_in_obj(worker_config) - parsed: Final = safe_json_loads(worker_config, default=None) + parsed: Final[JsonValue] = safe_json_loads(worker_config, default=None) if isinstance(parsed, dict): return safe_dumps(_redact_secret_values_in_obj(parsed)) return worker_config @@ -1466,11 +1469,13 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState ) _module_path, _func_name = _hook_spec.rsplit(":", 1) _module = importlib.import_module(_module_path) - _hook_fn = getattr(_module, _func_name) + _hook_fn = _UNTYPED_VALUE.validate_python(getattr(_module, _func_name)) if inspect.iscoroutinefunction(_hook_fn): await _hook_fn() - else: + elif callable(_hook_fn): _hook_fn() + else: + raise TypeError(f"'{type(_hook_fn).__name__}' object is not callable") verbose_proxy_logger.info("Worker startup hook '%s' executed successfully", _hook_spec) except Exception as e: verbose_proxy_logger.error("Worker startup hook '%s' failed: %s", _hook_spec, e) @@ -1918,15 +1923,22 @@ def _strip_operation_id_method_suffix(operation_id: str) -> str: return operation_id +class _SupportsValues(Protocol): + def values(self) -> Iterable[JsonValue]: ... + + +def _openapi_path_items(openapi_schema: JsonObject) -> Iterable[JsonObject]: + paths: Final = require_attributes(openapi_schema.get("paths", {}), _SupportsValues, "values") + return (path_item for path_item in paths.values() if isinstance(path_item, dict)) + + def ensure_unique_openapi_operation_ids( - openapi_schema: dict[str, Any], + openapi_schema: JsonObject, reserved_operation_ids: set[str] | None = None, -) -> dict[str, Any]: +) -> JsonObject: operation_entries: Final = [] operation_id_counts: Final[dict[str, int]] = {} - for path_item in openapi_schema.get("paths", {}).values(): - if not isinstance(path_item, dict): - continue + for path_item in _openapi_path_items(openapi_schema): for method, operation in path_item.items(): if method not in _OPENAPI_HTTP_METHODS or not isinstance(operation, dict): continue @@ -2003,6 +2015,19 @@ from starlette.routing import BaseRoute from litellm.proxy.common_utils.openapi_route import OpenAPISchema, serve_openapi_off_the_event_loop +class _OpenAPIQueryParam(Protocol): + @property + def name(self) -> object: ... + + @property + def required(self) -> object: ... + + +def _query_param_name_and_required(param: object) -> tuple[object, object]: + fields: Final = require_attributes(param, _OpenAPIQueryParam, "name", "required") + return fields.name, fields.required + + def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_routes: list) -> dict: """ Add a synthetic GET stub for each WebSocket route so it appears in Swagger UI. @@ -2019,14 +2044,15 @@ def _inject_websocket_stubs_into_openapi_schema(openapi_schema: dict, websocket_ try: if hasattr(route, "dependant") and route.dependant is not None: # Handle both FastAPI <0.120 and >=0.120 - query_params = getattr(route.dependant, "query_params", []) + query_params = _UNTYPED_VALUE.validate_python(getattr(route.dependant, "query_params", [])) if query_params: - for param in query_params: + for param in iter_items(query_params): + name, required = _query_param_name_and_required(param) parameters.append( { - "name": param.name, + "name": name, "in": "query", - "required": param.required, + "required": required, "schema": {"type": "string"}, } ) @@ -5576,6 +5602,13 @@ def _log_ignored_cost_map_copy(model_id: str, fields: tuple[str, ...]) -> None: _DB_CONFIG_CALLBACK_EVENT_TYPES: Final[Mapping[str, tuple[Literal["success", "failure"], ...]]] = MappingProxyType( {"success_callback": ("success",), "failure_callback": ("failure",), "callbacks": ("success", "failure")} ) +_DB_CONFIG_CALLBACK_LISTS: Final[Mapping[str, Callable[[], Container[object]]]] = MappingProxyType( + { + "success_callback": lambda: litellm.success_callback, + "failure_callback": lambda: litellm.failure_callback, + "callbacks": lambda: litellm.callbacks, + } +) _CALLBACK_LIST_NAMES: Final = ( "callbacks", "success_callback", @@ -5586,7 +5619,9 @@ _CALLBACK_LIST_NAMES: Final = ( def _callback_list_entries(list_name: str) -> tuple[tuple[str, object], ...]: - return tuple((list_name, entry) for entry in getattr(litellm, list_name)) + return tuple( + (list_name, entry) for entry in iter_items(_UNTYPED_VALUE.validate_python(getattr(litellm, list_name))) + ) def _registered_callback_entries() -> tuple[tuple[str, object], ...]: @@ -5615,7 +5650,7 @@ def _restore_displaced_callback(list_name: str, entry: object) -> None: def _unregister_db_callback(registration: _DbCallbackRegistration) -> None: for list_name, entry in registration.added: litellm.logging_callback_manager.remove_callback_from_list_by_object( - getattr(litellm, list_name), entry, require_self=False + _UNTYPED_VALUE.validate_python(getattr(litellm, list_name)), entry, require_self=False ) for list_name, entry in registration.displaced: _restore_displaced_callback(list_name, entry) @@ -7824,7 +7859,7 @@ class ProxyConfig: self, callback: str, event_types: list[Literal["success", "failure"]], - existing_callbacks: list, + existing_callbacks: Container[object], ) -> None: """ Helper method to add a single callback to litellm for specified event types. @@ -7867,7 +7902,7 @@ class ProxyConfig: self._add_callback_from_db_to_in_memory_litellm_callbacks( callback=callback, event_types=list(_DB_CONFIG_CALLBACK_EVENT_TYPES[setting_key]), - existing_callbacks=getattr(litellm, setting_key), + existing_callbacks=_DB_CONFIG_CALLBACK_LISTS[setting_key](), ) after: Final = _registered_callback_entries() return _DbCallbackRegistration( @@ -8362,8 +8397,12 @@ class ProxyConfig: environment_read: Final = next(self._environment_read_numbers) sections: Final = tuple(self._settings_stores) responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections)) + response: _ConfigParamValueRow | None for section, response in zip(sections, responses): - if response is None or (param_value := getattr(response, "param_value", None)) is None: + if response is None: + continue + param_value: JsonValue = getattr(response, "param_value", None) + if param_value is None: continue verbose_proxy_logger.debug( @@ -8390,7 +8429,10 @@ class ProxyConfig: self, prisma_client: PrismaClient, row: object, environment_before_db: Mapping[str, str], read: int ) -> None: current: Final = await self._environment_row_not_behind_the_applied_one(prisma_client, row) - if current is None or (param_value := getattr(current, "param_value", None)) is None: + if current is None: + return + param_value: Final[object] = getattr(current, "param_value", None) + if param_value is None: return self._apply_db_environment_variables(param_value, _config_row_revision(current), environment_before_db, read) @@ -15997,7 +16039,23 @@ async def _get_caller_byok_team_scope( return key_team_scope | set(user_row.teams or []) -def _byok_row_outside_caller_teams(model_info_dict: dict[str, JsonValue], allowed_team_ids: set[str] | None) -> bool: +class _SupportsGet(Protocol): + def get(self, key: str, default: object = None, /) -> object: ... + + +def _with_get(value: object) -> _SupportsGet: + return require_attributes(value, _SupportsGet, "get") + + +class _SupportsLower(Protocol): + def lower(self) -> object: ... + + +def _lowered(value: object) -> object: + return require_attributes(value, _SupportsLower, "lower").lower() + + +def _byok_row_outside_caller_teams(model_info_dict: object, allowed_team_ids: set[str] | None) -> bool: """Whether a team BYOK row belongs to a team the caller is not a member of. `team_id` is only set on team BYOK rows; non-team rows fall through @@ -16005,7 +16063,7 @@ def _byok_row_outside_caller_teams(model_info_dict: dict[str, JsonValue], allowe """ if allowed_team_ids is None: return False - team_id: Final = model_info_dict.get("team_id") + team_id: Final = _with_get(model_info_dict).get("team_id") if team_id is None: return False return team_id not in allowed_team_ids @@ -16106,7 +16164,7 @@ async def _fetch_db_models_for_search( async def _apply_search_filter_to_models( - all_models: list[dict[str, Any]], + all_models: list[dict[str, object]], search: str, prisma_client: PrismaClient | None, proxy_config: ProxyConfig, @@ -16158,7 +16216,7 @@ async def _apply_search_filter_to_models( prisma_client=prisma_client, ) - def _is_byok_outside_caller_teams(model_info_dict: dict[str, JsonValue]) -> bool: + def _is_byok_outside_caller_teams(model_info_dict: object) -> bool: return _byok_row_outside_caller_teams(model_info_dict, allowed_team_ids) def _model_matches_search(m: dict[str, Any]) -> bool: @@ -16195,7 +16253,7 @@ async def _apply_search_filter_to_models( db_model_ids_in_router: Final = set() for m in filtered_router_models: - model_info = m.get("model_info", {}) + model_info = _with_get(m.get("model_info", {})) is_db_model = model_info.get("db_model", False) model_id = model_info.get("id") @@ -16306,7 +16364,7 @@ def _sort_models( reverse: Final = sort_order.lower() == "desc" def get_sort_key(model: dict[str, Any]) -> Any: - model_info: Final = model.get("model_info", {}) + model_info: Final = _with_get(_UNTYPED_VALUE.validate_python(model.get("model_info", {}))) if sort_by == "model_name": # Team BYOK models persist an internal `model_name` (e.g. @@ -16318,7 +16376,7 @@ def _sort_models( team_public_model_name: Final = model_info.get("team_public_model_name") if team_public_model_name: return str(team_public_model_name).lower() - return model.get("model_name", "").lower() + return _lowered(_UNTYPED_VALUE.validate_python(model.get("model_name", ""))) elif sort_by == "created_at": created_at: Final = model_info.get("created_at") @@ -16344,8 +16402,8 @@ def _sort_models( return normalized_dt elif sort_by == "costs": - input_cost: Final = model_info.get("input_cost_per_token", 0) or 0 - output_cost: Final = model_info.get("output_cost_per_token", 0) or 0 + input_cost: Final = model.get("model_info", {}).get("input_cost_per_token", 0) or 0 + output_cost: Final = model.get("model_info", {}).get("output_cost_per_token", 0) or 0 total_cost: Final = input_cost + output_cost # Put 0 or None costs at the end for asc, at the start for desc if total_cost == 0: @@ -16580,7 +16638,7 @@ async def _authorize_team_id_query( async def _filter_models_by_team_id( - all_models: list[dict[str, Any]], + all_models: list[dict[str, object]], team_id: str, prisma_client: PrismaClient, llm_router: Router, @@ -16624,7 +16682,7 @@ async def _filter_models_by_team_id( # every public model the admin can call. filtered_models: Final = [] for _model in all_models: - model_info = _model.get("model_info", {}) + model_info = _with_get(_model.get("model_info", {})) model_id = model_info.get("id", None) # BYOK rows owned by this team are always accessible to it, even if diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index cd634e490b6..e58149fba0b 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -107,10 +107,14 @@ def _features(row: ModelGroupInfoProxy) -> tuple[str, ...]: The hub's feature control has always been a multi-select over the `supports_*` flags. One boolean filter per flag would AND them, which is the opposite of what it does. """ + return _enabled_features(row.model_dump()) + + +def _enabled_features(fields: Mapping[str, object]) -> tuple[str, ...]: return tuple( sorted( name.removeprefix(FEATURE_PREFIX) - for name, value in row.model_dump().items() + for name, value in fields.items() if name.startswith(FEATURE_PREFIX) and value is True ) ) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 803df9f83ca..0b91e57f01e 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -728,7 +728,9 @@ async def _get_team_member_budget_counter( membership_cache_key: Final = team_membership_reservation_cache_key( user_id=valid_token.user_id, team_id=team_object.team_id ) - cached_team_membership: Final = await user_api_key_cache.async_get_cache(key=membership_cache_key) + cached_team_membership: Final = _CACHED_VALUE.validate_python( + await user_api_key_cache.async_get_cache(key=membership_cache_key) + ) team_membership: LiteLLM_TeamMembership | None = None if isinstance(cached_team_membership, LiteLLM_TeamMembership): team_membership = cached_team_membership diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index ea543aeae6e..4abc79febe1 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -21,6 +21,8 @@ from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol +from pydantic import TypeAdapter + from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import ( @@ -135,6 +137,9 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: return str(getattr(row, "model_name", "") or "") +_DECODED_JSON: Final = TypeAdapter(object) + + def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """A deployment's model_info as a mapping, decoding a JSON string, else None. @@ -143,7 +148,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """ if isinstance(raw, str): try: - decoded: Final[object] = json.loads(raw) + decoded: Final = _DECODED_JSON.validate_python(json.loads(raw)) except (TypeError, ValueError): return None return decoded if isinstance(decoded, dict) else None diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 84278f4619d..3c65fc0bf06 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3045,7 +3045,7 @@ async def ui_view_spend_logs( ) sql_params.extend([page_size, skip]) - data: Final = await prisma_client.db.query_raw(sql_query, *sql_params) + data: Final[Sequence[Mapping[str, object]]] = await _query_raw(prisma_client, sql_query, *sql_params) if request_id is not None and not is_v2 and not is_admin_view: await _assert_user_owns_fetched_spend_rows( @@ -3064,7 +3064,7 @@ async def ui_view_spend_logs( return await _build_ui_spend_logs_response( prisma_client, - data, + list(data), total_records, page, page_size, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index efc83132568..6cf274884c7 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -19,6 +19,7 @@ from collections.abc import ( Awaitable, Callable, Coroutine, + Iterable, Iterator, Mapping, Sequence, @@ -51,7 +52,7 @@ from typing import ( import anyio import httpx -from typing_extensions import Never, ReadOnly, TypedDict +from typing_extensions import Never, NotRequired, ReadOnly, TypedDict from litellm import _custom_logger_compatible_callbacks_literal from litellm.constants import ( @@ -323,10 +324,22 @@ class _RelTuplesRow(TypedDict): reltuples: ReadOnly[int] +class _TeamMemberRow(TypedDict): + role: ReadOnly[Literal["admin", "user"]] + user_id: ReadOnly[NotRequired[str | None]] + user_email: ReadOnly[NotRequired[str | None]] + + +class _TeamMembershipRow(TypedDict): + user_id: ReadOnly[str | None] + team_members_with_roles: ReadOnly[Iterable[_TeamMemberRow] | None] + + _VIEW_SETUP_POLL_INTERVAL_SECONDS: Final = 5.0 _VIEW_SETUP_DEADLINE_SECONDS: Final = 15 * 60.0 _VIEW_SETUP_GATE_TABLE: Final = '"LiteLLM_SpendLogs"' _VIEW_SETUP_GATE_PROBE_ROWS: Final = TypeAdapter(tuple[Mapping[str, bool], ...]) +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) _ViewSetupOutcome: TypeAlias = Literal["ready", "timed_out"] _ViewSetupAttempt: TypeAlias = Literal["ready", "table_missing"] | Exception @@ -364,6 +377,47 @@ class _UpdateManyBatch(Protocol): async def commit(self) -> None: ... +def _update_many_batch(db: PrismaWrapper | RoutingPrismaWrapper) -> _UpdateManyBatch: + return db.batch_() + + +def _query_raw_rows( + db: PrismaWrapper | RoutingPrismaWrapper, query: str, *args: object +) -> Awaitable[Sequence[Mapping[str, object]]]: + return db.query_raw(query, *args) + + +def _query_first_row( + db: PrismaWrapper | RoutingPrismaWrapper, query: str, *args: object +) -> Awaitable[Mapping[str, object] | None]: + return db.query_first(query, *args) + + +def _team_member_for_user(row: _TeamMembershipRow) -> Member | None: + members_with_roles: Final = row["team_members_with_roles"] + if members_with_roles is None: + return None + user_id: Final = row["user_id"] + if user_id is None: + return None + matches: Final = [ + Member(**member) + for member in members_with_roles + if member.get("user_id") is not None and user_id == member.get("user_id") + ] + return matches[-1] if matches else None + + +class _EndUserSpendTransaction(Protocol): + def batch_(self) -> contextlib.AbstractAsyncContextManager[_EndUserSpendBatch]: ... + + +def _end_user_spend_transaction( + db: PrismaWrapper | RoutingPrismaWrapper, timeout: timedelta +) -> contextlib.AbstractAsyncContextManager[_EndUserSpendTransaction]: + return db.tx(timeout=timeout) + + unified_guardrail: Final = UnifiedLLMGuardrails() NON_OPENAI_STREAM_GUARDRAIL_TRANSLATION_CALL_TYPES: "frozenset[CallTypes]" = frozenset({CallTypes.anthropic_messages}) @@ -543,7 +597,7 @@ class _UpstreamStreamBoundary(Generic[_T]): self.failure: BaseException | None = None def __getattr__(self, name: str) -> object: - return getattr(self._source, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._source, name)) def __aiter__(self) -> "_UpstreamStreamBoundary[_T]": return self @@ -5494,7 +5548,9 @@ class PrismaClient: raise e - async def _query_first_with_cached_plan_fallback(self, sql_query: str, *args) -> dict | None: + async def _query_first_with_cached_plan_fallback( + self, sql_query: str, *args: object + ) -> Mapping[str, object] | None: """ Execute a query, recovering once from PostgreSQL's "cached plan must not change result type" error. @@ -5745,7 +5801,7 @@ class PrismaClient: LIMIT $1 OFFSET $2 """ - response = await self.db.query_raw(sql_query, limit, offset) + response = await _query_raw_rows(self.db, sql_query, limit, offset) return response elif table_name == "spend": verbose_proxy_logger.debug("PrismaClient: get_data: table_name == 'spend'") @@ -5889,7 +5945,7 @@ class PrismaClient: """ response = ( - await self.writer_db.query_first(sql_query, hashed_token) + await _query_first_row(self.writer_db, sql_query, hashed_token) if use_writer else await self._query_first_with_cached_plan_fallback(sql_query, hashed_token) ) @@ -5925,31 +5981,14 @@ class PrismaClient: if response["team_blocked"] is None: response["team_blocked"] = False - team_member: Member | None = None - if response["team_members_with_roles"] is not None and response["user_id"] is not None: - ## find the team member corresponding to user id - """ - [ - { - "role": "admin", - "user_id": "default_user_id", - "user_email": null - }, - { - "role": "user", - "user_id": null, - "user_email": "test@email.com" - } - ] - """ - for tm in response["team_members_with_roles"]: - if tm.get("user_id") is not None and response["user_id"] == tm.get("user_id"): - team_member = Member(**tm) + ## find the team member corresponding to user id + team_member: Final = _team_member_for_user(response) response["team_member"] = team_member - response = LiteLLM_VerificationTokenView(**response, last_refreshed_at=time.time()) + token_view: Final = LiteLLM_VerificationTokenView(**response, last_refreshed_at=time.time()) # for prisma we need to cast the expires time to str - if response.expires is not None and isinstance(response.expires, datetime): - response.expires = response.expires.isoformat() + if token_view.expires is not None and isinstance(token_view.expires, datetime): + token_view.expires = token_view.expires.isoformat() + response = token_view return response except Exception as e: import traceback @@ -6239,7 +6278,7 @@ class PrismaClient: """ Batch write update queries """ - batcher: _UpdateManyBatch = self.db.batch_() + batcher: _UpdateManyBatch = _update_many_batch(self.db) for idx, t in enumerate(data_list): # check if plain text or hash if t.token.startswith("sk-"): @@ -6264,7 +6303,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for user in sorted(data_list, key=attrgetter("user_id")): try: data_json = self.jsonify_object(data=user.model_dump(exclude_none=True)) @@ -6289,7 +6328,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for enduser in data_list: try: data_json = self.jsonify_object(data=enduser.model_dump(exclude_none=True)) @@ -6314,7 +6353,7 @@ class PrismaClient: """ Batch write update queries """ - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for budget in data_list: try: data_json = self.jsonify_object(data=budget.model_dump(exclude_none=True)) @@ -6337,7 +6376,7 @@ class PrismaClient: and isinstance(data_list, list) ): # Batch write update queries - batcher = self.db.batch_() + batcher = _update_many_batch(self.db) for idx, team in enumerate(data_list): try: data_json = self.jsonify_team_object(db_data=team.model_dump(exclude_none=True)) @@ -7002,7 +7041,7 @@ class PrismaClient: # caller's own retried query, a stronger check than SELECT 1, # and a reader that fails to come back sets `_reader_unavailable` # so reads fall through to the writer just recreated here. - await self.writer_db.query_raw("SELECT 1") + await _query_raw_rows(self.writer_db, "SELECT 1") # A recreate can decline: the optimistic-lock guard no-ops when # the writer generation moved since cycle entry, and the routing # wrapper then leaves the reader untouched as well. Callers that @@ -7350,7 +7389,7 @@ class PrismaClient: async def _view_setup_gate_table_present(self) -> bool: rows: Final = _VIEW_SETUP_GATE_PROBE_ROWS.validate_python( - await self.db.query_raw("SELECT to_regclass($1) IS NOT NULL AS present", _VIEW_SETUP_GATE_TABLE) + await _query_raw_rows(self.db, "SELECT to_regclass($1) IS NOT NULL AS present", _VIEW_SETUP_GATE_TABLE) ) return rows[0]["present"] @@ -7360,7 +7399,7 @@ class PrismaClient: await asyncio.sleep(self._db_health_watchdog_interval_seconds) async with db_span("db_health_watchdog", None): await asyncio.wait_for( - self.db.query_raw("SELECT 1"), + _query_raw_rows(self.db, "SELECT 1"), timeout=self._db_health_watchdog_probe_timeout_seconds, ) if isinstance(self.db, RoutingPrismaWrapper) and self.db.writer_unavailable: @@ -7420,7 +7459,7 @@ class PrismaClient: async def _writer_is_read_only(self, writer: PrismaWrapper) -> bool: """True iff the pooled writer session answers reads but rejects writes (SQLSTATE 25006).""" rows: Final = _WRITER_WRITABILITY_PROBE_ROWS.validate_python( - await writer.query_raw(_WRITER_WRITABILITY_PROBE_SQL) + await _query_raw_rows(writer, _WRITER_WRITABILITY_PROBE_SQL) ) return any(row.get("transaction_read_only") == "on" for row in rows) @@ -7447,7 +7486,7 @@ class PrismaClient: """ sql_query: Final = "SELECT 1" async with db_span("health_check", None): - response: Final[object] = await wrapper.query_raw(sql_query) + response: Final[object] = await _query_raw_rows(wrapper, sql_query) return response async def _probe_answers_now(self, wrapper: PrismaWrapper) -> bool: @@ -8000,7 +8039,7 @@ class ProxyUpdateSpend: try: async with ( db_span("update_end_user_spend", "LiteLLM_EndUserTable"), - prisma_client.db.tx(timeout=timedelta(seconds=60)) as transaction, + _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) as transaction, ): batcher: _EndUserSpendBatch async with transaction.batch_() as batcher: diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index d4643994043..6d9249d5572 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -4,6 +4,7 @@ from typing import Final from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile from litellm.proxy._types import * @@ -36,6 +37,7 @@ from litellm.types.videos.utils import ( ) router: Final = APIRouter() +_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) @router.post( @@ -582,7 +584,9 @@ async def video_create_character( if video_file: data["video"] = video_file[0] - target_model_name: Final = extract_model_from_target_model_names(data.get("target_model_names")) + target_model_name: Final = extract_model_from_target_model_names( + _OPAQUE_VALUE_ADAPTER.validate_python(data.get("target_model_names")) + ) if target_model_name and not data.get("model"): data["model"] = target_model_name diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index a2be42f1b76..56123e4bee3 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from litellm.router import Router -def extract_model_from_target_model_names(target_model_names: Any) -> str | None: +def extract_model_from_target_model_names(target_model_names: object) -> str | None: if isinstance(target_model_names, str): model_names: Final = [m.strip() for m in target_model_names.split(",") if m.strip()] return model_names[0] if model_names else None diff --git a/litellm/rag/ingestion/bedrock_ingestion.py b/litellm/rag/ingestion/bedrock_ingestion.py index 1e7aa6ac1c6..5520912dd41 100644 --- a/litellm/rag/ingestion/bedrock_ingestion.py +++ b/litellm/rag/ingestion/bedrock_ingestion.py @@ -106,11 +106,11 @@ class BedrockRAGIngestion(BaseRAGIngestion, BaseAWSLLM): ) self.embedding_model = self.vector_store_config.get("embedding_model") or "amazon.titan-embed-text-v2:0" - self.wait_for_ingestion = self.vector_store_config.get("wait_for_ingestion", False) + self.wait_for_ingestion: object = self.vector_store_config.get("wait_for_ingestion", False) self.ingestion_timeout: int = _get_int(self.vector_store_config.get("ingestion_timeout"), 300) # Get AWS region using BaseAWSLLM method - _aws_region: Final = self.vector_store_config.get("aws_region_name") + _aws_region: Final[object] = self.vector_store_config.get("aws_region_name") self.aws_region_name = self.get_aws_region_name_for_non_llm_api_calls( aws_region_name=str(_aws_region) if _aws_region else None ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 088dfb54155..7b9068ff95e 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -33,7 +33,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference -from ..litellm_core_utils.core_helpers import normalize_drop_params +from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -83,6 +83,13 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> return {**session, "model": model_name} +def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None: + session: Final = as_str_mapping(request_data.get("session")) + if not isinstance(session, dict): + return None + return _with_resolved_session_model(session, model_name) + + def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})} @@ -189,8 +196,9 @@ async def acreate_realtime_client_secret( custom_llm_provider=custom_llm_provider, ) request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - if isinstance(request_data.get("session"), dict): - request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + resolved_session: Final = _resolved_request_session(request_data, model_name) + if resolved_session is not None: + request_data["session"] = resolved_session return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( diff --git a/litellm/repositories/base_repository.py b/litellm/repositories/base_repository.py index 81fba770b70..92f8ba984f4 100644 --- a/litellm/repositories/base_repository.py +++ b/litellm/repositories/base_repository.py @@ -8,6 +8,7 @@ from typing import Any, Final, Generic, Protocol, TypeVar, runtime_checkable from pydantic import BaseModel +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.repositories.chunked_in import find_many_in from litellm.repositories.prisma_protocols import TableActions @@ -131,4 +132,4 @@ def is_unique_violation(exc: BaseException) -> bool: return "P2002" in str(exc) or "unique constraint" in str(exc).lower() if isinstance(exc, UniqueViolationError): return True - return getattr(exc, "code", None) == "P2002" + return attribute_or_none(exc, "code") == "P2002" diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py index 57f6fd33c11..b89d87aa354 100644 --- a/litellm/repositories/team_repository.py +++ b/litellm/repositories/team_repository.py @@ -75,6 +75,7 @@ class _PrismaClientView(Protocol): _MEMBERS_WITH_ROLES_ADAPTER: Final = TypeAdapter(list[Member]) +_RAW_COLUMN: Final = TypeAdapter(object) _JSON_ENCODED_TEAM_FIELDS: Final = ( "metadata", "model_spend", @@ -142,7 +143,7 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]): ) if not rows: return None - raw_value: Final[object] = rows[0]["members_with_roles"] + raw_value: Final = _RAW_COLUMN.validate_python(rows[0]["members_with_roles"]) parsed: Final[object] = json.loads(raw_value) if isinstance(raw_value, str) else raw_value if not parsed: return [] diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index f7e2616324f..65d9b89cf79 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -1,6 +1,6 @@ import asyncio import contextvars -from collections.abc import Awaitable +from collections.abc import Awaitable, Coroutine from functools import partial from typing import Final, Literal @@ -76,7 +76,7 @@ async def arerank( func_with_context: Final = partial(ctx.run, func) init_response: Final = await loop.run_in_executor(None, func_with_context) - if asyncio.iscoroutine(init_response): + if isinstance(init_response, Coroutine): response = await init_response else: response = init_response diff --git a/litellm/rerank_api/rerank_utils.py b/litellm/rerank_api/rerank_utils.py index f365ae749d1..4924625b011 100644 --- a/litellm/rerank_api/rerank_utils.py +++ b/litellm/rerank_api/rerank_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig @@ -8,7 +8,7 @@ def get_optional_rerank_params( model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py index 7121908517b..f29f6c83ef0 100644 --- a/litellm/responses/file_search/emulated_handler.py +++ b/litellm/responses/file_search/emulated_handler.py @@ -15,7 +15,7 @@ import json import time import uuid from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast # noqa: TID251 # see kwargs-ok / cast-ok markers +from typing import TYPE_CHECKING, Final, TypeAlias, cast # noqa: TID251 # see cast-ok markers from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -517,7 +517,7 @@ async def aresponses_with_emulated_file_search( model: str, tools: Iterable[ToolParam] | None = None, # Pass-through params — forwarded as-is to the underlying aresponses call - **kwargs: Any, # kwargs-ok: `object` would surface the caller's partially-unknown dict at its call site + **kwargs: object, # kwargs-ok: pass-through params forwarded as-is to the underlying aresponses call ) -> ResponsesAPIResponse: """ Emulated file_search for providers that don't support it natively. diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py index fcda6b31b00..98e1984550c 100644 --- a/litellm/responses/litellm_completion_transformation/custom_tools.py +++ b/litellm/responses/litellm_completion_transformation/custom_tools.py @@ -23,6 +23,7 @@ from typing import Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( ChatCompletionToolParam, @@ -30,6 +31,7 @@ from litellm.types.llms.openai import ( ) _MAX_ARGUMENTS_LEN: Final = 1_000_000 +_DECODED_JSON: Final = TypeAdapter(object) TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE: Final = MappingProxyType( {"function_call": "fc", "custom_tool_call": "ctc", "tool_search_call": "tsc"} @@ -111,8 +113,8 @@ def unwrap_custom_tool_arguments(arguments: str) -> str: if len(arguments) > _MAX_ARGUMENTS_LEN: return arguments try: - parsed: Final = json.loads(arguments) - if isinstance(parsed, dict) and "content" in parsed: + parsed: Final = as_str_mapping(_DECODED_JSON.validate_python(json.loads(arguments))) + if parsed is not None and "content" in parsed: return str(parsed["content"]) except (json.JSONDecodeError, TypeError, ValueError): pass diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c25dd4a405e..b5ef9e95741 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -5,6 +5,7 @@ from itertools import filterfalse from typing import Any, Final, cast import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.main import stream_chunk_builder @@ -84,14 +85,18 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) +def _is_signed_thinking_block(block: object) -> bool: + fields: Final = as_str_mapping(block) + return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data")) + + def _streams_compaction(chunk: ModelResponseStream) -> bool: delta_fields: Final = chunk.choices[0].delta.provider_specific_fields if chunk.choices else None return delta_fields is not None and "compaction_blocks" in delta_fields def _delta_has_signed_thinking_block(delta: object) -> bool: - blocks: Final = getattr(delta, "thinking_blocks", None) or () - return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) + return any(_is_signed_thinking_block(block) for block in attribute_items(delta, "thinking_blocks")) def _delta_carries_output(delta: ChatCompletionDelta) -> bool: @@ -1427,7 +1432,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _is_unstreamed_empty_message(self, item: object) -> bool: if getattr(item, "type", None) != "message" or self._message_output_index is not None: return False - return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ()) + return not any(getattr(part, "text", None) for part in attribute_items(item, "content")) def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: if not isinstance(self.litellm_model_response, ModelResponse): diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e7380c26529..1ff80b0d71d 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -191,6 +191,7 @@ def mock_responses_api_response( _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_FORWARDED_VALUE: Final = TypeAdapter(object) def _client_trace_id(request_kwargs: Mapping[str, object]) -> str | None: @@ -866,7 +867,7 @@ async def aresponses( stream: bool | None = None, temperature: float | None = None, text: Optional["ResponseText"] = None, - text_format: type["BaseModel"] | dict | None = None, + text_format: type["BaseModel"] | dict[str, object] | None = None, tool_choice: ToolChoice | None = None, tools: Iterable[ToolParam] | None = None, top_p: float | None = None, @@ -1355,7 +1356,7 @@ def _responses_try_dispatch_emulated_file_search( user: str | None, service_tier: str | None, safety_identifier: str | None, - text_format: type[BaseModel] | dict | None, + text_format: type[BaseModel] | dict[str, object] | None, allowed_openai_params: list[str] | None, extra_headers: dict[str, object] | None, extra_query: dict[str, object] | None, @@ -1390,7 +1391,7 @@ def _responses_try_dispatch_emulated_file_search( "background": background, "stream": stream, "temperature": temperature, - "text": text, + "text": _FORWARDED_VALUE.validate_python(text), "tool_choice": tool_choice, "top_p": top_p, "truncation": truncation, @@ -1435,7 +1436,7 @@ def responses( stream: bool | None = None, temperature: float | None = None, text: Optional["ResponseText"] = None, - text_format: type["BaseModel"] | dict | None = None, + text_format: type["BaseModel"] | dict[str, object] | None = None, tool_choice: ToolChoice | None = None, tools: Iterable[ToolParam] | None = None, top_p: float | None = None, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index b100858b306..cd072825ef5 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -4,6 +4,7 @@ import logging from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call @@ -24,6 +25,8 @@ from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +_DELEGATED_ATTRIBUTE: Final = TypeAdapter(object) + class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): """Extra keywords forwarded verbatim to ``litellm.acompletion``, which owns their contract.""" @@ -566,7 +569,7 @@ async def acompletion_with_mcp( def __getattr__(self, name: str) -> object: # Delegate all other attributes to original wrapper - return getattr(self._original_wrapper, name) + return _DELEGATED_ATTRIBUTE.validate_python(getattr(self._original_wrapper, name)) # Helper class to wrap async iterator for sync iteration class _SyncIteratorWrapper: diff --git a/litellm/responses/sse_output_recovery.py b/litellm/responses/sse_output_recovery.py index cdabcb504d3..e7d0a540022 100644 --- a/litellm/responses/sse_output_recovery.py +++ b/litellm/responses/sse_output_recovery.py @@ -11,9 +11,12 @@ import json from collections.abc import Mapping from typing import Final, SupportsInt, TypeAlias, cast # noqa: TID251 # int() re-checks the cast below at runtime +from pydantic import TypeAdapter + from litellm.constants import STREAM_SSE_DONE_STRING _MAX_CONTENT_INDEX: Final = 1024 +_DECODED_JSON: Final = TypeAdapter(object) _ConvertibleToInt: TypeAlias = SupportsInt | str @@ -33,7 +36,7 @@ def parse_sse_json_chunk(chunk: str) -> dict[str, object] | None: if not stripped_chunk or stripped_chunk == STREAM_SSE_DONE_STRING or stripped_chunk.startswith("event:"): return None try: - parsed_chunk: Final[object] = json.loads(stripped_chunk) + parsed_chunk: Final = _DECODED_JSON.validate_python(json.loads(stripped_chunk)) except json.JSONDecodeError: return None if not isinstance(parsed_chunk, dict): diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0d59c93786c..50ab3e4f75a 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -28,7 +28,7 @@ from typing import ( import httpx from openai._streaming import SSEDecoder -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs import litellm @@ -166,6 +166,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams: _SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache" _UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text" +_DECODED_JSON: Final = TypeAdapter(object) def _load_json_object(payload: str | bytes) -> dict[str, object]: @@ -175,7 +176,7 @@ def _load_json_object(payload: str | bytes) -> dict[str, object]: def _load_json_value(payload: str | bytes) -> object: """Parse a JSON payload whose top-level shape the caller narrows itself.""" - return json.loads(payload) + return _DECODED_JSON.validate_python(json.loads(payload)) def _model_id_from_metadata(litellm_metadata: Mapping[str, object] | None) -> str | None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index f256480d952..dc75e77d685 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -5,10 +5,12 @@ from functools import reduce from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload from pydantic import BaseModel, TypeAdapter +from typing_extensions import TypedDict as ExtraItemsTypedDict from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire payloads without a runtime conversion import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( @@ -33,6 +35,10 @@ from litellm.types.utils import ( ) +class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): + pass + + def _apply_nested_drop_params(params: dict[str, object], additional_drop_params: list[str] | None) -> dict[str, object]: nested_paths: Final = tuple(path for path in additional_drop_params or () if is_nested_path(path)) return reduce(lambda acc, path: delete_nested_value(acc, path), nested_paths, params) @@ -64,6 +70,28 @@ def _is_object_dict( return isinstance(value, dict) +def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrapper: + if _is_object_dict(details): + input_details: Final = InputTokensDetails.model_validate(details) + return PromptTokensDetailsWrapper( + **input_details.model_dump(exclude={"cache_write_tokens_details"}), + cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), + ) + return PromptTokensDetailsWrapper( + cached_tokens=getattr(details, "cached_tokens", None), + audio_tokens=getattr(details, "audio_tokens", None), + text_tokens=getattr(details, "text_tokens", None), + image_tokens=getattr(details, "image_tokens", None), + cached_tokens_details=getattr(details, "cached_tokens_details", None), + video_tokens=getattr(details, "video_tokens", None), + cache_write_tokens=getattr(details, "cache_write_tokens", None), + cache_creation_token_details=_cache_write_lifetimes(getattr(details, "cache_write_tokens_details", None)), + cache_type=getattr(details, "cache_type", None), + web_search_requests=getattr(details, "web_search_requests", None), + google_maps_grounding_requests=getattr(details, "google_maps_grounding_requests", None), + ) + + def normalize_responses_api_stream_options( stream_options: object, ) -> ResponsesAPIStreamOptions | None: @@ -1060,7 +1088,7 @@ class ResponsesAPIRequestUtils: @staticmethod def _update_container_ids_in_response( - responses_api_response: ResponsesAPIResponse | dict[str, Any], + responses_api_response: ResponsesAPIResponse | dict[str, object], custom_llm_provider: str | None, litellm_metadata: dict[str, Any] | None = None, ) -> ResponsesAPIResponse | dict[str, object]: @@ -1074,15 +1102,16 @@ class ResponsesAPIRequestUtils: model_id: Final = model_info.get("id") # Get the output list - if isinstance(responses_api_response, dict): - output = responses_api_response.get("output", []) - else: - output = getattr(responses_api_response, "output", []) + output: Final = ( + responses_api_response.get("output", []) + if isinstance(responses_api_response, dict) + else attribute_or_none(responses_api_response, "output") + ) if not output: return responses_api_response - for item in output: + for item in iter_items(output): ResponsesAPIRequestUtils.encode_container_id_on_output_item( item=item, custom_llm_provider=custom_llm_provider, @@ -1257,34 +1286,11 @@ class ResponseAPILoggingUtils: response_api_usage = usage_input prompt_tokens: Final[int] = response_api_usage.input_tokens or 0 completion_tokens: Final[int] = response_api_usage.output_tokens or 0 - prompt_tokens_details: PromptTokensDetailsWrapper | None = None - if response_api_usage.input_tokens_details: - if isinstance(response_api_usage.input_tokens_details, dict): - input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details) - prompt_tokens_details = PromptTokensDetailsWrapper( - **input_details.model_dump(exclude={"cache_write_tokens_details"}), - cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), - ) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), - audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), - text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), - image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), - video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), - cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), - cache_creation_token_details=_cache_write_lifetimes( - getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None) - ), - cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), - web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), - google_maps_grounding_requests=getattr( - response_api_usage.input_tokens_details, "google_maps_grounding_requests", None - ), - ) + prompt_tokens_details: Final = ( + _prompt_tokens_details_wrapper(response_api_usage.input_tokens_details) + if response_api_usage.input_tokens_details + else None + ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None output_tokens_details: Final[OutputTokensDetails | None] = getattr( response_api_usage, "output_tokens_details", None @@ -1308,20 +1314,22 @@ class ResponseAPILoggingUtils: audio_tokens=audio_tokens, ) - extra_usage_fields: Final = { - key: value - for key, value in (response_api_usage.model_extra or {}).items() - if key - not in ( - "input_token_details", - "output_token_details", - "prompt_tokens", - "completion_tokens", - "total_tokens", - "prompt_tokens_details", - "completion_tokens_details", - ) - } + extra_usage_fields: Final = _UsageExtraFields( + **{ + key: value + for key, value in (as_str_mapping(response_api_usage.model_extra) or {}).items() + if key + not in ( + "input_token_details", + "output_token_details", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "prompt_tokens_details", + "completion_tokens_details", + ) + } + ) chat_usage: Final = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, diff --git a/litellm/router.py b/litellm/router.py index 705accbcf8d..40c5c8afa6c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -934,7 +934,7 @@ MAX_HELD_PRE_OUTPUT_RESPONSES_EVENTS: Final = 200 def _retry_policy_ceiling(policy: RetryPolicy) -> int: """The most retries any error class under this policy can be granted.""" - retry_counts: Final[Iterable[object]] = policy.model_dump().values() + retry_counts: Final[tuple[object, ...]] = tuple(policy.model_dump().values()) return max((retries for retries in retry_counts if isinstance(retries, int)), default=0) @@ -6504,7 +6504,7 @@ class Router: has_generated_content: bool, buffered_lifecycle_chunks: tuple[bytes, ...], model: str, - initial_kwargs: dict[str, Any], # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it + initial_kwargs: dict[str, object], # mutable-ok: _aanthropic_messages_fallback_attempt mutates it wrapper: "FallbackAwareAnthropicMessagesStream", close_superseded: Callable[[], Awaitable[None]], ) -> AsyncGenerator[bytes, None]: diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 1d119375778..d8c52b4c8b6 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -47,13 +47,13 @@ class QualityRouter(CustomLogger): model_name: str, litellm_router_instance: "Router", default_model: str | None = None, - quality_router_config: dict[str, Any] | None = None, + quality_router_config: Mapping[str, object] | None = None, ): self.model_name = model_name self.litellm_router_instance = litellm_router_instance if quality_router_config: - self.config = QualityRouterConfig(**quality_router_config) + self.config = QualityRouterConfig.model_validate(dict(**quality_router_config)) else: self.config = QualityRouterConfig() @@ -185,7 +185,7 @@ class QualityRouter(CustomLogger): if isinstance(raw_prefs, RoutingPreferences): prefs = raw_prefs elif isinstance(raw_prefs, dict): - prefs = RoutingPreferences(**raw_prefs) + prefs = RoutingPreferences.model_validate(dict(**raw_prefs)) else: # A Pydantic object of some other shape — coerce via its dict. prefs = RoutingPreferences( diff --git a/litellm/sandbox/__init__.py b/litellm/sandbox/__init__.py index 1974859fb1e..bcbb7bc5743 100644 --- a/litellm/sandbox/__init__.py +++ b/litellm/sandbox/__init__.py @@ -7,7 +7,10 @@ They resolve lazily so `import litellm` does not pull in litellm.harness. import importlib from typing import Final +from pydantic import TypeAdapter + _HARNESS_SANDBOX_MODULE: Final = "litellm.harness.sandbox" +_HARNESS_EXPORT: Final = TypeAdapter(object) _HARNESS_EXPORTS: Final = frozenset( { "local", @@ -23,5 +26,5 @@ _HARNESS_EXPORTS: Final = frozenset( def __getattr__(name: str) -> object: if name in _HARNESS_EXPORTS: - return getattr(importlib.import_module(_HARNESS_SANDBOX_MODULE), name) + return _HARNESS_EXPORT.validate_python(getattr(importlib.import_module(_HARNESS_SANDBOX_MODULE), name)) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/secret_managers/base_secret_manager.py b/litellm/secret_managers/base_secret_manager.py index e537dd1d998..9c25086c451 100644 --- a/litellm/secret_managers/base_secret_manager.py +++ b/litellm/secret_managers/base_secret_manager.py @@ -1,6 +1,6 @@ import re from abc import ABC, abstractmethod -from typing import Any, Final +from typing import Final import httpx @@ -73,7 +73,7 @@ class BaseSecretManager(ABC): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Asynchronously write a secret to the secret manager. diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index a1ec7c0a386..abfa9649e91 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -1,7 +1,7 @@ import asyncio import base64 import os -from typing import Any, Final +from typing import Final from urllib.parse import quote import httpx @@ -278,7 +278,7 @@ class CyberArkSecretManager(BaseSecretManager): optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None, tags: dict | list | None = None, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Writes a secret to CyberArk Conjur using an async HTTPX client. diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index f4234e26c60..b33cf0e6b7e 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -24,6 +24,7 @@ from litellm.types.llms.base import LiteLLMBaseModel oidc_cache: Final = DualCache() _PARSED_LITERAL: Final = TypeAdapter(object) +_SECRET_MANAGER_CLIENT: Final = TypeAdapter(object) @lru_cache(maxsize=128) @@ -383,7 +384,7 @@ def get_secret( try: if _should_read_secret_from_secret_manager() and litellm.secret_manager_client is not None: try: - client: Final = litellm.secret_manager_client + client: Final = _SECRET_MANAGER_CLIENT.validate_python(litellm.secret_manager_client) key_manager = "local" if key_management_system is not None: key_manager = key_management_system.value diff --git a/litellm/utils.py b/litellm/utils.py index bb94ca6d40a..1db5f7129df 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1641,7 +1641,7 @@ def _deployment_provider(kwargs: Mapping[str, object]) -> str: return "" -async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]: +async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type: str) -> dict[str, Any]: """ Allow modifying the request just before it's sent to the deployment. @@ -4287,7 +4287,7 @@ def get_optional_params_embeddings( # retrieve all parameters passed to the function passed_params: Final = locals() - custom_llm_provider = passed_params.pop("custom_llm_provider", None) + passed_params.pop("custom_llm_provider", None) special_params: Final = passed_params.pop("kwargs") passed_params.pop("drop_params", None) @@ -4325,7 +4325,7 @@ def get_optional_params_embeddings( provider_config: BaseEmbeddingConfig | None = None optional_params = {} - if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): + if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_embedding_config( model=model, provider=LlmProviders(custom_llm_provider), @@ -9060,8 +9060,9 @@ _get_bundled_model_cost_map = get_bundled_model_cost_map def _get_model_cost_entry_for_provider_config( model: str, provider: LlmProviders, -) -> dict[str, object]: +) -> Mapping[str, object]: candidate_keys: Final = (model, f"{provider.value}/{model}") + model_info: Mapping[str, object] | None for model_key in candidate_keys: model_info = litellm.model_cost.get(model_key) if model_info is not None: @@ -10886,18 +10887,12 @@ def extract_duration_from_srt_or_vtt(srt_or_vtt_content: str) -> float | None: # Regular expression to match timestamps in the format "hh:mm:ss,ms" or "hh:mm:ss.ms" timestamp_pattern: Final = r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})" - timestamps: Final[Sequence[tuple[str, str, str, str]]] = re.findall(timestamp_pattern, srt_or_vtt_content) - - if not timestamps: - return None + timestamps: Final = re.finditer(timestamp_pattern, srt_or_vtt_content) # Convert timestamps to seconds and find the max (end time) - durations: Final = [] - match: tuple[str, str, str, str] - for match in timestamps: - hours, minutes, seconds, milliseconds = map(int, match) - total_seconds = hours * 3600 + minutes * 60 + seconds + milliseconds / 1000.0 - durations.append(total_seconds) + durations: Final = [ + int(match[1]) * 3600 + int(match[2]) * 60 + int(match[3]) + int(match[4]) / 1000.0 for match in timestamps + ] return max(durations) if durations else None diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 8b33b1ce12c..9f53accfa0b 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -5,7 +5,7 @@ LiteLLM SDK Functions for Creating and Searching Vector Stores import asyncio import builtins import contextvars -from collections.abc import Coroutine, Mapping +from collections.abc import Coroutine, Iterator, Mapping from functools import partial from typing import TYPE_CHECKING, Final @@ -53,6 +53,10 @@ def _direct_vector_store_embedding_executor( ) +def _items_without_key(values: Mapping[str, object], excluded_key: str) -> Iterator[tuple[str, object]]: + return ((key, value) for key, value in values.items() if key != excluded_key) + + def mock_vector_store_search_response( mock_results: builtins.list[VectorStoreSearchResult] | None = None, ): @@ -307,7 +311,7 @@ async def asearch( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: loop: Final = asyncio.get_event_loop() @@ -391,7 +395,7 @@ def search( embedding_executor: Final = _direct_vector_store_embedding_executor( kwargs.pop("_direct_vector_store_embedding_executor", None), router, kwargs ) - local_vars: Final = {key: value for key, value in locals().items() if key != "embedding_executor"} + local_vars: Final = dict(_items_without_key(locals(), "embedding_executor")) try: litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) @@ -694,7 +698,7 @@ def list( timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, -): +) -> Mapping[str, object] | Coroutine[object, object, Mapping[str, object]]: """ List vector stores. @@ -992,7 +996,7 @@ def delete( timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, **kwargs, -): +) -> Mapping[str, object] | Coroutine[object, object, Mapping[str, object]]: """ Delete a vector store. diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 7610d602fff..a5bb5e0b340 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -42,7 +42,7 @@ IGNORE_FUNCTIONS = [ "contains_encrypted_marker", # max depth set. "_rotate_guardrail_row", # bounded by attempts_left. "_delete_nested_value_custom", # max depth set (bounded by number of path segments). - "filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. + "_filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion. "__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion. "_validate_inheritance_chain", # max depth set (default 100) to prevent infinite recursion in policy inheritance validation. "_basic_json_schema_validate", # max depth set. diff --git a/tests/unit/a2a_protocol/test_utils.py b/tests/unit/a2a_protocol/test_utils.py index 8a219211c62..dfa8449f6e6 100644 --- a/tests/unit/a2a_protocol/test_utils.py +++ b/tests/unit/a2a_protocol/test_utils.py @@ -1,5 +1,8 @@ """Tests for litellm/a2a_protocol/utils.py token/usage extraction.""" +from types import SimpleNamespace +from typing import Final + import pytest pytest.importorskip("a2a.compat.v0_3.types") @@ -7,6 +10,7 @@ pytest.importorskip("a2a.compat.v0_3.types") from a2a.compat.v0_3.types import MessageSendParams, SendMessageRequest from litellm.a2a_protocol.utils import A2ARequestUtils +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder def _request(user_text: str) -> SendMessageRequest: @@ -39,3 +43,13 @@ def test_calculate_usage_counts_input_tokens_from_request_object(): assert prompt_tokens > 0 assert completion_tokens > 0 assert total_tokens == prompt_tokens + completion_tokens + + +def test_calculate_usage_counts_input_tokens_from_a_message_that_forwards_its_fields(): + request: Final = _request("count these input tokens please") + forwarded_request: Final = SimpleNamespace(params=SimpleNamespace(message=AttributeForwarder(request.params.message))) + response_dict: Final = {"result": {"kind": "message", "parts": [{"kind": "text", "text": "ok"}]}} + + assert A2ARequestUtils.calculate_usage_from_request_response( + request=forwarded_request, response_dict=response_dict + ) == A2ARequestUtils.calculate_usage_from_request_response(request=request, response_dict=response_dict) diff --git a/tests/unit/containers/test_container_proxy_ownership.py b/tests/unit/containers/test_container_proxy_ownership.py index 780ae9bf6d1..ac5adeee8c1 100644 --- a/tests/unit/containers/test_container_proxy_ownership.py +++ b/tests/unit/containers/test_container_proxy_ownership.py @@ -1,6 +1,7 @@ import json import sys from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -10,6 +11,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.container_endpoints import ownership from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.containers.main import ContainerListResponse, ContainerObject +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder @pytest.fixture(autouse=True) @@ -97,6 +99,23 @@ async def test_should_not_mutate_dict_container_response_when_recording_owner( assert file_obj["provider_container_id"] == "cntr_provider" +@pytest.mark.asyncio +async def test_should_store_the_full_container_of_a_response_that_forwards_its_fields(monkeypatch): + table: Final = AsyncMock() + table.find_unique.return_value = None + prisma_client: Final = SimpleNamespace(db=SimpleNamespace(litellm_managedobjecttable=table)) + monkeypatch.setattr(ownership, "_get_prisma_client", AsyncMock(return_value=prisma_client)) + + await ownership.record_container_owner( + response=AttributeForwarder(_container("cntr_provider")), + user_api_key_dict=UserAPIKeyAuth(user_id="user-1"), + custom_llm_provider="openai", + ) + + file_obj: Final = json.loads(table.create.await_args.kwargs["data"]["file_object"]) + assert file_obj["status"] == "active" + + @pytest.mark.asyncio async def test_should_record_team_owner_for_keys_without_user_id(monkeypatch): table = AsyncMock() diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 20ed8ac0c10..7f6e955e831 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -31,6 +31,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( encode_file_id_with_model, ) from litellm.types.llms.openai import OpenAIFileObject +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder class _InMemoryManagedFileTable: @@ -451,6 +452,20 @@ async def test_async_pre_call_deployment_hook_prefers_top_level_model_info(): ), "Should prefer top-level model_info over litellm_metadata" +@pytest.mark.asyncio +async def test_async_pre_call_deployment_hook_reads_model_info_that_forwards_its_fields(): + proxy_managed_files: Final = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()) + kwargs: Final = { + "input_file_id": "managed-file-abc", + "model_file_id_mapping": {"managed-file-abc": {"deployment-xyz": "file-provider-123"}}, + "model_info": AttributeForwarder({"id": "deployment-xyz"}), + } + + result: Final = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) + + assert result["input_file_id"] == "file-provider-123" + + @pytest.mark.asyncio async def test_async_pre_call_deployment_hook_no_model_info_leaves_file_id_unchanged(): """ diff --git a/tests/unit/integrations/dotprompt/test_prompt_manager.py b/tests/unit/integrations/dotprompt/test_prompt_manager.py index dbd4e4a4c4c..cbf19fe7985 100644 --- a/tests/unit/integrations/dotprompt/test_prompt_manager.py +++ b/tests/unit/integrations/dotprompt/test_prompt_manager.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient +from typing import Final from unittest.mock import MagicMock, Mock, patch import httpx @@ -362,6 +363,22 @@ def test_load_prompts_from_json_data(): assert rendered == "Dynamic prompt: test" +def test_load_prompts_from_json_data_skips_malformed_entries(): + manager: Final = PromptManager(prompt_directory=str(Path(__file__).parent)) + initial_count: Final = len(manager.prompts) + + manager.load_prompts_from_json_data( + { + "valid_prompt": {"content": "Valid: {{value}}", "metadata": {"model": "gpt-4"}}, + "string_entry": "not a prompt", + "null_entry": None, + } + ) + + assert len(manager.prompts) == initial_count + 1 + assert manager.render("valid_prompt", {"value": "ok"}) == "Valid: ok" + + def test_prompt_file_to_json_conversion(): """Test converting .prompt files to JSON format.""" # Create a temporary prompt file with frontmatter @@ -661,6 +678,27 @@ def test_prompt_initializer_registers_flat_db_prompt_under_base_id(): assert template.content == "AHOY {{name}}" +def test_prompt_initializer_accepts_params_without_dotprompt_content(): + from types import SimpleNamespace + + from litellm.integrations.dotprompt import prompt_initializer + from litellm.types.prompts.init_prompts import PromptLiteLLMParams, PromptSpec + + litellm_params: Final = SimpleNamespace( + prompt_data={"content": "AHOY {{name}}", "metadata": {"model": "gpt-4o-mini"}}, + ) + prompt_spec: Final = PromptSpec( + prompt_id="agent-prompt", + litellm_params=PromptLiteLLMParams(prompt_integration="dotprompt"), + ) + + dotprompt_manager: Final = prompt_initializer(litellm_params, prompt_spec) + + template: Final = dotprompt_manager.prompt_manager.get_prompt("agent-prompt") + assert template is not None + assert template.content == "AHOY {{name}}" + + def _swap_prompt_manager_and_spec(ignore_prompt_manager_model: bool) -> tuple[DotpromptManager, PromptSpec]: manager = DotpromptManager( prompt_data={"content": "You are a pirate assistant.", "metadata": {"model": "gpt-4o-mini"}}, diff --git a/tests/unit/integrations/newrelic/test_newrelic.py b/tests/unit/integrations/newrelic/test_newrelic.py index 00a115dd8f2..d09732c6eb2 100644 --- a/tests/unit/integrations/newrelic/test_newrelic.py +++ b/tests/unit/integrations/newrelic/test_newrelic.py @@ -1,6 +1,7 @@ import os import sys from datetime import datetime, timezone +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -491,6 +492,18 @@ class TestExtractAllMessagesTimestamps: input_msg = next(m for m in messages if not m.get("is_response")) assert input_msg["timestamp"] == int(1_000_000.0 * 1000.0) + def test_input_messages_read_a_start_time_that_only_exposes_timestamp(self): + class EpochSeconds: + def timestamp(self) -> float: + return 1_000_000.0 + + kwargs: Final = {**make_kwargs(messages=[{"role": "user", "content": "Hi"}]), "start_time": EpochSeconds()} + + messages: Final = self.logger._extract_all_messages(kwargs, make_response(), response_model="gpt-4", vendor="openai") + + input_msg: Final = next(m for m in messages if not m.get("is_response")) + assert input_msg["timestamp"] == int(1_000_000.0 * 1000.0) + def test_output_messages_get_end_time_timestamp(self): kwargs = make_kwargs(messages=[{"role": "user", "content": "Hi"}]) response = make_response() diff --git a/tests/unit/litellm_core_utils/attribute_forwarder.py b/tests/unit/litellm_core_utils/attribute_forwarder.py new file mode 100644 index 00000000000..3056057fc97 --- /dev/null +++ b/tests/unit/litellm_core_utils/attribute_forwarder.py @@ -0,0 +1,6 @@ +class AttributeForwarder: + def __init__(self, target: object) -> None: + self._target = target + + def __getattr__(self, name: str) -> object: + return getattr(self._target, name) diff --git a/tests/unit/llms/compactifai/test_compactifai.py b/tests/unit/llms/compactifai/test_compactifai.py index 1367c703fda..56d5374f383 100644 --- a/tests/unit/llms/compactifai/test_compactifai.py +++ b/tests/unit/llms/compactifai/test_compactifai.py @@ -2,7 +2,7 @@ import json import os import sys from unittest.mock import AsyncMock, patch -from typing import Optional +from typing import Final, Optional import httpx import pytest @@ -271,3 +271,38 @@ async def test_compactifai_async_completion(respx_mock): assert response.choices[0].message.content == "Async response from CompactifAI" assert response.usage.total_tokens == 23 + + +@pytest.mark.parametrize( + ("body", "json_mode", "error", "message"), + [ + ([], None, TypeError, r"argument after \*\* must be a mapping, not list"), + ({"choices": None}, True, TypeError, "'NoneType' object is not iterable"), + ({"choices": [{"message": "hi"}]}, True, AttributeError, "'str' object has no attribute 'get'"), + ({"choices": [{"message": {"tool_calls": [5]}}]}, True, TypeError, "'int' object is not subscriptable"), + ], +) +def test_compactifai_malformed_response_body_raises_the_error_of_the_failing_lookup(body, json_mode, error, message): + from unittest.mock import MagicMock + + from litellm.llms.compactifai.chat.transformation import CompactifAIChatConfig + + raw_response: Final = httpx.Response( + 200, json=body, request=httpx.Request("POST", "https://api.compactif.ai/v1/chat/completions") + ) + + with pytest.raises(error, match=message) as raised: + CompactifAIChatConfig().transform_response( + model="cai-llama-3-1-8b-slim", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + json_mode=json_mode, + ) + + assert raised.type is error diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index aff90549a7b..0951ba6ba92 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,6 +1,7 @@ import io import os import pathlib +from typing import Final from unittest.mock import MagicMock import httpx @@ -594,3 +595,24 @@ def test_transform_response_wraps_malformed_words_with_the_raw_body(words: objec DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + + +@pytest.mark.parametrize( + ("payload", "error"), + [ + ({"results": None}, "'NoneType' object is not subscriptable"), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), + ( + {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, + "'NoneType' object is not subscriptable", + ), + ], +) +def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): + raw_response: Final = httpx.Response(200, json=payload) + + with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: + DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert exc_info.type is ValueError + assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" diff --git a/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py b/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py index efdb428db32..ee6ee187d64 100644 --- a/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py +++ b/tests/unit/llms/edenai/embedding/test_edenai_embedding_transformation.py @@ -98,6 +98,12 @@ class TestEmbedding: class TestErrors: + def test_a_non_object_body_fails_as_an_invalid_response_object(self, eden_key, respx_mock): + respx_mock.post(EDEN_EMBEDDINGS_URL).mock(return_value=httpx.Response(200, json=["unexpected"])) + + with pytest.raises(litellm.APIConnectionError, match="Invalid response object"): + litellm.embedding(model=MODEL, input="hello") + def test_middleware_401_maps_to_authentication_error(self, eden_key, respx_mock): respx_mock.post(EDEN_EMBEDDINGS_URL).mock(return_value=httpx.Response(401, json={"detail": "Invalid token."})) diff --git a/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py b/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py index d7b32affc70..4e0673b8c27 100644 --- a/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py +++ b/tests/unit/llms/edenai/image_generation/test_edenai_image_generation_transformation.py @@ -117,6 +117,12 @@ class TestImageGeneration: class TestErrors: + def test_a_non_object_body_fails_as_an_invalid_response_object(self, eden_key, respx_mock): + respx_mock.post(EDEN_IMAGES_URL).mock(return_value=httpx.Response(200, json=["unexpected"])) + + with pytest.raises(litellm.APIConnectionError, match="Invalid response object"): + litellm.image_generation(model=MODEL, prompt="a red square") + def test_middleware_401_maps_to_authentication_error(self, eden_key, respx_mock): respx_mock.post(EDEN_IMAGES_URL).mock(return_value=httpx.Response(401, json={"detail": "Invalid token."})) diff --git a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py index da8ab138416..175ce040d9f 100644 --- a/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py +++ b/tests/unit/llms/gemini/google_genai/test_google_genai_transformation.py @@ -106,6 +106,26 @@ def test_generate_content_response_parses_function_call() -> None: } +@pytest.mark.parametrize( + ("body", "message"), + [ + ([], r"argument after \*\* must be a mapping, not list"), + (["candidates"], "list indices must be integers or slices, not str"), + ], +) +def test_generate_content_response_non_object_body_raises_the_error_of_the_failing_lookup( + body: JsonValue, message: str +) -> None: + with pytest.raises(TypeError, match=message) as raised: + GoogleGenAIConfig().transform_generate_content_response( + model="gemini-2.5-flash", + raw_response=httpx.Response(200, json=body), + logging_obj=SimpleNamespace(model_call_details={}), + ) + + assert raised.type is TypeError + + def test_generate_content_request_preserves_tool_declarations() -> None: tools: Final = [ { diff --git a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py index bd9b7006e58..c21dec9b6bb 100644 --- a/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py +++ b/tests/unit/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -241,6 +241,26 @@ class TestGeminiImageEditTransformation: ) assert logging_usage["completion_tokens_details"]["image_tokens"] == 1716 + @pytest.mark.parametrize( + ("body", "error", "message"), + [ + (["not", "an", "object"], AttributeError, "'list' object has no attribute 'get'"), + ({"candidates": None}, TypeError, "'NoneType' object is not iterable"), + ({"candidates": [{"content": {"parts": None}}]}, TypeError, "'NoneType' object is not iterable"), + ], + ) + def test_transform_image_edit_response_malformed_body_raises_the_error_of_the_failing_lookup( + self, body: object, error: type[Exception], message: str + ) -> None: + with pytest.raises(error, match=message) as raised: + self.config.transform_image_edit_response( + model=self.model, + raw_response=httpx.Response(200, json=body), + logging_obj=self.logging_obj, + ) + + assert raised.type is error + def test_transform_image_edit_request_without_image_raises(self) -> None: optional_params = {} diff --git a/tests/unit/llms/gemini/test_gemini_common_utils.py b/tests/unit/llms/gemini/test_gemini_common_utils.py index 196b912752b..20c36878fef 100644 --- a/tests/unit/llms/gemini/test_gemini_common_utils.py +++ b/tests/unit/llms/gemini/test_gemini_common_utils.py @@ -355,6 +355,25 @@ class TestGeminiModelInfo: assert result == expected +@pytest.mark.parametrize( + ("body", "error", "message"), + [ + ([], TypeError, "list indices must be integers or slices, not str"), + ({"models": None}, TypeError, "'NoneType' object is not iterable"), + ({"models": [{"name": 1}]}, AttributeError, "'int' object has no attribute 'replace'"), + ], +) +def test_get_models_malformed_listing_raises_the_error_of_the_failing_lookup( + body: JsonValue, error: type[Exception], message: str, respx_mock: respx.MockRouter +) -> None: + respx_mock.get("https://models.test/v1beta/models").respond(200, json=body) + + with pytest.raises(error, match=message) as raised: + GeminiModelInfo().get_models(api_key="test-key", api_base="https://models.test") + + assert raised.type is error + + class TestGoogleAIStudioTokenCounter: """Test suite for GoogleAIStudioTokenCounter class""" diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 0a5039791ee..8995b3cb068 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -8,6 +8,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.transformation impor from litellm.llms.openai.responses.count_tokens.transformation import ( OpenAICountTokensConfig, ) +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder def test_transform_basic_request(): @@ -207,6 +208,23 @@ def test_messages_to_responses_input_with_tool(): } +def test_messages_to_responses_input_reads_a_tool_call_that_forwards_its_fields(): + tool_call: Final = {"id": "call_123", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + messages: Final = [ + {"role": "user", "content": "What's the weather?"}, + {"role": "assistant", "content": None, "tool_calls": [AttributeForwarder(tool_call)]}, + ] + + input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages) + + assert input_items[1] == { + "type": "function_call", + "call_id": "call_123", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + } + + def test_messages_to_responses_input_preserves_images(): """An image block must survive the round trip, or OpenAI counts only the text. diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 2f10fd6fd0d..0d1b1aca9bc 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -47,6 +47,7 @@ from litellm.types.responses.main import ( OutputToolSearchCall, ) from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder class MockGuardrail(CustomGuardrail): @@ -1783,6 +1784,22 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing: assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]" assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]" + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_syncs_events_that_forward_their_fields(self): + handler: Final = OpenAIResponsesHandler() + plain_events: Final = self._ended_stream_events() + forwarded_delta: Final = AttributeForwarder(plain_events[0]) + events: Final = [forwarded_delta, *plain_events[1:]] + + await handler.process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._masking_guardrail(), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + + assert forwarded_delta.delta == "hello [MASKED]" + @staticmethod def _ended_function_call_stream_events() -> List[dict]: def item(arguments: str, status: str) -> dict: diff --git a/tests/unit/llms/runwayml/image_generation/__init__.py b/tests/unit/llms/runwayml/image_generation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py b/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py new file mode 100644 index 00000000000..7eec9541b98 --- /dev/null +++ b/tests/unit/llms/runwayml/image_generation/test_runway_image_generation_transformation.py @@ -0,0 +1,87 @@ +from typing import Final +from unittest.mock import MagicMock + +import httpx +import pytest + +from litellm.llms.runwayml.image_generation.transformation import RunwayMLImageGenerationConfig +from litellm.types.utils import ImageResponse + +TASK_URL = "https://api.dev.runwayml.com/v1/tasks/task_1" + + +@pytest.mark.respx() +def test_output_dict_with_non_string_values_still_yields_the_image(respx_mock): + respx_mock.get(TASK_URL).respond( + json={ + "id": "task_1", + "status": "SUCCEEDED", + "output": [{"url": "https://cdn.example/image.png", "b64_json": None, "width": 1024}], + }, + status_code=200, + ) + created: Final = httpx.Response( + 200, + json={"id": "task_1", "status": "PENDING"}, + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + image_response: Final = RunwayMLImageGenerationConfig().transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert [image.url for image in image_response.data] == ["https://cdn.example/image.png"] + + +def test_non_object_task_body_raises_the_base_error_class(): + created: Final = httpx.Response( + 200, + json=["unexpected"], + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + with pytest.raises(AttributeError): + RunwayMLImageGenerationConfig().transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +@pytest.mark.asyncio +async def test_async_non_object_task_body_raises_the_base_error_class(): + created: Final = httpx.Response( + 200, + json=["unexpected"], + request=httpx.Request( + "POST", "https://api.dev.runwayml.com/v1/text_to_image", headers={"Authorization": "Bearer k"} + ), + ) + + with pytest.raises(AttributeError): + await RunwayMLImageGenerationConfig().async_transform_image_generation_response( + model="gen4_image", + raw_response=created, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/tests/unit/proxy/auth/test_mcp_ip_filtering.py b/tests/unit/proxy/auth/test_mcp_ip_filtering.py index 4c219760762..c403daf733f 100644 --- a/tests/unit/proxy/auth/test_mcp_ip_filtering.py +++ b/tests/unit/proxy/auth/test_mcp_ip_filtering.py @@ -6,6 +6,7 @@ external callers only see servers with available_on_public_internet=True. """ import logging +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -176,6 +177,19 @@ class TestMCPClientIPExtraction: assert result == "203.0.113.5" + def test_non_boolean_use_x_forwarded_for_never_trusts_xff(self): + request: Final = _make_request("10.0.0.5", "203.0.113.9, 10.0.0.5") + + result: Final = IPAddressUtils.get_mcp_client_ip( + request, + general_settings={ + "use_x_forwarded_for": "false", + "mcp_trusted_proxy_ranges": ["10.0.0.0/8"], + }, + ) + + assert result == "10.0.0.5" + def _make_request(client_host, xff): request = MagicMock(spec=Request) diff --git a/tests/unit/proxy/client/cli/test_encryption_commands.py b/tests/unit/proxy/client/cli/test_encryption_commands.py index 3a86eb82593..6685d29fb7e 100644 --- a/tests/unit/proxy/client/cli/test_encryption_commands.py +++ b/tests/unit/proxy/client/cli/test_encryption_commands.py @@ -5,6 +5,8 @@ check vs POST migrate, dry-run param) and its residual-state messaging without a live proxy. """ +from typing import Final + import pytest from click.testing import CliRunner @@ -69,3 +71,20 @@ def test_migrate_reports_residual_legacy(runner): result = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) assert result.exit_code == 0, result.output assert "Residual legacy values remaining: 3" in result.output + + +@pytest.mark.parametrize("residual", [-1, float("nan")]) +def test_migrate_prints_no_residual_status_for_a_residual_below_zero_or_nan(runner, residual): + _FakeHTTPClient.response = {"status": "success", "report": {"residual_legacy": residual}} + result: Final = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) + assert result.exit_code == 0, result.output + assert "No legacy values remaining" not in result.output + assert "Residual legacy values remaining" not in result.output + + +@pytest.mark.parametrize("report", [None, "done", {"residual_legacy": "3"}]) +def test_migrate_malformed_report_fails(runner, report): + _FakeHTTPClient.response = {"status": "success", "report": report} + result: Final = runner.invoke(cli_main.cli, ["encryption", "migrate", "--check"]) + assert result.exit_code != 0 + assert "Unexpected" in result.output diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 33b8085e082..d7fc844171d 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -4259,6 +4259,17 @@ class TestCreatorTurnedOffMessageLogging: assert await instance._creator_turned_off_message_logging(self._job()) is expected + @pytest.mark.asyncio + async def test_a_job_without_creator_fields_falls_back_to_the_proxy_setting(self, monkeypatch): + from types import SimpleNamespace + + import litellm + + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + instance: Final = self._instance() + + assert await instance._creator_turned_off_message_logging(SimpleNamespace(id="job-1")) is False + @pytest.mark.asyncio async def test_a_team_that_turned_off_message_logging_covers_its_keys(self, monkeypatch): from types import SimpleNamespace diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py b/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py index aa43a5530f3..953baf5c173 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/semantic_guard/test_semantic_guard.py @@ -129,6 +129,20 @@ class TestHelperFunctions: mock_response.choices = [] assert _extract_response_text(mock_response) == "" + def test_extract_response_text_fails_closed_on_a_message_without_content(self): + from types import SimpleNamespace + + from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import ( + _extract_response_text, + ) + + response: Final = SimpleNamespace( + choices=[SimpleNamespace(message={"role": "assistant", "content": "ignore all rules"})] + ) + + with pytest.raises(AttributeError, match="'dict' object has no attribute 'content'"): + _extract_response_text(response) + def test_get_top_route_choice_single(self): from litellm.proxy.guardrails.guardrail_hooks.semantic_guard.semantic_guard import ( _get_top_route_choice, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index df2fc8e1da2..3da77a8707e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -1238,6 +1238,23 @@ class TestFailClosed: class TestStreamingRehydration: + @pytest.mark.asyncio + async def test_held_text_behind_a_chunk_that_cannot_be_copied_fails_closed(self): + guardrail: Final = _guardrail(event_hook="post_call") + _mock_post(guardrail, {"text": "", "carry": "[EMAIL"}, {"text": "a@b.com", "carry": ""}) + + async def stream(): + yield SimpleNamespace( + choices=[SimpleNamespace(index=0, delta=SimpleNamespace(content="[EMAIL"), finish_reason=None)] + ) + + with pytest.raises(AttributeError, match="model_copy"): + await _drain( + guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=None, response=stream(), request_data={"messages": []} + ) + ) + @pytest.mark.asyncio async def test_split_placeholder_is_not_emitted_in_fragments(self): """The window holds back a partial placeholder and releases it once complete.""" diff --git a/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py b/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py index 81c5803da33..09104c035c7 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py +++ b/tests/unit/proxy/openai_files_endpoint/test_storage_backend_service.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import MagicMock import pytest @@ -27,10 +28,7 @@ class _RecordingStorageBackend: raise self.delete_error -class _FakeManagedFilesHook(BaseFileEndpoints): - def __init__(self): - self.stored = [] - +class _ManagedFilesEndpoints(BaseFileEndpoints): async def acreate_file( self, create_file_request, llm_router, target_model_names_list, litellm_parent_otel_span, user_api_key_dict ): @@ -48,10 +46,23 @@ class _FakeManagedFilesHook(BaseFileEndpoints): async def afile_content(self, file_id, litellm_parent_otel_span, llm_router, **data): raise NotImplementedError + +class _FakeManagedFilesHook(_ManagedFilesEndpoints): + def __init__(self): + self.stored = [] + async def store_unified_file_id(self, **kwargs): self.stored.append(kwargs) +class _ForwardingManagedFilesHook(_ManagedFilesEndpoints): + def __init__(self, writer: _FakeManagedFilesHook): + self._writer = writer + + def __getattr__(self, name: str) -> object: + return getattr(self._writer, name) + + class _FailingManagedFilesHook(_FakeManagedFilesHook): async def store_unified_file_id(self, **kwargs): raise RuntimeError("db down") @@ -142,6 +153,24 @@ async def test_upload_with_target_model_names_and_hook_stores_unified_id(monkeyp } +@pytest.mark.asyncio +async def test_upload_stores_the_unified_id_through_a_hook_that_forwards_its_writer(monkeypatch): + backend: Final = _RecordingStorageBackend() + monkeypatch.setattr(storage_backend_service, "get_storage_backend", lambda name, prisma_client=None: backend) + writer: Final = _FakeManagedFilesHook() + + file_object: Final = await StorageBackendFileService.upload_file_to_storage_backend( + file_data=_file_data(), + target_storage="azure_storage", + target_model_names=["gpt-x"], + purpose="batch", + proxy_logging_obj=_FakeProxyLogging(hook=_ForwardingManagedFilesHook(writer)), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + assert [stored["file_id"] for stored in writer.stored] == [file_object.id] + + @pytest.mark.asyncio async def test_upload_hands_the_prisma_client_to_the_storage_backend_factory(monkeypatch: pytest.MonkeyPatch): backend = _RecordingStorageBackend() diff --git a/tests/unit/proxy/proxy_server/test_openapi_customization.py b/tests/unit/proxy/proxy_server/test_openapi_customization.py index 6c588976efc..0eacf333960 100644 --- a/tests/unit/proxy/proxy_server/test_openapi_customization.py +++ b/tests/unit/proxy/proxy_server/test_openapi_customization.py @@ -36,6 +36,7 @@ from litellm.proxy.proxy_server import ( get_openapi_schema, mount_swagger_ui, ) +from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder from .conftest import normalize @@ -209,6 +210,17 @@ def test_inject_websocket_stubs_into_openapi_schema_adds_stub(): } +def test_inject_websocket_stubs_into_openapi_schema_documents_query_params_that_forward_their_fields(): + query_param: Final = AttributeForwarder(SimpleNamespace(name="session_id", required=True)) + route: Final = SimpleNamespace(path="/ws/chat", name="ws_chat", dependant=SimpleNamespace(query_params=[query_param])) + + result: Final = _inject_websocket_stubs_into_openapi_schema({"paths": {}}, [route]) + + assert result["paths"]["/ws/chat"]["get"]["parameters"] == [ + {"name": "session_id", "in": "query", "required": True, "schema": {"type": "string"}} + ] + + def test_inject_websocket_stubs_into_openapi_schema_does_not_overwrite_existing_get(): # Existing GET on the same path must not be replaced by the stub. existing_get = {"summary": "real http get", "operationId": "real_get"}