diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 13e9e5093a8..41974c26158 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -2,10 +2,11 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked. """ +from collections.abc import Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast +from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -18,8 +19,8 @@ if TYPE_CHECKING: from prisma import models as prisma_models from litellm.integrations.prometheus import PrometheusLogger - from litellm.proxy._types import LiteLLM_ManagedObjectTable from litellm.proxy.utils import PrismaClient, ProxyLogging + from litellm.repositories.prisma_protocols import TableActions from litellm.router import Router from litellm.types.router import Deployment from litellm.types.utils import LiteLLMBatch @@ -41,6 +42,42 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = ( ) +class _ManagedObjectRow(Protocol): + @property + def id(self) -> str: ... + + @property + def unified_object_id(self) -> str: ... + + @property + def created_by(self) -> str | None: ... + + @property + def file_object(self) -> object: ... + + +def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": + table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable + return table + + +def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]": + table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable + return table + + +def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]": + table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = ( + prisma_client.db.litellm_verificationtoken + ) + return table + + +def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]": + table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable + return table + + class CheckBatchCost: def __init__( self, @@ -73,7 +110,7 @@ class CheckBatchCost: inline for a batch the first poll cycle then accounts again. """ try: - await self.prisma_client.db.litellm_managedobjecttable.find_first( + await _managed_object_table(self.prisma_client).find_first( where={"file_purpose": "batch", "batch_processed": False} ) except Exception as probe_err: @@ -97,10 +134,8 @@ class CheckBatchCost: if not user_id: return {} try: - user_row: prisma_models.LiteLLM_UserTable | None = ( - await self.prisma_client.db.litellm_usertable.find_unique( - where={"user_id": user_id} - ) + user_row: prisma_models.LiteLLM_UserTable | None = await _user_table(self.prisma_client).find_unique( + where={"user_id": user_id} ) if user_row is None: return {} @@ -117,11 +152,9 @@ class CheckBatchCost: if not api_key: return None try: - key_row: prisma_models.LiteLLM_VerificationToken | None = ( - await self.prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": api_key} - ) - ) + key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table( + self.prisma_client + ).find_unique(where={"token": api_key}) return getattr(key_row, "key_alias", None) if key_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}") @@ -132,17 +165,15 @@ class CheckBatchCost: if not team_id: return None try: - team_row: prisma_models.LiteLLM_TeamTable | None = ( - await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) + team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique( + where={"team_id": team_id} ) return getattr(team_row, "team_alias", None) if team_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}") return None - async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None: + async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None: org_id = getattr(job, "org_id", None) if org_id: return org_id @@ -150,11 +181,9 @@ class CheckBatchCost: team_id = getattr(job, "team_id", None) if api_key: try: - key_row: prisma_models.LiteLLM_VerificationToken | None = ( - await self.prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": api_key} - ) - ) + key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table( + self.prisma_client + ).find_unique(where={"token": api_key}) key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None if key_org_id: return key_org_id @@ -166,10 +195,8 @@ class CheckBatchCost: if not team_id: return None try: - team_row: prisma_models.LiteLLM_TeamTable | None = ( - await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) + team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique( + where={"team_id": team_id} ) return getattr(team_row, "organization_id", None) if team_row is not None else None except Exception as e: @@ -177,7 +204,7 @@ class CheckBatchCost: return None async def _build_creator_attribution_metadata( - self, job: "LiteLLM_ManagedObjectTable", batch_id: str + self, job: "_ManagedObjectRow", batch_id: str ) -> dict[str, object]: """ Rebuild the spend-tracking metadata for the key, team, and tags that created the @@ -225,7 +252,7 @@ class CheckBatchCost: should not be polled. """ cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + result: Final = await _managed_object_table(self.prisma_client).update_many( where={ "file_purpose": "batch", "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)}, @@ -244,7 +271,7 @@ class CheckBatchCost: # A row already in a terminal status is never rewritten by the sweep above, so # without this it keeps a poll-page slot forever and starves newer batches. - retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + retired: Final = await _managed_object_table(self.prisma_client).update_many( where={ "file_purpose": "batch", "batch_processed": False, @@ -259,9 +286,9 @@ class CheckBatchCost: f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed" ) - async def _fallback_find_jobs(self) -> list: + async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]": """Query batch jobs without the batch_processed filter (for older schemas).""" - return await self.prisma_client.db.litellm_managedobjecttable.find_many( + return await _managed_object_table(self.prisma_client).find_many( where={ "file_purpose": "batch", "status": { @@ -279,7 +306,7 @@ class CheckBatchCost: order={"created_at": "asc"}, ) - async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None: + async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None: """ Take a row that can never be costed out of the poll page. Leaving it selectable would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and @@ -292,7 +319,7 @@ class CheckBatchCost: else {"status": "stale_expired"} ) try: - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=data, ) @@ -306,7 +333,7 @@ class CheckBatchCost: "so it will no longer be polled" ) - async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool: + async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool: """ Atomically flip batch_processed from false to true, returning whether this pod won the row. Every pod and uvicorn worker schedules its own poller against the shared @@ -321,7 +348,7 @@ class CheckBatchCost: if not self._has_batch_processed_column: return True try: - claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + claimed: Final = await _managed_object_table(self.prisma_client).update_many( where={"id": job.id, "batch_processed": False}, data={"batch_processed": True}, ) @@ -332,7 +359,7 @@ class CheckBatchCost: return False return claimed > 0 - async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None: + async def _release_job_claim(self, job: "_ManagedObjectRow") -> None: """Give a claimed row back once billing it failed, so a later poll cycle retries it. Safe to match on batch_processed=True: while this poller is active the retrieve @@ -342,7 +369,7 @@ class CheckBatchCost: if not self._has_batch_processed_column: return try: - await self.prisma_client.db.litellm_managedobjecttable.update_many( + await _managed_object_table(self.prisma_client).update_many( where={"id": job.id, "batch_processed": True}, data={"batch_processed": False}, ) @@ -353,7 +380,7 @@ class CheckBatchCost: ) @staticmethod - def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool: + def _has_unified_id_without_model(job: "_ManagedObjectRow") -> bool: """A unified id that decodes but carries no model_id can never be routed.""" from litellm.proxy.openai_files_endpoints.common_utils import ( convert_b64_uid_to_unified_uid, @@ -402,7 +429,7 @@ class CheckBatchCost: return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error) async def _finalize_unbilled_terminal_job( - self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch" + self, job: "_ManagedObjectRow", response: "LiteLLMBatch" ) -> None: """Persist a terminal batch that has nothing billable, converting any raw provider file ids to managed ids, and take it out of the poll page.""" @@ -426,7 +453,7 @@ class CheckBatchCost: "file_object": response.model_dump_json(), **({"batch_processed": True} if self._has_batch_processed_column else {}), } - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=update_data, ) @@ -447,7 +474,7 @@ class CheckBatchCost: def _resolve_job_routing( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", prom_logger: Optional["PrometheusLogger"], ) -> Optional[Tuple[str, str]]: """ @@ -524,7 +551,7 @@ class CheckBatchCost: def _resolve_unmanaged_provider_routing( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", prom_logger: Optional["PrometheusLogger"], llm_provider: str, bare_model_name: str, @@ -620,7 +647,7 @@ class CheckBatchCost: @classmethod def _get_managed_file_model_name( cls, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", deployment_info: "Deployment", ) -> Optional[str]: """ @@ -640,7 +667,7 @@ class CheckBatchCost: ) @staticmethod - def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]: + def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]: import json from litellm.types.utils import LiteLLMBatch @@ -660,7 +687,7 @@ class CheckBatchCost: async def _track_completed_batch_cost( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", response: "LiteLLMBatch", model_id: str, batch_id: str, @@ -936,7 +963,7 @@ class CheckBatchCost: # endpoint may transition a batch to "complete" before # CheckBatchCost runs. The batch_processed=False filter # already prevents reprocessing finished batches. - jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many( + jobs = await _managed_object_table(self.prisma_client).find_many( where={ "file_purpose": "batch", "batch_processed": False, @@ -1038,7 +1065,7 @@ class CheckBatchCost: } if self._has_batch_processed_column: update_data["batch_processed"] = True - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=update_data, ) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 06cf5fcf82f..cdeea0d3d4b 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -6,7 +6,7 @@ same route are non-inference and free. """ from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Dict, Optional, cast +from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast import litellm from litellm._logging import verbose_proxy_logger @@ -22,11 +22,31 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging + from litellm.repositories.prisma_protocols import TableActions from litellm.router import Router TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"}) +class _ManagedObjectRow(Protocol): + @property + def id(self) -> str: ... + + @property + def unified_object_id(self) -> str: ... + + @property + def created_by(self) -> str | None: ... + + @property + def file_object(self) -> object: ... + + +def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": + table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable + return table + + class CheckResponsesCost: def __init__( self, @@ -128,7 +148,7 @@ class CheckResponsesCost: f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}" ) - jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many( + jobs = await _managed_object_table(self.prisma_client).find_many( where={ "status": {"in": ["queued", "in_progress"]}, "file_purpose": "response", @@ -138,7 +158,7 @@ class CheckResponsesCost: ) verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check") - completed_jobs = [] + completed_jobs: Final[list[_ManagedObjectRow]] = [] for job in jobs: unified_object_id = job.unified_object_id @@ -189,7 +209,7 @@ class CheckResponsesCost: # Mark completed jobs in the database if len(completed_jobs) > 0: - await self.prisma_client.db.litellm_managedobjecttable.update_many( + await _managed_object_table(self.prisma_client).update_many( where={"id": {"in": [job.id for job in completed_jobs]}}, data={"status": "completed"}, ) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 09cd0ed192f..5ac7c1e53c1 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -481,10 +481,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ if self.prisma_client is None: return - managed_object = ( - await self.prisma_client.db.litellm_managedobjecttable.find_first( - where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]} - ) + managed_object = await _managed_object_table(self.prisma_client).find_first( + where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]} ) if managed_object is None: return @@ -509,10 +507,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ if self.prisma_client is None: return - managed_file = ( - await self.prisma_client.db.litellm_managedfiletable.find_first( - where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]} - ) + managed_file = await _managed_file_table(self.prisma_client).find_first( + where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]} ) if managed_file is None: return @@ -535,8 +531,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): provider_file_ids = tuple( file_id for file_id in ( - getattr(response, "output_file_id", None), - getattr(response, "error_file_id", None), + response.output_file_id, + response.error_file_id, ) if file_id and not _is_base64_encoded_unified_file_id(file_id) ) @@ -544,10 +540,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return if self.prisma_client is None: return - batch_row = ( - await self.prisma_client.db.litellm_managedobjecttable.find_first( - where={"unified_object_id": response.id} - ) + batch_row = await _managed_object_table(self.prisma_client).find_first( + where={"unified_object_id": response.id} ) if batch_row is None or ( batch_row.created_by is None and batch_row.team_id is None diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py index b7546e1a2a1..20404e3702b 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py @@ -23,7 +23,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): params: dict[str, Any], api_base: str | None = None, **kwargs: Any, - ) -> dict[str, Any]: + ) -> dict[str, object]: """Handle non-streaming request to Pydantic AI agent.""" if api_base is None: raise ValueError("api_base is required for PydanticAIProviderConfig") @@ -41,7 +41,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): params: dict[str, Any], api_base: str | None = None, **kwargs, - ) -> AsyncIterator[dict[str, Any]]: + ) -> AsyncIterator[dict[str, object]]: """Handle streaming request with fake streaming.""" if not api_base: raise ValueError("api_base is required for Pydantic AI agents") diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index be82f5def1f..6a98b104221 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -81,7 +81,7 @@ class Cache: s3_aws_access_key_id: str | None = None, s3_aws_secret_access_key: str | None = None, s3_aws_session_token: str | None = None, - s3_config: Any | None = None, + s3_config: object | None = None, s3_path: str | None = None, gcs_bucket_name: str | None = None, gcs_path_service_account: str | None = None, diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 50426ea89ea..36c3b744a06 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -74,7 +74,7 @@ class CachingHandlerResponse(BaseModel): For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others """ - cached_result: Any | None = None + cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call @@ -722,7 +722,7 @@ class LLMCachingHandler: async def _retrieve_from_cache( self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...] - ) -> Any | None: + ) -> object | None: """ Internal method to - get cache key @@ -968,7 +968,7 @@ class LLMCachingHandler: def _convert_cached_stream_response( self, - cached_result: Any, + cached_result: dict[str, object], call_type: str, logging_obj: LiteLLMLoggingObj, model: str, @@ -997,7 +997,7 @@ class LLMCachingHandler: async def async_set_cache( self, - result: Any, + result: object, original_function: Callable, kwargs: dict[str, Any], args: tuple[object, ...] | None = None, @@ -1065,7 +1065,7 @@ class LLMCachingHandler: def sync_set_cache( self, - result: Any, + result: object, kwargs: dict[str, object], args: tuple[object, ...] | None = None, ): diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index a09aacb8a23..7cec84e0ebb 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -981,7 +981,7 @@ class RedisCache(BaseCache): client: object = None, ) -> object: async def execute() -> object: - executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache( + executor: Callable[..., Awaitable[object]] | None = litellm.in_memory_llm_clients_cache.get_cache( key=script_cache_key ) if executor is None: @@ -993,7 +993,7 @@ class RedisCache(BaseCache): return run_script - def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[Any]]: + def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[object]]: """ Register the script against the current event loop's Redis client. diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index f494d6610a1..642a78789b2 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -2,7 +2,7 @@ Handler for transforming /chat/completions api requests to litellm.responses requests """ -from collections.abc import Coroutine +from collections.abc import AsyncIterable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Final, Union from typing_extensions import TypedDict @@ -74,7 +74,7 @@ class ResponsesToCompletionBridgeHandler: existing.setdefault(key, value) return response - def _collect_response_from_stream(self, stream_iter: Any) -> "ResponsesAPIResponse": + def _collect_response_from_stream(self, stream_iter: Iterable[object]) -> "ResponsesAPIResponse": for _ in stream_iter: pass @@ -89,7 +89,7 @@ class ResponsesToCompletionBridgeHandler: raise ValueError("Stream completed response is invalid") return response - async def _collect_response_from_stream_async(self, stream_iter: Any) -> "ResponsesAPIResponse": + async def _collect_response_from_stream_async(self, stream_iter: AsyncIterable[object]) -> "ResponsesAPIResponse": async for _ in stream_iter: pass diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4024ce5360e..4ceb89bd83a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -6,7 +6,7 @@ import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast, get_args from openai.types.chat import ChatCompletion from openai.types.responses import Response @@ -52,7 +52,7 @@ from litellm.types.llms.openai import ( from litellm.types.utils import GenericStreamingChunk, ModelResponseStream if TYPE_CHECKING: - from openai.types.responses import ResponseInputImageParam + from openai.types.responses import ResponseInputImageParam, ResponseOutputItem from openai.types.responses.response_text_config_param import ( ResponseTextConfigParam as ResponseText, ) @@ -197,6 +197,9 @@ def _as_chat_reasoning_items( return cast(list[ChatCompletionReasoningItem], list(reasoning_items)) +_ToolChoiceT = TypeVar("_ToolChoiceT") + + def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]: if incomplete_reason == "content_filter": return "content_filter" @@ -291,7 +294,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def __init__(self): pass - def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any: + def _normalize_tool_choice_for_responses_api( + self, tool_choice: _ToolChoiceT + ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]: """Chat tool_choice nests the name under function/custom; Responses API expects top-level name.""" if not isinstance(tool_choice, dict): return tool_choice @@ -497,7 +502,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): responses_api_request["max_output_tokens"] = value elif key == "tools" and value is not None: responses_api_request["tools"] = self._convert_tools_to_responses_format( - cast(list[dict[str, Any]], value) + cast(list[dict[str, object]], value) ) elif key == "response_format": text_format = self._transform_response_format_to_text_format(value) @@ -828,7 +833,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): response_output: Final = response_payload.get("output") if not isinstance(response_output, list) or len(response_output) == 0: return None - return cast(list[dict[str, Any]], response_output) + return cast(list[dict[str, object]], response_output) @classmethod def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]: @@ -911,10 +916,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items = raw_response.output if len(output_items) == 0: - recovered_output_items: Final = self._recover_output_items_from_logging(logging_obj) + recovered_output_items: Final[list[ResponseOutputItem | dict[str, object]]] = [ + *self._recover_output_items_from_logging(logging_obj) + ] if recovered_output_items: - output_items = cast(Any, recovered_output_items) - raw_response.output = cast(Any, recovered_output_items) + output_items = recovered_output_items + raw_response.output = recovered_output_items verbose_logger.warning( "Recovered empty Responses API output from raw SSE for model=%s", model, @@ -1110,7 +1117,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): verbose_logger.debug("Chat provider: Other content type -> %s", result) return result - def _convert_tools_to_responses_format(self, tools: list[dict[str, Any]]) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: + def _convert_tools_to_responses_format( + self, tools: list[dict[str, object]] + ) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: """Convert chat completion tools to responses API tools format""" responses_tools: Final[list[ALL_RESPONSES_API_TOOL_PARAMS]] = [] for tool in tools: @@ -1126,12 +1135,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): description=function_tool.get("description"), ) ) - elif tool.get("type") == "custom" and isinstance(tool.get("custom"), dict): + elif tool.get("type") == "custom" and isinstance(custom_payload := tool.get("custom"), dict): from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_responses_shape, ) - custom_payload = tool["custom"] flat_custom = CustomToolParam(type="custom", name=custom_payload.get("name", "")) if custom_payload.get("description") is not None: flat_custom["description"] = custom_payload["description"] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 38758867a11..b317e356e1d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -353,7 +353,7 @@ def cost_per_token( data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") - response: Any | None = None, + response: object | None = None, ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection custom_model_info: OCRPricing | None = None, @@ -609,7 +609,7 @@ def cost_per_token( model=model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries or 1, - optional_params=(response._hidden_params if response and hasattr(response, "_hidden_params") else None), + optional_params=(getattr(response, "_hidden_params", None) if response else None), ) elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( @@ -999,7 +999,7 @@ def _is_known_usage_objects(usage_obj): ) -def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: Any) -> CallTypesLiteral | None: +def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: object) -> CallTypesLiteral | None: if call_type is not None: return call_type diff --git a/litellm/integrations/argilla.py b/litellm/integrations/argilla.py index 9a87a94cf0b..664ef8efda1 100644 --- a/litellm/integrations/argilla.py +++ b/litellm/integrations/argilla.py @@ -7,7 +7,8 @@ import json import os import random import types -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx from pydantic import BaseModel @@ -69,7 +70,7 @@ class ArgillaLogger(CustomBatchLogger): self.flush_lock = asyncio.Lock() super().__init__(**kwargs, flush_lock=self.flush_lock) - def validate_argilla_transformation_object(self, argilla_transformation_object: dict[str, Any]): + def validate_argilla_transformation_object(self, argilla_transformation_object: Mapping[str, object]): if not isinstance(argilla_transformation_object, dict): raise Exception("'argilla_transformation_object' must be a dictionary, to log your payload to Argilla.") @@ -115,7 +116,7 @@ class ArgillaLogger(CustomBatchLogger): ARGILLA_DATASET_NAME=_credentials_dataset_name, ) - def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, Any]]: + def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, object]]: payload_messages: Final = payload.get("messages", None) if payload_messages is None: diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index aaf72a0bc4e..501f5749ea4 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -139,13 +139,13 @@ class BraintrustLogger(CustomLogger): ): output = None elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse): - output = response_obj["choices"][0]["message"].json() + output = response_obj.choices[0].message.json() choices = response_obj["choices"] elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse): output = response_obj.choices[0].text choices = response_obj.choices elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse): - output = response_obj["data"] + output = response_obj.data litellm_params: Final = kwargs.get("litellm_params", {}) or {} dynamic_metadata: Final = litellm_params.get("metadata", {}) or {} @@ -264,13 +264,13 @@ class BraintrustLogger(CustomLogger): ): output = None elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse): - output = response_obj["choices"][0]["message"].json() + output = response_obj.choices[0].message.json() choices = response_obj["choices"] elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse): output = response_obj.choices[0].text choices = response_obj.choices elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse): - output = response_obj["data"] + output = response_obj.data litellm_params: Final = kwargs.get("litellm_params", {}) dynamic_metadata: Final = litellm_params.get("metadata", {}) or {} diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 3865be763ea..ffa0bc36f6b 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -150,7 +150,7 @@ class CustomGuardrail(CustomLogger): def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks super().__init_subclass__(**kwargs) - own_apply_guardrail: Final = cls.__dict__.get("apply_guardrail") + own_apply_guardrail: Final[object] = cls.__dict__.get("apply_guardrail") if own_apply_guardrail is None or LOGS_GUARDRAIL_INFORMATION_MARKER in vars(own_apply_guardrail): return cls.apply_guardrail = log_guardrail_information(own_apply_guardrail) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index c64a12c6d75..98aac7336bf 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -54,7 +54,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, ) -_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""} _MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024 _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset( @@ -154,7 +154,7 @@ def _guardrail_information_without_prompt_carriers( return tuple(_guardrail_entry_without_prompt_carriers(entry) for entry in _guardrail_entries(guardrail_information)) -def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]: +def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, object]) -> Mapping[str, object]: """The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text.""" return MappingProxyType( { @@ -237,7 +237,7 @@ def _declared_cost_tags(span_tags: Sequence[str]) -> tuple[str, ...]: return tuple(dimension for dimension in _COST_DIMENSIONS if dimension in present) -def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float: +def _reasoning_output_tokens(usage_object: Mapping[str, object] | None) -> float: """The provider's reasoning-token count, from either the chat or the responses spelling.""" if usage_object is None: return 0.0 @@ -254,20 +254,24 @@ def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float: ) -def _mapping_field(source: Mapping[str, Any], key: str) -> Mapping[str, Any]: +def _mapping_field(source: Mapping[str, object], key: str) -> Mapping[str, object]: """The value at `key` when it is a mapping, else an empty one.""" value: Final = source.get(key) return value if isinstance(value, dict) else _EMPTY_MAPPING -def _content_blocks(message: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: +def _text_field(source: Mapping[str, object], key: str, default: str = "") -> str: + return _safe_identifier(source.get(key, default)) + + +def _content_blocks(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: content: Final = message.get("content") if not isinstance(content, list): return () return tuple(block for block in content if isinstance(block, dict)) -def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str: +def _to_dd_arguments(raw_arguments: object) -> dict[str, object] | str: """ Arguments as the object LLM Obs types them as, or the raw string when they are not one. @@ -282,7 +286,7 @@ def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str: return parsed if isinstance(parsed, dict) else raw_arguments -def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: +def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: """ The tool calls a message carries, in LLM Obs' ToolCall schema, from either dialect. @@ -293,10 +297,10 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: raw_tool_calls: Final = message.get("tool_calls") openai_calls: Final = tuple( ToolCall( - name=function.get("name", ""), + name=_text_field(function, "name"), arguments=_to_dd_arguments(function.get("arguments", "")), - tool_id=tool_call.get("id", ""), - type=tool_call.get("type", "function"), + tool_id=_text_field(tool_call, "id"), + type=_text_field(tool_call, "type", "function"), ) for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) if isinstance(tool_call, dict) @@ -304,9 +308,9 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: ) anthropic_calls: Final = tuple( ToolCall( - name=block.get("name", ""), + name=_text_field(block, "name"), arguments=_to_dd_arguments(block.get("input") or {}), - tool_id=block.get("id", ""), + tool_id=_text_field(block, "id"), type="tool_use", ) for block in _content_blocks(message) @@ -315,7 +319,7 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: return openai_calls + anthropic_calls -def _to_dd_tool_results(message: Mapping[str, Any], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]: +def _to_dd_tool_results(message: Mapping[str, object], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]: """ The tool results a message carries, linked back to the call each answers. @@ -400,14 +404,14 @@ def _to_dd_messages(messages: object) -> tuple[Message, ...]: return tuple(_to_dd_message(message, tool_call_names) for message in messages) -def _to_dd_tool_definition(entry: Mapping[str, Any]) -> ToolDefinition | None: +def _to_dd_tool_definition(entry: Mapping[str, object]) -> ToolDefinition | None: function: Final = entry.get("function") - declared: Final[Mapping[str, Any]] = function if isinstance(function, dict) else entry - name: Final = declared.get("name") + declared: Final[Mapping[str, object]] = function if isinstance(function, dict) else entry + name: Final = _text_field(declared, "name") if not name: return None schema: Final = declared.get("parameters") or declared.get("input_schema") - description: Final = declared.get("description", "") + description: Final = _text_field(declared, "description") if not isinstance(schema, dict): return ToolDefinition(name=name, description=description) return ToolDefinition(name=name, description=description, schema=schema) @@ -683,7 +687,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): if callable(current_span_fn): current_span: Final = current_span_fn() if current_span is not None: - trace_id: Final = getattr(current_span, "trace_id", None) + trace_id: Final[object] = getattr(current_span, "trace_id", None) if trace_id is not None: return str(trace_id) except Exception: @@ -716,7 +720,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): def redacts_messages_itself(self) -> bool: return True - def _payload_logging_is_off(self, kwargs: Mapping[str, Any]) -> bool: + def _payload_logging_is_off(self, kwargs: Mapping[str, object]) -> bool: return ( bool(self.turn_off_message_logging) or self.message_logging is not True diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 38f5924a233..3fbbfe91ddf 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -3,12 +3,21 @@ import os import traceback -from typing import Any, Final +from collections.abc import Mapping +from typing import Final, Protocol import litellm from litellm._uuid import uuid +class _DynamoTable(Protocol): + def put_item(self, *, Item: Mapping[str, object]) -> object: ... + + +class _DynamoResource(Protocol): + def Table(self, name: str) -> _DynamoTable: ... + + class DyanmoDBLogger: # Class variables or attributes @@ -16,7 +25,7 @@ class DyanmoDBLogger: # Instance variables import boto3 - self.dynamodb: Any = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"]) + self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"]) if litellm.dynamodb_table_name is None: raise ValueError( "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=`" @@ -41,7 +50,7 @@ class DyanmoDBLogger: id: Final = response_obj.get("id", str(uuid.uuid4())) # Build the initial payload - payload: Final = { + payload: Final[dict[str, object]] = { "id": id, "call_type": call_type, "startTime": start_time, diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 657c7e0d264..891318f1c54 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Final +from typing import Final import polars as pl @@ -32,7 +32,7 @@ class FocusLiteLLMDatabase: client: Final = self._ensure_prisma_client() where_clauses: Final[list[str]] = [] - query_params: Final[list[Any]] = [] + query_params: Final[list[datetime | int]] = [] placeholder_index = 1 if start_time_utc: where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz") @@ -112,7 +112,7 @@ class FocusLiteLLMDatabase: except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc - async def get_table_info(self) -> dict[str, Any]: + async def get_table_info(self) -> dict[str, object]: """Return metadata about the spend table for diagnostics.""" client: Final = self._ensure_prisma_client() diff --git a/litellm/integrations/focus/destinations/vantage_destination.py b/litellm/integrations/focus/destinations/vantage_destination.py index 132f27779c2..68b0d399975 100644 --- a/litellm/integrations/focus/destinations/vantage_destination.py +++ b/litellm/integrations/focus/destinations/vantage_destination.py @@ -4,7 +4,8 @@ from __future__ import annotations import csv import io -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx # noqa: F401 - used at runtime (AsyncClient, HTTPStatusError) @@ -94,7 +95,7 @@ class FocusVantageDestination(FocusDestination): self, *, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, object] | None = None, ) -> None: config = config or {} api_key: Final = config.get("api_key") diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index b27618993a3..010f8ad8ef2 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -396,12 +396,13 @@ class GalileoObserve(CustomLogger): ) @staticmethod - def _log_v2_payload_validation(payload: dict[str, Any]) -> None: + def _log_v2_payload_validation(payload: dict[str, object]) -> None: missing_fields: Final[list[str]] = [] - traces: Final[Sequence[object]] = payload.get("traces", []) - if not traces: + traces_value: Final = payload.get("traces", []) + if not traces_value: missing_fields.append("traces") + traces: Final[Sequence[object]] = traces_value if isinstance(traces_value, list) else [] for trace_index, trace in enumerate(traces): if not isinstance(trace, dict): continue @@ -425,8 +426,8 @@ class GalileoObserve(CustomLogger): missing_fields, ) - def _log_flush_payload(self, url: str, payload: dict[str, Any]) -> None: - traces: Final[Sequence[object]] = payload.get("traces", []) + def _log_flush_payload(self, url: str, payload: dict[str, object]) -> None: + traces: Final = payload.get("traces") verbose_logger.debug( "Galileo Logger flush URL: %s trace_count=%s", url, diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 52d8d8c06f3..96d711337fb 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -4,7 +4,7 @@ import inspect import os import re import traceback -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from datetime import datetime from functools import lru_cache from types import MappingProxyType @@ -432,7 +432,7 @@ class LangFuseLogger: prompt: dict, level: str, status_message: str | None, - ) -> tuple[dict | None, str | dict | list | None]: + ) -> tuple[dict | None, str | dict | Sequence[object] | None]: """ Get the input and output content for Langfuse logging @@ -448,7 +448,7 @@ class LangFuseLogger: output: The output content for Langfuse logging """ input = None - output: str | dict | list[Any] | None = None + output: str | dict | Sequence[object] | None = None if level == "ERROR" and status_message is not None and isinstance(status_message, str): input = prompt output = status_message @@ -508,7 +508,7 @@ class LangFuseLogger: user_id: str | None, metadata: dict[str, object], litellm_params: dict, - output: str | dict | list | None, + output: str | dict | Sequence[object] | None, start_time: datetime | None, end_time: datetime | None, kwargs: dict, diff --git a/litellm/integrations/langfuse/langfuse_otel_attributes.py b/litellm/integrations/langfuse/langfuse_otel_attributes.py index 70fea1abb3b..1eb3ce1a9c2 100644 --- a/litellm/integrations/langfuse/langfuse_otel_attributes.py +++ b/litellm/integrations/langfuse/langfuse_otel_attributes.py @@ -5,6 +5,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764 """ import json +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel @@ -40,7 +41,7 @@ def get_output_content_by_type( | HttpxBinaryResponseContent | ResponsesAPIResponse | list, - kwargs: dict[str, Any] | None = None, + kwargs: Mapping[str, object] | None = None, ) -> str: """ Extract output content from response objects based on their type. diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 32664ed75d2..352fcdf90f3 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -77,9 +77,9 @@ class LangsmithLogger(CustomBatchLogger): if _batch_size: self.batch_size = int(_batch_size) self.log_queue: list[LangsmithQueueObject] = [] - self._flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() + self._flush_task: asyncio.Task[None] | None = self._start_periodic_flush_task() - def _start_periodic_flush_task(self) -> asyncio.Task[Any] | None: + def _start_periodic_flush_task(self) -> asyncio.Task[None] | None: """Start the periodic flush task only when an event loop is already running.""" try: loop: Final = asyncio.get_running_loop() @@ -154,9 +154,9 @@ class LangsmithLogger(CustomBatchLogger): return self._redact_metadata(extra_metadata) - def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> dict[str, Any]: + def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> dict[str, object]: response: Final = payload["response"] - outputs: dict[str, Any] + outputs: dict[str, object] if isinstance(response, dict): outputs = {**response} else: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index dd4247ad3d0..ad513968b45 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -36,7 +36,7 @@ model. They coincide on the SDK path, which is correct. from __future__ import annotations -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -61,7 +61,7 @@ class RequestIdentity: # The team's free-form metadata, carried raw (empty/missing -> None) and # filtered to an operator allowlist only at Baggage-promotion time, so an # unconfigured deployment never promotes any of it. - team_metadata: Mapping[str, Any] | None = None + team_metadata: Mapping[str, object] | None = None key_hash: str | None = None end_user: str | None = None # The model litellm dispatched to the provider. Only known once the call @@ -111,7 +111,7 @@ class RequestIdentity: snapshot) is flattened to dotted keys so ``requester_metadata.`` resolves too. """ - get: Final = lambda name: getattr(auth, name, None) # noqa: E731 + get: Final[Callable[[str], object]] = lambda name: getattr(auth, name, None) # noqa: E731 auth_meta: Final = tuple( (meta_key, str(value)) for meta_key, attr in ( @@ -228,7 +228,7 @@ class LLMCallEvent: trace: TraceControls @classmethod - def from_dict(cls, kwargs: Mapping[str, Any]) -> LLMCallEvent: + def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent: raw_payload: Final = kwargs.get("standard_logging_object") payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None operation: Final = resolve_operation(as_str(kwargs.get("call_type"))) @@ -251,7 +251,7 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: to the first streamed chunk (``completion_start_time``); ``None`` for non-streaming calls, where ``completion_start_time`` is backfilled with the end time and would not measure first-chunk latency.""" - optional_params: Final = cast(Mapping[str, Any], kwargs.get("optional_params") or {}) + optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None api_call_start: Final = to_seconds(kwargs.get("api_call_start_time")) @@ -312,7 +312,7 @@ def _metadata_dicts( ) -def _call_id(payload: StandardLoggingPayload | None, kwargs: Mapping[str, Any]) -> str | None: +def _call_id(payload: StandardLoggingPayload | None, kwargs: Mapping[str, object]) -> str | None: """The call id from the payload (when closed) or the bare kwargs (at pre_call).""" if payload is not None: call_id: Final = as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")) @@ -385,7 +385,7 @@ def _model_info_id(model_info: object) -> str | None: return None -def _team_metadata_dict(value: object) -> Mapping[str, Any] | None: +def _team_metadata_dict(value: object) -> Mapping[str, object] | None: """The team's free-form metadata as a raw mapping, or ``None`` when missing or empty. diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index ac647c2c4f6..9340f6e9e15 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -12,11 +12,14 @@ when the feature gate is off. """ import os -from typing import Any, Final +from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_logger from litellm.integrations.otel.model.config import is_otel_v2_enabled +if TYPE_CHECKING: + from fastapi import FastAPI + # Routes excluded from server-span tracing by default: high-frequency pollers and # static UI/docs assets, none of which are LLM traffic. Entries are substring-matched # against the request path (unanchored, so they survive a ``server_root_path`` prefix @@ -65,7 +68,15 @@ PASSTHROUGH_PREFIXES: Final = frozenset( ) -def _passthrough_span_name_hook(span: Any, scope: dict) -> None: +class _RenameableSpan(Protocol): + def is_recording(self) -> bool: ... + + def update_name(self, name: str) -> None: ... + + def set_attribute(self, key: str, value: str) -> None: ... + + +def _passthrough_span_name_hook(span: "_RenameableSpan | None", scope: dict) -> None: """FastAPI ``server_request_hook``: give passthrough server spans a useful name. The instrumentation matches the route at span creation, so both the span name @@ -88,7 +99,7 @@ def _passthrough_span_name_hook(span: Any, scope: dict) -> None: pass -def instrument_fastapi_app(app: Any) -> None: +def instrument_fastapi_app(app: "FastAPI") -> None: """Attach OTel server-span instrumentation to the proxy FastAPI app. Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` diff --git a/litellm/litellm_core_utils/coroutine_checker.py b/litellm/litellm_core_utils/coroutine_checker.py index 7b9a650c66b..99ba74dbabf 100644 --- a/litellm/litellm_core_utils/coroutine_checker.py +++ b/litellm/litellm_core_utils/coroutine_checker.py @@ -16,7 +16,7 @@ class CoroutineChecker: """ def __init__(self): - self._cache = WeakKeyDictionary() + self._cache: WeakKeyDictionary[object, bool] = WeakKeyDictionary() self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY def is_async_callable(self, callback: Any) -> bool: @@ -33,10 +33,10 @@ class CoroutineChecker: pass # Determine target - optimized path for common cases - target = callback + target: object = callback if not inspect.isfunction(target) and not inspect.ismethod(target): try: - call_attr: Final = getattr(target, "__call__", None) # noqa: B004 # value unwrap so iscoroutinefunction sees through functors + call_attr: Final[object] = getattr(target, "__call__", None) # noqa: B004 # value unwrap so iscoroutinefunction sees through functors if call_attr is not None: target = call_attr except Exception: diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 61e2698dd6f..425714d730f 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -4,7 +4,7 @@ import re import traceback from collections.abc import Mapping from types import MappingProxyType -from typing import Any, Final, Protocol, cast +from typing import Final, Protocol, cast import httpx @@ -194,7 +194,7 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None _response_headers: httpx.Headers | None = None try: _response_headers = getattr(original_exception, "headers", None) - error_response: Final = getattr(original_exception, "response", None) + error_response: Final[object] = getattr(original_exception, "response", None) if not _response_headers and error_response: _response_headers = getattr(error_response, "headers", None) if not _response_headers: @@ -211,7 +211,7 @@ def _accepted_init_kwargs(exception_class: type[Exception], candidates: Mapping[ def extract_and_raise_litellm_exception( - response: Any | None, + response: object | None, error_str: str, model: str, custom_llm_provider: str, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e24fa004448..c60e3089816 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone, tzinfo from types import MappingProxyType -from typing import Any, Final, Literal, TypedDict, cast +from typing import Final, Literal, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from typing_extensions import ReadOnly @@ -100,7 +100,7 @@ def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None -def get_web_search_requests(server_tool_use: Any) -> int | None: +def get_web_search_requests(server_tool_use: object) -> int | None: """ Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance, @@ -1653,7 +1653,7 @@ def calculate_image_response_cost_from_usage( if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: return None - input_tokens_details: Final = getattr(usage, "input_tokens_details", None) + input_tokens_details: Final[object] = getattr(usage, "input_tokens_details", None) prompt_tokens_details: PromptTokensDetailsWrapper | None = None if input_tokens_details is not None: # input_tokens_details may be a dict (e.g. OpenAI image edit responses) @@ -1666,9 +1666,12 @@ def calculate_image_response_cost_from_usage( cached_tokens=0, ) - output_tokens_details = getattr(usage, "completion_tokens_details", None) - if output_tokens_details is None: - output_tokens_details = getattr(usage, "output_tokens_details", None) + completion_tokens_details_attr: Final[object] = getattr(usage, "completion_tokens_details", None) + output_tokens_details: Final[object] = ( + getattr(usage, "output_tokens_details", None) + if completion_tokens_details_attr is None + else completion_tokens_details_attr + ) if output_tokens_details is None: completion_tokens_details = CompletionTokensDetailsWrapper( diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 93701b3c1e7..4a3c8de78c5 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -1,7 +1,7 @@ import datetime from collections.abc import Mapping from functools import reduce -from typing import Any, Final +from typing import Final import httpx @@ -106,7 +106,7 @@ class ResponseMetadata: Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses """ - def __init__(self, result: Any): + def __init__(self, result: object): self.result = result self._hidden_params: HiddenParams | dict = getattr(result, "_hidden_params", {}) or {} diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 7fedefa4025..4ba7c3966c0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,14 +13,6 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from openai.types.chat.chat_completion_custom_tool_param import ( - CustomFormatGrammar, - CustomFormatGrammarGrammar, -) -from openai.types.shared_params.custom_tool_input_format import ( - Grammar as ResponsesGrammarFormat, -) - import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -59,7 +51,7 @@ if TYPE_CHECKING: def handle_any_messages_to_chat_completion_str_messages_conversion( - messages: Any, + messages: object, ) -> list[dict[str, str]]: """ Handles any messages to chat completion str messages conversion @@ -804,7 +796,7 @@ def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None] """ filename: str | None = None content_type: str | None = None - file_content: Any = None + file_content: object = None if isinstance(file_data, tuple): if len(file_data) == 2: @@ -1002,7 +994,7 @@ def unpack_defs( # Use iterative approach with queue to avoid recursion # Each item in queue is (node, parent_container, key/index, active_defs, ref_chain) - queue: Final[deque[tuple[Any, dict | list | None, str | int | None, dict, set]]] = deque( + queue: Final[deque[tuple[object, dict | list | None, str | int | None, dict, set]]] = deque( [(schema, None, None, root_defs, set())] ) inlined_bytes = 0 @@ -1624,7 +1616,10 @@ def is_function_call(optional_params: dict) -> bool: return False -def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: +_CUSTOM_GRAMMAR_FIELDS: Final = ("definition", "syntax") + + +def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, object]) -> Mapping[str, object]: """ Responses API grammar formats are flat ({"type": "grammar", "definition", "syntax"}); Chat Completions wraps the same fields in a "grammar" object. Text formats are @@ -1632,15 +1627,11 @@ def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, Any]) -> M """ if format_obj.get("type") != "grammar" or "grammar" in format_obj: return format_obj - grammar: Final = CustomFormatGrammarGrammar() - if "definition" in format_obj: - grammar["definition"] = format_obj["definition"] - if "syntax" in format_obj: - grammar["syntax"] = format_obj["syntax"] - return CustomFormatGrammar(type="grammar", grammar=grammar) + grammar: Final[Mapping[str, object]] = {key: format_obj[key] for key in _CUSTOM_GRAMMAR_FIELDS if key in format_obj} + return {"type": "grammar", "grammar": grammar} -def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: +def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, object]) -> Mapping[str, object]: """ Inverse of convert_custom_tool_format_to_chat_shape: unwrap the Chat Completions "grammar" object into the flat Responses API grammar shape. @@ -1648,12 +1639,10 @@ def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, Any]) grammar: Final = format_obj.get("grammar") if format_obj.get("type") != "grammar" or not isinstance(grammar, dict): return format_obj - flat: Final = ResponsesGrammarFormat(type="grammar") - if "definition" in grammar: - flat["definition"] = grammar["definition"] - if "syntax" in grammar: - flat["syntax"] = grammar["syntax"] - return flat + return { + "type": "grammar", + **{key: grammar[key] for key in _CUSTOM_GRAMMAR_FIELDS if key in grammar}, + } def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: diff --git a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py index 3878b36cd91..8f8228d6dfd 100644 --- a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py +++ b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py @@ -1,6 +1,8 @@ import json from datetime import datetime -from typing import Any, Final +from typing import Any, Final, Literal + +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -9,6 +11,20 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.types.llms.custom_http import httpxSpecialProvider +class _TokenizerConfigResult(TypedDict): + """Outcome of a tokenizer_config.json fetch, carrying the parsed document when the fetch succeeded.""" + + status: ReadOnly[Literal["success", "failure"]] + tokenizer: NotRequired[ReadOnly[object]] + + +class _ChatTemplateFileResult(TypedDict): + """Outcome of a chat template file fetch, carrying the template body when the fetch succeeded.""" + + status: ReadOnly[Literal["success", "failure"]] + chat_template: NotRequired[ReadOnly[str]] + + def strftime_now(fmt: str) -> str: """ Custom function for templates that need current date/time formatting (e.g., gpt-oss) @@ -22,7 +38,7 @@ def strftime_now(fmt: str) -> str: return datetime.now().strftime(fmt) -def _get_tokenizer_config(hf_model_name: str) -> dict[str, Any]: +def _get_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: """ Fetch tokenizer_config.json from HuggingFace (sync) @@ -45,7 +61,7 @@ def _get_tokenizer_config(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -async def _aget_tokenizer_config(hf_model_name: str) -> dict[str, Any]: +async def _aget_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: """ Fetch tokenizer_config.json from HuggingFace (async) @@ -70,7 +86,7 @@ async def _aget_tokenizer_config(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -def _get_chat_template_file(hf_model_name: str) -> dict[str, Any]: +def _get_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult: """ Fetch chat template from separate .jinja file (sync) @@ -98,7 +114,7 @@ def _get_chat_template_file(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -async def _aget_chat_template_file(hf_model_name: str) -> dict[str, Any]: +async def _aget_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult: """ Fetch chat template from separate .jinja file (async) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index b7bd0a1498b..b83ecc6929b 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -93,13 +93,13 @@ class SensitiveDataMasker: def _mask_sequence( self, - values: list[Any], + values: Sequence[object], depth: int, max_depth: int, excluded_keys: set[str] | None, key_is_sensitive: bool, - ) -> list[Any]: - masked_items: Final[list[Any]] = [] + ) -> Sequence[object]: + masked_items: Final[list[object]] = [] if depth >= max_depth: return values @@ -222,7 +222,7 @@ class _PayloadWalker: return [self.walk(item, key_is_sensitive, depth + 1) for item in node] -def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dict[str, Any]: +def mask_sensitive_keys(data: Mapping[str, object], sensitive_fields: set[str]) -> dict[str, object]: """Return a new dict with values masked for keys listed in ``sensitive_fields``. Unlike :meth:`SensitiveDataMasker.mask_dict`, this does exact key-name @@ -234,7 +234,7 @@ def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dic range and are replaced with a fixed-length all-mask string, so a short credential is never returned verbatim. """ - masked: Final[dict[str, Any]] = {} + masked: Final[dict[str, object]] = {} mask_char: Final = _default_masker.mask_char min_visible: Final = _default_masker.visible_prefix + _default_masker.visible_suffix for key, value in data.items(): diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index 5c30ff4747a..92dc49ea9c1 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -125,7 +125,7 @@ class A2AGuardrailHandler(BaseTranslation): litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, user_api_key_dict: Optional["UserAPIKeyAuth"] = None, request_data: dict | None = None, - ) -> Any: + ) -> object: """ Process A2A output response by applying guardrails to text content. diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1f90d375bc2..545e920156e 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, cast import httpx -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -151,7 +151,7 @@ class _AnthropicToolResultBlock(TypedDict, total=False): content: ReadOnly[object] -_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[object], bool]]] = MappingProxyType( +_ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType( { "null": lambda v: v is None, "boolean": lambda v: isinstance(v, bool), @@ -164,7 +164,7 @@ _ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[object], bool]]] = MappingProxyT ) -def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool: +def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool: """Whether ``schema``'s ``enum`` cannot match its declared ``type``.""" enum_values: Final = schema.get("enum") declared_type: Final = schema.get("type") @@ -659,7 +659,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return result - def get_json_schema_from_pydantic_object(self, response_format: Any | dict | None) -> dict | None: + def get_json_schema_from_pydantic_object(self, response_format: type[BaseModel] | dict | None) -> dict | None: return type_to_response_format_param( response_format, ref_template="/$defs/{model}" ) # Relevant issue: https://github.com/BerriAI/litellm/issues/7755 @@ -1072,7 +1072,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def _sanitize_tool_names_in_request( - optional_params: dict[str, Any], + optional_params: dict[str, object], ) -> tuple[dict[str, str], dict[str, str]]: """Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']`` in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``. @@ -1119,7 +1119,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # so a caller reusing the same tool list/dicts across requests # doesn't see its inputs permanently rewritten (which would also # drop the original key from `forward` on the next request). - new_tools: Final[list[Any]] = [] + new_tools: Final[list[object]] = [] for t in tools: if ( isinstance(t, dict) @@ -1442,7 +1442,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): entry_type = entry.get("type") if entry_type == "compaction": - anthropic_edit: dict[str, Any] = {"type": "compact_20260112"} + anthropic_edit: dict[str, object] = {"type": "compact_20260112"} compact_threshold = entry.get("compact_threshold") # Rewrite to 'trigger' with correct nesting if threshold exists if compact_threshold is not None and isinstance(compact_threshold, (int, float)): @@ -2442,9 +2442,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: try: - args = json.loads(tc.get("function", {}).get("arguments", "{}")) + args: object = json.loads(tc.get("function", {}).get("arguments", "{}")) + if not isinstance(args, Mapping): + continue call_id = tc.get("id") - command = args.get("command", "") + command: object = args.get("command", "") if isinstance(call_id, str): code_by_id[call_id] = command if isinstance(command, str) else "" except Exception: @@ -2514,8 +2516,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results: Sequence[_AnthropicToolResultBlock] | None, compaction_blocks: Sequence[object] | None, tool_calls: list[ChatCompletionToolCallChunk], - ) -> dict[str, Any]: - provider_specific_fields: Final[dict[str, Any]] = { + ) -> dict[str, object]: + provider_specific_fields: Final[dict[str, object]] = { "citations": citations, "thinking_blocks": thinking_blocks, } diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 98e2f6d5bde..c6015e7884e 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping, MutableMapping, Sequence from datetime import datetime, timezone from types import MappingProxyType -from typing import Any, Final, Literal +from typing import Any, Final, Literal, TypeVar import httpx from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError @@ -40,6 +40,8 @@ from litellm.types.llms.anthropic import ( from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.model_listing import ModelInfoResponse +_MessageT = TypeVar("_MessageT") + DROP_FORCED_TOOL_CHOICE_WARNING: Final = ( "Downgrading forced tool_choice to 'auto' for model=%s (drop_params=True): this model rejects tool_choice type " "'any'/'tool' with a 400 because thinking is always on and a forced call would skip it." @@ -1121,7 +1123,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): return AnthropicTokenCounter() -def strip_advisor_blocks_from_messages(messages: list[Any], replace_with_text: bool = False) -> list[Any]: +def strip_advisor_blocks_from_messages(messages: list[_MessageT], replace_with_text: bool = False) -> list[_MessageT]: """ Remove (or replace) server_tool_use (name='advisor') and advisor_tool_result blocks from assistant message content. @@ -1228,7 +1230,7 @@ def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool: return "must contain thinking" in lower -def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[Any]: +def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]: """ Return a new message list with thinking / redacted_thinking content blocks removed from each message. Used to recover from invalid thinking signatures on retry. @@ -1236,7 +1238,7 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A Messages whose content is a list and becomes empty after stripping are omitted, since Anthropic rejects empty content arrays. """ - out: Final[list[Any]] = [] + out: Final[list[object]] = [] for m in messages: if not isinstance(m, dict): out.append(m) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py index 171f5156594..306041d9949 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py @@ -25,6 +25,9 @@ from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0 SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = ( @@ -182,7 +185,7 @@ class AgenticAnthropicStreamingIterator: http_handler: Any, model: str, messages: list[dict], - anthropic_messages_provider_config: Any, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig", anthropic_messages_optional_request_params: dict, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, @@ -402,7 +405,7 @@ class AgenticAnthropicStreamingIterator: @staticmethod def _rebuild_anthropic_response_from_sse( raw_bytes: list[bytes], - ) -> dict[str, Any] | None: + ) -> dict[str, object] | None: """ Parse collected SSE bytes into an Anthropic Messages response dict. @@ -416,17 +419,18 @@ class AgenticAnthropicStreamingIterator: """ events: Final = _parse_sse_events(b"".join(raw_bytes)) - response: Final[dict[str, Any]] = { + content: Final[list[dict[str, object]]] = [] + response: Final[dict[str, object]] = { "id": "", "type": "message", "role": "assistant", "model": "", - "content": [], + "content": content, "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0}, } - content_blocks: Final[dict[int, dict[str, Any]]] = {} + content_blocks: Final[dict[int, dict[str, object]]] = {} saw_message_start = False for event_type, data in events: @@ -448,6 +452,6 @@ class AgenticAnthropicStreamingIterator: for idx in sorted(content_blocks.keys()): block = content_blocks[idx] block.pop("_partial_json", None) - response["content"].append(block) + content.append(block) return response diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index dfd62ca575b..e4c75a704ec 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -185,7 +185,11 @@ class AnthropicFilesHandler: if not line.strip(): continue - anthropic_result = json.loads(line) + anthropic_result: object = json.loads(line) + if not isinstance(anthropic_result, dict): + raise TypeError( + f"Anthropic batch result line is not a JSON object: {type(anthropic_result).__name__}" + ) custom_id = anthropic_result.get("custom_id", "") result = anthropic_result.get("result", {}) result_type = result.get("type", "") diff --git a/litellm/llms/azure/assistants.py b/litellm/llms/azure/assistants.py index f7b419405ac..a4742a25a87 100644 --- a/litellm/llms/azure/assistants.py +++ b/litellm/llms/azure/assistants.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine, Iterable -from typing import Any, Final, Literal, TypedDict +from typing import Final, Literal, TypedDict import httpx from openai import AsyncAzureOpenAI, AzureOpenAI @@ -715,7 +715,8 @@ class AzureAssistantsAPI(BaseAzureLLM): event_handler: AssistantEventHandler | None, litellm_params: dict | None = None, ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data: Final[dict[str, Any]] = { + stream_fn: Final = client.beta.threads.runs.stream + base_data: Final[_RunThreadStreamData] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -725,8 +726,8 @@ class AzureAssistantsAPI(BaseAzureLLM): "tools": tools, } if event_handler is not None: - data["event_handler"] = event_handler - return client.beta.threads.runs.stream(**data) + return stream_fn(**base_data, event_handler=event_handler) + return stream_fn(**base_data) def run_thread_stream( self, diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py index d8ccf26ce60..eed7a3178ca 100644 --- a/litellm/llms/azure/text_to_speech/transformation.py +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -19,6 +19,7 @@ from litellm.secret_managers.main import get_secret_str if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.llms.openai import HttpxBinaryResponseContent else: LiteLLMLoggingObj = Any @@ -67,15 +68,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): litellm_params_dict: dict, logging_obj: "LiteLLMLoggingObj", timeout: float | httpx.Timeout, - extra_headers: dict[str, Any] | None, - base_llm_http_handler: Any, + extra_headers: dict[str, object] | None, + base_llm_http_handler: "BaseLLMHTTPHandler", aspeech: bool, api_base: str | None, api_key: str | None, - **kwargs: Any, + **kwargs: object, ) -> Union[ "HttpxBinaryResponseContent", - Coroutine[Any, Any, "HttpxBinaryResponseContent"], + Coroutine[object, object, "HttpxBinaryResponseContent"], ]: """ Dispatch method to handle Azure AVA TTS requests diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 3cc90823af9..36d5a56db0d 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -33,7 +33,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): litellm_params: dict[str, Any] | None = None, timeout: float | httpx.Timeout | None = None, tools: list[dict[str, Any]] | None = None, - system: Any | None = None, + system: object = None, ) -> dict[str, Any]: """ Handle a CountTokens request using httpx with Azure authentication. diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index a765d493347..4f94cec0973 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -180,7 +180,7 @@ class BaseVideoConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video remix request into a URL and data @@ -207,7 +207,7 @@ class BaseVideoConfig(ABC): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video list request into a URL and params @@ -355,8 +355,8 @@ class BaseVideoConfig(ABC): litellm_params: GenericLiteLLMParams, headers: dict, video_file: FileContent | None = None, - extra_body: dict[str, Any] | None = None, - prefetched_source_data: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, + prefetched_source_data: dict[str, object] | None = None, ) -> tuple[str, Mapping[str, object], RequestFiles | None]: """ Transform the video edit request into a URL plus either JSON data or @@ -386,7 +386,7 @@ class BaseVideoConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video extension request into a URL and JSON data. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3e412b5ad24..4f9b1f56a5b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1126,7 +1126,7 @@ class AmazonConverseConfig(BaseConfig): return optional_params - def _map_request_metadata_param(self, value: Any, optional_params: dict) -> None: + def _map_request_metadata_param(self, value: object, optional_params: dict) -> None: if value is not None and isinstance(value, dict): self._validate_request_metadata(value) optional_params["requestMetadata"] = value diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 9c5211ed072..2d02b152c61 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -2,6 +2,7 @@ Bedrock Token Counter implementation using the CountTokens API. """ +from collections.abc import Mapping, Sequence from typing import Any, Final from litellm._logging import verbose_logger @@ -26,12 +27,12 @@ class BedrockTokenCounter(BaseTokenCounter): async def count_tokens( self, model_to_use: str, - messages: list[dict[str, Any]] | None, - contents: list[dict[str, Any]] | None, + messages: Sequence[Mapping[str, object]] | None, + contents: Sequence[Mapping[str, object]] | None, deployment: dict[str, Any] | None = None, request_model: str = "", - tools: list[dict[str, Any]] | None = None, - system: Any | None = None, + tools: Sequence[Mapping[str, object]] | None = None, + system: object | None = None, ) -> TokenCountResponse | None: """ Count tokens using AWS Bedrock's CountTokens API. @@ -56,7 +57,7 @@ class BedrockTokenCounter(BaseTokenCounter): litellm_params: Final = deployment.get("litellm_params", {}) # Build request data in the format expected by BedrockCountTokensHandler - request_data: Final[dict[str, Any]] = { + request_data: Final[dict[str, object]] = { "model": model_to_use, "messages": messages, } diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index ac80ecb26b8..a7486dd4de0 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -375,7 +375,7 @@ def _listed_managed_file( ) -def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Response) -> int: +def _uploaded_object_size(litellm_params: Mapping[str, object], response_headers: Mapping[str, str]) -> int: """ S3 answers PutObject with an empty body, so the stored object size comes from the signed request recorded by `transform_create_file_request`, not the response headers. @@ -383,7 +383,7 @@ def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Re uploaded_size: Final = litellm_params.get(UPLOAD_CONTENT_LENGTH_PARAM) if isinstance(uploaded_size, int): return uploaded_size - response_content_length: Final = raw_response.headers.get("Content-Length", "0") + response_content_length: Final = response_headers.get("Content-Length", "0") return int(response_content_length) if response_content_length.isdigit() else 0 @@ -1277,7 +1277,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): filename=filename, created_at=int(time.time()), # Current timestamp status="uploaded", - bytes=_uploaded_object_size(litellm_params=litellm_params, raw_response=raw_response), + bytes=_uploaded_object_size(litellm_params=litellm_params, response_headers=raw_response.headers), object="file", ) diff --git a/litellm/llms/bedrock/image_edit/stability_transformation.py b/litellm/llms/bedrock/image_edit/stability_transformation.py index bc9a64f587a..01e25f4671e 100644 --- a/litellm/llms/bedrock/image_edit/stability_transformation.py +++ b/litellm/llms/bedrock/image_edit/stability_transformation.py @@ -125,7 +125,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig): } # Create a copy to not mutate original - convert TypedDict to regular dict - mapped_params: Final[dict[str, Any]] = dict(image_edit_optional_params) + mapped_params: Final[dict[str, object]] = dict(image_edit_optional_params) for k, v in image_edit_optional_params.items(): if k in param_mapping: @@ -172,7 +172,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig): Returns the request body dict that will be JSON-encoded by the handler. """ # Build Bedrock Stability request - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "output_format": "png", # Default to PNG } diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index b87f6196e51..eac8afd767c 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -14,6 +14,9 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.pass_through.guardrail_translation.handler import ( + PassThroughEndpointHandler, + ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -27,7 +30,7 @@ def _is_converse_endpoint(endpoint: str) -> bool: return bool(parts) and parts[-1] in _CONVERSE_ACTIONS -def _generic_passthrough_handler() -> BaseTranslation: +def _generic_passthrough_handler() -> "PassThroughEndpointHandler": """ Fallback for non-Converse Bedrock routes (e.g. invoke). The generic handler scans the full request/response payload so blocking guardrails diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 86e20e31d7f..b04029e4c74 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -16,8 +16,8 @@ BaseAWSLLM._sign_request after the request body is finalized. """ import json -from collections.abc import Mapping -from typing import Any, Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict +from collections.abc import Mapping, Sequence +from typing import Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict import httpx from typing_extensions import ReadOnly, TypedDict @@ -142,9 +142,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI return False @staticmethod - def _filter_unsupported_tools(tools: list[Any]) -> list[Any]: + def _filter_unsupported_tools(tools: "Sequence[object]") -> "list[object]": """Keep only tool types Mantle's Responses API accepts.""" - kept: Final[list[Any]] = [] + kept: Final[list[object]] = [] dropped_types: Final[list[str]] = [] for tool in tools: if not isinstance(tool, dict): diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index 35e32e4172f..fe33219f110 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -268,7 +268,7 @@ def _normalize_litellm_params(litellm_params: Any | None) -> dict: return {} -def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: +def get_chatgpt_session_id(litellm_params: object) -> str | None: params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_session_id", "session_id"): value = params.get(key) @@ -286,5 +286,5 @@ def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: return None -def ensure_chatgpt_session_id(litellm_params: Any | None) -> str: +def ensure_chatgpt_session_id(litellm_params: object) -> str: return get_chatgpt_session_id(litellm_params) or str(uuid4()) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index b96e06be3d8..9774b762396 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -1,5 +1,8 @@ +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final +import httpx + from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -13,6 +16,7 @@ from litellm.responses.sse_output_recovery import ( record_output_text_chunk, ) from litellm.types.llms.openai import ( + ResponseInputParam, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) @@ -64,7 +68,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_responses_api_request( self, model: str, - input: Any, + input: str | ResponseInputParam, response_api_optional_request_params: dict, litellm_params: GenericLiteLLMParams, headers: dict, @@ -109,9 +113,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_response_api_response( self, model: str, - raw_response: Any, + raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", - ): + ) -> ResponsesAPIResponse: body_text: Final = raw_response.text or "" if not self._should_parse_as_sse(raw_response=raw_response, body_text=body_text): return super().transform_response_api_response( @@ -135,7 +139,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): self._attach_response_headers(completed_response=completed_response, raw_response=raw_response) return completed_response - def _should_parse_as_sse(self, raw_response: Any, body_text: str) -> bool: + def _should_parse_as_sse(self, raw_response: httpx.Response, body_text: str) -> bool: content_type: Final = (raw_response.headers or {}).get("content-type", "") if "text/event-stream" in content_type.lower(): return True @@ -150,8 +154,8 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def _extract_completed_response_from_sse(self, body_text: str) -> tuple[ResponsesAPIResponse | None, str | None]: completed_response = None error_message = None - streamed_output_items: Final[dict[int, dict]] = {} - text_only_output_items: Final[dict[int, dict]] = {} + streamed_output_items: Final[dict[int, dict[str, object]]] = {} + text_only_output_items: Final[dict[int, dict[str, object]]] = {} for chunk in body_text.splitlines(): parsed_chunk = parse_sse_json_chunk(chunk) if parsed_chunk is None: @@ -178,7 +182,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): # output_index, but text-only items at indices without a # matching OUTPUT_ITEM_DONE must still be preserved (e.g. # providers that emit only OUTPUT_TEXT_DONE for some indices). - merged_items: dict[int, dict] = {**text_only_output_items} + merged_items: dict[int, dict[str, object]] = {**text_only_output_items} merged_items.update(streamed_output_items) completed_response = self._build_completed_response_from_chunk( parsed_chunk=parsed_chunk, @@ -197,7 +201,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): return completed_response, error_message def _build_completed_response_from_chunk( - self, parsed_chunk: dict[str, Any], streamed_output_items: dict[int, dict] + self, parsed_chunk: Mapping[str, object], streamed_output_items: Mapping[int, dict[str, object]] ) -> ResponsesAPIResponse | None: response_payload = parsed_chunk.get("response") if not isinstance(response_payload, dict): @@ -223,7 +227,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def _attach_response_headers( self, completed_response: ResponsesAPIResponse, - raw_response: Any, + raw_response: httpx.Response, ) -> None: raw_headers: Final = dict(raw_response.headers) processed_headers: Final = process_response_headers(raw_headers) diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index 319603b0dad..fa46bd7f6cf 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -110,7 +110,7 @@ class CohereChatConfig(BaseConfig): tool_results: list | None = None, seed: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/cohere/embed/v1_transformation.py b/litellm/llms/cohere/embed/v1_transformation.py index ee40464362d..b35fae5a1ac 100644 --- a/litellm/llms/cohere/embed/v1_transformation.py +++ b/litellm/llms/cohere/embed/v1_transformation.py @@ -2,7 +2,8 @@ Legacy /v1/embedding transformation logic for Bedrock Cohere. """ -from typing import Any, Final +from collections.abc import Sized +from typing import Final, Protocol import httpx @@ -16,6 +17,12 @@ from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, U from litellm.utils import is_base64_encoded +class _SupportsEncode(Protocol): + """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" + + def encode(self, text: str, /) -> Sized: ... + + class CohereEmbeddingConfig: """ Reference: https://docs.cohere.com/v2/reference/embed @@ -61,7 +68,7 @@ class CohereEmbeddingConfig: return transformed_request - def _calculate_usage(self, input: list[str], encoding: Any, meta: dict) -> Usage: + def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: input_tokens = 0 text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") @@ -97,7 +104,7 @@ class CohereEmbeddingConfig: data: dict | CohereEmbeddingRequest, model_response: EmbeddingResponse, model: str, - encoding: Any, + encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: response_json: Final = response.json() @@ -121,7 +128,7 @@ class CohereEmbeddingConfig: response_json: dict, model_response: EmbeddingResponse, model: str, - encoding: Any, + encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: """ diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 6b90394043f..b7b2477e85c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -479,6 +479,11 @@ def _safe_get_response_text(response: httpx.Response) -> str: return "" +def header_value(headers: Mapping[str, str], name: str) -> str | None: + """Read one header as ``str | None``; ``httpx.Headers.get`` itself is typed ``Any``.""" + return headers.get(name) + + async def _safe_aread_response(response: httpx.Response, timeout: float | None = None) -> bytes: """Safely read async response body, falling back to empty bytes on errors.""" try: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 82c3b5d91d3..dd257cd68b0 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/chat/completions` to Databricks' `/chat/completion """ import os -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx @@ -67,7 +67,7 @@ def _is_bare_assistant_message(message_dict: Mapping[str, object]) -> bool: ) -def _sanitize_empty_content(message_dict: dict[str, Any]) -> None: +def _sanitize_empty_content(message_dict: dict[str, object]) -> None: """ Remove or filter content so empty text blocks are not sent. Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks. @@ -430,7 +430,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, list[AllMessageValues]]: ... + ) -> Coroutine[object, object, list[AllMessageValues]]: ... @overload def _transform_messages( @@ -442,7 +442,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False - ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: + ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """ Databricks does not support: - 'name' in user message. @@ -564,7 +564,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): @staticmethod def extract_citations( content: AllDatabricksContentValues | None, - ) -> list[Any] | None: + ) -> Sequence[Sequence[Mapping[str, object]]] | None: if content is None: return None citations: Final = [] diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index b6c2b379d66..28ebb39a303 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,6 +1,6 @@ import json from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, cast import httpx @@ -759,7 +759,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "FireworksAIChatCompletionStreamingHandler": return FireworksAIChatCompletionStreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index 03037512551..6eac3ac79cd 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -7,14 +7,32 @@ import os import re import threading from collections.abc import Callable -from typing import Any, Final, Protocol +from typing import Final, Protocol from urllib.parse import urlsplit +from typing_extensions import ReadOnly, TypedDict, Unpack + import litellm from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues +class _OpenAIGPTConfigOptions(TypedDict, total=False): + """The sampling defaults ``OpenAIGPTConfig.__init__`` accepts and stashes on the class.""" + + frequency_penalty: ReadOnly[int | None] + function_call: ReadOnly[str | dict[str, object] | None] + functions: ReadOnly[list[object] | None] + logit_bias: ReadOnly[dict[str, object] | None] + max_tokens: ReadOnly[int | None] + n: ReadOnly[int | None] + presence_penalty: ReadOnly[int | None] + stop: ReadOnly[str | list[object] | None] + temperature: ReadOnly[int | None] + top_p: ReadOnly[int | None] + response_format: ReadOnly[dict[str, object] | None] + + class _GDCHAudienceCredentials(Protocol): """A GDCH service account credential already bound to an audience, ready to mint a bearer token.""" @@ -32,7 +50,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig): _GDCH_CREDENTIAL_TYPE: Final[str] = "gdch_service_account" _PATH_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-zA-Z0-9_-]+$") - def __init__(self, **kwargs: Any) -> None: + def __init__(self, **kwargs: Unpack[_OpenAIGPTConfigOptions]) -> None: super().__init__(**kwargs) self._creds_lock = threading.Lock() self._gdch_creds_cache: dict[tuple[str, str], _GDCHAudienceCredentials] = {} diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index cb2be2c860e..c2f0ef473ae 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -84,7 +84,7 @@ class GoogleAIStudioTokenCounter: api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, - **kwargs, + **kwargs: object, ) -> dict[str, Any]: """ Count tokens using Google Gen AI Studio countTokens endpoint. diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index e6c22dc60b4..31d3963c70c 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -1,4 +1,5 @@ import base64 +from collections.abc import Mapping from io import BufferedReader, BytesIO from typing import TYPE_CHECKING, Any, Final, cast @@ -44,7 +45,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): image_edit_optional_params: ImageEditOptionalRequestParams, model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: return map_openai_image_params_to_gemini( params=image_edit_optional_params, model=model, @@ -87,10 +88,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): model: str, prompt: str | None, image: FileTypes | None, - image_edit_optional_request_params: dict[str, Any], + image_edit_optional_request_params: Mapping[str, object], litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[dict[str, Any], RequestFiles | None]: + ) -> tuple[dict[str, object], RequestFiles | None]: inline_parts: Final = self._prepare_inline_image_parts(image) if image else [] if not inline_parts: raise ValueError("Gemini image edit requires at least one image.") @@ -106,7 +107,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): } ] - request_body: Final[dict[str, Any]] = {"contents": contents} + request_body: Final[dict[str, object]] = {"contents": contents} request_body["generationConfig"] = get_gemini_image_generation_config( model=model, @@ -153,14 +154,14 @@ class GeminiImageEditConfig(BaseImageEditConfig): model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"]) return model_response - def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, Any]]: + def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]: images: list[FileTypes] if isinstance(image, list): images = image else: images = [image] - inline_parts: Final[list[dict[str, Any]]] = [] + inline_parts: Final[list[dict[str, object]]] = [] for img in images: if img is None: continue diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 89920ebd27b..d9250ea8836 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -81,9 +81,17 @@ class GigaChatConfig(BaseConfig): repetition_penalty: float | None = None, profanity_check: bool | None = None, ) -> None: - locals_: Final = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: + config_params: Final[Mapping[str, float | int | bool | None]] = MappingProxyType( + { + "temperature": temperature, + "top_p": top_p, + "max_tokens": max_tokens, + "repetition_penalty": repetition_penalty, + "profanity_check": profanity_check, + } + ) + for key, value in config_params.items(): + if value is not None: setattr(self.__class__, key, value) # Instance variables for current request context self._current_credentials: str | None = None diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 5a4bb798851..8b85b668cba 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -19,6 +19,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, + ResponsesAPIStreamingResponse, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -129,7 +130,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): model: str, parsed_chunk: dict, logging_obj: LiteLLMLoggingObj, - ) -> Any: + ) -> ResponsesAPIStreamingResponse: parsed_chunk = self._normalize_stream_item_id(parsed_chunk) return super().transform_streaming_response( model=model, @@ -262,7 +263,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Return the responses endpoint return f"{effective_api_base}/responses" - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, object]) -> dict[str, object]: """ Handle reasoning items for GitHub Copilot, preserving encrypted_content. @@ -280,7 +281,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Filter out None values for known problematic fields, # but preserve encrypted_content even if it exists - filtered_item: Final[dict[str, Any]] = {} + filtered_item: Final[dict[str, object]] = {} for k, v in item.items(): # Always include encrypted_content if present (even if None) if k == "encrypted_content": diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 29dc485732f..32c60bd01b5 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -4,7 +4,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` import json from collections.abc import Coroutine -from typing import Any, Final, Literal, cast, overload +from typing import Final, Literal, cast, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, @@ -28,12 +28,12 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class HostedVLLMChatConfig(OpenAIGPTConfig): - def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]: """ vLLM chat completions currently accepts only OpenAI function tools. Convert custom tools into function tools so request validation does not fail. """ - converted_tools: Final[list[dict[str, Any]]] = [] + converted_tools: Final[list[dict[str, object]]] = [] for idx, tool in enumerate(tools): if not isinstance(tool, dict): converted_tools.append(tool) @@ -63,17 +63,14 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): "required": ["input"], } - function_tool: dict[str, Any] = { - "type": "function", - "function": { - "name": str(tool_name), - "parameters": tool_parameters, - }, + function_definition: dict[str, object] = { + "name": str(tool_name), + "parameters": tool_parameters, } if isinstance(tool_description, str): - function_tool["function"]["description"] = tool_description + function_definition["description"] = tool_description - converted_tools.append(function_tool) + converted_tools.append({"type": "function", "function": function_definition}) return converted_tools @@ -148,7 +145,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, list[AllMessageValues]]: ... + ) -> Coroutine[object, object, list[AllMessageValues]]: ... @overload def _transform_messages( @@ -160,7 +157,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False - ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: + ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """ Support translating: - video files from file_id or file_data to video_url diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index f6fe7f2fa10..33b0e21e326 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -84,13 +84,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig): typical_p: float | None = None, watermark: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod - def get_config(cls): + def get_config(cls) -> dict[str, object]: return super().get_config() def get_special_options_params(self): @@ -352,17 +352,17 @@ class HuggingFaceEmbeddingConfig(BaseConfig): model: str, data: dict, api_key: str | None = None, - ) -> list[dict[str, Any]]: + ) -> list[dict[str, str]]: streamed_response: Final = CustomStreamWrapper( completion_stream=response.iter_lines(), model=model, custom_llm_provider="huggingface", logging_obj=logging_obj, ) - content = "" + content: str = "" for chunk in streamed_response: content += chunk["choices"][0]["delta"]["content"] - completion_response: Final[list[dict[str, Any]]] = [{"generated_text": content}] + completion_response: Final[list[dict[str, str]]] = [{"generated_text": content}] ## LOGGING logging_obj.post_call( input=data, diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index d188fac8704..bea77a6761c 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -27,7 +27,6 @@ without the optional STT extras installed. import asyncio import inspect from collections.abc import Callable, Iterable -from types import ModuleType from typing import TYPE_CHECKING, Any, Final, Protocol from litellm.litellm_core_utils.audio_utils.utils import ( @@ -95,11 +94,37 @@ class _AudioEncoding(Protocol): def LINEAR_PCM(self) -> object: ... -def _auth_factory(riva_module: ModuleType) -> Callable[..., _RivaAuth]: +class _RivaClientModule(Protocol): + """The ``riva.client`` entry points this handler calls.""" + + @property + def Auth(self) -> Callable[..., _RivaAuth]: ... + + @property + def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ... + + +class _RivaAsrModule(Protocol): + """The protobuf constructors this handler calls, from whichever module exposes them.""" + + @property + def AudioEncoding(self) -> _AudioEncoding: ... + + @property + def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ... + + @property + def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ... + + @property + def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ... + + +def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]: return riva_module.Auth -def _audio_encoding(riva_asr_module: ModuleType) -> _AudioEncoding: +def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding: return riva_asr_module.AudioEncoding @@ -317,7 +342,7 @@ class NvidiaRivaAudioTranscription: def _construct_auth( self, - riva_module: ModuleType, + riva_module: _RivaClientModule, api_base: str, api_key: str | None, optional_params: dict, @@ -349,7 +374,7 @@ class NvidiaRivaAudioTranscription: return _auth_factory(riva_module)(None, use_ssl, api_base, metadata) def _build_recognition_config_proto( - self, riva_asr_module: ModuleType, recognition_config_dict: dict[str, Any] + self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any] ) -> _RecognitionConfig: encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper() encoding_enum: Final[object] = getattr( @@ -436,7 +461,7 @@ class NvidiaRivaAudioTranscription: return final_results -def _import_riva() -> tuple[ModuleType, ModuleType]: +def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]: """ Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``. diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 181894646e3..257ebf921d9 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -124,7 +124,7 @@ class OllamaChatConfig(BaseConfig): setattr(self.__class__, key, value) @classmethod - def get_config(cls): + def get_config(cls) -> dict[str, object]: return super().get_config() def get_supported_openai_params(self, model: str): diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index a1340ba1952..3eb2c833094 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -231,7 +231,7 @@ class OllamaConfig(BaseConfig): model: str, api_base: str | None = None, api_key: str | None = None, - ) -> Any: + ) -> dict[str, object] | None: """ curl http://localhost:11434/api/show -d '{ "name": "mistral" diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 2b895049743..fa5512e7bfe 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -17,7 +17,7 @@ This pattern can be replicated for other message formats (e.g., Anthropic). import json import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast @@ -269,7 +269,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _extract_inputs( self, - message: dict[str, Any], + message: Mapping[str, object], msg_idx: int, texts_to_check: list[str], images_to_check: list[str], @@ -330,7 +330,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_input_texts( self, - messages: list[dict[str, Any]], + messages: list[dict[str, object]], responses: list[str], task_mappings: list[tuple[int, int | None]], ) -> None: @@ -355,12 +355,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): elif isinstance(content, list) and content_idx_optional is not None: # Replace specific text item in list content - messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response + content[content_idx_optional]["text"] = guardrail_response async def _apply_guardrail_responses_to_input_tool_calls( self, - messages: list[dict[str, Any]], - tool_calls: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], + tool_calls: Sequence[Mapping[str, object]], task_mappings: list[tuple[int, int]], ) -> None: """ @@ -412,7 +412,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] - tool_calls_to_check: Final[list[dict[str, Any]]] = [] + tool_calls_to_check: Final[list[dict[str, object]]] = [] text_task_mappings: Final[list[tuple[int, int | None]]] = [] tool_call_task_mappings: Final[list[tuple[int, int]]] = [] # text_task_mappings: Track (choice_index, content_index) for each text @@ -461,8 +461,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") - guardrailed_tool_calls: Final[list[dict[str, Any]]] = ( - cast(list[dict[str, Any]], returned_tool_calls) + guardrailed_tool_calls: Final[list[dict[str, object]]] = ( + cast(list[dict[str, object]], returned_tool_calls) if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) else tool_calls_to_check ) @@ -939,7 +939,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): choice_idx: int, texts_to_check: list[str], images_to_check: list[str], - tool_calls_to_check: list[dict[str, Any]], + tool_calls_to_check: list[dict[str, object]], text_task_mappings: list[tuple[int, int | None]], tool_call_task_mappings: list[tuple[int, int]], ) -> None: diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index cb6a5e4e96a..b47edee9976 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -10,7 +10,7 @@ import ssl import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional +from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Optional from urllib.parse import urlsplit import httpx @@ -88,8 +88,8 @@ class OpenAIError(BaseLLMException): ################################################################### def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, - data: dict[str, Any], -) -> dict[str, Any]: + data: Mapping[str, object], +) -> dict[str, object]: """ Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message. diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5f04ebe0c01..869ad387c5a 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,7 +1,7 @@ import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -2756,7 +2756,12 @@ class OpenAIAssistantsAPI(BaseLLM): message_thread: Final = await openai_client.beta.threads.create(**data) - return Thread(**message_thread.dict()) + return Thread( + id=message_thread.id, + created_at=message_thread.created_at, + metadata=message_thread.metadata, + object=message_thread.object, + ) # fmt: off @@ -2842,7 +2847,12 @@ class OpenAIAssistantsAPI(BaseLLM): message_thread: Final = openai_client.beta.threads.create(**data) - return Thread(**message_thread.dict()) + return Thread( + id=message_thread.id, + created_at=message_thread.created_at, + metadata=message_thread.metadata, + object=message_thread.object, + ) async def async_get_thread( self, @@ -2865,7 +2875,12 @@ class OpenAIAssistantsAPI(BaseLLM): response: Final = await openai_client.beta.threads.retrieve(thread_id=thread_id) - return Thread(**response.dict()) + return Thread( + id=response.id, + created_at=response.created_at, + metadata=response.metadata, + object=response.object, + ) # fmt: off @@ -2931,7 +2946,12 @@ class OpenAIAssistantsAPI(BaseLLM): response: Final = openai_client.beta.threads.retrieve(thread_id=thread_id) - return Thread(**response.dict()) + return Thread( + id=response.id, + created_at=response.created_at, + metadata=response.metadata, + object=response.object, + ) def delete_thread(self): pass @@ -2988,18 +3008,27 @@ class OpenAIAssistantsAPI(BaseLLM): tools: Iterable[AssistantToolParam] | None, event_handler: AssistantEventHandler | None, ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data: Final[dict[str, Any]] = { - "thread_id": thread_id, - "assistant_id": assistant_id, - "additional_instructions": additional_instructions, - "instructions": instructions, - "metadata": metadata, - "model": model, - "tools": tools, - } + runs_stream: Final = client.beta.threads.runs.stream if event_handler is not None: - data["event_handler"] = event_handler - return client.beta.threads.runs.stream(**data) + return runs_stream( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + return runs_stream( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) def run_thread_stream( self, diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 94dc30f41e5..9a4b030993f 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -237,7 +237,7 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video remix request for OpenAI API. @@ -252,7 +252,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url: Final = f"{api_base.rstrip('/')}/{encoded_video_id}/remix" # Prepare the request data - data: Final = {"prompt": prompt} + data: Final[dict[str, object]] = {"prompt": prompt} # Add any extra body parameters if extra_body: @@ -305,7 +305,7 @@ class OpenAIVideoConfig(BaseVideoConfig): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video list request for OpenAI API. diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index b01c25aad0c..3d46277a69e 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -90,20 +90,21 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): drop_params: bool, ) -> dict: supported_params: Final = self.get_supported_openai_params(model) - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} + image_config: Final[dict[str, str]] = {} for key, value in image_edit_optional_params.items(): if key in supported_params: if key == "size": if "image_config" not in mapped_params: - mapped_params["image_config"] = {} - mapped_params["image_config"]["aspect_ratio"] = self._map_size_to_aspect_ratio(cast(str, value)) + mapped_params["image_config"] = image_config + image_config["aspect_ratio"] = self._map_size_to_aspect_ratio(cast(str, value)) elif key == "quality": image_size = self._map_quality_to_image_size(cast(str, value)) if image_size: if "image_config" not in mapped_params: - mapped_params["image_config"] = {} - mapped_params["image_config"]["image_size"] = image_size + mapped_params["image_config"] = image_config + image_config["image_size"] = image_size else: mapped_params[key] = value diff --git a/litellm/llms/perplexity/embedding/transformation.py b/litellm/llms/perplexity/embedding/transformation.py index a911fa62719..c93206db2bb 100644 --- a/litellm/llms/perplexity/embedding/transformation.py +++ b/litellm/llms/perplexity/embedding/transformation.py @@ -130,7 +130,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): if isinstance(embedding_value, str): raw_bytes: Final = base64.b64decode(embedding_value) count: Final = len(raw_bytes) - int8_values: Final = struct.unpack(f"{count}b", raw_bytes) + int8_values: Final[tuple[int, ...]] = struct.unpack(f"{count}b", raw_bytes) return [float(v) / 127.0 for v in int8_values] return embedding_value diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index f65b0876202..aeff902f655 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -315,7 +315,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): for msg in messages: if isinstance(msg, dict): role = msg.get("role", "") - content: Any = msg.get("content", "") + content: object = msg.get("content", "") msg_cache_control: object = msg.get("cache_control") else: role = getattr(msg, "role", "") @@ -463,7 +463,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): return body - def _transform_tool_choice_to_anthropic(self, tool_choice: Any) -> dict[str, Any]: + def _transform_tool_choice_to_anthropic(self, tool_choice: object) -> Mapping[str, object]: """ Convert tool_choice from OpenAI format to Anthropic format. diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 0b6052ad593..94711d21b50 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -74,7 +74,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): } # Create a copy to not mutate original - convert TypedDict to regular dict - mapped_params: Final[dict[str, Any]] = dict(image_edit_optional_params) + mapped_params: Final[dict[str, object]] = dict(image_edit_optional_params) for k, v in image_edit_optional_params.items(): if k in param_mapping: @@ -182,7 +182,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): # Build Stability request # Populate multipart form-data as separate text fields (data) and files. # Stability expects prompt/output_format/etc. as normal form fields, not file parts. - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "output_format": "png", # Default to PNG } diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 98a68ba2c36..3c868b3a96f 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -4,7 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` endpoint to Triton's `/generate` import json from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal from httpx import Headers, Response @@ -172,7 +172,7 @@ class TritonConfig(BaseConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "TritonResponseIterator": return TritonResponseIterator( streaming_response=streaming_response, sync_stream=sync_stream, @@ -195,14 +195,14 @@ class TritonGenerateConfig(TritonConfig): ) -> dict: inference_params: Final = optional_params.copy() stream: Final = inference_params.pop("stream", False) - data_for_triton: Final[dict[str, Any]] = { + data_for_triton: Final[dict[str, object]] = { "text_input": prompt_factory(model=model, messages=messages), "parameters": { "max_tokens": int(optional_params.get("max_tokens", DEFAULT_MAX_TOKENS_FOR_TRITON)), + **inference_params, }, "stream": bool(stream), } - data_for_triton["parameters"].update(inference_params) return data_for_triton def transform_response( diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 7ecc5e8ff3d..c79b6ffce43 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -280,7 +280,7 @@ class VertexFineTuningAPI(VertexLLM): vertex_location: str, vertex_credentials: str, request_route: str, - ): + ) -> object: _auth_header, vertex_project = await self._ensure_access_token_async( credentials=vertex_credentials, project_id=vertex_project, @@ -341,5 +341,4 @@ class VertexFineTuningAPI(VertexLLM): f"Error creating fine tuning job. Status code: {response.status_code}. Response: {response.text}" ) - response_json: Final = response.json() - return response_json + return response.json() diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 13e2238fdf6..e3cc3bbb2dc 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -179,7 +179,7 @@ def _apply_gemini_metadata( part: PartType, model: str | None, media_resolution_enum: dict[str, str] | None, - video_metadata: dict[str, Any] | None, + video_metadata: Mapping[str, object] | None, ) -> PartType: """ Apply media_resolution and video_metadata parameters to a Gemini part. @@ -480,7 +480,7 @@ def _process_gemini_media( format: str | None = None, media_resolution_enum: dict[str, str] | None = None, model: str | None = None, - video_metadata: dict[str, Any] | None = None, + video_metadata: Mapping[str, object] | None = None, vertex_project: str | None = None, vertex_credentials: object = None, ) -> PartType: diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index c6ad5928b74..fddc075bfc6 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -1,6 +1,7 @@ import base64 import json import os +from collections.abc import Mapping from io import BufferedRandom, BufferedReader, BytesIO from pathlib import Path from typing import TYPE_CHECKING, Any, Final, cast @@ -47,11 +48,11 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): image_edit_optional_params: ImageEditOptionalRequestParams, model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) filtered_params = {key: value for key, value in image_edit_optional_params.items() if key in supported_params} - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} # Map OpenAI parameters to Imagen format if "n" in filtered_params: @@ -148,10 +149,10 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): model: str, prompt: str | None, image: FileTypes | None, - image_edit_optional_request_params: dict[str, Any], + image_edit_optional_request_params: Mapping[str, object], litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[dict[str, Any], RequestFiles | None]: + ) -> tuple[dict[str, object], RequestFiles | None]: # Prepare reference images in the correct Imagen format if image is None: raise ValueError("Vertex AI Imagen image edit requires at least one reference image.") @@ -182,14 +183,14 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): parameters["guidanceScale"] = 7.5 # Default guidance scale parameters["seed"] = None # Let Vertex AI choose random seed - request_body: Final[dict[str, Any]] = { + request_body: Final[dict[str, object]] = { "instances": instances, "parameters": parameters, } - payload: Final[Any] = json.dumps(request_body) + payload: Final = json.dumps(request_body) empty_files: Final = cast(RequestFiles, []) - return cast(tuple[dict[str, Any], RequestFiles | None], (payload, empty_files)) + return cast(tuple[dict[str, object], RequestFiles | None], (payload, empty_files)) def transform_image_edit_response( self, @@ -237,8 +238,8 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): def _prepare_reference_images( self, image: FileTypes | list[FileTypes], - image_edit_optional_request_params: dict[str, Any], - ) -> list[dict[str, Any]]: + image_edit_optional_request_params: Mapping[str, object], + ) -> list[dict[str, object]]: """ Prepare reference images in the correct Imagen API format """ @@ -248,7 +249,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): else: images = [image] - reference_images: Final[list[dict[str, Any]]] = [] + reference_images: Final[list[dict[str, object]]] = [] for idx, img in enumerate(images): if img is None: @@ -258,7 +259,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): base64_data = base64.b64encode(image_bytes).decode("utf-8") # Create reference image structure - reference_image = { + reference_image: dict[str, object] = { "referenceType": "REFERENCE_TYPE_RAW", "referenceId": idx + 1, "referenceImage": {"bytesBase64Encoded": base64_data}, @@ -272,7 +273,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): mask_bytes: Final = self._read_all_bytes(mask_image) mask_base64: Final = base64.b64encode(mask_bytes).decode("utf-8") - mask_reference: Final = { + mask_reference: Final[dict[str, object]] = { "referenceType": "REFERENCE_TYPE_MASK", "referenceId": len(reference_images) + 1, "referenceImage": {"bytesBase64Encoded": mask_base64}, diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index d7a2491c04a..b2c52c53580 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -218,10 +218,10 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): contents: Final = [{"role": "user", "parts": [{"text": prompt}]}] # Prepare generation config - generation_config: Final[dict[str, Any]] = {"responseModalities": ["IMAGE"]} + generation_config: Final[dict[str, object]] = {"responseModalities": ["IMAGE"]} # Seed from user-supplied imageConfig dict; flat params are overlaid for backward compat. - image_config: Final[dict[str, Any]] = dict(optional_params.get("imageConfig") or {}) + image_config: Final[dict[str, object]] = dict(optional_params.get("imageConfig") or {}) if "aspectRatio" in optional_params: image_config["aspectRatio"] = optional_params["aspectRatio"] @@ -242,7 +242,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): elif "n" in optional_params: generation_config["candidateCount"] = optional_params["n"] - request_body: Final[dict[str, Any]] = { + request_body: Final[dict[str, object]] = { "contents": contents, "generationConfig": generation_config, } diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index b0c6add69fd..dce4d2f2a87 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -7,7 +7,7 @@ Why separate file? Make it easy to see how transformation works import math import uuid from collections.abc import Mapping -from typing import Any, Final +from typing import Final import httpx @@ -232,7 +232,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 279035c455d..89a5b8a570e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -1,6 +1,6 @@ import types from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -95,7 +95,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "VertexAILlama3StreamingHandler": return VertexAILlama3StreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 58cf7c7e702..67b01c2dc43 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: import tiktoken from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.base_model_iterator import MockResponseIterator class VertexGemmaConfig(OpenAIGPTConfig): @@ -56,7 +57,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): self, model_response: ModelResponse, stream: bool, - ) -> ModelResponse | Any: + ) -> "ModelResponse | MockResponseIterator": """ Helper method to return fake stream iterator if streaming is requested. @@ -138,7 +139,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): client: HTTPHandler | httpx.Client | None, api_base: str, headers: dict[str, str], # mutable-ok: forwarded to post(headers: dict | None) - request_data: dict[str, Any], # mutable-ok: forwarded to post(json: dict | ...) + request_data: dict[str, object], # mutable-ok: forwarded to post(json: dict | ...) timeout: float | httpx.Timeout | None, ) -> httpx.Response: if isinstance(client, HTTPHandler): @@ -173,7 +174,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): client: AsyncHTTPHandler | httpx.AsyncClient | None, api_base: str, headers: dict[str, str], # mutable-ok: forwarded to post(headers: dict | None) - request_data: dict[str, Any], # mutable-ok: forwarded to post(json: dict | ...) + request_data: dict[str, object], # mutable-ok: forwarded to post(json: dict | ...) timeout: float | httpx.Timeout | None, ) -> httpx.Response: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index 091b9dfd334..7c626c66e9d 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -3,7 +3,8 @@ Volcengine Embedding Transformation Transforms OpenAI embedding requests to Volcengine format """ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx @@ -83,11 +84,11 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): def map_openai_params( self, - non_default_params: dict[str, Any], - optional_params: dict[str, Any], + non_default_params: Mapping[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Map OpenAI embedding parameters to Volcengine format. diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 0f57ac11028..b48efe229b3 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -4,8 +4,8 @@ Transformation logic for Voyage AI's /v1/rerank endpoint. Docs - https://docs.voyageai.com/docs/reranker """ -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final import httpx @@ -34,7 +34,7 @@ class VoyageRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: Sequence[str | Mapping[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index bd6b23ff2be..ff95f14951a 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -5,8 +5,8 @@ Docs - https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank """ import uuid -from collections.abc import Mapping -from typing import Any, Final, cast +from collections.abc import Mapping, Sequence +from typing import Final, cast import httpx @@ -96,7 +96,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: Sequence[str | Mapping[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, @@ -178,7 +178,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): transformed_results: Final = [] for result in _results: - transformed_result: dict[str, Any] = { + transformed_result: dict[str, object] = { "index": result["index"], "relevance_score": result["score"], } diff --git a/litellm/llms/xai/realtime/transformation.py b/litellm/llms/xai/realtime/transformation.py index e9d16daad7c..5efe125ee60 100644 --- a/litellm/llms/xai/realtime/transformation.py +++ b/litellm/llms/xai/realtime/transformation.py @@ -16,7 +16,7 @@ construction time (see ``handler.py``) so all normalization is isolated here and ``RealTimeStreaming`` stays provider-agnostic. """ -from typing import Any, Final +from typing import Final class XAIRealtimeNormalizer: @@ -58,7 +58,7 @@ class XAIRealtimeNormalizer: # Cache content-part objects keyed by (response_id, item_id, content_index) # so that ``response.content_part.done`` events missing ``part`` can be # back-filled from earlier ``content_part.added`` / delta-done events. - self._content_part_by_key: dict[tuple, dict[str, Any]] = {} + self._content_part_by_key: dict[tuple, dict[str, object]] = {} # --------------------------------------------------------------------------- # Public interface consumed by RealTimeStreaming @@ -140,7 +140,7 @@ class XAIRealtimeNormalizer: } self._content_part_by_key[key] = updated - def _resolve_content_part(self, event: dict) -> dict[str, Any]: + def _resolve_content_part(self, event: dict) -> dict[str, object]: part: Final = event.get("part") if isinstance(part, dict): return part @@ -214,7 +214,7 @@ class XAIRealtimeNormalizer: needs_content: Final = event_type in self._EVENTS_NEEDING_CONTENT_INDEX if not needs_output and not needs_content: return event - patch: Final[dict[str, Any]] = {} + patch: Final[dict[str, object]] = {} if needs_output and "output_index" not in event: patch["output_index"] = 0 if needs_content and "content_index" not in event: @@ -228,8 +228,8 @@ class XAIRealtimeNormalizer: # --------------------------------------------------------------------------- @staticmethod - def _default_ga_usage() -> dict[str, Any]: - default_details: Final[dict[str, Any]] = { + def _default_ga_usage() -> dict[str, object]: + default_details: Final[dict[str, int]] = { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, @@ -243,7 +243,7 @@ class XAIRealtimeNormalizer: } @staticmethod - def _normalize_usage(usage: object, *, empty_as_null: bool) -> dict[str, Any] | None: + def _normalize_usage(usage: object, *, empty_as_null: bool) -> dict[str, object] | None: """Coerce a usage object into the full OpenAI GA shape. ``empty_as_null=True`` for ``response.created`` (usage optional). @@ -253,12 +253,12 @@ class XAIRealtimeNormalizer: return None if not usage: return None if empty_as_null else XAIRealtimeNormalizer._default_ga_usage() - default_details: Final[dict[str, Any]] = { + default_details: Final[dict[str, int]] = { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, } - normalized: Final[dict[str, Any]] = { + normalized: Final[dict[str, object]] = { "total_tokens": usage.get("total_tokens", 0), "input_tokens": usage.get("input_tokens", 0), "output_tokens": usage.get("output_tokens", 0), diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index a04e2f5c9b8..30ee8b7a4fc 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -46,6 +46,7 @@ from litellm.repositories.table_repositories import ( MCPServerOAuthClientRepository, MCPServerRepository, MCPUserCredentialsRepository, + PrismaTableRepository, ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.verification_token_repository import ( @@ -535,11 +536,14 @@ def _user_credential_actions( return table +class _MCPUserEnvVarsRepository(PrismaTableRepository["prisma_db_models.LiteLLM_MCPUserEnvVars"]): + table_name = "litellm_mcpuserenvvars" + + def _user_env_var_actions( prisma_client: PrismaClient, ) -> "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]": - table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars - return table + return _MCPUserEnvVarsRepository(prisma_client).table async def _db_find_user_credential_row( diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index e49638e8e9c..b49bac8a4cd 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -5612,7 +5612,7 @@ class MCPServerManager: async def pre_call_tool_check( self, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, server_name: str, user_api_key_auth: UserAPIKeyAuth | None, proxy_logging_obj: ProxyLogging | None, diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 0cdf40ae8d3..1247ff1ac28 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -49,6 +49,7 @@ from litellm.litellm_core_utils.url_utils import async_safe_get from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, + header_value, httpxSpecialProvider, ) from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -457,7 +458,7 @@ def _raise_for_upstream_failure( if response.status_code == 401 and relays_upstream_auth: raise MCPUpstreamAuthError( status_code=response.status_code, - www_authenticate=response.headers.get("www-authenticate"), + www_authenticate=header_value(response.headers, "www-authenticate"), server_name=upstream, ) raise MCPOpenApiUpstreamError(response.status_code, upstream) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py index f7b92df5ba3..5503d19211b 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py @@ -18,6 +18,7 @@ TTL ``MCP_SSO_ASSERTION_CACHE_TTL_SECONDS``; invalidation also guards against st from __future__ import annotations import json +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol @@ -29,6 +30,8 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE, MCP_SSO_ASSERTION_CACHE_TTL_SECONDS if TYPE_CHECKING: + from prisma.models import LiteLLM_SSOIdentityAssertion + from litellm.proxy.utils import PrismaClient _ASSERTION_DECRYPT_LOG_KEY: Final = "sso_identity_assertion" @@ -36,6 +39,34 @@ _STR_ADAPTER: Final[TypeAdapter[str]] = TypeAdapter(str) _MAYBE_STR_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +class _SSOAssertionTable(Protocol): + """The ``LiteLLM_SSOIdentityAssertion`` table operations this store calls.""" + + async def find_unique(self, *, where: Mapping[str, str]) -> LiteLLM_SSOIdentityAssertion | None: ... + + async def find_many(self) -> Sequence[LiteLLM_SSOIdentityAssertion]: ... + + async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> object: ... + + async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> object: ... + + +class _MCPServerTable(Protocol): + """The ``LiteLLM_MCPServerTable`` lookup the retention gate calls.""" + + async def find_first(self, *, where: Mapping[str, str]) -> object | None: ... + + +def _assertion_table(prisma_client: PrismaClient) -> _SSOAssertionTable: + """The SSO assertion table, typed so the untyped prisma client surface stops here.""" + return prisma_client.db.litellm_ssoidentityassertion + + +def _mcp_server_table(prisma_client: PrismaClient) -> _MCPServerTable: + """The MCP server table, typed so the untyped prisma client surface stops here.""" + return prisma_client.db.litellm_mcpservertable + + class SSOIdentityAssertion(BaseModel): """The IdP material an EMA exchange needs: ``id_token`` is the RFC 8693 subject token, ``expires_at`` bounds its usefulness, and the refresh token renews it without re-login.""" @@ -163,9 +194,7 @@ async def ema_assertion_retention_enabled() -> bool: return True if prisma_client is None: return False - row: Final = await prisma_client.db.litellm_mcpservertable.find_first( - where={"auth_type": MCPAuth.oauth2_id_jag.value} - ) + row: Final = await _mcp_server_table(prisma_client).find_first(where={"auth_type": MCPAuth.oauth2_id_jag.value}) return row is not None @@ -184,7 +213,7 @@ async def persist_sso_identity_assertion( **({"expires_at": assertion.expires_at.isoformat()} if assertion.expires_at else {}), } encoded: Final = _STR_ADAPTER.validate_python(encrypt_value_helper(json.dumps(payload))) - await prisma_client.db.litellm_ssoidentityassertion.upsert( + await _assertion_table(prisma_client).upsert( where={"user_id": user_id}, data={ "create": {"user_id": user_id, "assertion_b64": encoded}, @@ -200,7 +229,7 @@ async def _read_assertion_from_db(user_id: str) -> SSOIdentityAssertion | None: if prisma_client is None: return None - row: Final = await prisma_client.db.litellm_ssoidentityassertion.find_unique(where={"user_id": user_id}) + row: Final = await _assertion_table(prisma_client).find_unique(where={"user_id": user_id}) if row is None: return None raw: Final = _MAYBE_STR_ADAPTER.validate_python( @@ -310,13 +339,13 @@ async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient, re_encrypted: Final = _STR_ADAPTER.validate_python( encrypt_value_helper(plaintext, new_encryption_key=new_master_key) ) - await prisma_client.db.litellm_ssoidentityassertion.update( + await _assertion_table(prisma_client).update( where={"user_id": row.user_id}, data={"assertion_b64": re_encrypted}, ) return True - rows: Final = await prisma_client.db.litellm_ssoidentityassertion.find_many() + rows: Final = await _assertion_table(prisma_client).find_many() outcomes: Final = [await _rotate_row(row) for row in rows] verbose_proxy_logger.info( "rotate_sso_identity_assertions_master_key: rotated %d row(s), skipped %d", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index d2bf7e2a3a5..17c85bbdbca 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -15,7 +15,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final from starlette.routing import BaseRoute, Match -from starlette.types import Receive, Scope, Send +from starlette.types import ASGIApp, Receive, Scope, Send from litellm._logging import verbose_proxy_logger from litellm.proxy.route_priority import hot_routes_first @@ -304,7 +304,7 @@ class LazyFeatureMiddleware: def __init__( self, - app, + app: ASGIApp, fastapi_app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY_FEATURES, ): diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py index 902e3fb3db3..5de3b610782 100644 --- a/litellm/proxy/analytics_endpoints/cache_activity.py +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -2,19 +2,29 @@ import asyncio import json from collections.abc import Sequence from datetime import datetime -from typing import TYPE_CHECKING, Final +from typing import Final, Protocol from pydantic import BaseModel, TypeAdapter from litellm.proxy._types import LiteLLMRoutes -if TYPE_CHECKING: - from litellm.proxy.utils import PrismaClient - UNKNOWN_CALL_TYPE: Final = "Unknown" INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value) +class _SupportsQueryRaw(Protocol): + """The single database operation the cache-activity queries issue.""" + + async def query_raw(self, query: str, *args: object) -> Sequence[object]: ... + + +class _SupportsRawQueryDb(Protocol): + """A prisma client handle, narrowed to the raw-query surface used here.""" + + @property + def db(self) -> _SupportsQueryRaw: ... + + class CacheActivityGroup(BaseModel): call_type: str api_requests: int @@ -150,7 +160,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: async def get_cache_activity( - prisma_client: "PrismaClient", + prisma_client: _SupportsRawQueryDb, start_date: datetime, end_date: datetime, key_aliases: Sequence[str], diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 3372145e66c..ee012e65ab1 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -167,7 +167,7 @@ def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool: def _is_param_allowed( param: str, - request_body_value: Any, + request_body_value: object, configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, ) -> bool: """ @@ -190,7 +190,7 @@ def _is_param_allowed( def _allow_model_level_clientside_configurable_parameters( - model: str, param: str, request_body_value: Any, llm_router: Router | None + model: str, param: str, request_body_value: object, llm_router: Router | None ) -> bool: """ Check if model is allowed to use configurable client-side params @@ -533,7 +533,7 @@ def is_request_body_safe(request_body: dict, general_settings: dict, llm_router: return True -def _coerce_metadata_to_dict(value: Any) -> dict[str, Any] | None: +def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None: """Return ``value`` as a dict, parsing it from JSON if delivered as a string. Multipart/form-data and ``extra_body`` callers send ``litellm_metadata`` @@ -892,7 +892,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: return True -async def check_response_size_is_safe(response: Any) -> bool: +async def check_response_size_is_safe(response: object) -> bool: """ Enterprise Only: - Checks if the response size is within the limit @@ -1525,7 +1525,7 @@ def get_customer_user_header_from_mapping(user_id_mapping) -> list | None: def _get_customer_id_from_standard_headers( - request_headers: dict | None, + request_headers: Mapping[str, object] | None, ) -> str | None: """ Check standard customer ID headers for a customer/end-user ID. @@ -1551,7 +1551,7 @@ def _get_customer_id_from_standard_headers( return None -def _coerce_user_id_to_str(value: Any) -> str | None: +def _coerce_user_id_to_str(value: object) -> str | None: """Return a usable end-user identifier string, or None if the value isn't one. Always drops non-string structured values (dict/list/tuple/set) because @@ -1578,7 +1578,7 @@ def _coerce_user_id_to_str(value: Any) -> str | None: # behind the flag preserves backwards compatibility for deployments # that intentionally pass JSON-encoded user identifiers. if litellm.validate_end_user_id_in_db and stripped[:1] in ("{", "["): - parsed: Final = safe_json_loads(stripped) + parsed: Final[object] = safe_json_loads(stripped) if isinstance(parsed, (dict, list)): return None return stripped @@ -1586,7 +1586,9 @@ def _coerce_user_id_to_str(value: Any) -> str | None: return None -def get_end_user_id_from_request_body(request_body: dict, request_headers: dict | None = None) -> str | None: +def get_end_user_id_from_request_body( + request_body: Mapping[str, object], request_headers: Mapping[str, object] | None = None +) -> str | None: # Import general_settings here to avoid potential circular import issues at module level # and to ensure it's fetched at runtime. from litellm.proxy.proxy_server import general_settings @@ -1635,7 +1637,7 @@ def get_end_user_id_from_request_body(request_body: dict, request_headers: dict if user_id_str: return user_id_str - def _as_dict(value: Any) -> dict: + def _as_dict(value: object) -> dict: # metadata / litellm_metadata can arrive as JSON strings from # multipart/form-data or extra_body; coerce so string-encoded # payloads can't evade end-user attribution. @@ -1720,11 +1722,11 @@ _MODEL_ROUTING_ID_FIELDS: Final = ( ) -def _append_model_candidates(candidates: list[str], value: Any) -> None: +def _append_model_candidates(candidates: list[str], value: object) -> None: if value is None: return - values: Final = value if isinstance(value, (list, tuple, set)) else [value] + values: Final[tuple[object, ...]] = tuple(value) if isinstance(value, (list, tuple, set)) else (value,) for item in values: if item is None: continue @@ -1765,7 +1767,7 @@ def _route_uses_model_routing_sources(route: str) -> bool: def _extract_models_from_managed_resource_id( - resource_id: Any, + resource_id: object, resource_id_field: str | None = None, llm_router: Router | None = None, ) -> list[str]: diff --git a/litellm/proxy/client/cli/commands/configure.py b/litellm/proxy/client/cli/commands/configure.py index 2878ae0e9f8..eca7ba86496 100644 --- a/litellm/proxy/client/cli/commands/configure.py +++ b/litellm/proxy/client/cli/commands/configure.py @@ -98,13 +98,15 @@ def _preflight(target: str) -> None: raise click.ClickException(str(e)) from e -def _start(ctx: click.Context, api_key: str | None, target: str = _CLAUDE_TARGET) -> tuple[StaticToken, _Listing]: +def _start( + ctx: click.Context, base_url: str, api_key: str | None, target: str = _CLAUDE_TARGET +) -> tuple[StaticToken, _Listing]: _preflight(target) try: credential: Final = resolve_credential(ctx, api_key) except ClaudeSettingsError as e: raise click.ClickException(str(e)) - return credential, _listed_models(ctx.obj["base_url"], credential.token, target) + return credential, _listed_models(base_url, credential.token, target) def _listing_error(base_url: str, error: PiSyncError, target: str) -> str: @@ -147,9 +149,7 @@ def _validated_model(model: str | None, listing: _Listing, base_url: str) -> str return starting -def _apply_claude(ctx: click.Context, credential: StaticToken, listing: _Listing, model: str | None) -> None: - ctx_obj: Final[CliContextObj] = ctx.obj - base_url: Final = ctx_obj["base_url"] +def _apply_claude(base_url: str, credential: StaticToken, listing: _Listing, model: str | None) -> None: listed: Final = listing.ids starting: Final = _validated_model(model, listing, base_url) settings_path: Final = claude_settings_path(os.environ) @@ -214,8 +214,7 @@ def _pick_codex_model(listed: Sequence[str]) -> str: return str(inquirer.fuzzy(message="Model Codex starts on (type to filter):", choices=choices).execute()) -def _apply_codex(ctx: click.Context, credential: StaticToken, listing: _Listing, model: str) -> None: - base_url: Final[str] = ctx.obj["base_url"] +def _apply_codex(base_url: str, credential: StaticToken, listing: _Listing, model: str) -> None: _validated_model(model, listing, base_url) settings_path: Final = codex_config_path(os.environ) try: @@ -237,13 +236,12 @@ class _Setup: def _choose_setup( - ctx: click.Context, + base_url: str, target: str, credential: StaticToken, pick_model: Callable[[Sequence[str]], str | None], pick_codex_model: Callable[[Sequence[str]], str], ) -> _Setup: - base_url: Final[str] = ctx.obj["base_url"] listing: Final = _listed_models(base_url, credential.token, target) model: Final = ( pick_model(tuple(item.source_model or item.id for item in listing.models)) @@ -270,12 +268,15 @@ def interactive_configure( credential: Final = resolve_credential(ctx, None) except ClaudeSettingsError as e: raise click.ClickException(str(e)) from e - setups: Final = tuple(_choose_setup(ctx, target, credential, pick_model, pick_codex_model) for target in targets) + base_url: Final[str] = ctx.obj["base_url"] + setups: Final = tuple( + _choose_setup(base_url, target, credential, pick_model, pick_codex_model) for target in targets + ) for setup in setups: if setup.target == _CLAUDE_TARGET: - _apply_claude(ctx, credential, setup.listing, setup.model) + _apply_claude(base_url, credential, setup.listing, setup.model) elif setup.model is not None: - _apply_codex(ctx, credential, setup.listing, setup.model) + _apply_codex(base_url, credential, setup.listing, setup.model) class _ConnectionOptions(BaseModel): @@ -283,7 +284,8 @@ class _ConnectionOptions(BaseModel): gateway_url: str | None = None -def _connection_context(ctx: click.Context, api_key: str | None, gateway_url: str | None) -> click.Context: +def _connection_settings(ctx: click.Context, api_key: str | None, gateway_url: str | None) -> CliContextObj: + """The context object a subcommand runs with: its own --api-key / --gateway-url over the group's, over `lite`'s.""" ctx_obj: Final[CliContextObj] = ctx.obj group: Final = ( _ConnectionOptions.model_validate(ctx.parent.params) @@ -300,7 +302,11 @@ def _connection_context(ctx: click.Context, api_key: str | None, gateway_url: st "api_key": key if key is not None else ctx_obj.get("api_key"), "api_key_from_token_file": False if key is not None else ctx_obj.get("api_key_from_token_file", False), } - return click.Context(ctx.command, parent=ctx.parent, obj=connection) + return connection + + +def _connection_context(ctx: click.Context, settings: CliContextObj) -> click.Context: + return click.Context(ctx.command, parent=ctx.parent, obj=settings) @click.group(name="configure", invoke_without_command=True) @@ -316,19 +322,19 @@ def configure_group(ctx: click.Context, api_key: str | None, gateway_url: str | """ if ctx.invoked_subcommand is not None: return - connection: Final = _connection_context(ctx, api_key, gateway_url) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + connection: Final = _connection_context(ctx, settings) if not sys.stdin.isatty(): raise click.ClickException( "`lite configure` asks questions, so it needs a terminal. Non-interactively, run " "`lite configure claude --api-key --model ` or " "`lite configure codex --api-key --model `." ) - prompted: Final = ( - connection - if connection.obj.get("base_url_explicit") - else _connection_context(connection, None, click.prompt("Gateway URL", default=connection.obj["base_url"])) - ) - interactive_configure(prompted) + if settings.get("base_url_explicit"): + interactive_configure(connection) + return + prompted: Final = _connection_settings(connection, None, click.prompt("Gateway URL", default=settings["base_url"])) + interactive_configure(_connection_context(connection, prompted)) @click.group(name="unconfigure") @@ -356,9 +362,9 @@ def configure_claude(ctx: click.Context, api_key: str | None, model: str | None, setting is kept, and what changed is recorded so `lite unconfigure claude` can put it back. Assumes the proxy is already running. """ - connection: Final = _connection_context(ctx, api_key, gateway_url) - credential, listing = _start(connection, api_key) - _apply_claude(connection, credential, listing, model) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + credential, listing = _start(_connection_context(ctx, settings), settings["base_url"], api_key) + _apply_claude(settings["base_url"], credential, listing, model) @configure_group.command(name="codex") @@ -368,9 +374,9 @@ def configure_claude(ctx: click.Context, api_key: str | None, model: str | None, @click.pass_context def configure_codex(ctx: click.Context, api_key: str | None, gateway_url: str | None, model: str) -> None: """Route plain `codex` through the gateway until `lite unconfigure codex`.""" - connection: Final = _connection_context(ctx, api_key, gateway_url) - credential, listing = _start(connection, api_key, _CODEX_TARGET) - _apply_codex(connection, credential, listing, model) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + credential, listing = _start(_connection_context(ctx, settings), settings["base_url"], api_key, _CODEX_TARGET) + _apply_codex(settings["base_url"], credential, listing, model) @unconfigure_group.command(name="codex") diff --git a/litellm/proxy/client/cli/commands/model_groups.py b/litellm/proxy/client/cli/commands/model_groups.py index c904e5bed49..367c2063b6b 100644 --- a/litellm/proxy/client/cli/commands/model_groups.py +++ b/litellm/proxy/client/cli/commands/model_groups.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final, Literal import click @@ -5,10 +6,17 @@ import rich import rich.table from ... import Client +from ._cli_context import cli_context_values def create_client(ctx: click.Context) -> Client: - return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"]) + context: Final = cli_context_values(ctx) + return Client(base_url=context["base_url"], api_key=context["api_key"]) + + +def _rendered_field(group: Mapping[str, object], key: str, default: str) -> str: + """The rendered value of one model group field, or ``default`` when the group omits it.""" + return str(group.get(key, default)) @click.group(name="model-groups") @@ -46,10 +54,10 @@ def list_model_groups(ctx: click.Context, output_format: Literal["table", "json" for group in groups: table.add_row( - str(group.get("model_group", "")), - str(group.get("mode", "chat")), - str(group.get("input_cost_per_token", "")), - str(group.get("output_cost_per_token", "")), + _rendered_field(group, "model_group", ""), + _rendered_field(group, "mode", "chat"), + _rendered_field(group, "input_cost_per_token", ""), + _rendered_field(group, "output_cost_per_token", ""), ) rich.print(table) diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index f2624797a5f..00e8b0a3a76 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -166,7 +166,8 @@ def up(ctx: click.Context) -> None: is already running (this does not start one for you). Cursor is not supported: it has no equivalent file-based config to patch. """ - base_url: Final = ctx.obj["base_url"] + ctx_obj: Final[CliContextObj] = ctx.obj + base_url: Final = ctx_obj["base_url"] try: ensure_fresh_login(ctx) diff --git a/litellm/proxy/client/users.py b/litellm/proxy/client/users.py index 3f11fe94043..503c92228a8 100644 --- a/litellm/proxy/client/users.py +++ b/litellm/proxy/client/users.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final import requests @@ -50,7 +51,7 @@ class UsersManagementClient: response.raise_for_status() return response.json() - def create_user(self, user_data: dict[str, Any]) -> dict[str, Any]: + def create_user(self, user_data: Mapping[str, object]) -> dict[str, Any]: """Create a new user (POST /user/new)""" url: Final = f"{self.base_url}/user/new" response: Final = requests.post(url, headers=self._get_headers(), json=user_data, timeout=self.timeout) diff --git a/litellm/proxy/common_utils/cache_pydantic_utils.py b/litellm/proxy/common_utils/cache_pydantic_utils.py index 725c2b61145..3703cf7c916 100644 --- a/litellm/proxy/common_utils/cache_pydantic_utils.py +++ b/litellm/proxy/common_utils/cache_pydantic_utils.py @@ -37,7 +37,7 @@ class CacheCodec: """ @staticmethod - def serialize(value: Any, model_type: type[T] | None = None) -> Any: + def serialize(value: object, model_type: type[T] | None = None) -> object: """ Encode a value for DualCache / Redis (``json.dumps``-safe). diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index bdf45ad46f8..cb8b51d092e 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -714,7 +714,7 @@ def strip_callback_config(metadata: dict[str, object] | None) -> dict[str, objec return {k: v for k, v in metadata.items() if k not in _CALLBACK_CONFIG_SLOTS} -def encrypt_callback_vars(metadata: Any) -> Any: +def encrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values encrypted at rest. Idempotent: a value that already decrypts cleanly is left unchanged so @@ -723,7 +723,7 @@ def encrypt_callback_vars(metadata: Any) -> Any: return _transform_callback_vars(metadata, _encrypt_if_plaintext) -def decrypt_callback_vars(metadata: Any) -> Any: +def decrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values decrypted. Legacy plaintext rows pass through unchanged (decrypt failure → original). @@ -731,7 +731,7 @@ def decrypt_callback_vars(metadata: Any) -> Any: return _transform_callback_vars(metadata, _decrypt_or_passthrough) -def _transform_callback_vars(metadata: object, transform: Callable[[str, Any], Any]) -> object: +def _transform_callback_vars(metadata: object, transform: Callable[[str, object], object]) -> object: if not isinstance(metadata, dict): return metadata out: Final = copy.deepcopy(metadata) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c17c46e5af..1c2bd7ea217 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -56,14 +56,18 @@ def _unqualified(annotation: object) -> object: return _unqualified(qualified[0]) +def _union_members(annotation: object) -> tuple[object, ...]: + """The non-``None`` members of a union annotation, or the annotation itself when it is not a union.""" + if get_origin(annotation) not in (Union, UnionType): + return (annotation,) + members: Final[tuple[object, ...]] = get_args(annotation) + return tuple(arg for arg in members if arg is not type(None)) + + def _numeric_form_type(annotation: object) -> type[int] | type[float] | None: """The scalar to parse an ``int``/``float``-typed field as, else ``None``.""" unwrapped: Final = _unqualified(annotation) - candidates: Final = ( - tuple(arg for arg in get_args(unwrapped) if arg is not type(None)) - if get_origin(unwrapped) in (Union, UnionType) - else (unwrapped,) - ) + candidates: Final = _union_members(unwrapped) if len(candidates) != 1: return None if candidates[0] is int: diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index 888a6d077ad..b028e7fda20 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -66,7 +66,7 @@ def map_v3_rate_limit_type( return None -def _coerce_message(detail: Any) -> str: +def _coerce_message(detail: object) -> str: """Best-effort, JSON-friendly stringification of an HTTPException-style detail.""" if detail is None: return "" @@ -144,7 +144,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): def __init__( self, detail: Any, - headers: Mapping[str, Any] | None = None, + headers: Mapping[str, object] | None = None, category: str | RateLimitErrorCategory = RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type: str | RateLimitType | None = None, model: str | None = None, diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 343461fa105..3efc189a475 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -49,7 +49,7 @@ from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManage from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.organization_repository import OrganizationRepository -from litellm.repositories.prisma_protocols import SpendLinkedTable +from litellm.repositories.prisma_protocols import PrismaBatch, SpendLinkedTable from litellm.repositories.project_repository import ProjectRepository from litellm.repositories.table_repositories import ( EndUserRepository, @@ -478,6 +478,11 @@ class ResetBudgetJob: self.reset_settings: BudgetResetSettings = reset_settings or get_budget_reset_settings() self.pod_lock_manager: PodLockManager | None = pod_lock_manager + @property + def _new_batch(self) -> Callable[[], PrismaBatch]: + new_batch: Final[Callable[[], PrismaBatch]] = self.prisma_client.db.batch_ + return new_batch + async def _lease_is_held(self, lock_manager: PodLockManager) -> bool: """True only when the lease is readable and someone holds it. @@ -837,7 +842,7 @@ class ResetBudgetJob: ) async def _commit_budget_cascade_once(self, cascade: _BudgetCascade) -> None: - async with budget_cascade_unit_of_work(self.prisma_client.db.batch_) as uow: + async with budget_cascade_unit_of_work(self._new_batch) as uow: _queue_budget_linked_resets(uow.team_memberships, cascade) _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE) _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE) @@ -959,7 +964,7 @@ class ResetBudgetJob: ) async def _write_key_reset_updates_once(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for k in updated_keys: if k.row.token is None: continue @@ -983,7 +988,7 @@ class ResetBudgetJob: ) async def _write_user_reset_updates_once(self, updated_users: Sequence[_RowReset[LiteLLM_UserTable]]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for u in updated_users: uow.users.queue_spend_reset( user_id=u.row.user_id, @@ -1005,7 +1010,7 @@ class ResetBudgetJob: ) async def _write_team_reset_updates_once(self, updated_teams: Sequence[_RowReset[LiteLLM_TeamTable]]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for t in updated_teams: uow.teams.queue_spend_reset( team_id=t.row.team_id, diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index e852eb5d6f9..c1407979f29 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -106,7 +106,7 @@ async def create_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response: Final = await processor.base_process_llm_request( + response: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -216,7 +216,7 @@ async def list_containers( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "query_params": query_params, "model": query_params.get("model"), "order": order, @@ -341,7 +341,7 @@ async def retrieve_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + container: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -366,6 +366,7 @@ async def retrieve_container( proxy_logging_obj=proxy_logging_obj, version=version, ) + return container @router.delete( @@ -446,7 +447,7 @@ async def delete_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + deleted_container: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -471,6 +472,7 @@ async def delete_container( proxy_logging_obj=proxy_logging_obj, version=version, ) + return deleted_container # Register JSON-configured container file endpoints diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 0c88cc23042..d9c8b271646 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -15,7 +15,7 @@ import traceback from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, overload from urllib.parse import quote, unquote from pydantic import TypeAdapter @@ -136,6 +136,25 @@ class _SpendBatch(Protocol): litellm_modelaccessgroupbudgettable: BatchTable +_EntitySpendTable: TypeAlias = Literal[ + "litellm_tagtable", "litellm_agentstable", "litellm_modelaccessgroupbudgettable", "litellm_projecttable" +] + + +_ENTITY_SPEND_TABLES: Final[Mapping[_EntitySpendTable, Callable[[_SpendBatch], BatchTable]]] = MappingProxyType( + { + "litellm_tagtable": lambda batcher: batcher.litellm_tagtable, + "litellm_agentstable": lambda batcher: batcher.litellm_agentstable, + "litellm_modelaccessgroupbudgettable": lambda batcher: batcher.litellm_modelaccessgroupbudgettable, + "litellm_projecttable": lambda batcher: batcher.litellm_projecttable, + } +) + + +def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) -> BatchTable: + return _ENTITY_SPEND_TABLES[table_accessor](batcher) + + class _SpendBatchManager(Protocol): async def __aenter__(self) -> _SpendBatch: ... @@ -2159,9 +2178,7 @@ class DBSpendUpdateWriter: async def _update_entity_spend_in_db( entity_name: str, transactions: dict[str, float] | None, - table_accessor: Literal[ - "litellm_tagtable", "litellm_agentstable", "litellm_modelaccessgroupbudgettable", "litellm_projecttable" - ], + table_accessor: _EntitySpendTable, where_field: str, n_retry_times: int, prisma_client: PrismaClient, @@ -2195,7 +2212,7 @@ class DBSpendUpdateWriter: entity_id, response_cost, ) - getattr(batcher, table_accessor).update_many( + _entity_spend_table(batcher, table_accessor).update_many( where={where_field: entity_id}, data={"spend": {"increment": response_cost}}, ) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 9146f234570..f25c2787252 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,6 +1,6 @@ import re from collections.abc import Awaitable, Callable, Iterator -from typing import Any, Final, TypeVar +from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError @@ -446,8 +446,20 @@ def _coerce_timeout(value: object, fallback: float) -> float: _ReadResultT: Final = TypeVar("_ReadResultT") +class _DBReconnectClient(Protocol): + """The one method `call_with_db_reconnect_retry` needs from a Prisma client.""" + + async def attempt_db_reconnect( + self, + *, + reason: str, + timeout_seconds: float | None = None, + lock_timeout_seconds: float | None = None, + ) -> bool: ... + + async def call_with_db_reconnect_retry( - prisma_client: Any, + prisma_client: _DBReconnectClient, coro_factory: Callable[[], Awaitable[_ReadResultT]], *, reason: str, diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index acd01b0e99e..0524d015047 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -13,7 +13,7 @@ import urllib import urllib.parse from collections.abc import Callable from datetime import datetime, timedelta -from typing import Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final, Protocol from litellm._logging import verbose_proxy_logger from litellm.proxy.db.db_url_settings import add_missing_query_params, token_refresh_params_from_url @@ -28,6 +28,9 @@ from litellm.proxy.db.token_auth import ( ) from litellm.secret_managers.main import str_to_bool +if TYPE_CHECKING: + from prisma import Prisma + __all__ = ( "IAMEndpoint", "PrismaManager", @@ -243,7 +246,7 @@ class PrismaWrapper: def _write_engine(prisma_client: _PrismaClient, engine: _PrismaEngine) -> None: prisma_client._Prisma__engine = engine - def _instrument_prisma_client(self, prisma_client: _PrismaClient) -> _PrismaDrainTracker | None: + def _instrument_prisma_client(self, prisma_client: "Prisma | _PrismaClient") -> _PrismaDrainTracker | None: from prisma.errors import ClientNotConnectedError try: @@ -256,7 +259,7 @@ class PrismaWrapper: self._write_engine(prisma_client, _TrackedPrismaEngine(engine, tracker)) return tracker - def _get_engine_pid(self, prisma_client: _PrismaClient | None = None) -> int: + def _get_engine_pid(self, prisma_client: "Prisma | _PrismaClient | None" = None) -> int: """Get the PID of the current Prisma engine subprocess, or 0 if unavailable. Must never raise: it runs inside the reconnect path, where the client diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index c01fe15fc09..6d012c64b95 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -162,4 +162,4 @@ async def flush_tool_usage_transactions( except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise - await asyncio.sleep(2**attempt + random.uniform(0, 1)) + await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py index 2c27531cea1..a7f45a37ae6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py @@ -11,10 +11,11 @@ import asyncio import json import os from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal import httpx from fastapi import HTTPException +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -25,12 +26,34 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.guardrails import GuardrailEventHooks, Mode from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel + +class _CustomGuardrailKwargs(TypedDict): + """Keyword arguments forwarded verbatim to CustomGuardrail.__init__.""" + + guardrail_name: NotRequired[ReadOnly[str | None]] + event_hook: NotRequired[ReadOnly[GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None]] + default_on: NotRequired[ReadOnly[bool]] + mask_request_content: NotRequired[ReadOnly[bool]] + mask_response_content: NotRequired[ReadOnly[bool]] + violation_message_template: NotRequired[ReadOnly[str | None]] + end_session_after_n_fails: NotRequired[ReadOnly[int | None]] + on_violation: NotRequired[ReadOnly[str | None]] + realtime_violation_message: NotRequired[ReadOnly[str | None]] + on_sensitive_data: NotRequired[ReadOnly[str | None]] + sensitive_data_route_to_model: NotRequired[ReadOnly[str | None]] + sticky_session_routing: NotRequired[ReadOnly[bool]] + run_in_parallel: NotRequired[ReadOnly[bool]] + scan_raw_request: NotRequired[ReadOnly[bool]] + only_scan_new_messages: NotRequired[ReadOnly[bool]] + supported_event_hooks: NotRequired[ReadOnly[list[GuardrailEventHooks]]] + + HTTP_PROXY_PATH: Final = "/api/http-proxy" AKTO_CONNECTOR_NAME: Final = "litellm" DEFAULT_GUARDRAIL_TIMEOUT: Final = 5 @@ -66,7 +89,7 @@ class AktoGuardrail(CustomGuardrail): akto_vxlan_id: str | None = None, unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", guardrail_timeout: int | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailKwargs], ) -> None: """Initialize the Akto guardrail. @@ -96,8 +119,11 @@ class AktoGuardrail(CustomGuardrail): self.akto_account_id = akto_account_id or os.environ.get("AKTO_ACCOUNT_ID", "1000000") self.akto_vxlan_id = akto_vxlan_id or os.environ.get("AKTO_VXLAN_ID", "0") - kwargs["supported_event_hooks"] = list(self.get_supported_event_hooks()) - super().__init__(**kwargs) + init_kwargs: Final[_CustomGuardrailKwargs] = { + **kwargs, + "supported_event_hooks": list(self.get_supported_event_hooks()), + } + super().__init__(**init_kwargs) verbose_proxy_logger.debug( "Akto guardrail initialized: base_url=%s fallback=%s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py index bf2aa1f76e0..2b697671eda 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py @@ -8,9 +8,10 @@ confidence scoring and a tunable threshold (only block when confidence >= thresh import re from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -314,6 +315,10 @@ def _confidence_for_block( return 0.0 +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class BlockCodeExecutionGuardrail(CustomGuardrail): """ Guardrail that detects fenced code blocks (markdown ```) and blocks or masks them @@ -332,7 +337,7 @@ class BlockCodeExecutionGuardrail(CustomGuardrail): detect_execution_intent: bool = True, event_hook: Literal["pre_call", "post_call", "during_call"] | list[str] | None = None, default_on: bool = False, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: # Normalize to type expected by CustomGuardrail _event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | None = None diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 176c308eda6..2d203c31974 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -264,7 +264,7 @@ class CatoNetworksGuardrail(CustomGuardrail): stack.extend(reversed(node)) @classmethod - def _extra_inspection_sources(cls, data: Mapping[str, Any]) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: + def _extra_inspection_sources(cls, data: Mapping[str, object]) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: """Text the proxy forwards to the model outside chat ``messages``: Responses-API ``input`` and ``instructions``, legacy completion ``prompt`` and tool/function/``response_format`` schema strings. Returned @@ -336,7 +336,7 @@ class CatoNetworksGuardrail(CustomGuardrail): ) raise HTTPException(status_code=400, detail=detection_message) - def _anonymize_request(self, res: Any, data: dict) -> dict: + def _anonymize_request(self, res: _CatoAnalyzeResponse, data: dict) -> dict: verbose_proxy_logger.info("Cato: anonymize action") redacted_chat: Final = res.get("redacted_chat") if not redacted_chat: @@ -379,7 +379,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return data @classmethod - def _apply_extra_redaction(cls, data: dict, field: str, redacted: list) -> bool: + def _apply_extra_redaction(cls, data: dict, field: str, redacted: Sequence[Mapping[str, object]]) -> bool: if field == "input": input_only: Final = {"input": data["input"]} if not redacted: @@ -400,7 +400,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return True @classmethod - def _apply_schema_string_redaction(cls, data: dict, redacted: list) -> None: + def _apply_schema_string_redaction(cls, data: dict, redacted: Sequence[Mapping[str, object]]) -> None: redactions: Final = iter(redacted) for container, key in cls._iter_schema_string_refs(data): replacement = next(redactions, None) @@ -408,7 +408,7 @@ class CatoNetworksGuardrail(CustomGuardrail): container[key] = replacement["content"] @staticmethod - def _apply_prompt_redaction(data: dict, redacted: list) -> None: + def _apply_prompt_redaction(data: dict, redacted: Sequence[Mapping[str, object]]) -> None: contents: Final = [m.get("content") for m in redacted if isinstance(m, dict)] prompt: Final = data.get("prompt") if isinstance(prompt, str): diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index facb822d00d..017ef6e09f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal import httpx from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm import DualCache from litellm._logging import verbose_proxy_logger @@ -111,6 +112,10 @@ class CiscoAIDefenseGuardrailAPIError(Exception): """Raised when there is an error talking to the Cisco AI Defense API.""" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class CiscoAIDefenseGuardrail(_CiscoAIDefenseMcpMixin, CustomGuardrail): """ Cisco AI Defense guardrail integration. @@ -144,7 +149,7 @@ class CiscoAIDefenseGuardrail(_CiscoAIDefenseMcpMixin, CustomGuardrail): on_flagged_action: str | None = None, fallback_on_error: str | None = None, timeout: float | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: resolved_api_key: Final = api_key or os.environ.get("CISCO_AI_DEFENSE_API_KEY") if not resolved_api_key: diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 93d859066b0..1ecdb1b0f63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -22,7 +22,7 @@ import json import time from collections import Counter, OrderedDict from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, Final, Literal, TypeGuard from urllib.parse import urlparse import httpx @@ -64,6 +64,9 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) from litellm.types.proxy.guardrails.guardrail_hooks.base import ( GuardrailConfigModel, ) @@ -1049,7 +1052,7 @@ class CompresrGuardrail(CustomGuardrail): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -1069,8 +1072,8 @@ class CompresrGuardrail(CustomGuardrail): tools: dict, model: str, messages: list[dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: BaseAnthropicMessagesConfig | None, anthropic_messages_optional_request_params: dict, logging_obj: LiteLLMLoggingObj | None, stream: bool, diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index 924bbd2bc1a..3d4aba4ac02 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Optiona from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, Field, ValidationError -from typing_extensions import Any, override +from typing_extensions import override from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -79,7 +79,7 @@ class _GuardChatCompletionsResult(BaseModel): """Whether or not the prompt triggered a block detection.""" transformed: bool | None = None """Whether or not the original input was transformed.""" - detectors: dict[str, Any] | None = None + detectors: dict[str, object] | None = None """Result of the policy analyzing and input prompt.""" @@ -147,8 +147,8 @@ def _extract_text_from_message(message: _Message) -> str: return "\n".join(part.text for part in content if isinstance(part, _TextContentPart)) -def _merge_metadata_bags(request_data: Mapping[str, Any]) -> Mapping[str, Any] | None: - merged: Final[dict[str, Any]] = {} +def _merge_metadata_bags(request_data: Mapping[str, object]) -> Mapping[str, object] | None: + merged: Final[dict[str, object]] = {} present = False for bag in (request_data.get("metadata"), request_data.get("litellm_metadata")): if isinstance(bag, Mapping): @@ -325,7 +325,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): self._set_streaming_params(streaming_params_from_litellm_params(litellm_params)) async def _call_crowdstrike_aidr_guard( - self, payload: dict[str, Any], hook_name: str + self, payload: dict[str, object], hook_name: str ) -> _GuardChatCompletionsResult: """ Makes the API call to the CrowdStrike AIDR AI Guard endpoint. @@ -435,7 +435,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): return [_extract_text_from_message(msg) for msg in tail] async def _call_or_fail_open( - self, payload: dict[str, Any], hook_name: str, request_data: dict[str, object] + self, payload: dict[str, object], hook_name: str, request_data: dict[str, object] ) -> _GuardChatCompletionsResult: start_time: Final = time.time() try: @@ -518,7 +518,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): event_type = "output" hook_name = "apply_guardrail (response)" - ai_guard_payload: Final[dict[str, Any]] = { + ai_guard_payload: Final[dict[str, object]] = { "guard_input": guard_input.model_dump(mode="json"), "event_type": event_type, } @@ -533,7 +533,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): if user_id: ai_guard_payload["user_id"] = user_id - extra_info: Final[dict[str, str]] = {} + extra_info: Final[dict[str, object]] = {} user_email: Final = metadata.get("user_api_key_user_email") if user_email: extra_info["user_name"] = user_email diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index e0291975699..ea26eafccae 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -38,9 +38,10 @@ import asyncio import threading import time from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import ModifyResponseException @@ -79,6 +80,10 @@ class CustomCodeExecutionError(CustomCodeGuardrailError): """Raised when custom code fails during execution.""" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class CustomCodeGuardrailConfigModel(GuardrailConfigModel): """Configuration parameters for the custom code guardrail.""" @@ -114,7 +119,7 @@ class CustomCodeGuardrail(CustomGuardrail): self, custom_code: str, guardrail_name: str | None = "custom_code", - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: """ Initialize the custom code guardrail. diff --git a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py index 214d4b486d4..539dc1ea1e9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py +++ b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py @@ -7,10 +7,10 @@ import os from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol import httpx -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version @@ -56,7 +56,13 @@ class DeepKeepFirewallResponse(TypedDict): class _DeepKeepInitKwargsView(TypedDict): """Typed read of the guardrail name carried in the untyped base-guardrail kwargs.""" - guardrail_name: ReadOnly[str] + guardrail_name: ReadOnly[str | None] + + +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + guardrail_name: ReadOnly[str | None] class _DeepKeepMetadataSource(TypedDict, total=False): @@ -110,7 +116,7 @@ class DeepKeepGuardrail(CustomGuardrail): firewall_id: str | None = None, unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", extra_headers: Mapping[str, str] | list[str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ): self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py index 89afecafb0f..efe959bd186 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py @@ -6,7 +6,7 @@ # +-------------------------------------------------------------+ import os -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, AsyncIterable from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional @@ -465,7 +465,7 @@ class EnkryptAIGuardrails(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: Any, + response: AsyncIterable[ModelResponseStream], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index fa113aa4d33..eb62b896784 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -7,7 +7,7 @@ import time import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeGuard import httpx from fastapi import HTTPException @@ -50,6 +50,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -878,9 +879,9 @@ class HeadroomGuardrail(CustomGuardrail): async def async_pre_call_deployment_hook( self, - kwargs: dict[str, Any], + kwargs: dict[str, object], call_type: CallTypes | None, - ) -> dict[str, Any] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict + ) -> dict[str, object] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type) effective: Final = base_result if base_result is not None else kwargs if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES: @@ -897,7 +898,7 @@ class HeadroomGuardrail(CustomGuardrail): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -919,8 +920,8 @@ class HeadroomGuardrail(CustomGuardrail): tools: dict, model: str, messages: list[dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: BaseAnthropicMessagesConfig | None, anthropic_messages_optional_request_params: dict, logging_obj: LiteLLMLoggingObj | None, stream: bool, diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index cf5da27e9ca..63821428c62 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -121,7 +121,7 @@ class LassoGuardrail(CustomGuardrail): super().__init__(**kwargs) @staticmethod - def _get_field(obj: Any, field: str, default: object = None) -> Any: + def _get_field(obj: object, field: str, default: object = None) -> object: """Get a field from either a dict or a Pydantic object.""" if isinstance(obj, dict): return obj.get(field, default) @@ -130,7 +130,7 @@ class LassoGuardrail(CustomGuardrail): @staticmethod def _extract_tool_call_fields( call: object, - ) -> tuple[str | None, str | None, dict[str, object] | None]: + ) -> tuple[object, object, dict[str, object] | None]: """Extract (call_id, name, parsed_input) from a tool call. Handles both dict-style and Pydantic object-style tool_calls. @@ -146,7 +146,7 @@ class LassoGuardrail(CustomGuardrail): input_data: dict[str, object] | None = None if args_str: try: - parsed = json.loads(args_str) + parsed = json.loads(args_str) if isinstance(args_str, (str, bytes, bytearray)) else None except (json.JSONDecodeError, TypeError): parsed = None if isinstance(parsed, dict): @@ -488,7 +488,7 @@ class LassoGuardrail(CustomGuardrail): while preserving the original structure. """ # Index masked content by type so we can look up by id without caring about order. - masked_tool_use: Final[dict[str, dict[str, object]]] = {} + masked_tool_use: Final[dict[object, dict[str, object]]] = {} masked_tool_result: Final[dict[str, str]] = {} masked_text: Final[list[str]] = [] @@ -565,7 +565,7 @@ class LassoGuardrail(CustomGuardrail): def _update_tool_calls_from_masked( self, tool_calls: list[object], - masked_tool_use: dict[str, dict[str, object]], + masked_tool_use: Mapping[object, Mapping[str, object]], ) -> list[object]: """Replace tool_call arguments with masked values returned by Lasso.""" updated: Final = [] @@ -922,7 +922,7 @@ class LassoGuardrail(CustomGuardrail): ) -> None: """Apply masking to the actual model response when mask=True and masked content is available.""" # Index masked tool_use blocks by id for O(1) lookup. - masked_tool_use: Final[dict[str, dict[str, object]]] = {} + masked_tool_use: Final[dict[object, dict[str, object]]] = {} masked_text: Final[list[str]] = [] for masked_msg in masked_messages: content = masked_msg.get("content") diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index 8eac6b2ee53..efec15144b8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -3,11 +3,11 @@ from collections.abc import Callable, Mapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeVar +from typing import TYPE_CHECKING, Final, Generic, Literal, Optional, TypeVar from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, ValidationError -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack import litellm from litellm._logging import verbose_logger @@ -177,6 +177,23 @@ def _build_judge_prompt( ) +class _CustomGuardrailOptions(TypedDict, total=False): + """The ``CustomGuardrail`` options this guardrail accepts and forwards untouched.""" + + mask_request_content: ReadOnly[bool] + mask_response_content: ReadOnly[bool] + violation_message_template: ReadOnly[str | None] + end_session_after_n_fails: ReadOnly[int | None] + on_violation: ReadOnly[str | None] + realtime_violation_message: ReadOnly[str | None] + on_sensitive_data: ReadOnly[str | None] + sensitive_data_route_to_model: ReadOnly[str | None] + sticky_session_routing: ReadOnly[bool] + run_in_parallel: ReadOnly[bool] + scan_raw_request: ReadOnly[bool] + only_scan_new_messages: ReadOnly[bool] + + class LLMAsAJudgeGuardrail(CustomGuardrail): """Guardrail that judges request (pre_call/during_call) or response (post_call) quality via an LLM.""" @@ -190,7 +207,7 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): event_hook: JudgeModeParam = None, default_on: bool = False, router_provider: "Callable[[], Router | None] | None" = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: super().__init__( guardrail_name=guardrail_name, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index a7c93e63b32..75e875c2384 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -377,7 +377,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): else: return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}} - def _should_block_content(self, armor_response: Mapping[str, Any], allow_sanitization: bool = False) -> bool: + def _should_block_content(self, armor_response: Mapping[str, object], allow_sanitization: bool = False) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" for filt in self._filter_result_items(armor_response): # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before @@ -446,7 +446,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return filter_results return [] - def _has_deidentify_match(self, armor_response: Mapping[str, Any]) -> bool: + def _has_deidentify_match(self, armor_response: Mapping[str, object]) -> bool: """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction.""" for filter_entry in self._filter_result_items(armor_response): sdp = filter_entry.get("sdpFilterResult") @@ -456,7 +456,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): def _resolve_streaming_outcome( self, - armor_response: Mapping[str, Any], + armor_response: Mapping[str, object], assembled_response: object, content: str, ) -> tuple[bool, str | None]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 7ef0a9f73f3..edd78e0bbc6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -9,13 +9,13 @@ import asyncio import json import os import warnings -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, AsyncIterable from datetime import datetime from typing import ( TYPE_CHECKING, - Any, Final, Literal, + TypeVar, ) from urllib.parse import urljoin @@ -39,9 +39,7 @@ from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( CallTypes, CallTypesLiteral, - EmbeddingResponse, GuardrailStatus, - ImageResponse, ModelResponseStream, TextCompletionResponse, ) @@ -53,7 +51,8 @@ SENSITIVE_DATA_DETECTOR_KEYS: Final[list[str]] = ["sensitiveData", "dataDetector # Type aliases MessageRole = Literal["user", "assistant"] -LLMResponse = Any | ModelResponse | EmbeddingResponse | ImageResponse +LLMResponse = object +_LLMResponseT: Final = TypeVar("_LLMResponseT") _LEGACY_NOMA_DEPRECATION_WARNED = False if TYPE_CHECKING: @@ -709,10 +708,10 @@ class NomaGuardrail(CustomGuardrail): async def _check_llm_response( self, request_data: dict, - response: LLMResponse, + response: _LLMResponseT, user_auth: UserAPIKeyAuth, event_type: GuardrailEventHooks | None = None, - ) -> Any: + ) -> _LLMResponseT: """Check LLM response for policy violations""" content: Final = await self._process_llm_response_check(request_data, response, user_auth, event_type) if not content: @@ -798,7 +797,7 @@ class NomaGuardrail(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: Any, + response: AsyncIterable[ModelResponseStream], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: """Process streaming response chunks with Noma guardrail.""" diff --git a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py index b31ed4b0f4a..c69b24c0553 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py +++ b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py @@ -10,6 +10,7 @@ import os from typing import TYPE_CHECKING, Any, Final, Literal import httpx +from typing_extensions import ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -33,6 +34,12 @@ BLOCKED_BY_OVALIX_FALLBACK_MESSAGE: Final = "This message was blocked by Ovalix" BLOCKED_ACTION_TYPE: Final = "block" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + supported_event_hooks: ReadOnly[list[GuardrailEventHooks]] + + class OvalixGuardrailMissingSecrets(Exception): """Raised when required Ovalix config (API base, key, application/checkpoint IDs) is missing.""" @@ -80,7 +87,7 @@ class OvalixGuardrail(CustomGuardrail): application_id: str | None = None, pre_checkpoint_id: str | None = None, post_checkpoint_id: str | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ): self._tracker_api_base = tracker_api_base or os.environ.get("OVALIX_TRACKER_API_BASE") self._tracker_api_key = tracker_api_key or os.environ.get("OVALIX_TRACKER_API_KEY") @@ -88,10 +95,9 @@ class OvalixGuardrail(CustomGuardrail): self._pre_checkpoint_id = pre_checkpoint_id or os.environ.get("OVALIX_PRE_CHECKPOINT_ID") self._post_checkpoint_id = post_checkpoint_id or os.environ.get("OVALIX_POST_CHECKPOINT_ID") - if "supported_event_hooks" not in kwargs: - kwargs["supported_event_hooks"] = [] + supported_event_hooks: Final = kwargs.get("supported_event_hooks", []) - self._validate_config(kwargs["supported_event_hooks"]) + self._validate_config(supported_event_hooks) self._tracker_headers = httpx.Headers( { @@ -103,7 +109,8 @@ class OvalixGuardrail(CustomGuardrail): self._async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) - super().__init__(**kwargs) + forwarded: Final[_CustomGuardrailOptions] = {**kwargs, "supported_event_hooks": supported_event_hooks} + super().__init__(**forwarded) verbose_proxy_logger.debug( "Ovalix Guardrail initialized: tracker=%s, application_id=%s, pre_checkpoint_id=%s, post_checkpoint_id=%s", self._tracker_api_base, diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index 9002e2aea07..df5a265bb72 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -801,7 +801,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): }, ) - def _prepare_metadata_from_request(self, data: dict[str, Any]) -> dict[str, Any]: + def _prepare_metadata_from_request(self, data: dict[str, Any]) -> dict[str, object]: """ Extract and prepare metadata from request data for PANW API call. @@ -817,7 +817,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): """ user_metadata: Final = data.get("metadata", {}) or {} requester_meta: Final = user_metadata.get("requester_metadata", {}) or {} - metadata: Final = { + metadata: Final[dict[str, object]] = { "user": data.get("user") or "litellm_user", "model": data.get("model") or "unknown", } diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index d82944c44ed..eceb54681f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -383,7 +383,7 @@ class QualifireGuardrail(CustomGuardrail): result: Final = response.json() # Extract response info for logging - qualifire_response: Final = { + qualifire_response: Final[dict[str, object]] = { "score": result.get("score"), "status": result.get("status"), } diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index 8d5923d1302..b2139779925 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -6,6 +6,7 @@ then builds a SemanticRouter for prompt matching. """ import os +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import yaml @@ -66,7 +67,7 @@ class SemanticGuardRouteLoader: cls, route_templates: list[str] | None, custom_routes_file: str | None, - custom_routes: list[dict[str, Any]] | None, + custom_routes: Sequence[Mapping[str, object]] | None, global_threshold: float = DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD, ) -> list["Route"]: """Build semantic-router Route objects from templates + custom config.""" diff --git a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py index a5945a39589..e807da7079e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py @@ -3,7 +3,7 @@ from json import JSONDecodeError from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeAlias, cast import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -18,7 +18,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.types.llms.openai import ChatCompletionToolCallChunk +from litellm.types.utils import ChatCompletionMessageToolCall, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -53,6 +54,7 @@ _METADATA_ALLOWLIST: Final = ( _FallbackMode: TypeAlias = Literal["fail_closed", "fail_open"] _MetadataValue: TypeAlias = str | int | float | Sequence[str | int | float] +_ToolCalls: TypeAlias = list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] class _AnalyzePayload(TypedDict): @@ -70,6 +72,12 @@ class _AnalysisView(TypedDict): analysis: ReadOnly[Mapping[str, object]] +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + supported_event_hooks: ReadOnly[list[GuardrailEventHooks]] + + class _AsyncPostHandler(Protocol): def post( self, @@ -93,7 +101,7 @@ class VigilGuardGuardrail(CustomGuardrail): unreachable_fallback: str | None = None, timeout: float | None = None, async_handler: _AsyncPostHandler | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: resolved_base: Final = api_base or get_secret_str("VIGIL_GUARD_URL") if not resolved_base: @@ -122,9 +130,12 @@ class VigilGuardGuardrail(CustomGuardrail): llm_provider=httpxSpecialProvider.GuardrailCallback, ) - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) + forwarded: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } - super().__init__(**kwargs) + super().__init__(**forwarded) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -264,7 +275,7 @@ class VigilGuardGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, source: str, final_texts: list[str], - final_tool_calls: Any, + final_tool_calls: _ToolCalls | None, ) -> GenericGuardrailAPIInputs: if self.unreachable_fallback == "fail_open": verbose_proxy_logger.error( diff --git a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py index 831df43692b..f4330ad6aa9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py @@ -196,9 +196,9 @@ class XecGuardGuardrail(CustomGuardrail): async def async_logging_hook( self, kwargs: dict, - result: Any, + result: object, call_type: str, - ) -> tuple[dict, Any]: + ) -> tuple[dict, object]: """Observe-only scan for logging_only mode. Never blocks, never raises - all errors are swallowed. Records a @@ -275,9 +275,9 @@ class XecGuardGuardrail(CustomGuardrail): def logging_hook( self, kwargs: dict, - result: Any, + result: object, call_type: str, - ) -> tuple[dict, Any]: + ) -> tuple[dict, object]: """Sync counterpart to ``async_logging_hook``. Runs the async version on an available loop, swallowing every @@ -433,7 +433,7 @@ class XecGuardGuardrail(CustomGuardrail): return {"role": role, "content": ""} @staticmethod - def _synthesize_user_from_inputs(inputs: Any) -> dict | None: + def _synthesize_user_from_inputs(inputs: object) -> dict | None: if not isinstance(inputs, dict): return None texts: Final = inputs.get("texts") @@ -490,7 +490,7 @@ class XecGuardGuardrail(CustomGuardrail): return None @staticmethod - def _content_to_text(content: Any) -> str | None: + def _content_to_text(content: object) -> str | None: if isinstance(content, str) and content: return content if isinstance(content, list): diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 556b6a4e919..fc237bd55c4 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -45,6 +45,7 @@ _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) _ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3}) _T = TypeVar("_T") +_MetricsRowT = TypeVar("_MetricsRowT", bound="_DailyMetricsRow") _USAGE_MAX_RANGE_DAYS: Final = 366 @@ -360,10 +361,12 @@ def _trend_from_comparison(current_fail: float, previous_fail: float) -> str: return "stable" -def _aggregate_daily_metrics(metrics: "Sequence[_DailyMetricsRow]", id_attr: str) -> Mapping[str, _MetricTotals]: +def _aggregate_daily_metrics( + metrics: "Sequence[_MetricsRowT]", id_of: "Callable[[_MetricsRowT], str]" +) -> Mapping[str, _MetricTotals]: agg: Final[dict[str, _MetricTotals]] = {} for m in metrics: - gid: str = getattr(m, id_attr) + gid: str = id_of(m) if gid not in agg: agg[gid] = {"requests": 0, "passed": 0, "blocked": 0, "flagged": 0} agg[gid]["requests"] += int(m.requests_evaluated or 0) @@ -373,10 +376,12 @@ def _aggregate_daily_metrics(metrics: "Sequence[_DailyMetricsRow]", id_attr: str return agg -def _prev_fail_rates(metrics_prev: "Sequence[_DailyMetricsRow]", id_attr: str) -> Mapping[str, float]: +def _prev_fail_rates( + metrics_prev: "Sequence[_MetricsRowT]", id_of: "Callable[[_MetricsRowT], str]" +) -> Mapping[str, float]: prev_agg_raw: Final[dict[str, _PrevPeriodCounts]] = {} for m in metrics_prev: - gid: str = getattr(m, id_attr) + gid: str = id_of(m) r, b = int(m.requests_evaluated or 0), int(m.blocked_count or 0) if gid not in prev_agg_raw: prev_agg_raw[gid] = {"req": 0, "blocked": 0} @@ -429,7 +434,7 @@ def _field_str(mapping: Mapping[str, object], key: str, default: str) -> str: return str(mapping.get(key, default)) -def _get_guardrail_attrs(g: "_DbOrConfigGuardrail") -> tuple[Any, str]: +def _get_guardrail_attrs(g: "_DbOrConfigGuardrail") -> tuple[str | None, str]: """Get (guardrail_id, display_name) from guardrail - handles Prisma model or dict.""" gid: Final = _get_guardrail_field(g, "guardrail_id") name: Final = _get_guardrail_field(g, "guardrail_name") @@ -592,8 +597,8 @@ async def guardrails_usage_overview( Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits] ] = await _find_daily_guardrail_usage_units(prisma_client, where=units_where) - agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id") - prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id") + agg: Final = _aggregate_daily_metrics(metrics, lambda m: m.guardrail_id) + prev_agg: Final = _prev_fail_rates(metrics_prev, lambda m: m.guardrail_id) units_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_counter_units) cost_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_tracked_cost) untracked_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_untracked_units) @@ -811,7 +816,7 @@ def _usage_log_entry_from_row( ) -def _snippet(text: Any, max_len: int = 200) -> str | None: +def _snippet(text: object, max_len: int = 200) -> str | None: if text is None: return None if isinstance(text, str): @@ -964,8 +969,8 @@ async def policies_usage_overview( } }, ) - agg: Final = _aggregate_daily_metrics(metrics, "policy_id") - prev_agg: Final = _prev_fail_rates(metrics_prev, "policy_id") + agg: Final = _aggregate_daily_metrics(metrics, lambda m: m.policy_id) + prev_agg: Final = _prev_fail_rates(metrics_prev, lambda m: m.policy_id) chart: Final = _chart_from_metrics(metrics) total_requests: Final = sum(a["requests"] for a in agg.values()) total_blocked: Final = sum(a["blocked"] for a in agg.values()) diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index b313cb64c3f..d41acadc4dd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any @@ -75,7 +75,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): current: dict | None, request_count_api_key: str, rate_limit_type: Literal["key", "model_per_key", "user", "customer", "team"], - values_to_update_in_cache: list[tuple[Any, Any]], + values_to_update_in_cache: list[tuple[str, object]], ) -> dict: verbose_proxy_logger.info("Current Usage of %s in this minute: %s", rate_limit_type, current) if current is None: @@ -266,7 +266,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): rpm_limit = sys.maxsize values_to_update_in_cache: list[ - tuple[Any, Any] + tuple[str, object] ] = [] # values that need to get updated in cache, will run a batch_set_cache after this function # ------------ diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index b38fb856215..08e8e4f8c10 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -2,7 +2,7 @@ import asyncio import traceback from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import litellm from litellm._logging import verbose_proxy_logger @@ -659,18 +659,36 @@ def _get_request_tags_for_cost_tracking( return None +class _IncrementSpendCounters(Protocol): + """The ``increment_spend_counters`` coroutine :func:`_update_database_and_spend_counters` awaits.""" + + async def __call__( + self, + token: str | None, + team_id: str | None, + user_id: str | None, + response_cost: float | None, + org_id: str | None = None, + budget_reservation: dict[str, object] | None = None, + end_user_id: str | None = None, + tags: list[str] | None = None, + request_started_at: datetime | None = None, + model_access_groups: Sequence[str] | None = None, + ) -> None: ... + + async def _update_database_and_spend_counters( proxy_logging_obj: "ProxyLogging", - increment_spend_counters: Any, + increment_spend_counters: _IncrementSpendCounters, user_api_key: str | None, user_id: str | None, end_user_id: str | None, team_id: str | None, org_id: str | None, kwargs: dict, - completion_response: litellm.ModelResponse | Any | None, - start_time: Any, - end_time: Any, + completion_response: object, + start_time: datetime | None, + end_time: datetime | None, response_cost: float, budget_reservation: dict | None, request_tags: list[str] | None = None, diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 768da79451f..d8054a8dc4a 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -8,7 +8,6 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone from itertools import chain, groupby -from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol from uuid import uuid4 @@ -1283,6 +1282,10 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: ) +def _leg_group_id(leg: "_LegRow") -> str: + return leg.group_id + + class _LegRow(BaseModel): """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is one target's leg of a job; the legs of a job share group_id and identical config, @@ -1787,10 +1790,7 @@ async def list_shadow_eval_jobs( or () ) by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType( - { - group_id: tuple(group) - for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id")) - } + {group_id: tuple(group) for group_id, group in groupby(sorted(legs, key=_leg_group_id), key=_leg_group_id)} ) newest_first: Final = sorted( by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 5a19d743105..bf8e7bc15fb 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -19,6 +19,7 @@ from litellm.proxy.spend_tracking.key_metadata_recovery import ( ) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled from litellm.proxy.utils import PrismaClient +from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import DeletedVerificationTokenRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, @@ -1305,8 +1306,10 @@ async def get_daily_activity( include_current_utc_day=include_current_utc_day, ) + spend_table: Final[TableActions[DailySpendRecord]] = getattr(prisma_client.db, table_name) + # Get total count for pagination - total_count: Final[int] = await getattr(prisma_client.db, table_name).count(where=where_conditions) + total_count: Final[int] = await spend_table.count(where=where_conditions) # Fetch paginated results. # ``date`` alone is not a unique sort key -- a busy tenant has many @@ -1318,7 +1321,7 @@ async def get_daily_activity( # total. Adding ``id`` (the row's UUID primary key, present on both # LiteLLM_DailyUserSpend and LiteLLM_DailyTeamSpend) as a tiebreaker # gives every page a stable cursor (#30164). - daily_spend_data: Final[Sequence[DailySpendRecord]] = await getattr(prisma_client.db, table_name).find_many( + daily_spend_data: Final[Sequence[DailySpendRecord]] = await spend_table.find_many( where=where_conditions, order=[ {"date": "desc"}, diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index a48130a4f22..e960bdfe337 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -540,7 +540,7 @@ async def get_all_access_groups_from_db( deployments: Final = await ModelRepository(prisma_client).table.find_many() # Build access group map - access_group_map: Final[dict[str, dict[str, Any]]] = {} + model_names_by_group: Final[dict[str, list[str]]] = {} for deployment in deployments: model_info = deployment.model_info or {} @@ -550,25 +550,20 @@ async def get_all_access_groups_from_db( model_name = deployment.model_name for access_group in access_groups: - if access_group not in access_group_map: - access_group_map[access_group] = { - "model_names": set(), - "deployment_count": 0, - } + if access_group not in model_names_by_group: + model_names_by_group[access_group] = [] - access_group_map[access_group]["model_names"].add(model_name) - access_group_map[access_group]["deployment_count"] += 1 + model_names_by_group[access_group].append(model_name) # Convert to AccessGroupInfo objects - result: Final = {} - for access_group, data in access_group_map.items(): - result[access_group] = AccessGroupInfo( + return { + access_group: AccessGroupInfo( access_group=access_group, - model_names=sorted(list(data["model_names"])), - deployment_count=data["deployment_count"], + model_names=sorted(frozenset(model_names)), + deployment_count=len(model_names), ) - - return result + for access_group, model_names in model_names_by_group.items() + } @router.post( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 2ff77b5008b..10a0a2f3104 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -654,7 +654,6 @@ async def _auto_router_capability_slot( ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" -_REQUIRED_RATE_LIMIT_FIELDS: Final = ("rpm", "tpm") def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLMParams, enforced: bool) -> None: @@ -669,8 +668,8 @@ def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLM return missing: Final = tuple( field - for field in _REQUIRED_RATE_LIMIT_FIELDS - if (value := getattr(litellm_params, field)) is None or value <= 0 + for field, value in (("rpm", litellm_params.rpm), ("tpm", litellm_params.tpm)) + if value is None or value <= 0 ) if not missing: return diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index ac13b6150b7..4091d69e44e 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -517,7 +517,7 @@ async def delete_team_callback( raise _callback_error(404, f"callback_name = {callback_name} is not registered for team_id = {team_id}.") updated_metadata: Final = {**team_metadata, "logging": remaining_callbacks} # mutable-ok: persisted as JSON - encrypted_metadata: Final = encrypt_callback_vars(updated_metadata) + encrypted_metadata: Final[object] = encrypt_callback_vars(updated_metadata) team_metadata_json: Final = json.dumps(encrypted_metadata) updated_team: Final = await TeamRepository(prisma_client).table.update( @@ -654,8 +654,8 @@ async def disable_team_logging( # _get_dynamic_logging_metadata stops at metadata["logging"], where the API # and Admin UI register callbacks, without ever reading callback_settings. team_metadata["logging"] = [] # mutable-ok: the disabled state is persisted as an empty JSON array - team_metadata = encrypt_callback_vars(team_metadata) - team_metadata_json: Final = json.dumps(team_metadata) + encrypted_metadata: Final[object] = encrypt_callback_vars(team_metadata) + team_metadata_json: Final = json.dumps(encrypted_metadata) # Update team in database updated_team: Final = await TeamRepository(prisma_client).table.update( @@ -687,7 +687,7 @@ async def disable_team_logging( await _emit_team_callback_audit_log( team_id=team_id, before_metadata=before_metadata, - after_metadata=team_metadata, + after_metadata=encrypted_metadata, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, ) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index da4ddbd0aac..1265da99d89 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -6,7 +6,7 @@ usage/spend data by querying the aggregated daily activity endpoints. import json from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import date -from typing import Any, Final, Literal, NamedTuple, Protocol, cast, overload +from typing import Final, Literal, NamedTuple, Protocol, cast, overload from typing_extensions import ReadOnly, TypedDict @@ -16,6 +16,7 @@ from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) +from litellm.types.utils import ChatCompletionMessageToolCall # --------------------------------------------------------------------------- # Constants @@ -489,19 +490,19 @@ async def _execute_tool_call( async def _process_tool_call( - tc: Any, + tc: ChatCompletionMessageToolCall, chat_messages: list[Mapping[str, object]], user_id: str | None, is_admin: bool, ) -> AsyncIterator[str]: """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final[str] = tc.function.name + fn_name: Final = tc.function.name fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) + handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None - if fn_name not in allowed_names or not handler: + if fn_name is None or fn_name not in allowed_names or not handler: chat_messages.append( { "role": "tool", diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index 981581919e4..0f997fcd745 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -2,7 +2,7 @@ import json from collections.abc import Mapping -from typing import Any, Final, cast +from typing import Final, cast import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, UploadFile @@ -40,7 +40,7 @@ def _build_document_from_upload( ) -def _with_request_format(data: Mapping[str, Any], request: Request) -> Mapping[str, Any]: +def _with_request_format(data: Mapping[str, object], request: Request) -> Mapping[str, object]: """ Resolve the requested response format from the body or the `x-req-format` header. @@ -82,7 +82,7 @@ def _native_response(response: object, fastapi_response: Response) -> Response | ) -async def _parse_multipart_form(request: Request) -> dict[str, Any]: +async def _parse_multipart_form(request: Request) -> dict[str, object]: """ Extract OCR data from a multipart form request. @@ -124,7 +124,7 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: content_type=uploaded_file.content_type, ) - data: Final[dict[str, Any]] = {"document": document} + data: Final[dict[str, object]] = {"document": document} for field_name, field_value in form.items(): if field_name in ("file", "document"): @@ -148,12 +148,12 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: return data -async def _parse_ocr_request(request: Request) -> Mapping[str, Any]: +async def _parse_ocr_request(request: Request) -> Mapping[str, object]: """Parse an OCR request and apply the `x-req-format` header, if any.""" return _with_request_format(await _parse_ocr_request_body(request), request) -async def _parse_ocr_request_body(request: Request) -> dict[str, Any]: +async def _parse_ocr_request_body(request: Request) -> dict[str, object]: """ Parse an OCR request, supporting both JSON and multipart form data. @@ -314,7 +314,7 @@ async def ocr( # Process request using ProxyBaseLLMRequestProcessing processor = ProxyBaseLLMRequestProcessing(data=data) - response: Final = await processor.base_process_llm_request( + response: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 9f12b6faa61..ea2cad558c2 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -575,8 +575,8 @@ async def create_file( # Parse expires_after if provided expires_after: FileExpiresAfter | None = None form_data_raw: Final = await request.form() - form_data_dict: Final[dict[str, Any]] = dict(form_data_raw) - extracted_litellm_metadata: Final[dict[str, Any] | None] = extract_nested_form_metadata( + form_data_dict: Final[Mapping[str, object]] = dict(form_data_raw) + extracted_litellm_metadata: Final[Mapping[str, object] | None] = extract_nested_form_metadata( form_data=form_data_dict, prefix="litellm_metadata[" ) expires_after_anchor: Final = form_data_raw.get("expires_after[anchor]") diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index 66dbcd87c0b..53f48d93aa2 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -7,7 +7,7 @@ storage backends (e.g., Azure Blob Storage) and managing associated metadata. import base64 import time -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import Any, Final, cast from litellm._logging import verbose_proxy_logger @@ -18,7 +18,7 @@ from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose -from litellm.types.utils import SpecialEnums +from litellm.types.utils import ExtractedFileData, SpecialEnums class StorageBackendFileService: @@ -34,7 +34,7 @@ class StorageBackendFileService: @staticmethod async def upload_file_to_storage_backend( - file_data: Mapping[str, Any], + file_data: ExtractedFileData, target_storage: str, target_model_names: Sequence[str], purpose: OpenAIFilesPurpose, @@ -183,7 +183,7 @@ class StorageBackendFileService: @staticmethod def _create_unified_file_id( - file_type: str, + file_type: str | None, target_model_names: Sequence[str], file_id: str, ) -> str: @@ -213,7 +213,7 @@ class StorageBackendFileService: @staticmethod async def _store_in_managed_files( file_object: OpenAIFileObject, - file_data: Mapping[str, Any], + file_data: ExtractedFileData, target_model_names: Sequence[str], target_storage: str, storage_url: str, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index cd226e80c6e..48c1ced47ae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio optional_str, request_tags_from_metadata, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import ( Choices, EmbeddingResponse, @@ -52,8 +53,6 @@ else: PassThroughEndpointLogging = Any LiteLLMBatch = Any -EndpointType = Any - _VERTEX_INTERACTIONS_PATH: Final = re.compile(r"/projects/[^/]+/locations/[^/]+/interactions/?$") _INTERACTIONS_RESPONSE_BODY: Final = TypeAdapter(dict[str, object]) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 73ab7e5213f..36b7a3a4a8a 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1365,7 +1365,7 @@ async def _read_ws_model_from_first_frame( return model, first_message -def _extract_model_from_first_ws_event(first_event: Any) -> str | None: +def _extract_model_from_first_ws_event(first_event: object) -> str | None: """Extract model from a response.create WS event, handling flat and nested formats. Flat: {"type": "response.create", "model": "gpt-4o", ...} diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index 0e6412a2c64..b8af432029f 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -31,11 +31,31 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.ptu_pricing import ptu_terms from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled +from litellm.repositories.model_repository import ModelRepository +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository if TYPE_CHECKING: + from prisma import models as prisma_models + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient + +class _DailyTeamSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyTeamSpend"]): + table_name = "litellm_dailyteamspend" + + +def _daily_team_spend_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_DailyTeamSpend]": + """The sentinel rows this rollup writes, reads back and prunes.""" + return _DailyTeamSpendRepository(prisma_client).table + + +def _proxy_model_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_ProxyModelTable]": + """The stored deployments the rollup scans for PTU config.""" + return ModelRepository(prisma_client).table + + _HOURS_PER_DAY: Final = 24 _PRUNE_ID_CHUNK_SIZE: Final = 5_000 _UPSERT_ATTEMPTS: Final = 3 @@ -97,7 +117,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """ if isinstance(raw, str): try: - decoded: Final = json.loads(raw) + decoded: Final[object] = json.loads(raw) except (TypeError, ValueError): return None return decoded if isinstance(decoded, dict) else None @@ -240,7 +260,7 @@ async def _upsert_ptu_daily_row( } } now: Final = datetime.now(timezone.utc) - await prisma_client.db.litellm_dailyteamspend.upsert( + await _daily_team_spend_table(prisma_client).upsert( where=where, data={ # mutable-ok: prisma upsert data payload "create": { # mutable-ok: prisma create payload @@ -353,7 +373,7 @@ async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | No The router is handed in rather than read off the proxy module, so a run prices exactly the deployments its caller declares and nothing a co-resident process left behind. """ - rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many() + rows: Final = await _proxy_model_table(prisma_client).find_many() db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or ""))) config_records: Final = _config_deployments(router, owned_by_db=db_ids) models: Final = tuple( @@ -503,7 +523,7 @@ async def _existing_sentinel_keys( survives a rename. Nothing here reads the display name. """ date_range: Final = {"gte": start.isoformat(), "lte": end.isoformat()} # mutable-ok: prisma range filter - rows: Final = await prisma_client.db.litellm_dailyteamspend.find_many( + rows: Final = await _daily_team_spend_table(prisma_client).find_many( where={"api_key": PTU_SENTINEL_API_KEY, "date": date_range} # mutable-ok: prisma find filter ) return frozenset( @@ -771,7 +791,7 @@ async def _prune_unrefreshed_sentinel_rows( ) filters: Final = tuple(_prune_filter(date_str=date_str, cutoff=cutoff, chunk=chunk) for chunk in chunks) deletions: Final = tuple( - [await prisma_client.db.litellm_dailyteamspend.delete_many(where=where) for where in filters] + [await _daily_team_spend_table(prisma_client).delete_many(where=where) for where in filters] ) deleted: Final = sum(deletions) if deleted: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c6ea360858b..de5f545f109 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4052,7 +4052,7 @@ class _ConfigRow: __slots__ = ("param_name", "param_value") - def __init__(self, param_name: str, param_value: Any) -> None: + def __init__(self, param_name: str, param_value: object) -> None: self.param_name = param_name self.param_value = param_value @@ -4065,7 +4065,7 @@ def _pack_config_row(row: Any) -> dict[str, object]: return {"param_name": row.param_name, "param_value": row.param_value} -def _unpack_config_row(cached: Any) -> _ConfigRow | None: +def _unpack_config_row(cached: object) -> _ConfigRow | None: if cached is None or cached == _CONFIG_CACHE_MISS: return None if isinstance(cached, dict): @@ -4218,6 +4218,7 @@ class PrismaClient: verbose_proxy_logger.debug("Creating Prisma Client..") try: from prisma import Prisma + from prisma.types import DatasourceOverride except Exception as e: verbose_proxy_logger.error("Failed to import Prisma client: %s", e) verbose_proxy_logger.error("This usually means 'prisma generate' hasn't been run yet.") @@ -4272,11 +4273,11 @@ class PrismaClient: token_refresh_params_from_url(read_replica_url), ) os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url - reader_kwargs: Final[dict[str, Any]] = {"datasource": {"url": read_replica_url}} + reader_datasource: Final = DatasourceOverride(url=read_replica_url) if http_client is not None: - reader_prisma = Prisma(http=http_client, **reader_kwargs) + reader_prisma = Prisma(http=http_client, datasource=reader_datasource) else: - reader_prisma = Prisma(**reader_kwargs) + reader_prisma = Prisma(datasource=reader_datasource) reader_wrapper: Final = PrismaWrapper( original_prisma=reader_prisma, token_auth=token_auth, diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 1feda0b0bb5..f21c294e5a2 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -244,7 +244,7 @@ async def vector_store_create( ) # Create vector store across multiple models - response: Final = await managed_vector_stores.acreate_vector_store( + response: Final[object] = await managed_vector_stores.acreate_vector_store( create_request=data, llm_router=llm_router, target_model_names_list=target_model_names_list, diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 0ca2c4c8865..2fb6813a471 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -88,7 +88,7 @@ def _redact_sensitive_litellm_params(litellm_params: object, _depth: int = 0) -> return None if isinstance(litellm_params, str): try: - parsed: Final = json.loads(litellm_params) + parsed: Final[object] = json.loads(litellm_params) except (TypeError, ValueError): return REDACTED_BY_LITELM_STRING return json.dumps(_redact_sensitive_litellm_params(parsed, _depth + 1)) @@ -589,7 +589,8 @@ async def update_vector_store( try: update_data: Final = data.model_dump(exclude_unset=True) - vector_store_id: Final[str] = update_data.pop("vector_store_id") + vector_store_id: Final[str] = data.vector_store_id + update_data.pop("vector_store_id") # Per-store access control: anyone authenticated who passes the # premium-feature gate could otherwise update *any* vector store — diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 97367e59023..8a8e43abd79 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -526,7 +526,7 @@ async def vector_store_file_create( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -729,7 +729,7 @@ async def vector_store_file_retrieve( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -836,7 +836,7 @@ async def vector_store_file_content( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -946,7 +946,7 @@ async def vector_store_file_update( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -1053,7 +1053,7 @@ async def vector_store_file_delete( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 66071c05b4f..fe966c2e31a 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -89,7 +89,7 @@ async def video_generation( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + generated: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -114,6 +114,8 @@ async def video_generation( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return generated @router.get( @@ -174,7 +176,7 @@ async def video_list( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + listed: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -199,6 +201,8 @@ async def video_list( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return listed @router.get( @@ -272,7 +276,7 @@ async def video_status( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + status: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -297,6 +301,8 @@ async def video_status( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return status @router.get( @@ -478,7 +484,7 @@ async def video_remix( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + remixed: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -503,6 +509,8 @@ async def video_remix( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return remixed @router.post( @@ -571,7 +579,7 @@ async def video_create_character( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -678,7 +686,7 @@ async def video_get_character( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -789,7 +797,7 @@ async def video_edit( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + edited: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -814,6 +822,8 @@ async def video_edit( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return edited @router.post( @@ -884,7 +894,7 @@ async def video_extension( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + extended: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -909,3 +919,5 @@ async def video_extension( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return extended diff --git a/litellm/proxy_auth/credentials.py b/litellm/proxy_auth/credentials.py index a4e29241959..f8814954a7a 100644 --- a/litellm/proxy_auth/credentials.py +++ b/litellm/proxy_auth/credentials.py @@ -7,7 +7,7 @@ It follows the same TokenCredential protocol used by Azure SDK. import time from dataclasses import dataclass -from typing import Any, Final, Protocol, runtime_checkable +from typing import Final, Protocol, runtime_checkable @dataclass @@ -50,6 +50,22 @@ class TokenCredential(Protocol): ... +class _AzureAccessToken(Protocol): + """The two attributes :class:`AzureADCredential` reads off an azure-identity token.""" + + @property + def token(self) -> str: ... + + @property + def expires_on(self) -> int: ... + + +class _AzureTokenCredential(Protocol): + """The single method :class:`AzureADCredential` calls on the credential it wraps.""" + + def get_token(self, *scopes: str) -> _AzureAccessToken: ... + + class AzureADCredential: """ Wrapper for Azure Identity credentials. @@ -71,7 +87,7 @@ class AzureADCredential: cred = AzureADCredential(credential=azure_cred) """ - def __init__(self, credential: Any | None = None): + def __init__(self, credential: _AzureTokenCredential | None = None): """ Initialize with an optional Azure credential. @@ -79,7 +95,7 @@ class AzureADCredential: credential: An azure-identity credential object. If None, DefaultAzureCredential will be used on first token request. """ - self._credential: Any = credential + self._credential: _AzureTokenCredential | None = credential self._initialized = credential is not None def get_token(self, scope: str) -> AccessToken: @@ -95,20 +111,30 @@ class AzureADCredential: Raises: ImportError: If azure-identity is not installed. """ - if not self._initialized: - try: - from azure.identity import DefaultAzureCredential - - self._credential = DefaultAzureCredential() - self._initialized = True - except ImportError: - raise ImportError( - "azure-identity is required for AzureADCredential. Install it with: pip install azure-identity" - ) - - result: Final = self._credential.get_token(scope) + result: Final = self._resolve_credential().get_token(scope) return AccessToken(token=result.token, expires_on=result.expires_on) + def _resolve_credential(self) -> _AzureTokenCredential: + """Return the wrapped credential, building the Azure default chain on first use. + + Raises: + ImportError: If azure-identity is not installed. + """ + existing: Final = self._credential + if existing is not None: + return existing + try: + from azure.identity import DefaultAzureCredential + + created: Final = DefaultAzureCredential() + except ImportError: + raise ImportError( + "azure-identity is required for AzureADCredential. Install it with: pip install azure-identity" + ) + self._credential = created + self._initialized = True + return created + class GenericOAuth2Credential: """ @@ -228,7 +254,7 @@ class ProxyAuthHandler: self._cached_token = self.credential.get_token(self.scope) return self._cached_token - def get_auth_headers(self) -> dict: + def get_auth_headers(self) -> dict[str, str]: """ Get HTTP headers for authentication. diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 73a0159fc9f..1cf5db549e4 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Final, cast from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, + header_value, httpxSpecialProvider, ) from litellm.llms.gemini.common_utils import GeminiModelInfo @@ -277,7 +278,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): raise Exception(error_msg) verbose_logger.debug("Initiate resumable upload response: %s", response.headers) # Extract upload URL from response headers - upload_url: Final = response.headers.get("x-goog-upload-url") + upload_url: Final = header_value(response.headers, "x-goog-upload-url") if not upload_url: raise Exception("No upload URL returned in response headers") diff --git a/litellm/rag/rag_query.py b/litellm/rag/rag_query.py index 255faf94402..9325547c17d 100644 --- a/litellm/rag/rag_query.py +++ b/litellm/rag/rag_query.py @@ -124,9 +124,9 @@ class RAGQuery: @staticmethod def extract_documents_from_search( search_response: Any, - ) -> list[str | dict[str, Any]]: + ) -> list[str | dict[str, object]]: """Extract text documents from vector store search response.""" - documents: Final[list[str | dict[str, Any]]] = [] + documents: Final[list[str | dict[str, object]]] = [] search_data: Final[_SearchDataView] = {"results": search_response.get("data", [])} for result in search_data["results"]: content_list = result.get("content", []) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 0e83edab5e1..acc42c44c04 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -55,11 +55,11 @@ bedrock_realtime: Final = BedrockRealtime() xai_realtime: Final = XAIRealtime() vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() -_EMPTY_MODEL_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) -def _model_params_with_stored_credentials(model_params: Mapping[str, Any]) -> Mapping[str, Any]: +def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: credential_name: Final = model_params.get("litellm_credential_name") credential_values: Final = ( CredentialAccessor.get_credential_values(credential_name) diff --git a/litellm/repositories/budget_repository.py b/litellm/repositories/budget_repository.py index 62632ffb5f6..205646c8393 100644 --- a/litellm/repositories/budget_repository.py +++ b/litellm/repositories/budget_repository.py @@ -2,7 +2,8 @@ Budget repository for database operations on LiteLLM_BudgetTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.budget import LiteLLM_BudgetTable from litellm.repositories.base_repository import BaseRepository @@ -12,12 +13,27 @@ if TYPE_CHECKING: from prisma import models as prisma_models +class _BudgetDb(Protocol): + """The single Prisma table this repository reaches for on ``prisma_client.db``.""" + + @property + def litellm_budgettable(self) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: ... + + +class _PrismaClientView(Protocol): + """The one attribute this repository reads off the untyped Prisma client wrapper.""" + + @property + def db(self) -> _BudgetDb: ... + + class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): """Repository for budget database operations.""" @property def table(self) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: - return self.prisma_client.db.litellm_budgettable + client: Final[_PrismaClientView] = self.prisma_client + return client.db.litellm_budgettable @property def model_class(self) -> type[LiteLLM_BudgetTable]: @@ -34,12 +50,12 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): max_parallel_requests: int | None = None, tpm_limit: int | None = None, rpm_limit: int | None = None, - model_max_budget: dict[str, Any] | None = None, + model_max_budget: Mapping[str, object] | None = None, budget_duration: str | None = None, allowed_models: list[str] | None = None, ) -> LiteLLM_BudgetTable: """Create a new budget record.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "created_by": created_by, "updated_by": created_by, } @@ -71,12 +87,12 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): max_parallel_requests: int | None = None, tpm_limit: int | None = None, rpm_limit: int | None = None, - model_max_budget: dict[str, Any] | None = None, + model_max_budget: Mapping[str, object] | None = None, budget_duration: str | None = None, allowed_models: list[str] | None = None, ) -> LiteLLM_BudgetTable | None: """Update an existing budget record.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if max_budget is not None: data["max_budget"] = max_budget if soft_budget is not None: diff --git a/litellm/repositories/model_repository.py b/litellm/repositories/model_repository.py index d24eb8ffc62..8ee76b93923 100644 --- a/litellm/repositories/model_repository.py +++ b/litellm/repositories/model_repository.py @@ -4,29 +4,23 @@ Model repository for database operations on LiteLLM_ProxyModelTable. import json from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final from litellm.models.model import LiteLLM_ProxyModelTable -from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) from litellm.repositories.base_repository import BaseRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository if TYPE_CHECKING: from prisma import models as prisma_models -class _PrismaModelDb(Protocol): - @property - def litellm_proxymodeltable(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]: ... - - -class _PrismaClientView(Protocol): - @property - def db(self) -> _PrismaModelDb: ... +class _ProxyModelTableRepository(PrismaTableRepository["prisma_models.LiteLLM_ProxyModelTable"]): + table_name = "litellm_proxymodeltable" class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @@ -38,11 +32,7 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @property def table(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]: - client: Final[_PrismaClientView] = self.prisma_client - return wrap_table_actions_for_config_sync( - actions=client.db.litellm_proxymodeltable, - table_name="litellm_proxymodeltable", - ) + return _ProxyModelTableRepository(self._prisma_client).table @property def model_class(self) -> type[LiteLLM_ProxyModelTable]: diff --git a/litellm/repositories/organization_repository.py b/litellm/repositories/organization_repository.py index 5a9bd3724e0..47eb8f4a609 100644 --- a/litellm/repositories/organization_repository.py +++ b/litellm/repositories/organization_repository.py @@ -2,7 +2,8 @@ Organization repository for database operations on LiteLLM_OrganizationTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.organization import LiteLLM_OrganizationTable from litellm.repositories.base_repository import BaseRepository @@ -12,12 +13,27 @@ if TYPE_CHECKING: from prisma import models as prisma_models +class _OrganizationDb(Protocol): + """The single Prisma table this repository reaches for on ``prisma_client.db``.""" + + @property + def litellm_organizationtable(self) -> TableActions["prisma_models.LiteLLM_OrganizationTable"]: ... + + +class _PrismaClientView(Protocol): + """The one attribute this repository reads off the untyped Prisma client wrapper.""" + + @property + def db(self) -> _OrganizationDb: ... + + class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): """Repository for organization database operations.""" @property def table(self) -> TableActions["prisma_models.LiteLLM_OrganizationTable"]: - return self.prisma_client.db.litellm_organizationtable + client: Final[_PrismaClientView] = self.prisma_client + return client.db.litellm_organizationtable @property def model_class(self) -> type[LiteLLM_OrganizationTable]: @@ -39,12 +55,12 @@ class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): budget_id: str, created_by: str, organization_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_OrganizationTable: """Create a new organization.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "organization_alias": organization_alias, "budget_id": budget_id, "created_by": created_by, @@ -67,12 +83,12 @@ class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): updated_by: str, organization_alias: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_OrganizationTable | None: """Update an organization.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if organization_alias is not None: data["organization_alias"] = organization_alias if budget_id is not None: diff --git a/litellm/repositories/project_repository.py b/litellm/repositories/project_repository.py index 48e55efd258..905e813f35e 100644 --- a/litellm/repositories/project_repository.py +++ b/litellm/repositories/project_repository.py @@ -2,7 +2,8 @@ Project repository for database operations on LiteLLM_ProjectTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm.models.project import LiteLLM_ProjectTable from litellm.repositories.base_repository import BaseRepository @@ -43,14 +44,14 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): description: str | None = None, team_id: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, model_rpm_limit: dict[str, int] | None = None, model_tpm_limit: dict[str, int] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_ProjectTable: """Create a new project.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "created_by": created_by, "updated_by": created_by, } @@ -85,7 +86,7 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): description: str | None = None, team_id: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, model_rpm_limit: dict[str, int] | None = None, model_tpm_limit: dict[str, int] | None = None, @@ -93,7 +94,7 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): object_permission_id: str | None = None, ) -> LiteLLM_ProjectTable | None: """Update a project.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if project_alias is not None: data["project_alias"] = project_alias if description is not None: diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py index 5ff07d76b5d..cbe263699c9 100644 --- a/litellm/repositories/team_repository.py +++ b/litellm/repositories/team_repository.py @@ -5,6 +5,7 @@ Team repository for database operations on LiteLLM_TeamTable. import json from collections.abc import Mapping, Sequence from datetime import datetime +from types import TracebackType from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter @@ -40,6 +41,36 @@ def _team_arrays(team: LiteLLM_TeamTable) -> _TeamArrays: return team +class _TeamTables(Protocol): + """The two team tables this repository reads and writes.""" + + @property + def litellm_teamtable(self) -> TableActions["prisma_models.LiteLLM_TeamTable"]: ... + + @property + def litellm_deletedteamtable(self) -> TableActions["prisma_models.LiteLLM_DeletedTeamTable"]: ... + + +class _TeamTransactionManager(Protocol): + async def __aenter__(self) -> _TeamTables: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + +class _PrismaTeamDb(_TeamTables, Protocol): + def tx(self) -> _TeamTransactionManager: ... + + +class _PrismaClientView(Protocol): + @property + def db(self) -> _PrismaTeamDb: ... + + _MEMBERS_WITH_ROLES_ADAPTER: Final = TypeAdapter(list[Member]) _JSON_ENCODED_TEAM_FIELDS: Final = ( "metadata", @@ -54,13 +85,18 @@ _JSON_ENCODED_TEAM_FIELDS: Final = ( class TeamRepository(BaseRepository[LiteLLM_TeamTable]): """Repository for team database operations.""" + @property + def _db(self) -> _PrismaTeamDb: + client: Final[_PrismaClientView] = self.prisma_client + return client.db + @property def table(self) -> TableActions["prisma_models.LiteLLM_TeamTable"]: - return self.prisma_client.db.litellm_teamtable + return self._db.litellm_teamtable @property def deleted_table(self) -> TableActions["prisma_models.LiteLLM_DeletedTeamTable"]: - return self.prisma_client.db.litellm_deletedteamtable + return self._db.litellm_deletedteamtable @property def model_class(self) -> type[LiteLLM_TeamTable]: @@ -256,7 +292,7 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]): archive_data["litellm_changed_by"] = litellm_changed_by archive_data["deleted_at"] = datetime.utcnow() - async with self.prisma_client.db.tx() as tx: + async with self._db.tx() as tx: await tx.litellm_deletedteamtable.create(data=archive_data) await tx.litellm_teamtable.delete(where={"team_id": team_id}) diff --git a/litellm/repositories/verification_token_repository.py b/litellm/repositories/verification_token_repository.py index c0e59f9b975..d02c2114136 100644 --- a/litellm/repositories/verification_token_repository.py +++ b/litellm/repositories/verification_token_repository.py @@ -5,7 +5,8 @@ VerificationToken repository for database operations on LiteLLM_VerificationToke import json from collections.abc import Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Final +from types import TracebackType +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.verification_token import ( LiteLLM_VerificationToken, @@ -25,7 +26,36 @@ if TYPE_CHECKING: LiteLLM_VerificationToken as PrismaVerificationToken, ) - from litellm.proxy.utils import PrismaClient + +class _VerificationTokenTables(Protocol): + """The two verification token tables this repository reads and writes.""" + + @property + def litellm_verificationtoken(self) -> TableActions["PrismaVerificationToken"]: ... + + @property + def litellm_deletedverificationtoken(self) -> TableActions["PrismaDeletedVerificationToken"]: ... + + +class _VerificationTokenTransactionManager(Protocol): + async def __aenter__(self) -> _VerificationTokenTables: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + +class _PrismaVerificationTokenDb(_VerificationTokenTables, Protocol): + def tx(self) -> _VerificationTokenTransactionManager: ... + + +class _PrismaClientView(Protocol): + @property + def db(self) -> _PrismaVerificationTokenDb: ... + _JSON_ENCODED_TOKEN_FIELDS: Final = ( "aliases", @@ -44,17 +74,17 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): """Repository for verification token (API key) database operations.""" @property - def prisma_client(self) -> "PrismaClient": - prisma_client: Final[PrismaClient] = super().prisma_client - return prisma_client + def _db(self) -> _PrismaVerificationTokenDb: + client: Final[_PrismaClientView] = self.prisma_client + return client.db @property def table(self) -> TableActions["PrismaVerificationToken"]: - return self.prisma_client.db.litellm_verificationtoken + return self._db.litellm_verificationtoken @property def deleted_table(self) -> TableActions["PrismaDeletedVerificationToken"]: - return self.prisma_client.db.litellm_deletedverificationtoken + return self._db.litellm_deletedverificationtoken @property def model_class(self) -> type[LiteLLM_VerificationToken]: @@ -325,7 +355,7 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): archive_data["litellm_changed_by"] = litellm_changed_by archive_data["deleted_at"] = datetime.utcnow() - async with self.prisma_client.db.tx() as tx: + async with self._db.tx() as tx: await tx.litellm_deletedverificationtoken.create(data=archive_data) await tx.litellm_verificationtoken.delete(where={"token": token}) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 10cb615dd08..88a2b92c680 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1268,14 +1268,14 @@ class LiteLLM_Proxy_MCP_Handler: return tool_execution_events @staticmethod - def _prepare_initial_call_params(call_params: dict[str, Any], should_auto_execute: bool) -> dict[str, Any]: + def _prepare_initial_call_params(call_params: Mapping[str, object], should_auto_execute: bool) -> dict[str, Any]: """ Prepare call parameters for the initial LLM call. For auto-execute scenarios, we need to disable streaming for the initial call so we can process the tool calls before streaming the final response. """ - initial_params: Final = call_params.copy() + initial_params: Final = dict(call_params) if should_auto_execute: # Disable streaming for initial call when auto-executing tools @@ -1284,14 +1284,16 @@ class LiteLLM_Proxy_MCP_Handler: return initial_params @staticmethod - def _prepare_follow_up_call_params(call_params: dict[str, Any], original_stream_setting: bool) -> dict[str, Any]: + def _prepare_follow_up_call_params( + call_params: Mapping[str, object], original_stream_setting: bool + ) -> dict[str, Any]: """ Prepare call parameters for the follow-up LLM call after tool execution. Restores the original streaming setting and removes tool_choice since we're now providing tool results, not requesting tool calls. """ - follow_up_params: Final = call_params.copy() + follow_up_params: Final = dict(call_params) # Restore original streaming setting for follow-up call follow_up_params["stream"] = original_stream_setting diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 195214b077c..59655800af6 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2627,7 +2627,7 @@ class ManagedResponsesWebSocketHandler: await self.websocket.send_text(serialized) @staticmethod - def _build_base_call_kwargs(msg_obj: _MutableJsonObject) -> dict[str, Any]: + def _build_base_call_kwargs(msg_obj: _MutableJsonObject) -> dict[str, object]: """ Extract Responses API params from the event, handling both wire formats: Nested: {"type": "response.create", "response": {"input": [...], ...}} diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 7f376b46a8d..9aa881fc4c9 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -16,6 +16,7 @@ from __future__ import annotations import asyncio import time from collections import OrderedDict +from collections.abc import Mapping from dataclasses import asdict, dataclass from typing import Any, Final, cast @@ -122,7 +123,7 @@ class AdaptiveRouter: prefs = self.model_to_prefs.get(model) or _default_prefs() self._cells[(rt, model)] = initial_cell(prefs, rt) - async def load_state_from_db(self, prisma_client: Any) -> None: + async def load_state_from_db(self, prisma_client: object) -> None: """Add each row's persisted delta to a freshly computed cold-start prior. A row holds an accumulated delta, not a full posterior, and can be one-sided @@ -237,7 +238,7 @@ class AdaptiveRouter: cost_weight=self.config.weights.cost, ) - async def get_state_snapshot(self) -> dict[str, Any]: + async def get_state_snapshot(self) -> dict[str, object]: """In-memory snapshot for the introspection endpoint. Cheap; no DB hit.""" cells: Final = [] for (rt, model), cell in sorted(self._cells.items(), key=lambda kv: (kv[0][0].value, kv[0][1])): @@ -278,7 +279,7 @@ class AdaptiveRouter: @staticmethod def _extract_min_quality_tier( - request_kwargs: dict[str, Any], + request_kwargs: Mapping[str, object], ) -> int | None: """Pull `min_quality_tier` from request headers or metadata. @@ -486,7 +487,7 @@ class AdaptiveRouter: return combined_delta @staticmethod - def _persistable_session_snapshot(state: SessionState) -> dict[str, Any]: + def _persistable_session_snapshot(state: SessionState) -> dict[str, object]: snapshot: Final = asdict(state) for sensitive in ( "last_user_content", diff --git a/litellm/router_strategy/adaptive_router/signals.py b/litellm/router_strategy/adaptive_router/signals.py index c28613b54eb..72e8d27d2bf 100644 --- a/litellm/router_strategy/adaptive_router/signals.py +++ b/litellm/router_strategy/adaptive_router/signals.py @@ -92,7 +92,7 @@ class Turn: user_content: str | None = None assistant_content: str | None = None - tool_calls: list[dict[str, Any]] = field(default_factory=list) + tool_calls: Sequence[Mapping[str, object]] = field(default_factory=list[Mapping[str, object]]) tool_results: Sequence[Mapping[str, object]] = field(default_factory=list) response_status: int | None = None @@ -174,7 +174,7 @@ def _detect_failure(tool_results: Sequence[Mapping[str, object]]) -> bool: return False -def _signature(call: dict[str, Any]) -> str: +def _signature(call: Mapping[str, Any]) -> str: """Stable signature for loop detection: name + sorted JSON-ish args.""" name: Final = call.get("name") or call.get("function", {}).get("name", "") call_args = call.get("arguments") @@ -185,7 +185,7 @@ def _signature(call: dict[str, Any]) -> str: return f"{name}({call_args})" -def _detect_loop(history: list[str], new_calls: list[dict[str, Any]]) -> bool: +def _detect_loop(history: list[str], new_calls: Sequence[Mapping[str, object]]) -> bool: """Fires if any new call's signature appears >= LOOP_REPEAT_THRESHOLD-1 times in recent history (so this call would be the Nth).""" if not new_calls: @@ -238,7 +238,7 @@ def detect_response_signals( previous_assistant_content: str | None, current_assistant_content: str | None, tool_call_history: list[str], - tool_calls: list[dict[str, Any]], + tool_calls: Sequence[Mapping[str, object]], tool_results: Sequence[Mapping[str, object]], response_status: int | None, ) -> SignalDelta: diff --git a/litellm/router_strategy/adaptive_router/update_queue.py b/litellm/router_strategy/adaptive_router/update_queue.py index 1b9fce284ac..e28f2379f9c 100644 --- a/litellm/router_strategy/adaptive_router/update_queue.py +++ b/litellm/router_strategy/adaptive_router/update_queue.py @@ -19,7 +19,8 @@ to the in-memory aggregator). Flush is async and batched. from __future__ import annotations import asyncio -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from litellm._logging import verbose_router_logger from litellm.repositories.table_repositories import ( @@ -39,7 +40,7 @@ class AdaptiveRouterUpdateQueue: def __init__(self) -> None: self._state_agg: dict[StateKey, dict[str, float]] = {} - self._session_agg: dict[SessionKey, dict[str, Any]] = {} + self._session_agg: dict[SessionKey, Mapping[str, object]] = {} self._lock = asyncio.Lock() self._max_state_size_seen = 0 self._max_session_size_seen = 0 @@ -77,7 +78,7 @@ class AdaptiveRouterUpdateQueue: session_id: str, router_name: str, model_name: str, - state_dict: dict[str, Any], + state_dict: Mapping[str, object], ) -> None: """ Last-write-wins per session row. The state_dict is a snapshot of the @@ -91,7 +92,7 @@ class AdaptiveRouterUpdateQueue: # ---- Flushers (called by background task) ---------------------------- - async def flush_state_to_db(self, prisma_client: Any) -> int: + async def flush_state_to_db(self, prisma_client: object) -> int: """ Drain state aggregator and apply to LiteLLM_AdaptiveRouterState. Returns number of cells flushed. @@ -147,7 +148,7 @@ class AdaptiveRouterUpdateQueue: return len(batch) - async def flush_session_to_db(self, prisma_client: Any) -> int: + async def flush_session_to_db(self, prisma_client: object) -> int: """ Drain session aggregator and upsert into LiteLLM_AdaptiveRouterSession. Returns number of session rows flushed. diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 64f3600af18..27affc09337 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -401,7 +401,7 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: list[str] return [*base_keywords, *deduped_custom.values()] -def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]: +def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, Any]: kwargs: Final = request_kwargs or {} return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} @@ -1286,7 +1286,7 @@ class ComplexityRouter(CustomLogger): self, model_name: str, litellm_router_instance: Router, - complexity_router_config: dict[str, Any] | None = None, + complexity_router_config: Mapping[str, object] | None = None, default_model: str | None = None, derive_savings_baseline: bool = True, jev_client: JevClassifierClient | None = None, @@ -1913,7 +1913,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, ) -> ClassificationOutcome: """Score locally, and only pay for the classifier call when the scorer did not confidently @@ -1946,7 +1946,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, ) -> ClassificationOutcome: """Score locally, and only pay for the classifier when the score sits near a tier boundary. @@ -2061,7 +2061,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, scored: ClassificationOutcome | None = None, ) -> ClassificationOutcome: @@ -2254,8 +2254,8 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to resolve_structured_messages as-is - raw_messages: list[dict[str, Any]] | None, # mutable-ok: same shape _run_routing_plugins receives + request_kwargs: dict[str, object] | None, # mutable-ok: handed to resolve_structured_messages as-is + raw_messages: list[dict[str, object]] | None, # mutable-ok: same shape _run_routing_plugins receives ) -> ClassificationOutcome: from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages from litellm.types.router import RoutingContext @@ -2827,8 +2827,8 @@ class ComplexityRouter(CustomLogger): async def _pick_model_for_tier( self, tier: ComplexityTier | str, - raw_messages: list[dict[str, Any]] | None, - resolved_messages: list[dict[str, Any]] | None, + raw_messages: list[dict[str, object]] | None, + resolved_messages: list[dict[str, object]] | None, request_kwargs: dict, allowed_models: tuple[str, ...] | None = None, retained_pin: _SessionAffinityPin | None = None, @@ -2979,7 +2979,7 @@ class ComplexityRouter(CustomLogger): self, classified_tier: ComplexityTier | str, user_message: str, - request_kwargs: dict[str, Any] | None = None, + request_kwargs: dict[str, object] | None = None, hard_floor: ComplexityTier | str | None = None, hard_ceiling: ComplexityTier | str | None = None, fit_filter: frozenset[str] | None = None, @@ -3492,7 +3492,7 @@ class ComplexityRouter(CustomLogger): async def _gate_response_modality( self, response: PreRoutingHookResponse, - messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: dict, # mutable-ok: same shape the hook receives context_fit: _RequestContextFit | None = None, @@ -3685,7 +3685,7 @@ class ComplexityRouter(CustomLogger): async def _gate_response_health( self, response: PreRoutingHookResponse, - messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: dict, # mutable-ok: same shape the hook receives @@ -4006,9 +4006,9 @@ class ComplexityRouter(CustomLogger): def _resolve_messages( self, - messages: list[dict[str, Any]] | None, + messages: list[dict[str, object]] | None, request_kwargs: dict, - ) -> list[dict[str, Any]] | None: + ) -> list[dict[str, object]] | None: """ Resolve messages from the request, converting from other formats if needed. @@ -4023,7 +4023,7 @@ class ComplexityRouter(CustomLogger): @staticmethod def _extract_user_message_and_system_prompt( - messages: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], ) -> tuple[str | None, str | None]: """ Deprecated: use _extract_current_ask_and_system_prompt instead. @@ -4368,7 +4368,7 @@ class ComplexityRouter(CustomLogger): self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, conversation_continuing: bool = True, diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index d0abaed4d3a..3c5795a81e0 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -275,7 +275,7 @@ def get_pre_routing_selection(kwargs: Mapping[str, object]) -> str | None: DISABLE_FALLBACKS_METADATA_KEY: Final = "_disable_fallbacks" -def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bool) -> None: +def record_disable_fallbacks(request_kwargs: Mapping[str, object] | None, disabled: bool) -> None: """ Write-or-clear the request's disable_fallbacks verdict into the router-internal metadata bucket. The wrapper pops the raw kwarg before any downstream frame runs, so the refusal @@ -295,7 +295,7 @@ def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bucket.pop(DISABLE_FALLBACKS_METADATA_KEY, None) -def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: +def fallbacks_disabled_for_request(kwargs: Mapping[str, object]) -> bool: """True when this request opted out of fallbacks, read from the raw kwarg (pre-pop snapshots keep it) or the router-internal bucket the wrapper stamps after popping it.""" if kwargs.get("disable_fallbacks") is True: @@ -670,7 +670,7 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or verbose_router_logger.error("Error in log_failure_fallback_event: %s", e) -def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: +def _check_non_standard_fallback_format(fallbacks: Sequence[object] | None) -> bool: """ Checks if the fallbacks list is a list of strings or a list of dictionaries. @@ -684,8 +684,9 @@ def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: return False if all(isinstance(item, str) for item in fallbacks): return True - elif all(isinstance(item, dict) for item in fallbacks): - for item in fallbacks: + dict_entries: Final = tuple(item for item in fallbacks if isinstance(item, dict)) + if len(dict_entries) == len(fallbacks): + for item in dict_entries: for key in LiteLLMParamsTypedDict.__annotations__: if key in item: # If the value is a list, it's likely a standard fallback model group mapping diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 309894957ea..1cfb311d796 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -10,18 +10,16 @@ import traceback from collections.abc import Callable from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import Any, Final, Protocol from litellm._logging import verbose_router_logger - -if TYPE_CHECKING: - from litellm.types.router import SearchToolTypedDict +from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict class _SearchToolsRouter(Protocol): """The one router attribute the search-tool helpers read and replace.""" - search_tools: "list[SearchToolTypedDict]" + search_tools: list[SearchToolTypedDict] class SearchAPIRouter: @@ -34,7 +32,7 @@ class SearchAPIRouter: @staticmethod def _resolve_search_provider_credentials( *, - tool_litellm_params: dict[str, Any], + tool_litellm_params: SearchToolLiteLLMParams, ) -> tuple[str | None, str | None]: """ Resolve search provider credentials from tool configuration ONLY. @@ -65,8 +63,6 @@ class SearchAPIRouter: search_tools: List of search tool configurations from the database """ try: - from litellm.types.router import SearchToolTypedDict - verbose_router_logger.debug("Adding %s search tools to router", len(search_tools)) # Convert search tools to the format expected by the router diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index d75375a01cc..0fb59105b5f 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -16,7 +16,7 @@ Requires: import json import os -from typing import Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -35,6 +35,9 @@ from litellm.types.secret_managers.main import KeyManagementSettings from .base_secret_manager import BaseSecretManager +if TYPE_CHECKING: + from botocore.awsrequest import HTTPHeaders + class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): def __init__( @@ -536,7 +539,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): secret_value: str | None = None, optional_params: dict | None = None, request_data: dict | None = None, - ) -> tuple[str, Any, bytes]: + ) -> tuple[str, "HTTPHeaders", bytes]: """Prepare the AWS Secrets Manager request""" try: from botocore.auth import SigV4Auth diff --git a/litellm/types/containers/main.py b/litellm/types/containers/main.py index 6a339fd2eac..62ef524a435 100644 --- a/litellm/types/containers/main.py +++ b/litellm/types/containers/main.py @@ -140,7 +140,7 @@ class ContainerFileObject(BaseModel): created_at: int path: str source: str - _hidden_params: dict[str, Any] = {} + _hidden_params: dict[str, builtins.object] = {} def __contains__(self, key: str) -> bool: return hasattr(self, key) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 172edf136fd..579a3f6322f 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -2,10 +2,10 @@ from collections.abc import Mapping from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from typing_extensions import Required, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import ( @@ -1050,7 +1050,7 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters for generic guardrail APIs", ) @@ -1274,7 +1274,7 @@ class GuardrailEventHooks(str, Enum): class DynamicGuardrailParams(TypedDict): - extra_body: dict[str, Any] + extra_body: ReadOnly[dict[str, object]] class GUARDRAIL_DEFINITION_LOCATION(str, Enum): @@ -1305,7 +1305,7 @@ class GuardrailUIAddGuardrailSettings(BaseModel): supported_modes: list[str] supported_modes_by_provider: dict[str, list[str]] pii_entity_categories: list[PiiEntityCategoryMap] - content_filter_settings: dict[str, Any] | None = None + content_filter_settings: dict[str, object] | None = None class PresidioPerRequestConfig(BaseModel): @@ -1323,8 +1323,8 @@ class ApplyGuardrailRequest(BaseModel): language: str | None = None entities: list[PiiEntityType] | None = None input_type: str = "request" - messages: list[dict[str, Any]] | None = None - metadata: dict[str, Any] | None = None + messages: list[dict[str, object]] | None = None + metadata: dict[str, object] | None = None class ApplyGuardrailResponse(BaseModel): @@ -1334,4 +1334,4 @@ class ApplyGuardrailResponse(BaseModel): class PatchGuardrailRequest(BaseModel): guardrail_name: str | None = None litellm_params: BaseLitellmParams | None = None - guardrail_info: dict[str, Any] | None = None + guardrail_info: dict[str, object] | None = None diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index f279c614cb4..f4893e857d1 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -44,7 +44,7 @@ def _sanitize_prometheus_label_name(label: str) -> str: _PROMETHEUS_LABEL_VALUE_TRANSLATE_V1: Final = str.maketrans("\n", " ", "\r\u2028\u2029") -def _sanitize_prometheus_label_value(value: Any | None) -> str | None: +def _sanitize_prometheus_label_value(value: object | None) -> str | None: """ Same semantics as :func:`_sanitize_prometheus_label_value`, implemented with ``str.translate`` plus a single escape pass instead of chained ``replace``. @@ -1066,7 +1066,7 @@ class UserAPIKeyLabelValues: ``hashed_api_key``. This supports ``**standard_logging_payload`` in tests. """ field_names: Final = {f.name for f in fields(self)} - merged: Final[dict[str, Any]] = {} + merged: Final[dict[str, object]] = {} for f in fields(self): if f.default_factory is not MISSING: merged[f.name] = f.default_factory() @@ -1103,9 +1103,9 @@ class UserAPIKeyLabelValues: # stays cheap. (Dataclass default `str()` delegates to `__repr__`.) return "" - def model_dump(self) -> dict[str, Any]: + def model_dump(self) -> dict[str, object]: """Same shape as the former Pydantic ``model_dump()`` (plain dict, list tags).""" - d: Final[dict[str, Any]] = {f.name: getattr(self, f.name) for f in fields(self)} + d: Final[dict[str, object]] = {f.name: getattr(self, f.name) for f in fields(self)} d["tags"] = list(self.tags) d["custom_metadata_labels"] = dict(self.custom_metadata_labels) return d diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 30db794c96e..855cccc8ddd 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -78,15 +78,15 @@ class RealtimeSessionConfig(BaseModel): type: str | None = None model: str | None = None instructions: str | None = None - audio: dict[str, Any] | None = None + audio: dict[str, object] | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None - tool_choice: Any | None = None - tools: list[dict[str, Any]] | None = None - tracing: Any | None = None - truncation: Any | None = None - prompt: dict[str, Any] | None = None + tool_choice: object | None = None + tools: list[dict[str, object]] | None = None + tracing: object | None = None + truncation: object | None = None + prompt: dict[str, object] | None = None class RealtimeClientSecretRequest(BaseModel): @@ -114,7 +114,7 @@ class RealtimeClientSecretResponse(BaseModel): expires_at: int | None = None value: str - session: dict[str, Any] | None = None + session: dict[str, object] | None = None class RealtimeTranscriptionSessionRequest(BaseModel): @@ -151,7 +151,7 @@ class RealtimeTranscriptionSessionResponse(BaseModel): model_config = {"extra": "allow"} - client_secret: dict[str, Any] | None = None + client_secret: dict[str, object] | None = None class RealtimeErrorDetail(TypedDict): diff --git a/litellm/types/router.py b/litellm/types/router.py index 8ad01206811..8ca27a9fb66 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -252,7 +252,7 @@ class ModelInfo(MirroredPricingParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -366,7 +366,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) merge_reasoning_content_in_choices: bool | None = False model_info: dict | None = None - mock_response: str | ModelResponse | Exception | Any | None = None + mock_response: str | ModelResponse | Exception | object | None = None # tag-based routing tags: list[str] | None = None @@ -443,7 +443,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -468,7 +468,7 @@ class LiteLLM_Params(GenericLiteLLMParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -1108,11 +1108,11 @@ class RoutingContext(BaseModel): plugins that need the exact original payload can read `raw_messages`. """ - raw_messages: list[dict[str, Any]] - structured_messages: list[dict[str, Any]] + raw_messages: list[dict[str, object]] + structured_messages: list[dict[str, object]] candidate_models: list[str] - metadata: dict[str, Any] = Field(default_factory=dict) - signals: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, object] = Field(default_factory=dict) + signals: dict[str, object] = Field(default_factory=dict) @runtime_checkable diff --git a/litellm/vector_store_files/utils.py b/litellm/vector_store_files/utils.py index 94ad5c0ecdf..8b4bff921f8 100644 --- a/litellm/vector_store_files/utils.py +++ b/litellm/vector_store_files/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Final, cast, get_type_hints +from collections.abc import Mapping +from typing import Final, cast, get_type_hints from litellm.types.vector_store_files import ( VectorStoreFileCreateRequest, @@ -11,25 +12,25 @@ class VectorStoreFileRequestUtils: """Helper utilities for constructing vector store file requests.""" @staticmethod - def _filter_params(params: dict[str, Any], model: Any) -> dict[str, Any]: + def _filter_params(params: Mapping[str, object], model: type[object]) -> dict[str, object]: valid_keys: Final = get_type_hints(model).keys() return {key: value for key, value in params.items() if key in valid_keys and value is not None} @staticmethod def get_create_request_params( - params: dict[str, Any], + params: Mapping[str, object], ) -> VectorStoreFileCreateRequest: filtered: Final = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileCreateRequest) return cast(VectorStoreFileCreateRequest, filtered) @staticmethod - def get_list_query_params(params: dict[str, Any]) -> VectorStoreFileListQueryParams: + def get_list_query_params(params: Mapping[str, object]) -> VectorStoreFileListQueryParams: filtered = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileListQueryParams) return cast(VectorStoreFileListQueryParams, filtered) @staticmethod def get_update_request_params( - params: dict[str, Any], + params: Mapping[str, object], ) -> VectorStoreFileUpdateRequest: filtered: Final = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileUpdateRequest) return cast(VectorStoreFileUpdateRequest, filtered) diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index b71d6784873..c7aed77286c 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -112,9 +112,8 @@ class VectorStoreRegistry: Dynamically extracts all parameters defined in VECTOR_STORE_OPENAI_PARAMS. """ # Get the list of supported param names from the Literal type - supported_params: Final = tuple( - param for param in get_args(VECTOR_STORE_OPENAI_PARAMS) if isinstance(param, str) - ) + declared_params: Final[tuple[object, ...]] = get_args(VECTOR_STORE_OPENAI_PARAMS) + supported_params: Final = tuple(param for param in declared_params if isinstance(param, str)) # Extract only the params that exist in the tool kwargs: Final = {param: tool.get(param) for param in supported_params if param in tool} diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index a2f81091893..77d2518696c 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,23 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] +def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": {"nested": "call_1"}, "type": 7, "function": {"name": ["get_weather"], "arguments": "{}"}} + ], + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + {"name": "", "arguments": {}, "tool_id": "", "type": ""} + ] + + def test_tool_calls_are_not_duplicated_into_metadata(logger: DataDogLLMObsLogger) -> None: """The flat `output_tool_calls.*` keys were a second copy of a fact that now has its own field.""" payload = build(