From fa966ca2d0bc9bca963112c8befbda5997f306ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:10:30 +0000 Subject: [PATCH 01/11] refactor(types): replace Any with real types across 20 backend files Fifth round of basedpyright Any reduction. Every change is typing-only and leaves runtime behavior identical. Prisma table access now goes through the PrismaTableRepository and TableActions protocols the repo already has, instead of reading untyped attributes off prisma_client.db. Payload and parameter annotations move from dict[str, Any] to dict[str, object] or Mapping[str, object]. Guardrail constructors that took **kwargs: Any now take Unpack of a PEP 728 TypedDict, the same _CustomGuardrailOptions shape three other guardrails already use. Calls into the OpenAI and Azure assistants SDKs pass explicit keywords rather than splatting a dict, so the arguments are checked against the real SDK signatures. --- litellm/llms/azure/assistants.py | 9 +-- litellm/llms/ollama/chat/transformation.py | 2 +- .../llms/ollama/completion/transformation.py | 2 +- litellm/llms/openai/openai.py | 61 ++++++++++++++----- litellm/llms/vertex_ai/fine_tuning/handler.py | 5 +- .../vertex_imagen_transformation.py | 25 ++++---- .../vertex_gemini_transformation.py | 6 +- litellm/proxy/_experimental/mcp_server/db.py | 8 ++- .../proxy/common_utils/reset_budget_job.py | 15 +++-- .../cisco_ai_defense/cisco_ai_defense.py | 7 ++- .../guardrail_hooks/deepkeep/deepkeep.py | 14 +++-- .../guardrail_hooks/headroom/headroom.py | 13 ++-- .../model_armor/model_armor.py | 6 +- .../auto_router_endpoints.py | 10 +-- .../spend_tracking/ptu_flat_cost_rollup.py | 30 +++++++-- litellm/proxy/utils.py | 11 ++-- litellm/realtime_api/main.py | 2 +- litellm/repositories/model_repository.py | 20 ++---- litellm/repositories/team_repository.py | 42 ++++++++++++- .../router_utils/fallback_event_handlers.py | 11 ++-- 20 files changed, 199 insertions(+), 100 deletions(-) diff --git a/litellm/llms/azure/assistants.py b/litellm/llms/azure/assistants.py index f7b419405ac..a4742a25a87 100644 --- a/litellm/llms/azure/assistants.py +++ b/litellm/llms/azure/assistants.py @@ -1,5 +1,5 @@ from collections.abc import Coroutine, Iterable -from typing import Any, Final, Literal, TypedDict +from typing import Final, Literal, TypedDict import httpx from openai import AsyncAzureOpenAI, AzureOpenAI @@ -715,7 +715,8 @@ class AzureAssistantsAPI(BaseAzureLLM): event_handler: AssistantEventHandler | None, litellm_params: dict | None = None, ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data: Final[dict[str, Any]] = { + stream_fn: Final = client.beta.threads.runs.stream + base_data: Final[_RunThreadStreamData] = { "thread_id": thread_id, "assistant_id": assistant_id, "additional_instructions": additional_instructions, @@ -725,8 +726,8 @@ class AzureAssistantsAPI(BaseAzureLLM): "tools": tools, } if event_handler is not None: - data["event_handler"] = event_handler - return client.beta.threads.runs.stream(**data) + return stream_fn(**base_data, event_handler=event_handler) + return stream_fn(**base_data) def run_thread_stream( self, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 181894646e3..257ebf921d9 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -124,7 +124,7 @@ class OllamaChatConfig(BaseConfig): setattr(self.__class__, key, value) @classmethod - def get_config(cls): + def get_config(cls) -> dict[str, object]: return super().get_config() def get_supported_openai_params(self, model: str): diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index dccc83efed4..a9bbfedafd6 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -227,7 +227,7 @@ class OllamaConfig(BaseConfig): model: str, api_base: str | None = None, api_key: str | None = None, - ) -> Any: + ) -> dict[str, object] | None: """ curl http://localhost:11434/api/show -d '{ "name": "mistral" diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index edc8d64d9c2..15c543b94f0 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,7 +1,7 @@ import time import types from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -2754,7 +2754,12 @@ class OpenAIAssistantsAPI(BaseLLM): message_thread: Final = await openai_client.beta.threads.create(**data) - return Thread(**message_thread.dict()) + return Thread( + id=message_thread.id, + created_at=message_thread.created_at, + metadata=message_thread.metadata, + object=message_thread.object, + ) # fmt: off @@ -2840,7 +2845,12 @@ class OpenAIAssistantsAPI(BaseLLM): message_thread: Final = openai_client.beta.threads.create(**data) - return Thread(**message_thread.dict()) + return Thread( + id=message_thread.id, + created_at=message_thread.created_at, + metadata=message_thread.metadata, + object=message_thread.object, + ) async def async_get_thread( self, @@ -2863,7 +2873,12 @@ class OpenAIAssistantsAPI(BaseLLM): response: Final = await openai_client.beta.threads.retrieve(thread_id=thread_id) - return Thread(**response.dict()) + return Thread( + id=response.id, + created_at=response.created_at, + metadata=response.metadata, + object=response.object, + ) # fmt: off @@ -2929,7 +2944,12 @@ class OpenAIAssistantsAPI(BaseLLM): response: Final = openai_client.beta.threads.retrieve(thread_id=thread_id) - return Thread(**response.dict()) + return Thread( + id=response.id, + created_at=response.created_at, + metadata=response.metadata, + object=response.object, + ) def delete_thread(self): pass @@ -2986,18 +3006,27 @@ class OpenAIAssistantsAPI(BaseLLM): tools: Iterable[AssistantToolParam] | None, event_handler: AssistantEventHandler | None, ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]: - data: Final[dict[str, Any]] = { - "thread_id": thread_id, - "assistant_id": assistant_id, - "additional_instructions": additional_instructions, - "instructions": instructions, - "metadata": metadata, - "model": model, - "tools": tools, - } + runs_stream: Final = client.beta.threads.runs.stream if event_handler is not None: - data["event_handler"] = event_handler - return client.beta.threads.runs.stream(**data) + return runs_stream( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + event_handler=event_handler, + ) + return runs_stream( + thread_id=thread_id, + assistant_id=assistant_id, + additional_instructions=additional_instructions, + instructions=instructions, + metadata=metadata, + model=model, + tools=tools, + ) def run_thread_stream( self, diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 7ecc5e8ff3d..c79b6ffce43 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -280,7 +280,7 @@ class VertexFineTuningAPI(VertexLLM): vertex_location: str, vertex_credentials: str, request_route: str, - ): + ) -> object: _auth_header, vertex_project = await self._ensure_access_token_async( credentials=vertex_credentials, project_id=vertex_project, @@ -341,5 +341,4 @@ class VertexFineTuningAPI(VertexLLM): f"Error creating fine tuning job. Status code: {response.status_code}. Response: {response.text}" ) - response_json: Final = response.json() - return response_json + return response.json() diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index c6ad5928b74..fddc075bfc6 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -1,6 +1,7 @@ import base64 import json import os +from collections.abc import Mapping from io import BufferedRandom, BufferedReader, BytesIO from pathlib import Path from typing import TYPE_CHECKING, Any, Final, cast @@ -47,11 +48,11 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): image_edit_optional_params: ImageEditOptionalRequestParams, model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) filtered_params = {key: value for key, value in image_edit_optional_params.items() if key in supported_params} - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} # Map OpenAI parameters to Imagen format if "n" in filtered_params: @@ -148,10 +149,10 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): model: str, prompt: str | None, image: FileTypes | None, - image_edit_optional_request_params: dict[str, Any], + image_edit_optional_request_params: Mapping[str, object], litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[dict[str, Any], RequestFiles | None]: + ) -> tuple[dict[str, object], RequestFiles | None]: # Prepare reference images in the correct Imagen format if image is None: raise ValueError("Vertex AI Imagen image edit requires at least one reference image.") @@ -182,14 +183,14 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): parameters["guidanceScale"] = 7.5 # Default guidance scale parameters["seed"] = None # Let Vertex AI choose random seed - request_body: Final[dict[str, Any]] = { + request_body: Final[dict[str, object]] = { "instances": instances, "parameters": parameters, } - payload: Final[Any] = json.dumps(request_body) + payload: Final = json.dumps(request_body) empty_files: Final = cast(RequestFiles, []) - return cast(tuple[dict[str, Any], RequestFiles | None], (payload, empty_files)) + return cast(tuple[dict[str, object], RequestFiles | None], (payload, empty_files)) def transform_image_edit_response( self, @@ -237,8 +238,8 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): def _prepare_reference_images( self, image: FileTypes | list[FileTypes], - image_edit_optional_request_params: dict[str, Any], - ) -> list[dict[str, Any]]: + image_edit_optional_request_params: Mapping[str, object], + ) -> list[dict[str, object]]: """ Prepare reference images in the correct Imagen API format """ @@ -248,7 +249,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): else: images = [image] - reference_images: Final[list[dict[str, Any]]] = [] + reference_images: Final[list[dict[str, object]]] = [] for idx, img in enumerate(images): if img is None: @@ -258,7 +259,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): base64_data = base64.b64encode(image_bytes).decode("utf-8") # Create reference image structure - reference_image = { + reference_image: dict[str, object] = { "referenceType": "REFERENCE_TYPE_RAW", "referenceId": idx + 1, "referenceImage": {"bytesBase64Encoded": base64_data}, @@ -272,7 +273,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): mask_bytes: Final = self._read_all_bytes(mask_image) mask_base64: Final = base64.b64encode(mask_bytes).decode("utf-8") - mask_reference: Final = { + mask_reference: Final[dict[str, object]] = { "referenceType": "REFERENCE_TYPE_MASK", "referenceId": len(reference_images) + 1, "referenceImage": {"bytesBase64Encoded": mask_base64}, diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index d7a2491c04a..b2c52c53580 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -218,10 +218,10 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): contents: Final = [{"role": "user", "parts": [{"text": prompt}]}] # Prepare generation config - generation_config: Final[dict[str, Any]] = {"responseModalities": ["IMAGE"]} + generation_config: Final[dict[str, object]] = {"responseModalities": ["IMAGE"]} # Seed from user-supplied imageConfig dict; flat params are overlaid for backward compat. - image_config: Final[dict[str, Any]] = dict(optional_params.get("imageConfig") or {}) + image_config: Final[dict[str, object]] = dict(optional_params.get("imageConfig") or {}) if "aspectRatio" in optional_params: image_config["aspectRatio"] = optional_params["aspectRatio"] @@ -242,7 +242,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): elif "n" in optional_params: generation_config["candidateCount"] = optional_params["n"] - request_body: Final[dict[str, Any]] = { + request_body: Final[dict[str, object]] = { "contents": contents, "generationConfig": generation_config, } diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 082a90fdcfb..980a67b30d6 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -37,6 +37,7 @@ from litellm.repositories.table_repositories import ( MCPServerOAuthClientRepository, MCPServerRepository, MCPUserCredentialsRepository, + PrismaTableRepository, ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.verification_token_repository import ( @@ -522,11 +523,14 @@ def _user_credential_actions( return table +class _MCPUserEnvVarsRepository(PrismaTableRepository["prisma_db_models.LiteLLM_MCPUserEnvVars"]): + table_name = "litellm_mcpuserenvvars" + + def _user_env_var_actions( prisma_client: PrismaClient, ) -> "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]": - table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars - return table + return _MCPUserEnvVarsRepository(prisma_client).table async def _db_find_user_credential_row( diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index f2648c8466e..dc29a8377c9 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -48,7 +48,7 @@ from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManage from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.organization_repository import OrganizationRepository -from litellm.repositories.prisma_protocols import SpendLinkedTable +from litellm.repositories.prisma_protocols import PrismaBatch, SpendLinkedTable from litellm.repositories.table_repositories import ( EndUserRepository, ModelAccessGroupBudgetRepository, @@ -435,6 +435,11 @@ class ResetBudgetJob: self.reset_settings: BudgetResetSettings = reset_settings or get_budget_reset_settings() self.pod_lock_manager: PodLockManager | None = pod_lock_manager + @property + def _new_batch(self) -> Callable[[], PrismaBatch]: + new_batch: Final[Callable[[], PrismaBatch]] = self.prisma_client.db.batch_ + return new_batch + async def _lease_is_held(self, lock_manager: PodLockManager) -> bool: """True only when the lease is readable and someone holds it. @@ -721,7 +726,7 @@ class ResetBudgetJob: ) async def _commit_budget_cascade_once(self, cascade: _BudgetCascade) -> None: - async with budget_cascade_unit_of_work(self.prisma_client.db.batch_) as uow: + async with budget_cascade_unit_of_work(self._new_batch) as uow: _queue_budget_linked_resets(uow.team_memberships, cascade) _queue_budget_linked_resets(uow.keys, cascade, extra=_LINKED_KEYS_WHERE) _queue_budget_linked_resets(uow.organizations, cascade, extra=_SPENT_ROWS_WHERE) @@ -861,7 +866,7 @@ class ResetBudgetJob: ) async def _write_key_reset_updates_once(self, updated_keys: list[LiteLLM_VerificationToken]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for k in updated_keys: if k.token is None: continue @@ -885,7 +890,7 @@ class ResetBudgetJob: ) async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for u in updated_users: uow.users.queue_spend_reset( user_id=u.user_id, @@ -907,7 +912,7 @@ class ResetBudgetJob: ) async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None: - async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: + async with spend_reset_unit_of_work(self._new_batch) as uow: for t in updated_teams: uow.teams.queue_spend_reset( team_id=t.team_id, diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index facb822d00d..017ef6e09f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal import httpx from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm import DualCache from litellm._logging import verbose_proxy_logger @@ -111,6 +112,10 @@ class CiscoAIDefenseGuardrailAPIError(Exception): """Raised when there is an error talking to the Cisco AI Defense API.""" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class CiscoAIDefenseGuardrail(_CiscoAIDefenseMcpMixin, CustomGuardrail): """ Cisco AI Defense guardrail integration. @@ -144,7 +149,7 @@ class CiscoAIDefenseGuardrail(_CiscoAIDefenseMcpMixin, CustomGuardrail): on_flagged_action: str | None = None, fallback_on_error: str | None = None, timeout: float | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: resolved_api_key: Final = api_key or os.environ.get("CISCO_AI_DEFENSE_API_KEY") if not resolved_api_key: diff --git a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py index 214d4b486d4..539dc1ea1e9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py +++ b/litellm/proxy/guardrails/guardrail_hooks/deepkeep/deepkeep.py @@ -7,10 +7,10 @@ import os from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol import httpx -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version @@ -56,7 +56,13 @@ class DeepKeepFirewallResponse(TypedDict): class _DeepKeepInitKwargsView(TypedDict): """Typed read of the guardrail name carried in the untyped base-guardrail kwargs.""" - guardrail_name: ReadOnly[str] + guardrail_name: ReadOnly[str | None] + + +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + guardrail_name: ReadOnly[str | None] class _DeepKeepMetadataSource(TypedDict, total=False): @@ -110,7 +116,7 @@ class DeepKeepGuardrail(CustomGuardrail): firewall_id: str | None = None, unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", extra_headers: Mapping[str, str] | list[str] | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ): self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index fa113aa4d33..eb62b896784 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -7,7 +7,7 @@ import time import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeGuard import httpx from fastapi import HTTPException @@ -50,6 +50,7 @@ from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import BaseAnthropicMessagesConfig from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -878,9 +879,9 @@ class HeadroomGuardrail(CustomGuardrail): async def async_pre_call_deployment_hook( self, - kwargs: dict[str, Any], + kwargs: dict[str, object], call_type: CallTypes | None, - ) -> dict[str, Any] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict + ) -> dict[str, object] | None: # mutable-ok: overrides CustomLogger hook whose contract is a plain dict base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type) effective: Final = base_result if base_result is not None else kwargs if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES: @@ -897,7 +898,7 @@ class HeadroomGuardrail(CustomGuardrail): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -919,8 +920,8 @@ class HeadroomGuardrail(CustomGuardrail): tools: dict, model: str, messages: list[dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: BaseAnthropicMessagesConfig | None, anthropic_messages_optional_request_params: dict, logging_obj: LiteLLMLoggingObj | None, stream: bool, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index fde40111d49..88c7f21ad72 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -360,7 +360,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): else: return {"modelResponseData": {"byteItem": {"byteDataType": file_type, "byteData": base64_data}}} - def _should_block_content(self, armor_response: Mapping[str, Any], allow_sanitization: bool = False) -> bool: + def _should_block_content(self, armor_response: Mapping[str, object], allow_sanitization: bool = False) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" for filt in self._filter_result_items(armor_response): # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before @@ -429,7 +429,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return filter_results return [] - def _has_deidentify_match(self, armor_response: Mapping[str, Any]) -> bool: + def _has_deidentify_match(self, armor_response: Mapping[str, object]) -> bool: """Whether an SDP de-identify filter matched, i.e. Model Armor owes this response a redaction.""" for filter_entry in self._filter_result_items(armor_response): sdp = filter_entry.get("sdpFilterResult") @@ -439,7 +439,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): def _resolve_streaming_outcome( self, - armor_response: Mapping[str, Any], + armor_response: Mapping[str, object], assembled_response: object, content: str, ) -> tuple[bool, str | None]: diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index bbc914a772a..12d23550cb9 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -8,7 +8,6 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone from itertools import chain, groupby -from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol from uuid import uuid4 @@ -1094,6 +1093,10 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: ) +def _leg_group_id(leg: "_LegRow") -> str: + return leg.group_id + + class _LegRow(BaseModel): """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is one target's leg of a job; the legs of a job share group_id and identical config, @@ -1598,10 +1601,7 @@ async def list_shadow_eval_jobs( or () ) by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType( - { - group_id: tuple(group) - for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id")) - } + {group_id: tuple(group) for group_id, group in groupby(sorted(legs, key=_leg_group_id), key=_leg_group_id)} ) newest_first: Final = sorted( by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index 0e6412a2c64..b8af432029f 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -31,11 +31,31 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.ptu_pricing import ptu_terms from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled +from litellm.repositories.model_repository import ModelRepository +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository if TYPE_CHECKING: + from prisma import models as prisma_models + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient + +class _DailyTeamSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyTeamSpend"]): + table_name = "litellm_dailyteamspend" + + +def _daily_team_spend_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_DailyTeamSpend]": + """The sentinel rows this rollup writes, reads back and prunes.""" + return _DailyTeamSpendRepository(prisma_client).table + + +def _proxy_model_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_ProxyModelTable]": + """The stored deployments the rollup scans for PTU config.""" + return ModelRepository(prisma_client).table + + _HOURS_PER_DAY: Final = 24 _PRUNE_ID_CHUNK_SIZE: Final = 5_000 _UPSERT_ATTEMPTS: Final = 3 @@ -97,7 +117,7 @@ def _decode_model_info(raw: object) -> "Mapping[str, object] | None": """ if isinstance(raw, str): try: - decoded: Final = json.loads(raw) + decoded: Final[object] = json.loads(raw) except (TypeError, ValueError): return None return decoded if isinstance(decoded, dict) else None @@ -240,7 +260,7 @@ async def _upsert_ptu_daily_row( } } now: Final = datetime.now(timezone.utc) - await prisma_client.db.litellm_dailyteamspend.upsert( + await _daily_team_spend_table(prisma_client).upsert( where=where, data={ # mutable-ok: prisma upsert data payload "create": { # mutable-ok: prisma create payload @@ -353,7 +373,7 @@ async def _load_ptu_models(prisma_client: "PrismaClient", *, router: object | No The router is handed in rather than read off the proxy module, so a run prices exactly the deployments its caller declares and nothing a co-resident process left behind. """ - rows: Final = await prisma_client.db.litellm_proxymodeltable.find_many() + rows: Final = await _proxy_model_table(prisma_client).find_many() db_ids: Final = frozenset(model_id for row in rows if (model_id := str(getattr(row, "model_id", "") or ""))) config_records: Final = _config_deployments(router, owned_by_db=db_ids) models: Final = tuple( @@ -503,7 +523,7 @@ async def _existing_sentinel_keys( survives a rename. Nothing here reads the display name. """ date_range: Final = {"gte": start.isoformat(), "lte": end.isoformat()} # mutable-ok: prisma range filter - rows: Final = await prisma_client.db.litellm_dailyteamspend.find_many( + rows: Final = await _daily_team_spend_table(prisma_client).find_many( where={"api_key": PTU_SENTINEL_API_KEY, "date": date_range} # mutable-ok: prisma find filter ) return frozenset( @@ -771,7 +791,7 @@ async def _prune_unrefreshed_sentinel_rows( ) filters: Final = tuple(_prune_filter(date_str=date_str, cutoff=cutoff, chunk=chunk) for chunk in chunks) deletions: Final = tuple( - [await prisma_client.db.litellm_dailyteamspend.delete_many(where=where) for where in filters] + [await _daily_team_spend_table(prisma_client).delete_many(where=where) for where in filters] ) deleted: Final = sum(deletions) if deleted: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddf31cb1d8a..21328a4962a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3401,7 +3401,7 @@ class _ConfigRow: __slots__ = ("param_name", "param_value") - def __init__(self, param_name: str, param_value: Any) -> None: + def __init__(self, param_name: str, param_value: object) -> None: self.param_name = param_name self.param_value = param_value @@ -3414,7 +3414,7 @@ def _pack_config_row(row: Any) -> dict[str, object]: return {"param_name": row.param_name, "param_value": row.param_value} -def _unpack_config_row(cached: Any) -> _ConfigRow | None: +def _unpack_config_row(cached: object) -> _ConfigRow | None: if cached is None or cached == _CONFIG_CACHE_MISS: return None if isinstance(cached, dict): @@ -3557,6 +3557,7 @@ class PrismaClient: verbose_proxy_logger.debug("Creating Prisma Client..") try: from prisma import Prisma + from prisma.types import DatasourceOverride except Exception as e: verbose_proxy_logger.error("Failed to import Prisma client: %s", e) verbose_proxy_logger.error("This usually means 'prisma generate' hasn't been run yet.") @@ -3607,11 +3608,11 @@ class PrismaClient: reader_token: Final = mint_database_token(token_auth, reader_iam_endpoint) read_replica_url = reader_iam_endpoint.build_url(reader_token) os.environ["DATABASE_URL_READ_REPLICA"] = read_replica_url - reader_kwargs: Final[dict[str, Any]] = {"datasource": {"url": read_replica_url}} + reader_datasource: Final = DatasourceOverride(url=read_replica_url) if http_client is not None: - reader_prisma = Prisma(http=http_client, **reader_kwargs) + reader_prisma = Prisma(http=http_client, datasource=reader_datasource) else: - reader_prisma = Prisma(**reader_kwargs) + reader_prisma = Prisma(datasource=reader_datasource) reader_wrapper: Final = PrismaWrapper( original_prisma=reader_prisma, token_auth=token_auth, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index b824a5928c6..9d81c725c70 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -53,7 +53,7 @@ bedrock_realtime: Final = BedrockRealtime() xai_realtime: Final = XAIRealtime() vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() -_EMPTY_MODEL_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: diff --git a/litellm/repositories/model_repository.py b/litellm/repositories/model_repository.py index d24eb8ffc62..8ee76b93923 100644 --- a/litellm/repositories/model_repository.py +++ b/litellm/repositories/model_repository.py @@ -4,29 +4,23 @@ Model repository for database operations on LiteLLM_ProxyModelTable. import json from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final from litellm.models.model import LiteLLM_ProxyModelTable -from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) from litellm.repositories.base_repository import BaseRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.table_repositories import PrismaTableRepository if TYPE_CHECKING: from prisma import models as prisma_models -class _PrismaModelDb(Protocol): - @property - def litellm_proxymodeltable(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]: ... - - -class _PrismaClientView(Protocol): - @property - def db(self) -> _PrismaModelDb: ... +class _ProxyModelTableRepository(PrismaTableRepository["prisma_models.LiteLLM_ProxyModelTable"]): + table_name = "litellm_proxymodeltable" class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @@ -38,11 +32,7 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]): @property def table(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]: - client: Final[_PrismaClientView] = self.prisma_client - return wrap_table_actions_for_config_sync( - actions=client.db.litellm_proxymodeltable, - table_name="litellm_proxymodeltable", - ) + return _ProxyModelTableRepository(self._prisma_client).table @property def model_class(self) -> type[LiteLLM_ProxyModelTable]: diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py index 5ff07d76b5d..cbe263699c9 100644 --- a/litellm/repositories/team_repository.py +++ b/litellm/repositories/team_repository.py @@ -5,6 +5,7 @@ Team repository for database operations on LiteLLM_TeamTable. import json from collections.abc import Mapping, Sequence from datetime import datetime +from types import TracebackType from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter @@ -40,6 +41,36 @@ def _team_arrays(team: LiteLLM_TeamTable) -> _TeamArrays: return team +class _TeamTables(Protocol): + """The two team tables this repository reads and writes.""" + + @property + def litellm_teamtable(self) -> TableActions["prisma_models.LiteLLM_TeamTable"]: ... + + @property + def litellm_deletedteamtable(self) -> TableActions["prisma_models.LiteLLM_DeletedTeamTable"]: ... + + +class _TeamTransactionManager(Protocol): + async def __aenter__(self) -> _TeamTables: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + +class _PrismaTeamDb(_TeamTables, Protocol): + def tx(self) -> _TeamTransactionManager: ... + + +class _PrismaClientView(Protocol): + @property + def db(self) -> _PrismaTeamDb: ... + + _MEMBERS_WITH_ROLES_ADAPTER: Final = TypeAdapter(list[Member]) _JSON_ENCODED_TEAM_FIELDS: Final = ( "metadata", @@ -54,13 +85,18 @@ _JSON_ENCODED_TEAM_FIELDS: Final = ( class TeamRepository(BaseRepository[LiteLLM_TeamTable]): """Repository for team database operations.""" + @property + def _db(self) -> _PrismaTeamDb: + client: Final[_PrismaClientView] = self.prisma_client + return client.db + @property def table(self) -> TableActions["prisma_models.LiteLLM_TeamTable"]: - return self.prisma_client.db.litellm_teamtable + return self._db.litellm_teamtable @property def deleted_table(self) -> TableActions["prisma_models.LiteLLM_DeletedTeamTable"]: - return self.prisma_client.db.litellm_deletedteamtable + return self._db.litellm_deletedteamtable @property def model_class(self) -> type[LiteLLM_TeamTable]: @@ -256,7 +292,7 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]): archive_data["litellm_changed_by"] = litellm_changed_by archive_data["deleted_at"] = datetime.utcnow() - async with self.prisma_client.db.tx() as tx: + async with self._db.tx() as tx: await tx.litellm_deletedteamtable.create(data=archive_data) await tx.litellm_teamtable.delete(where={"team_id": team_id}) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 7fda5d96fb0..bf1838e50f7 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -266,7 +266,7 @@ def get_pre_routing_selection(kwargs: Mapping[str, object]) -> str | None: DISABLE_FALLBACKS_METADATA_KEY: Final = "_disable_fallbacks" -def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bool) -> None: +def record_disable_fallbacks(request_kwargs: Mapping[str, object] | None, disabled: bool) -> None: """ Write-or-clear the request's disable_fallbacks verdict into the router-internal metadata bucket. The wrapper pops the raw kwarg before any downstream frame runs, so the refusal @@ -286,7 +286,7 @@ def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bucket.pop(DISABLE_FALLBACKS_METADATA_KEY, None) -def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: +def fallbacks_disabled_for_request(kwargs: Mapping[str, object]) -> bool: """True when this request opted out of fallbacks, read from the raw kwarg (pre-pop snapshots keep it) or the router-internal bucket the wrapper stamps after popping it.""" if kwargs.get("disable_fallbacks") is True: @@ -639,7 +639,7 @@ async def log_failure_fallback_event(original_model_group: str, kwargs: dict, or verbose_router_logger.error("Error in log_failure_fallback_event: %s", e) -def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: +def _check_non_standard_fallback_format(fallbacks: Sequence[object] | None) -> bool: """ Checks if the fallbacks list is a list of strings or a list of dictionaries. @@ -653,8 +653,9 @@ def _check_non_standard_fallback_format(fallbacks: list[Any] | None) -> bool: return False if all(isinstance(item, str) for item in fallbacks): return True - elif all(isinstance(item, dict) for item in fallbacks): - for item in fallbacks: + dict_entries: Final = tuple(item for item in fallbacks if isinstance(item, dict)) + if len(dict_entries) == len(fallbacks): + for item in dict_entries: for key in LiteLLMParamsTypedDict.__annotations__: if key in item: # If the value is a list, it's likely a standard fallback model group mapping From f747bc67048f2b38e6e99bd57ca284fe6f77bdb5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:14:07 +0000 Subject: [PATCH 02/11] refactor(types): replace Any with real types across 29 more backend files Second batch of the fifth basedpyright Any reduction round. Every change is typing-only and leaves runtime behavior identical. Guardrail hooks and file, vector store and usage endpoints move their payload, header and response annotations from Any to object, Mapping[str, object] or the concrete response model the call site already produces. Two private aggregation helpers in the guardrail usage endpoints take a key accessor function instead of an attribute name string, so the key they read is checked against the row type. The verification token repository reaches its two tables through Protocols that name the handles it calls, rather than reading them off an untyped prisma client, and the Azure AD credential wrapper describes the azure-identity credential it wraps the same way. --- .../litellm_core_utils/llm_cost_calc/utils.py | 15 +++-- .../azure/text_to_speech/transformation.py | 9 +-- .../image_edit/stability_transformation.py | 4 +- .../responses/transformation.py | 14 ++--- .../llms/cohere/embed/v1_transformation.py | 15 +++-- litellm/llms/gdc/chat/transformation.py | 22 ++++++- .../llama3/transformation.py | 4 +- litellm/llms/voyage/rerank/transformation.py | 6 +- litellm/proxy/client/users.py | 3 +- .../proxy/common_utils/http_parsing_utils.py | 14 +++-- litellm/proxy/db/prisma_client.py | 9 ++- .../block_code_execution.py | 9 ++- .../cato_networks/cato_networks.py | 10 ++-- .../guardrail_hooks/compresr/compresr.py | 11 ++-- .../llm_as_a_judge/__init__.py | 23 +++++++- .../guardrails/guardrail_hooks/noma/noma.py | 15 +++-- .../panw_prisma_airs/panw_prisma_airs.py | 4 +- litellm/proxy/guardrails/usage_endpoints.py | 25 ++++---- .../proxy/hooks/proxy_track_cost_callback.py | 28 +++++++-- ...model_access_group_management_endpoints.py | 25 ++++---- .../usage_endpoints/ai_usage_chat.py | 11 ++-- .../openai_files_endpoints/files_endpoints.py | 4 +- .../storage_backend_service.py | 9 ++- .../proxy/vector_store_endpoints/endpoints.py | 2 +- .../vector_store_files_endpoints/endpoints.py | 10 ++-- litellm/proxy_auth/credentials.py | 58 ++++++++++++++----- litellm/rag/rag_query.py | 4 +- .../verification_token_repository.py | 46 ++++++++++++--- litellm/router_utils/search_api_router.py | 4 +- 29 files changed, 274 insertions(+), 139 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 68dc27ec25e..52dac92ee22 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone, tzinfo from types import MappingProxyType -from typing import Any, Final, Literal, TypedDict, cast +from typing import Final, Literal, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import litellm @@ -89,7 +89,7 @@ def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None -def get_web_search_requests(server_tool_use: Any) -> int | None: +def get_web_search_requests(server_tool_use: object) -> int | None: """ Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance, @@ -1494,7 +1494,7 @@ def calculate_image_response_cost_from_usage( if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: return None - input_tokens_details: Final = getattr(usage, "input_tokens_details", None) + input_tokens_details: Final[object] = getattr(usage, "input_tokens_details", None) prompt_tokens_details: PromptTokensDetailsWrapper | None = None if input_tokens_details is not None: # input_tokens_details may be a dict (e.g. OpenAI image edit responses) @@ -1507,9 +1507,12 @@ def calculate_image_response_cost_from_usage( cached_tokens=0, ) - output_tokens_details = getattr(usage, "completion_tokens_details", None) - if output_tokens_details is None: - output_tokens_details = getattr(usage, "output_tokens_details", None) + completion_tokens_details_attr: Final[object] = getattr(usage, "completion_tokens_details", None) + output_tokens_details: Final[object] = ( + getattr(usage, "output_tokens_details", None) + if completion_tokens_details_attr is None + else completion_tokens_details_attr + ) if output_tokens_details is None: completion_tokens_details = CompletionTokensDetailsWrapper( diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py index d8ccf26ce60..eed7a3178ca 100644 --- a/litellm/llms/azure/text_to_speech/transformation.py +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -19,6 +19,7 @@ from litellm.secret_managers.main import get_secret_str if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.llms.openai import HttpxBinaryResponseContent else: LiteLLMLoggingObj = Any @@ -67,15 +68,15 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): litellm_params_dict: dict, logging_obj: "LiteLLMLoggingObj", timeout: float | httpx.Timeout, - extra_headers: dict[str, Any] | None, - base_llm_http_handler: Any, + extra_headers: dict[str, object] | None, + base_llm_http_handler: "BaseLLMHTTPHandler", aspeech: bool, api_base: str | None, api_key: str | None, - **kwargs: Any, + **kwargs: object, ) -> Union[ "HttpxBinaryResponseContent", - Coroutine[Any, Any, "HttpxBinaryResponseContent"], + Coroutine[object, object, "HttpxBinaryResponseContent"], ]: """ Dispatch method to handle Azure AVA TTS requests diff --git a/litellm/llms/bedrock/image_edit/stability_transformation.py b/litellm/llms/bedrock/image_edit/stability_transformation.py index bc9a64f587a..01e25f4671e 100644 --- a/litellm/llms/bedrock/image_edit/stability_transformation.py +++ b/litellm/llms/bedrock/image_edit/stability_transformation.py @@ -125,7 +125,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig): } # Create a copy to not mutate original - convert TypedDict to regular dict - mapped_params: Final[dict[str, Any]] = dict(image_edit_optional_params) + mapped_params: Final[dict[str, object]] = dict(image_edit_optional_params) for k, v in image_edit_optional_params.items(): if k in param_mapping: @@ -172,7 +172,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig): Returns the request body dict that will be JSON-encoded by the handler. """ # Build Bedrock Stability request - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "output_format": "png", # Default to PNG } diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index bbbda4d14b6..3d2eab8fcee 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -16,8 +16,8 @@ BaseAWSLLM._sign_request after the request body is finalized. """ import json -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final import httpx from typing_extensions import ReadOnly, TypedDict @@ -142,9 +142,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI return False @staticmethod - def _filter_unsupported_tools(tools: list[Any]) -> list[Any]: + def _filter_unsupported_tools(tools: "Sequence[object]") -> "list[object]": """Keep only tool types Mantle's Responses API accepts.""" - kept: Final[list[Any]] = [] + kept: Final[list[object]] = [] dropped_types: Final[list[str]] = [] for tool in tools: if not isinstance(tool, dict): @@ -217,11 +217,11 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI ) @staticmethod - def _is_codex_additional_tools_item(item: Any) -> bool: + def _is_codex_additional_tools_item(item: object) -> bool: return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE @staticmethod - def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]": + def _tools_of_additional_tools_item(item: "Mapping[str, object]") -> "list[object]": tools: Final = item.get("tools") return tools if isinstance(tools, list) else [] @@ -229,7 +229,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI def _hoist_codex_additional_tools( cls, input: "str | ResponseInputParam", - ) -> "tuple[str | ResponseInputParam, list[Any]]": + ) -> "tuple[str | ResponseInputParam, list[object]]": """Codex's "responses lite" wire mode ships tool definitions inside `input` as {"type": "additional_tools", "role": "developer", "tools": [...]} items. api.openai.com accepts that item type; Mantle diff --git a/litellm/llms/cohere/embed/v1_transformation.py b/litellm/llms/cohere/embed/v1_transformation.py index ee40464362d..b35fae5a1ac 100644 --- a/litellm/llms/cohere/embed/v1_transformation.py +++ b/litellm/llms/cohere/embed/v1_transformation.py @@ -2,7 +2,8 @@ Legacy /v1/embedding transformation logic for Bedrock Cohere. """ -from typing import Any, Final +from collections.abc import Sized +from typing import Final, Protocol import httpx @@ -16,6 +17,12 @@ from litellm.types.utils import EmbeddingResponse, PromptTokensDetailsWrapper, U from litellm.utils import is_base64_encoded +class _SupportsEncode(Protocol): + """Tokenizer handle: the embedding usage path only encodes text to measure its token length.""" + + def encode(self, text: str, /) -> Sized: ... + + class CohereEmbeddingConfig: """ Reference: https://docs.cohere.com/v2/reference/embed @@ -61,7 +68,7 @@ class CohereEmbeddingConfig: return transformed_request - def _calculate_usage(self, input: list[str], encoding: Any, meta: dict) -> Usage: + def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage: input_tokens = 0 text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens") @@ -97,7 +104,7 @@ class CohereEmbeddingConfig: data: dict | CohereEmbeddingRequest, model_response: EmbeddingResponse, model: str, - encoding: Any, + encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: response_json: Final = response.json() @@ -121,7 +128,7 @@ class CohereEmbeddingConfig: response_json: dict, model_response: EmbeddingResponse, model: str, - encoding: Any, + encoding: _SupportsEncode, input: list, ) -> EmbeddingResponse: """ diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index 03037512551..6eac3ac79cd 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -7,14 +7,32 @@ import os import re import threading from collections.abc import Callable -from typing import Any, Final, Protocol +from typing import Final, Protocol from urllib.parse import urlsplit +from typing_extensions import ReadOnly, TypedDict, Unpack + import litellm from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues +class _OpenAIGPTConfigOptions(TypedDict, total=False): + """The sampling defaults ``OpenAIGPTConfig.__init__`` accepts and stashes on the class.""" + + frequency_penalty: ReadOnly[int | None] + function_call: ReadOnly[str | dict[str, object] | None] + functions: ReadOnly[list[object] | None] + logit_bias: ReadOnly[dict[str, object] | None] + max_tokens: ReadOnly[int | None] + n: ReadOnly[int | None] + presence_penalty: ReadOnly[int | None] + stop: ReadOnly[str | list[object] | None] + temperature: ReadOnly[int | None] + top_p: ReadOnly[int | None] + response_format: ReadOnly[dict[str, object] | None] + + class _GDCHAudienceCredentials(Protocol): """A GDCH service account credential already bound to an audience, ready to mint a bearer token.""" @@ -32,7 +50,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig): _GDCH_CREDENTIAL_TYPE: Final[str] = "gdch_service_account" _PATH_ID_PATTERN: Final[re.Pattern[str]] = re.compile(r"^[a-zA-Z0-9_-]+$") - def __init__(self, **kwargs: Any) -> None: + def __init__(self, **kwargs: Unpack[_OpenAIGPTConfigOptions]) -> None: super().__init__(**kwargs) self._creds_lock = threading.Lock() self._gdch_creds_cache: dict[tuple[str, str], _GDCHAudienceCredentials] = {} diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 279035c455d..89a5b8a570e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -1,6 +1,6 @@ import types from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -95,7 +95,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "VertexAILlama3StreamingHandler": return VertexAILlama3StreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index fea8452d934..6f8d024f0b1 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -4,8 +4,8 @@ Transformation logic for Voyage AI's /v1/rerank endpoint. Docs - https://docs.voyageai.com/docs/reranker """ -from collections.abc import Mapping -from typing import Any, Final +from collections.abc import Mapping, Sequence +from typing import Final import httpx @@ -33,7 +33,7 @@ class VoyageRerankConfig(BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: Sequence[str | Mapping[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/proxy/client/users.py b/litellm/proxy/client/users.py index 3f11fe94043..503c92228a8 100644 --- a/litellm/proxy/client/users.py +++ b/litellm/proxy/client/users.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final import requests @@ -50,7 +51,7 @@ class UsersManagementClient: response.raise_for_status() return response.json() - def create_user(self, user_data: dict[str, Any]) -> dict[str, Any]: + def create_user(self, user_data: Mapping[str, object]) -> dict[str, Any]: """Create a new user (POST /user/new)""" url: Final = f"{self.base_url}/user/new" response: Final = requests.post(url, headers=self._get_headers(), json=user_data, timeout=self.timeout) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 54a0f18fd63..2bae3e946f7 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -52,14 +52,18 @@ def _unqualified(annotation: object) -> object: return _unqualified(qualified[0]) +def _union_members(annotation: object) -> tuple[object, ...]: + """The non-``None`` members of a union annotation, or the annotation itself when it is not a union.""" + if get_origin(annotation) not in (Union, UnionType): + return (annotation,) + members: Final[tuple[object, ...]] = get_args(annotation) + return tuple(arg for arg in members if arg is not type(None)) + + def _numeric_form_type(annotation: object) -> type[int] | type[float] | None: """The scalar to parse an ``int``/``float``-typed field as, else ``None``.""" unwrapped: Final = _unqualified(annotation) - candidates: Final = ( - tuple(arg for arg in get_args(unwrapped) if arg is not type(None)) - if get_origin(unwrapped) in (Union, UnionType) - else (unwrapped,) - ) + candidates: Final = _union_members(unwrapped) if len(candidates) != 1: return None if candidates[0] is int: diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 2190ae55fd2..2f2ebfdf2bb 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -13,7 +13,7 @@ import urllib import urllib.parse from collections.abc import Callable from datetime import datetime, timedelta -from typing import Any, Final, Protocol +from typing import TYPE_CHECKING, Any, Final, Protocol from litellm._logging import verbose_proxy_logger from litellm.proxy.db.token_auth import ( @@ -27,6 +27,9 @@ from litellm.proxy.db.token_auth import ( ) from litellm.secret_managers.main import str_to_bool +if TYPE_CHECKING: + from prisma import Prisma + __all__ = ( "IAMEndpoint", "PrismaManager", @@ -242,7 +245,7 @@ class PrismaWrapper: def _write_engine(prisma_client: _PrismaClient, engine: _PrismaEngine) -> None: prisma_client._Prisma__engine = engine - def _instrument_prisma_client(self, prisma_client: _PrismaClient) -> _PrismaDrainTracker | None: + def _instrument_prisma_client(self, prisma_client: "Prisma | _PrismaClient") -> _PrismaDrainTracker | None: from prisma.errors import ClientNotConnectedError try: @@ -255,7 +258,7 @@ class PrismaWrapper: self._write_engine(prisma_client, _TrackedPrismaEngine(engine, tracker)) return tracker - def _get_engine_pid(self, prisma_client: _PrismaClient | None = None) -> int: + def _get_engine_pid(self, prisma_client: "Prisma | _PrismaClient | None" = None) -> int: """Get the PID of the current Prisma engine subprocess, or 0 if unavailable. Must never raise: it runs inside the reconnect path, where the client diff --git a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py index bf2aa1f76e0..2b697671eda 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py +++ b/litellm/proxy/guardrails/guardrail_hooks/block_code_execution/block_code_execution.py @@ -8,9 +8,10 @@ confidence scoring and a tunable threshold (only block when confidence >= thresh import re from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -314,6 +315,10 @@ def _confidence_for_block( return 0.0 +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class BlockCodeExecutionGuardrail(CustomGuardrail): """ Guardrail that detects fenced code blocks (markdown ```) and blocks or masks them @@ -332,7 +337,7 @@ class BlockCodeExecutionGuardrail(CustomGuardrail): detect_execution_intent: bool = True, event_hook: Literal["pre_call", "post_call", "during_call"] | list[str] | None = None, default_on: bool = False, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: # Normalize to type expected by CustomGuardrail _event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | None = None diff --git a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py index 176c308eda6..2d203c31974 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cato_networks/cato_networks.py @@ -264,7 +264,7 @@ class CatoNetworksGuardrail(CustomGuardrail): stack.extend(reversed(node)) @classmethod - def _extra_inspection_sources(cls, data: Mapping[str, Any]) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: + def _extra_inspection_sources(cls, data: Mapping[str, object]) -> Sequence[tuple[str, Sequence[Mapping[str, str]]]]: """Text the proxy forwards to the model outside chat ``messages``: Responses-API ``input`` and ``instructions``, legacy completion ``prompt`` and tool/function/``response_format`` schema strings. Returned @@ -336,7 +336,7 @@ class CatoNetworksGuardrail(CustomGuardrail): ) raise HTTPException(status_code=400, detail=detection_message) - def _anonymize_request(self, res: Any, data: dict) -> dict: + def _anonymize_request(self, res: _CatoAnalyzeResponse, data: dict) -> dict: verbose_proxy_logger.info("Cato: anonymize action") redacted_chat: Final = res.get("redacted_chat") if not redacted_chat: @@ -379,7 +379,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return data @classmethod - def _apply_extra_redaction(cls, data: dict, field: str, redacted: list) -> bool: + def _apply_extra_redaction(cls, data: dict, field: str, redacted: Sequence[Mapping[str, object]]) -> bool: if field == "input": input_only: Final = {"input": data["input"]} if not redacted: @@ -400,7 +400,7 @@ class CatoNetworksGuardrail(CustomGuardrail): return True @classmethod - def _apply_schema_string_redaction(cls, data: dict, redacted: list) -> None: + def _apply_schema_string_redaction(cls, data: dict, redacted: Sequence[Mapping[str, object]]) -> None: redactions: Final = iter(redacted) for container, key in cls._iter_schema_string_refs(data): replacement = next(redactions, None) @@ -408,7 +408,7 @@ class CatoNetworksGuardrail(CustomGuardrail): container[key] = replacement["content"] @staticmethod - def _apply_prompt_redaction(data: dict, redacted: list) -> None: + def _apply_prompt_redaction(data: dict, redacted: Sequence[Mapping[str, object]]) -> None: contents: Final = [m.get("content") for m in redacted if isinstance(m, dict)] prompt: Final = data.get("prompt") if isinstance(prompt, str): diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 93d859066b0..1ecdb1b0f63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -22,7 +22,7 @@ import json import time from collections import Counter, OrderedDict from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Final, Literal, TypeGuard +from typing import TYPE_CHECKING, Final, Literal, TypeGuard from urllib.parse import urlparse import httpx @@ -64,6 +64,9 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) from litellm.types.proxy.guardrails.guardrail_hooks.base import ( GuardrailConfigModel, ) @@ -1049,7 +1052,7 @@ class CompresrGuardrail(CustomGuardrail): async def async_should_run_agentic_loop( self, - response: Any, + response: object, model: str, messages: list[dict], tools: list[dict] | None, @@ -1069,8 +1072,8 @@ class CompresrGuardrail(CustomGuardrail): tools: dict, model: str, messages: list[dict], - response: Any, - anthropic_messages_provider_config: Any, + response: object, + anthropic_messages_provider_config: BaseAnthropicMessagesConfig | None, anthropic_messages_optional_request_params: dict, logging_obj: LiteLLMLoggingObj | None, stream: bool, diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index 172b1440ca3..806ab5161e8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -2,10 +2,10 @@ from collections.abc import Callable, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeVar +from typing import TYPE_CHECKING, Final, Generic, Literal, Optional, TypeVar from fastapi import HTTPException -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack import litellm from litellm._logging import verbose_logger @@ -106,6 +106,23 @@ def _build_judge_prompt( ) +class _CustomGuardrailOptions(TypedDict, total=False): + """The ``CustomGuardrail`` options this guardrail accepts and forwards untouched.""" + + mask_request_content: ReadOnly[bool] + mask_response_content: ReadOnly[bool] + violation_message_template: ReadOnly[str | None] + end_session_after_n_fails: ReadOnly[int | None] + on_violation: ReadOnly[str | None] + realtime_violation_message: ReadOnly[str | None] + on_sensitive_data: ReadOnly[str | None] + sensitive_data_route_to_model: ReadOnly[str | None] + sticky_session_routing: ReadOnly[bool] + run_in_parallel: ReadOnly[bool] + scan_raw_request: ReadOnly[bool] + only_scan_new_messages: ReadOnly[bool] + + class LLMAsAJudgeGuardrail(CustomGuardrail): """Post-call guardrail that judges response quality via an LLM.""" @@ -119,7 +136,7 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | None = None, default_on: bool = False, router_provider: "Callable[[], Router | None] | None" = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: _event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | None = None if event_hook is not None: diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 7ef0a9f73f3..edd78e0bbc6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -9,13 +9,13 @@ import asyncio import json import os import warnings -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, AsyncIterable from datetime import datetime from typing import ( TYPE_CHECKING, - Any, Final, Literal, + TypeVar, ) from urllib.parse import urljoin @@ -39,9 +39,7 @@ from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( CallTypes, CallTypesLiteral, - EmbeddingResponse, GuardrailStatus, - ImageResponse, ModelResponseStream, TextCompletionResponse, ) @@ -53,7 +51,8 @@ SENSITIVE_DATA_DETECTOR_KEYS: Final[list[str]] = ["sensitiveData", "dataDetector # Type aliases MessageRole = Literal["user", "assistant"] -LLMResponse = Any | ModelResponse | EmbeddingResponse | ImageResponse +LLMResponse = object +_LLMResponseT: Final = TypeVar("_LLMResponseT") _LEGACY_NOMA_DEPRECATION_WARNED = False if TYPE_CHECKING: @@ -709,10 +708,10 @@ class NomaGuardrail(CustomGuardrail): async def _check_llm_response( self, request_data: dict, - response: LLMResponse, + response: _LLMResponseT, user_auth: UserAPIKeyAuth, event_type: GuardrailEventHooks | None = None, - ) -> Any: + ) -> _LLMResponseT: """Check LLM response for policy violations""" content: Final = await self._process_llm_response_check(request_data, response, user_auth, event_type) if not content: @@ -798,7 +797,7 @@ class NomaGuardrail(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: Any, + response: AsyncIterable[ModelResponseStream], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: """Process streaming response chunks with Noma guardrail.""" diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index b73d3adb99e..86ad2f9db5f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -793,7 +793,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): }, ) - def _prepare_metadata_from_request(self, data: dict[str, Any]) -> dict[str, Any]: + def _prepare_metadata_from_request(self, data: dict[str, Any]) -> dict[str, object]: """ Extract and prepare metadata from request data for PANW API call. @@ -809,7 +809,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): """ user_metadata: Final = data.get("metadata", {}) or {} requester_meta: Final = user_metadata.get("requester_metadata", {}) or {} - metadata: Final = { + metadata: Final[dict[str, object]] = { "user": data.get("user") or "litellm_user", "model": data.get("model") or "unknown", } diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 6259efb6654..3f7c36bbbf2 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -45,6 +45,7 @@ _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) _ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"passed": 0, "flagged": 1, "blocked": 2}) _T = TypeVar("_T") +_MetricsRowT = TypeVar("_MetricsRowT", bound="_DailyMetricsRow") _USAGE_MAX_RANGE_DAYS: Final = 366 @@ -360,10 +361,12 @@ def _trend_from_comparison(current_fail: float, previous_fail: float) -> str: return "stable" -def _aggregate_daily_metrics(metrics: "Sequence[_DailyMetricsRow]", id_attr: str) -> Mapping[str, _MetricTotals]: +def _aggregate_daily_metrics( + metrics: "Sequence[_MetricsRowT]", id_of: "Callable[[_MetricsRowT], str]" +) -> Mapping[str, _MetricTotals]: agg: Final[dict[str, _MetricTotals]] = {} for m in metrics: - gid: str = getattr(m, id_attr) + gid: str = id_of(m) if gid not in agg: agg[gid] = {"requests": 0, "passed": 0, "blocked": 0, "flagged": 0} agg[gid]["requests"] += int(m.requests_evaluated or 0) @@ -373,10 +376,12 @@ def _aggregate_daily_metrics(metrics: "Sequence[_DailyMetricsRow]", id_attr: str return agg -def _prev_fail_rates(metrics_prev: "Sequence[_DailyMetricsRow]", id_attr: str) -> Mapping[str, float]: +def _prev_fail_rates( + metrics_prev: "Sequence[_MetricsRowT]", id_of: "Callable[[_MetricsRowT], str]" +) -> Mapping[str, float]: prev_agg_raw: Final[dict[str, _PrevPeriodCounts]] = {} for m in metrics_prev: - gid: str = getattr(m, id_attr) + gid: str = id_of(m) r, b = int(m.requests_evaluated or 0), int(m.blocked_count or 0) if gid not in prev_agg_raw: prev_agg_raw[gid] = {"req": 0, "blocked": 0} @@ -429,7 +434,7 @@ def _field_str(mapping: Mapping[str, object], key: str, default: str) -> str: return str(mapping.get(key, default)) -def _get_guardrail_attrs(g: "_DbOrConfigGuardrail") -> tuple[Any, str]: +def _get_guardrail_attrs(g: "_DbOrConfigGuardrail") -> tuple[str | None, str]: """Get (guardrail_id, display_name) from guardrail - handles Prisma model or dict.""" gid: Final = _get_guardrail_field(g, "guardrail_id") name: Final = _get_guardrail_field(g, "guardrail_name") @@ -592,8 +597,8 @@ async def guardrails_usage_overview( Sequence[prisma_models.LiteLLM_DailyGuardrailUsageUnits] ] = await _find_daily_guardrail_usage_units(prisma_client, where=units_where) - agg: Final = _aggregate_daily_metrics(metrics, "guardrail_id") - prev_agg: Final = _prev_fail_rates(metrics_prev, "guardrail_id") + agg: Final = _aggregate_daily_metrics(metrics, lambda m: m.guardrail_id) + prev_agg: Final = _prev_fail_rates(metrics_prev, lambda m: m.guardrail_id) units_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_counter_units) cost_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_tracked_cost) untracked_agg: Final = _by(units_rows, lambda r: r.guardrail_id, _sum_untracked_units) @@ -811,7 +816,7 @@ def _usage_log_entry_from_row( ) -def _snippet(text: Any, max_len: int = 200) -> str | None: +def _snippet(text: object, max_len: int = 200) -> str | None: if text is None: return None if isinstance(text, str): @@ -964,8 +969,8 @@ async def policies_usage_overview( } }, ) - agg: Final = _aggregate_daily_metrics(metrics, "policy_id") - prev_agg: Final = _prev_fail_rates(metrics_prev, "policy_id") + agg: Final = _aggregate_daily_metrics(metrics, lambda m: m.policy_id) + prev_agg: Final = _prev_fail_rates(metrics_prev, lambda m: m.policy_id) chart: Final = _chart_from_metrics(metrics) total_requests: Final = sum(a["requests"] for a in agg.values()) total_blocked: Final = sum(a["blocked"] for a in agg.values()) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c4fba8ecf9e..e6ada41f062 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -2,7 +2,7 @@ import asyncio import traceback from collections.abc import Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Protocol, cast import litellm from litellm._logging import verbose_proxy_logger @@ -578,18 +578,36 @@ def _get_request_tags_for_cost_tracking( return None +class _IncrementSpendCounters(Protocol): + """The ``increment_spend_counters`` coroutine :func:`_update_database_and_spend_counters` awaits.""" + + async def __call__( + self, + token: str | None, + team_id: str | None, + user_id: str | None, + response_cost: float | None, + org_id: str | None = None, + budget_reservation: dict[str, object] | None = None, + end_user_id: str | None = None, + tags: list[str] | None = None, + request_started_at: datetime | None = None, + model_access_groups: Sequence[str] | None = None, + ) -> None: ... + + async def _update_database_and_spend_counters( proxy_logging_obj: "ProxyLogging", - increment_spend_counters: Any, + increment_spend_counters: _IncrementSpendCounters, user_api_key: str | None, user_id: str | None, end_user_id: str | None, team_id: str | None, org_id: str | None, kwargs: dict, - completion_response: litellm.ModelResponse | Any | None, - start_time: Any, - end_time: Any, + completion_response: object, + start_time: datetime | None, + end_time: datetime | None, response_cost: float, budget_reservation: dict | None, request_tags: list[str] | None = None, diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index a48130a4f22..e960bdfe337 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -540,7 +540,7 @@ async def get_all_access_groups_from_db( deployments: Final = await ModelRepository(prisma_client).table.find_many() # Build access group map - access_group_map: Final[dict[str, dict[str, Any]]] = {} + model_names_by_group: Final[dict[str, list[str]]] = {} for deployment in deployments: model_info = deployment.model_info or {} @@ -550,25 +550,20 @@ async def get_all_access_groups_from_db( model_name = deployment.model_name for access_group in access_groups: - if access_group not in access_group_map: - access_group_map[access_group] = { - "model_names": set(), - "deployment_count": 0, - } + if access_group not in model_names_by_group: + model_names_by_group[access_group] = [] - access_group_map[access_group]["model_names"].add(model_name) - access_group_map[access_group]["deployment_count"] += 1 + model_names_by_group[access_group].append(model_name) # Convert to AccessGroupInfo objects - result: Final = {} - for access_group, data in access_group_map.items(): - result[access_group] = AccessGroupInfo( + return { + access_group: AccessGroupInfo( access_group=access_group, - model_names=sorted(list(data["model_names"])), - deployment_count=data["deployment_count"], + model_names=sorted(frozenset(model_names)), + deployment_count=len(model_names), ) - - return result + for access_group, model_names in model_names_by_group.items() + } @router.post( diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py index da4ddbd0aac..1265da99d89 100644 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py @@ -6,7 +6,7 @@ usage/spend data by querying the aggregated daily activity endpoints. import json from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from datetime import date -from typing import Any, Final, Literal, NamedTuple, Protocol, cast, overload +from typing import Final, Literal, NamedTuple, Protocol, cast, overload from typing_extensions import ReadOnly, TypedDict @@ -16,6 +16,7 @@ from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) +from litellm.types.utils import ChatCompletionMessageToolCall # --------------------------------------------------------------------------- # Constants @@ -489,19 +490,19 @@ async def _execute_tool_call( async def _process_tool_call( - tc: Any, + tc: ChatCompletionMessageToolCall, chat_messages: list[Mapping[str, object]], user_id: str | None, is_admin: bool, ) -> AsyncIterator[str]: """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final[str] = tc.function.name + fn_name: Final = tc.function.name fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) + handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None - if fn_name not in allowed_names or not handler: + if fn_name is None or fn_name not in allowed_names or not handler: chat_messages.append( { "role": "tool", diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index bf07f4748ef..4d3a397e519 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -485,8 +485,8 @@ async def create_file( # Parse expires_after if provided expires_after: FileExpiresAfter | None = None form_data_raw: Final = await request.form() - form_data_dict: Final[dict[str, Any]] = dict(form_data_raw) - extracted_litellm_metadata: Final[dict[str, Any] | None] = extract_nested_form_metadata( + form_data_dict: Final[Mapping[str, object]] = dict(form_data_raw) + extracted_litellm_metadata: Final[Mapping[str, object] | None] = extract_nested_form_metadata( form_data=form_data_dict, prefix="litellm_metadata[" ) expires_after_anchor: Final = form_data_raw.get("expires_after[anchor]") diff --git a/litellm/proxy/openai_files_endpoints/storage_backend_service.py b/litellm/proxy/openai_files_endpoints/storage_backend_service.py index e766f335071..0407499cbcc 100644 --- a/litellm/proxy/openai_files_endpoints/storage_backend_service.py +++ b/litellm/proxy/openai_files_endpoints/storage_backend_service.py @@ -7,7 +7,6 @@ storage backends (e.g., Azure Blob Storage) and managing associated metadata. import base64 import time -from collections.abc import Mapping from typing import Any, Final, cast from litellm._logging import verbose_proxy_logger @@ -17,7 +16,7 @@ from litellm.llms.base_llm.files.transformation import BaseFileEndpoints from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose -from litellm.types.utils import SpecialEnums +from litellm.types.utils import ExtractedFileData, SpecialEnums class StorageBackendFileService: @@ -33,7 +32,7 @@ class StorageBackendFileService: @staticmethod async def upload_file_to_storage_backend( - file_data: Mapping[str, Any], + file_data: ExtractedFileData, target_storage: str, target_model_names: list[str], purpose: OpenAIFilesPurpose, @@ -163,7 +162,7 @@ class StorageBackendFileService: @staticmethod def _create_unified_file_id( - file_type: str, + file_type: str | None, target_model_names: list[str], file_id: str, ) -> str: @@ -193,7 +192,7 @@ class StorageBackendFileService: @staticmethod async def _store_in_managed_files( file_object: OpenAIFileObject, - file_data: Mapping[str, Any], + file_data: ExtractedFileData, target_model_names: list[str], target_storage: str, storage_url: str, diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 1feda0b0bb5..f21c294e5a2 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -244,7 +244,7 @@ async def vector_store_create( ) # Create vector store across multiple models - response: Final = await managed_vector_stores.acreate_vector_store( + response: Final[object] = await managed_vector_stores.acreate_vector_store( create_request=data, llm_router=llm_router, target_model_names_list=target_model_names_list, diff --git a/litellm/proxy/vector_store_files_endpoints/endpoints.py b/litellm/proxy/vector_store_files_endpoints/endpoints.py index 957ed9fd0b9..3ddea288ab8 100644 --- a/litellm/proxy/vector_store_files_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_files_endpoints/endpoints.py @@ -553,7 +553,7 @@ async def vector_store_file_create( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -756,7 +756,7 @@ async def vector_store_file_retrieve( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -863,7 +863,7 @@ async def vector_store_file_content( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -973,7 +973,7 @@ async def vector_store_file_update( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -1080,7 +1080,7 @@ async def vector_store_file_delete( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy_auth/credentials.py b/litellm/proxy_auth/credentials.py index a4e29241959..f8814954a7a 100644 --- a/litellm/proxy_auth/credentials.py +++ b/litellm/proxy_auth/credentials.py @@ -7,7 +7,7 @@ It follows the same TokenCredential protocol used by Azure SDK. import time from dataclasses import dataclass -from typing import Any, Final, Protocol, runtime_checkable +from typing import Final, Protocol, runtime_checkable @dataclass @@ -50,6 +50,22 @@ class TokenCredential(Protocol): ... +class _AzureAccessToken(Protocol): + """The two attributes :class:`AzureADCredential` reads off an azure-identity token.""" + + @property + def token(self) -> str: ... + + @property + def expires_on(self) -> int: ... + + +class _AzureTokenCredential(Protocol): + """The single method :class:`AzureADCredential` calls on the credential it wraps.""" + + def get_token(self, *scopes: str) -> _AzureAccessToken: ... + + class AzureADCredential: """ Wrapper for Azure Identity credentials. @@ -71,7 +87,7 @@ class AzureADCredential: cred = AzureADCredential(credential=azure_cred) """ - def __init__(self, credential: Any | None = None): + def __init__(self, credential: _AzureTokenCredential | None = None): """ Initialize with an optional Azure credential. @@ -79,7 +95,7 @@ class AzureADCredential: credential: An azure-identity credential object. If None, DefaultAzureCredential will be used on first token request. """ - self._credential: Any = credential + self._credential: _AzureTokenCredential | None = credential self._initialized = credential is not None def get_token(self, scope: str) -> AccessToken: @@ -95,20 +111,30 @@ class AzureADCredential: Raises: ImportError: If azure-identity is not installed. """ - if not self._initialized: - try: - from azure.identity import DefaultAzureCredential - - self._credential = DefaultAzureCredential() - self._initialized = True - except ImportError: - raise ImportError( - "azure-identity is required for AzureADCredential. Install it with: pip install azure-identity" - ) - - result: Final = self._credential.get_token(scope) + result: Final = self._resolve_credential().get_token(scope) return AccessToken(token=result.token, expires_on=result.expires_on) + def _resolve_credential(self) -> _AzureTokenCredential: + """Return the wrapped credential, building the Azure default chain on first use. + + Raises: + ImportError: If azure-identity is not installed. + """ + existing: Final = self._credential + if existing is not None: + return existing + try: + from azure.identity import DefaultAzureCredential + + created: Final = DefaultAzureCredential() + except ImportError: + raise ImportError( + "azure-identity is required for AzureADCredential. Install it with: pip install azure-identity" + ) + self._credential = created + self._initialized = True + return created + class GenericOAuth2Credential: """ @@ -228,7 +254,7 @@ class ProxyAuthHandler: self._cached_token = self.credential.get_token(self.scope) return self._cached_token - def get_auth_headers(self) -> dict: + def get_auth_headers(self) -> dict[str, str]: """ Get HTTP headers for authentication. diff --git a/litellm/rag/rag_query.py b/litellm/rag/rag_query.py index 255faf94402..9325547c17d 100644 --- a/litellm/rag/rag_query.py +++ b/litellm/rag/rag_query.py @@ -124,9 +124,9 @@ class RAGQuery: @staticmethod def extract_documents_from_search( search_response: Any, - ) -> list[str | dict[str, Any]]: + ) -> list[str | dict[str, object]]: """Extract text documents from vector store search response.""" - documents: Final[list[str | dict[str, Any]]] = [] + documents: Final[list[str | dict[str, object]]] = [] search_data: Final[_SearchDataView] = {"results": search_response.get("data", [])} for result in search_data["results"]: content_list = result.get("content", []) diff --git a/litellm/repositories/verification_token_repository.py b/litellm/repositories/verification_token_repository.py index c0e59f9b975..d02c2114136 100644 --- a/litellm/repositories/verification_token_repository.py +++ b/litellm/repositories/verification_token_repository.py @@ -5,7 +5,8 @@ VerificationToken repository for database operations on LiteLLM_VerificationToke import json from collections.abc import Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Final +from types import TracebackType +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.verification_token import ( LiteLLM_VerificationToken, @@ -25,7 +26,36 @@ if TYPE_CHECKING: LiteLLM_VerificationToken as PrismaVerificationToken, ) - from litellm.proxy.utils import PrismaClient + +class _VerificationTokenTables(Protocol): + """The two verification token tables this repository reads and writes.""" + + @property + def litellm_verificationtoken(self) -> TableActions["PrismaVerificationToken"]: ... + + @property + def litellm_deletedverificationtoken(self) -> TableActions["PrismaDeletedVerificationToken"]: ... + + +class _VerificationTokenTransactionManager(Protocol): + async def __aenter__(self) -> _VerificationTokenTables: ... + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool | None: ... + + +class _PrismaVerificationTokenDb(_VerificationTokenTables, Protocol): + def tx(self) -> _VerificationTokenTransactionManager: ... + + +class _PrismaClientView(Protocol): + @property + def db(self) -> _PrismaVerificationTokenDb: ... + _JSON_ENCODED_TOKEN_FIELDS: Final = ( "aliases", @@ -44,17 +74,17 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): """Repository for verification token (API key) database operations.""" @property - def prisma_client(self) -> "PrismaClient": - prisma_client: Final[PrismaClient] = super().prisma_client - return prisma_client + def _db(self) -> _PrismaVerificationTokenDb: + client: Final[_PrismaClientView] = self.prisma_client + return client.db @property def table(self) -> TableActions["PrismaVerificationToken"]: - return self.prisma_client.db.litellm_verificationtoken + return self._db.litellm_verificationtoken @property def deleted_table(self) -> TableActions["PrismaDeletedVerificationToken"]: - return self.prisma_client.db.litellm_deletedverificationtoken + return self._db.litellm_deletedverificationtoken @property def model_class(self) -> type[LiteLLM_VerificationToken]: @@ -325,7 +355,7 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]): archive_data["litellm_changed_by"] = litellm_changed_by archive_data["deleted_at"] = datetime.utcnow() - async with self.prisma_client.db.tx() as tx: + async with self._db.tx() as tx: await tx.litellm_deletedverificationtoken.create(data=archive_data) await tx.litellm_verificationtoken.delete(where={"token": token}) diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 309894957ea..76e833563ba 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol from litellm._logging import verbose_router_logger if TYPE_CHECKING: - from litellm.types.router import SearchToolTypedDict + from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict class _SearchToolsRouter(Protocol): @@ -34,7 +34,7 @@ class SearchAPIRouter: @staticmethod def _resolve_search_provider_credentials( *, - tool_litellm_params: dict[str, Any], + tool_litellm_params: "SearchToolLiteLLMParams", ) -> tuple[str | None, str | None]: """ Resolve search provider credentials from tool configuration ONLY. From 1ec5083ab4d94c758227c22be052f69e2ae2cf49 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:41:07 +0000 Subject: [PATCH 03/11] refactor(types): replace Any with real types across 54 more backend files Third batch of the fifth basedpyright Any reduction round. Every change is typing-only and leaves runtime behavior identical. Provider transformation configs, video and rerank base classes, OTel metadata and the guardrail and realtime type modules move their payload, header and optional-parameter annotations from Any to object, Mapping[str, object] or the concrete model the call site already produces. Repositories and endpoints that reached Prisma through an untyped handle now name the actions they call with the repo's own TableActions protocol. The pydantic field retypes were checked against pydantic to confirm object and Any validate, serialize and generate JSON schema identically. --- .../providers/pydantic_ai_agents/config.py | 4 +-- litellm/integrations/argilla.py | 7 +++-- litellm/integrations/dynamodb.py | 19 +++++++++++-- litellm/integrations/focus/database.py | 6 ++-- .../focus/destinations/vantage_destination.py | 5 ++-- .../langfuse/langfuse_otel_attributes.py | 3 +- litellm/integrations/langsmith.py | 8 +++--- litellm/integrations/otel/model/metadata.py | 14 +++++----- litellm/integrations/otel/mount.py | 19 +++++++++++-- .../litellm_core_utils/coroutine_checker.py | 6 ++-- .../exception_mapping_utils.py | 6 ++-- .../huggingface_template_handler.py | 26 +++++++++++++---- .../anthropic/count_tokens/handler.py | 2 +- .../llms/base_llm/videos/transformation.py | 10 +++---- .../count_tokens/bedrock_token_counter.py | 11 ++++---- litellm/llms/bedrock/files/transformation.py | 6 ++-- .../guardrail_translation/handler.py | 5 +++- litellm/llms/chatgpt/common_utils.py | 4 +-- .../responses/transformation.py | 7 +++-- .../llms/hosted_vllm/chat/transformation.py | 21 ++++++-------- litellm/llms/openai/common_utils.py | 6 ++-- litellm/llms/snowflake/chat/transformation.py | 4 +-- .../stability/image_edit/transformations.py | 4 +-- .../llms/triton/completion/transformation.py | 8 +++--- .../llms/vertex_ai/rerank/transformation.py | 4 +-- .../volcengine/embedding/transformation.py | 9 +++--- litellm/llms/watsonx/rerank/transformation.py | 8 +++--- litellm/llms/xai/realtime/transformation.py | 18 ++++++------ .../analytics_endpoints/cache_activity.py | 20 +++++++++---- .../proxy/client/cli/commands/model_groups.py | 18 ++++++++---- litellm/proxy/client/cli/commands/up.py | 3 +- .../common_utils/cache_pydantic_utils.py | 2 +- .../common_utils/proxy_rate_limit_error.py | 4 +-- .../proxy/container_endpoints/endpoints.py | 10 ++++--- litellm/proxy/db/exception_handler.py | 16 +++++++++-- litellm/proxy/db/spend_log_tool_index.py | 2 +- .../crowdstrike_aidr/crowdstrike_aidr.py | 16 +++++------ .../guardrail_hooks/enkryptai/enkryptai.py | 4 +-- .../guardrail_hooks/qualifire/qualifire.py | 2 +- .../semantic_guard/route_loader.py | 3 +- .../proxy/hooks/parallel_request_limiter.py | 6 ++-- .../common_daily_activity.py | 7 +++-- litellm/proxy/ocr_endpoints/endpoints.py | 14 +++++----- litellm/repositories/budget_repository.py | 28 +++++++++++++++---- .../repositories/organization_repository.py | 28 +++++++++++++++---- litellm/repositories/project_repository.py | 11 ++++---- .../adaptive_router/adaptive_router.py | 9 +++--- .../adaptive_router/signals.py | 8 +++--- .../adaptive_router/update_queue.py | 11 ++++---- litellm/types/containers/main.py | 2 +- litellm/types/guardrails.py | 16 +++++------ litellm/types/integrations/prometheus.py | 8 +++--- litellm/types/realtime.py | 16 +++++------ litellm/vector_store_files/utils.py | 11 ++++---- 54 files changed, 323 insertions(+), 202 deletions(-) diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py index b7546e1a2a1..20404e3702b 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/config.py @@ -23,7 +23,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): params: dict[str, Any], api_base: str | None = None, **kwargs: Any, - ) -> dict[str, Any]: + ) -> dict[str, object]: """Handle non-streaming request to Pydantic AI agent.""" if api_base is None: raise ValueError("api_base is required for PydanticAIProviderConfig") @@ -41,7 +41,7 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig): params: dict[str, Any], api_base: str | None = None, **kwargs, - ) -> AsyncIterator[dict[str, Any]]: + ) -> AsyncIterator[dict[str, object]]: """Handle streaming request with fake streaming.""" if not api_base: raise ValueError("api_base is required for Pydantic AI agents") diff --git a/litellm/integrations/argilla.py b/litellm/integrations/argilla.py index 9a87a94cf0b..664ef8efda1 100644 --- a/litellm/integrations/argilla.py +++ b/litellm/integrations/argilla.py @@ -7,7 +7,8 @@ import json import os import random import types -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx from pydantic import BaseModel @@ -69,7 +70,7 @@ class ArgillaLogger(CustomBatchLogger): self.flush_lock = asyncio.Lock() super().__init__(**kwargs, flush_lock=self.flush_lock) - def validate_argilla_transformation_object(self, argilla_transformation_object: dict[str, Any]): + def validate_argilla_transformation_object(self, argilla_transformation_object: Mapping[str, object]): if not isinstance(argilla_transformation_object, dict): raise Exception("'argilla_transformation_object' must be a dictionary, to log your payload to Argilla.") @@ -115,7 +116,7 @@ class ArgillaLogger(CustomBatchLogger): ARGILLA_DATASET_NAME=_credentials_dataset_name, ) - def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, Any]]: + def get_chat_messages(self, payload: StandardLoggingPayload) -> list[dict[str, object]]: payload_messages: Final = payload.get("messages", None) if payload_messages is None: diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 38f5924a233..3401ced4efb 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -3,12 +3,25 @@ import os import traceback -from typing import Any, Final +from collections.abc import Mapping +from typing import Final, Protocol import litellm from litellm._uuid import uuid +class _DynamoTable(Protocol): + """The one boto3 DynamoDB table call this logger makes.""" + + def put_item(self, *, Item: Mapping[str, object]) -> object: ... + + +class _DynamoResource(Protocol): + """The one boto3 DynamoDB resource call this logger makes.""" + + def Table(self, name: str) -> _DynamoTable: ... + + class DyanmoDBLogger: # Class variables or attributes @@ -16,7 +29,7 @@ class DyanmoDBLogger: # Instance variables import boto3 - self.dynamodb: Any = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"]) + self.dynamodb: Final[_DynamoResource] = boto3.resource("dynamodb", region_name=os.environ["AWS_REGION_NAME"]) if litellm.dynamodb_table_name is None: raise ValueError( "LiteLLM Error, trying to use DynamoDB but not table name passed. Create a table and set `litellm.dynamodb_table_name=`" @@ -41,7 +54,7 @@ class DyanmoDBLogger: id: Final = response_obj.get("id", str(uuid.uuid4())) # Build the initial payload - payload: Final = { + payload: Final[dict[str, object]] = { "id": id, "call_type": call_type, "startTime": start_time, diff --git a/litellm/integrations/focus/database.py b/litellm/integrations/focus/database.py index 657c7e0d264..891318f1c54 100644 --- a/litellm/integrations/focus/database.py +++ b/litellm/integrations/focus/database.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Final +from typing import Final import polars as pl @@ -32,7 +32,7 @@ class FocusLiteLLMDatabase: client: Final = self._ensure_prisma_client() where_clauses: Final[list[str]] = [] - query_params: Final[list[Any]] = [] + query_params: Final[list[datetime | int]] = [] placeholder_index = 1 if start_time_utc: where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz") @@ -112,7 +112,7 @@ class FocusLiteLLMDatabase: except Exception as exc: raise RuntimeError(f"Error retrieving usage data: {exc}") from exc - async def get_table_info(self) -> dict[str, Any]: + async def get_table_info(self) -> dict[str, object]: """Return metadata about the spend table for diagnostics.""" client: Final = self._ensure_prisma_client() diff --git a/litellm/integrations/focus/destinations/vantage_destination.py b/litellm/integrations/focus/destinations/vantage_destination.py index 132f27779c2..68b0d399975 100644 --- a/litellm/integrations/focus/destinations/vantage_destination.py +++ b/litellm/integrations/focus/destinations/vantage_destination.py @@ -4,7 +4,8 @@ from __future__ import annotations import csv import io -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx # noqa: F401 - used at runtime (AsyncClient, HTTPStatusError) @@ -94,7 +95,7 @@ class FocusVantageDestination(FocusDestination): self, *, prefix: str, - config: dict[str, Any] | None = None, + config: Mapping[str, object] | None = None, ) -> None: config = config or {} api_key: Final = config.get("api_key") diff --git a/litellm/integrations/langfuse/langfuse_otel_attributes.py b/litellm/integrations/langfuse/langfuse_otel_attributes.py index 70fea1abb3b..1eb3ce1a9c2 100644 --- a/litellm/integrations/langfuse/langfuse_otel_attributes.py +++ b/litellm/integrations/langfuse/langfuse_otel_attributes.py @@ -5,6 +5,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764 """ import json +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel @@ -40,7 +41,7 @@ def get_output_content_by_type( | HttpxBinaryResponseContent | ResponsesAPIResponse | list, - kwargs: dict[str, Any] | None = None, + kwargs: Mapping[str, object] | None = None, ) -> str: """ Extract output content from response objects based on their type. diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 9607eccef52..2c3837b12ec 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -75,9 +75,9 @@ class LangsmithLogger(CustomBatchLogger): if _batch_size: self.batch_size = int(_batch_size) self.log_queue: list[LangsmithQueueObject] = [] - self._flush_task: asyncio.Task[Any] | None = self._start_periodic_flush_task() + self._flush_task: asyncio.Task[None] | None = self._start_periodic_flush_task() - def _start_periodic_flush_task(self) -> asyncio.Task[Any] | None: + def _start_periodic_flush_task(self) -> asyncio.Task[None] | None: """Start the periodic flush task only when an event loop is already running.""" try: loop: Final = asyncio.get_running_loop() @@ -152,9 +152,9 @@ class LangsmithLogger(CustomBatchLogger): return self._redact_metadata(extra_metadata) - def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> dict[str, Any]: + def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> dict[str, object]: response: Final = payload["response"] - outputs: dict[str, Any] + outputs: dict[str, object] if isinstance(response, dict): outputs = {**response} else: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index ee116aca46b..9eab3021d58 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -36,7 +36,7 @@ model. They coincide on the SDK path, which is correct. from __future__ import annotations -from collections.abc import Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast @@ -57,7 +57,7 @@ class RequestIdentity: # The team's free-form metadata, carried raw (empty/missing -> None) and # filtered to an operator allowlist only at Baggage-promotion time, so an # unconfigured deployment never promotes any of it. - team_metadata: Mapping[str, Any] | None = None + team_metadata: Mapping[str, object] | None = None key_hash: str | None = None end_user: str | None = None # The model litellm dispatched to the provider. Only known once the call @@ -103,7 +103,7 @@ class RequestIdentity: ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` promotes. """ - get: Final = lambda name: getattr(auth, name, None) # noqa: E731 + get: Final[Callable[[str], object]] = lambda name: getattr(auth, name, None) # noqa: E731 metadata: Final = { meta_key: str(value) for meta_key, attr in ( @@ -217,7 +217,7 @@ class LLMCallEvent: time_to_first_chunk_seconds: float | None @classmethod - def from_dict(cls, kwargs: Mapping[str, Any]) -> LLMCallEvent: + def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent: raw_payload: Final = kwargs.get("standard_logging_object") payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None operation: Final = resolve_operation(as_str(kwargs.get("call_type"))) @@ -239,7 +239,7 @@ def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: to the first streamed chunk (``completion_start_time``); ``None`` for non-streaming calls, where ``completion_start_time`` is backfilled with the end time and would not measure first-chunk latency.""" - optional_params: Final = cast(Mapping[str, Any], kwargs.get("optional_params") or {}) + optional_params: Final = cast(Mapping[str, object], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None api_call_start: Final = to_seconds(kwargs.get("api_call_start_time")) @@ -307,7 +307,7 @@ def _metadata_dicts( ) -def _call_id(payload: StandardLoggingPayload | None, kwargs: Mapping[str, Any]) -> str | None: +def _call_id(payload: StandardLoggingPayload | None, kwargs: Mapping[str, object]) -> str | None: """The call id from the payload (when closed) or the bare kwargs (at pre_call).""" if payload is not None: call_id: Final = as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")) @@ -351,7 +351,7 @@ def _model_info_id(model_info: object) -> str | None: return None -def _team_metadata_dict(value: object) -> Mapping[str, Any] | None: +def _team_metadata_dict(value: object) -> Mapping[str, object] | None: """The team's free-form metadata as a raw mapping, or ``None`` when missing or empty. diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index ac647c2c4f6..776d8722d14 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -12,11 +12,14 @@ when the feature gate is off. """ import os -from typing import Any, Final +from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_logger from litellm.integrations.otel.model.config import is_otel_v2_enabled +if TYPE_CHECKING: + from fastapi import FastAPI + # Routes excluded from server-span tracing by default: high-frequency pollers and # static UI/docs assets, none of which are LLM traffic. Entries are substring-matched # against the request path (unanchored, so they survive a ``server_root_path`` prefix @@ -65,7 +68,17 @@ PASSTHROUGH_PREFIXES: Final = frozenset( ) -def _passthrough_span_name_hook(span: Any, scope: dict) -> None: +class _RenameableSpan(Protocol): + """The span surface the passthrough naming hook drives.""" + + def is_recording(self) -> bool: ... + + def update_name(self, name: str) -> None: ... + + def set_attribute(self, key: str, value: str) -> None: ... + + +def _passthrough_span_name_hook(span: "_RenameableSpan | None", scope: dict) -> None: """FastAPI ``server_request_hook``: give passthrough server spans a useful name. The instrumentation matches the route at span creation, so both the span name @@ -88,7 +101,7 @@ def _passthrough_span_name_hook(span: Any, scope: dict) -> None: pass -def instrument_fastapi_app(app: Any) -> None: +def instrument_fastapi_app(app: "FastAPI") -> None: """Attach OTel server-span instrumentation to the proxy FastAPI app. Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` diff --git a/litellm/litellm_core_utils/coroutine_checker.py b/litellm/litellm_core_utils/coroutine_checker.py index 52fc44ba8dc..a5d4f8b968f 100644 --- a/litellm/litellm_core_utils/coroutine_checker.py +++ b/litellm/litellm_core_utils/coroutine_checker.py @@ -16,7 +16,7 @@ class CoroutineChecker: """ def __init__(self): - self._cache = WeakKeyDictionary() + self._cache: WeakKeyDictionary[object, bool] = WeakKeyDictionary() self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY def is_async_callable(self, callback: Any) -> bool: @@ -33,10 +33,10 @@ class CoroutineChecker: pass # Determine target - optimized path for common cases - target = callback + target: object = callback if not inspect.isfunction(target) and not inspect.ismethod(target): try: - call_attr: Final = getattr(target, "__call__", None) + call_attr: Final[object] = getattr(target, "__call__", None) if call_attr is not None: target = call_attr except Exception: diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 82708d412c9..92f07057da5 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -1,7 +1,7 @@ import json import re import traceback -from typing import Any, Final, Protocol, cast +from typing import Final, Protocol, cast import httpx @@ -191,7 +191,7 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None _response_headers: httpx.Headers | None = None try: _response_headers = getattr(original_exception, "headers", None) - error_response: Final = getattr(original_exception, "response", None) + error_response: Final[object] = getattr(original_exception, "response", None) if not _response_headers and error_response: _response_headers = getattr(error_response, "headers", None) if not _response_headers: @@ -203,7 +203,7 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None def extract_and_raise_litellm_exception( - response: Any | None, + response: object | None, error_str: str, model: str, custom_llm_provider: str, diff --git a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py index 3878b36cd91..8f8228d6dfd 100644 --- a/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py +++ b/litellm/litellm_core_utils/prompt_templates/huggingface_template_handler.py @@ -1,6 +1,8 @@ import json from datetime import datetime -from typing import Any, Final +from typing import Any, Final, Literal + +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -9,6 +11,20 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.types.llms.custom_http import httpxSpecialProvider +class _TokenizerConfigResult(TypedDict): + """Outcome of a tokenizer_config.json fetch, carrying the parsed document when the fetch succeeded.""" + + status: ReadOnly[Literal["success", "failure"]] + tokenizer: NotRequired[ReadOnly[object]] + + +class _ChatTemplateFileResult(TypedDict): + """Outcome of a chat template file fetch, carrying the template body when the fetch succeeded.""" + + status: ReadOnly[Literal["success", "failure"]] + chat_template: NotRequired[ReadOnly[str]] + + def strftime_now(fmt: str) -> str: """ Custom function for templates that need current date/time formatting (e.g., gpt-oss) @@ -22,7 +38,7 @@ def strftime_now(fmt: str) -> str: return datetime.now().strftime(fmt) -def _get_tokenizer_config(hf_model_name: str) -> dict[str, Any]: +def _get_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: """ Fetch tokenizer_config.json from HuggingFace (sync) @@ -45,7 +61,7 @@ def _get_tokenizer_config(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -async def _aget_tokenizer_config(hf_model_name: str) -> dict[str, Any]: +async def _aget_tokenizer_config(hf_model_name: str) -> _TokenizerConfigResult: """ Fetch tokenizer_config.json from HuggingFace (async) @@ -70,7 +86,7 @@ async def _aget_tokenizer_config(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -def _get_chat_template_file(hf_model_name: str) -> dict[str, Any]: +def _get_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult: """ Fetch chat template from separate .jinja file (sync) @@ -98,7 +114,7 @@ def _get_chat_template_file(hf_model_name: str) -> dict[str, Any]: return {"status": "failure"} -async def _aget_chat_template_file(hf_model_name: str) -> dict[str, Any]: +async def _aget_chat_template_file(hf_model_name: str) -> _ChatTemplateFileResult: """ Fetch chat template from separate .jinja file (async) diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 3cc90823af9..36d5a56db0d 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -33,7 +33,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): litellm_params: dict[str, Any] | None = None, timeout: float | httpx.Timeout | None = None, tools: list[dict[str, Any]] | None = None, - system: Any | None = None, + system: object = None, ) -> dict[str, Any]: """ Handle a CountTokens request using httpx with Azure authentication. diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index f725b295d0f..88cdcd61dd6 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -180,7 +180,7 @@ class BaseVideoConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video remix request into a URL and data @@ -207,7 +207,7 @@ class BaseVideoConfig(ABC): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video list request into a URL and params @@ -342,8 +342,8 @@ class BaseVideoConfig(ABC): litellm_params: GenericLiteLLMParams, headers: dict, video_file: FileContent | None = None, - extra_body: dict[str, Any] | None = None, - prefetched_source_data: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, + prefetched_source_data: dict[str, object] | None = None, ) -> tuple[str, Mapping[str, object], RequestFiles | None]: """ Transform the video edit request into a URL plus either JSON data or @@ -373,7 +373,7 @@ class BaseVideoConfig(ABC): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video extension request into a URL and JSON data. diff --git a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py index 9c5211ed072..2d02b152c61 100644 --- a/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py +++ b/litellm/llms/bedrock/count_tokens/bedrock_token_counter.py @@ -2,6 +2,7 @@ Bedrock Token Counter implementation using the CountTokens API. """ +from collections.abc import Mapping, Sequence from typing import Any, Final from litellm._logging import verbose_logger @@ -26,12 +27,12 @@ class BedrockTokenCounter(BaseTokenCounter): async def count_tokens( self, model_to_use: str, - messages: list[dict[str, Any]] | None, - contents: list[dict[str, Any]] | None, + messages: Sequence[Mapping[str, object]] | None, + contents: Sequence[Mapping[str, object]] | None, deployment: dict[str, Any] | None = None, request_model: str = "", - tools: list[dict[str, Any]] | None = None, - system: Any | None = None, + tools: Sequence[Mapping[str, object]] | None = None, + system: object | None = None, ) -> TokenCountResponse | None: """ Count tokens using AWS Bedrock's CountTokens API. @@ -56,7 +57,7 @@ class BedrockTokenCounter(BaseTokenCounter): litellm_params: Final = deployment.get("litellm_params", {}) # Build request data in the format expected by BedrockCountTokensHandler - request_data: Final[dict[str, Any]] = { + request_data: Final[dict[str, object]] = { "model": model_to_use, "messages": messages, } diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 33b27943ad8..f50d63c9f6f 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -247,7 +247,7 @@ def _validate_file_id_against_configured_buckets( return validate_against(configured_bucket_names[-1]) -def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Response) -> int: +def _uploaded_object_size(litellm_params: Mapping[str, object], response_headers: Mapping[str, str]) -> int: """ S3 answers PutObject with an empty body, so the stored object size comes from the signed request recorded by `transform_create_file_request`, not the response headers. @@ -255,7 +255,7 @@ def _uploaded_object_size(litellm_params: Mapping[str, object], raw_response: Re uploaded_size: Final = litellm_params.get(UPLOAD_CONTENT_LENGTH_PARAM) if isinstance(uploaded_size, int): return uploaded_size - response_content_length: Final = raw_response.headers.get("Content-Length", "0") + response_content_length: Final = response_headers.get("Content-Length", "0") return int(response_content_length) if response_content_length.isdigit() else 0 @@ -1161,7 +1161,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): filename=filename, created_at=int(time.time()), # Current timestamp status="uploaded", - bytes=_uploaded_object_size(litellm_params=litellm_params, raw_response=raw_response), + bytes=_uploaded_object_size(litellm_params=litellm_params, response_headers=raw_response.headers), object="file", ) diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index b87f6196e51..eac8afd767c 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -14,6 +14,9 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.pass_through.guardrail_translation.handler import ( + PassThroughEndpointHandler, + ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -27,7 +30,7 @@ def _is_converse_endpoint(endpoint: str) -> bool: return bool(parts) and parts[-1] in _CONVERSE_ACTIONS -def _generic_passthrough_handler() -> BaseTranslation: +def _generic_passthrough_handler() -> "PassThroughEndpointHandler": """ Fallback for non-Converse Bedrock routes (e.g. invoke). The generic handler scans the full request/response payload so blocking guardrails diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index 35e32e4172f..fe33219f110 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -268,7 +268,7 @@ def _normalize_litellm_params(litellm_params: Any | None) -> dict: return {} -def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: +def get_chatgpt_session_id(litellm_params: object) -> str | None: params: Final = _normalize_litellm_params(litellm_params) for key in ("litellm_session_id", "session_id"): value = params.get(key) @@ -286,5 +286,5 @@ def get_chatgpt_session_id(litellm_params: Any | None) -> str | None: return None -def ensure_chatgpt_session_id(litellm_params: Any | None) -> str: +def ensure_chatgpt_session_id(litellm_params: object) -> str: return get_chatgpt_session_id(litellm_params) or str(uuid4()) diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index 5a4bb798851..8b85b668cba 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -19,6 +19,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi from litellm.types.llms.openai import ( ResponseInputParam, ResponsesAPIOptionalRequestParams, + ResponsesAPIStreamingResponse, ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -129,7 +130,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): model: str, parsed_chunk: dict, logging_obj: LiteLLMLoggingObj, - ) -> Any: + ) -> ResponsesAPIStreamingResponse: parsed_chunk = self._normalize_stream_item_id(parsed_chunk) return super().transform_streaming_response( model=model, @@ -262,7 +263,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Return the responses endpoint return f"{effective_api_base}/responses" - def _handle_reasoning_item(self, item: dict[str, Any]) -> dict[str, Any]: + def _handle_reasoning_item(self, item: dict[str, object]) -> dict[str, object]: """ Handle reasoning items for GitHub Copilot, preserving encrypted_content. @@ -280,7 +281,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): # Filter out None values for known problematic fields, # but preserve encrypted_content even if it exists - filtered_item: Final[dict[str, Any]] = {} + filtered_item: Final[dict[str, object]] = {} for k, v in item.items(): # Always include encrypted_content if present (even if None) if k == "encrypted_content": diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 29dc485732f..92bc857e385 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -28,12 +28,12 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class HostedVLLMChatConfig(OpenAIGPTConfig): - def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, object]]: """ vLLM chat completions currently accepts only OpenAI function tools. Convert custom tools into function tools so request validation does not fail. """ - converted_tools: Final[list[dict[str, Any]]] = [] + converted_tools: Final[list[dict[str, object]]] = [] for idx, tool in enumerate(tools): if not isinstance(tool, dict): converted_tools.append(tool) @@ -63,17 +63,14 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): "required": ["input"], } - function_tool: dict[str, Any] = { - "type": "function", - "function": { - "name": str(tool_name), - "parameters": tool_parameters, - }, + function_definition: dict[str, object] = { + "name": str(tool_name), + "parameters": tool_parameters, } if isinstance(tool_description, str): - function_tool["function"]["description"] = tool_description + function_definition["description"] = tool_description - converted_tools.append(function_tool) + converted_tools.append({"type": "function", "function": function_definition}) return converted_tools @@ -148,7 +145,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, list[AllMessageValues]]: ... + ) -> Coroutine[object, object, list[AllMessageValues]]: ... @overload def _transform_messages( @@ -160,7 +157,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False - ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: + ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """ Support translating: - video files from file_id or file_data to video_url diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 2db6d78a218..e60b6145b44 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -10,7 +10,7 @@ import ssl import time import uuid from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional +from typing import TYPE_CHECKING, Final, Literal, NamedTuple, Optional from urllib.parse import urlsplit import httpx @@ -87,8 +87,8 @@ class OpenAIError(BaseLLMException): ################################################################### def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, - data: dict[str, Any], -) -> dict[str, Any]: + data: Mapping[str, object], +) -> dict[str, object]: """ Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message. diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index f65b0876202..aeff902f655 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -315,7 +315,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): for msg in messages: if isinstance(msg, dict): role = msg.get("role", "") - content: Any = msg.get("content", "") + content: object = msg.get("content", "") msg_cache_control: object = msg.get("cache_control") else: role = getattr(msg, "role", "") @@ -463,7 +463,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): return body - def _transform_tool_choice_to_anthropic(self, tool_choice: Any) -> dict[str, Any]: + def _transform_tool_choice_to_anthropic(self, tool_choice: object) -> Mapping[str, object]: """ Convert tool_choice from OpenAI format to Anthropic format. diff --git a/litellm/llms/stability/image_edit/transformations.py b/litellm/llms/stability/image_edit/transformations.py index 0b6052ad593..94711d21b50 100644 --- a/litellm/llms/stability/image_edit/transformations.py +++ b/litellm/llms/stability/image_edit/transformations.py @@ -74,7 +74,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): } # Create a copy to not mutate original - convert TypedDict to regular dict - mapped_params: Final[dict[str, Any]] = dict(image_edit_optional_params) + mapped_params: Final[dict[str, object]] = dict(image_edit_optional_params) for k, v in image_edit_optional_params.items(): if k in param_mapping: @@ -182,7 +182,7 @@ class StabilityImageEditConfig(BaseImageEditConfig): # Build Stability request # Populate multipart form-data as separate text fields (data) and files. # Stability expects prompt/output_format/etc. as normal form fields, not file parts. - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "output_format": "png", # Default to PNG } diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 98a68ba2c36..3c868b3a96f 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -4,7 +4,7 @@ Translates from OpenAI's `/v1/chat/completions` endpoint to Triton's `/generate` import json from collections.abc import AsyncIterator, Iterator -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal from httpx import Headers, Response @@ -172,7 +172,7 @@ class TritonConfig(BaseConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "TritonResponseIterator": return TritonResponseIterator( streaming_response=streaming_response, sync_stream=sync_stream, @@ -195,14 +195,14 @@ class TritonGenerateConfig(TritonConfig): ) -> dict: inference_params: Final = optional_params.copy() stream: Final = inference_params.pop("stream", False) - data_for_triton: Final[dict[str, Any]] = { + data_for_triton: Final[dict[str, object]] = { "text_input": prompt_factory(model=model, messages=messages), "parameters": { "max_tokens": int(optional_params.get("max_tokens", DEFAULT_MAX_TOKENS_FOR_TRITON)), + **inference_params, }, "stream": bool(stream), } - data_for_triton["parameters"].update(inference_params) return data_for_triton def transform_response( diff --git a/litellm/llms/vertex_ai/rerank/transformation.py b/litellm/llms/vertex_ai/rerank/transformation.py index 2ec4f2da79b..2b1ed3a29a8 100644 --- a/litellm/llms/vertex_ai/rerank/transformation.py +++ b/litellm/llms/vertex_ai/rerank/transformation.py @@ -5,7 +5,7 @@ Why separate file? Make it easy to see how transformation works """ from collections.abc import Mapping -from typing import Any, Final +from typing import Final import httpx @@ -227,7 +227,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index 091b9dfd334..7c626c66e9d 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -3,7 +3,8 @@ Volcengine Embedding Transformation Transforms OpenAI embedding requests to Volcengine format """ -from typing import Any, Final +from collections.abc import Mapping +from typing import Final import httpx @@ -83,11 +84,11 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): def map_openai_params( self, - non_default_params: dict[str, Any], - optional_params: dict[str, Any], + non_default_params: Mapping[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: """ Map OpenAI embedding parameters to Volcengine format. diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 293880b188d..0c81d50a5fe 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -5,8 +5,8 @@ Docs - https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank """ import uuid -from collections.abc import Mapping -from typing import Any, Final, cast +from collections.abc import Mapping, Sequence +from typing import Final, cast import httpx @@ -96,7 +96,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: Sequence[str | Mapping[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, @@ -178,7 +178,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): transformed_results: Final = [] for result in _results: - transformed_result: dict[str, Any] = { + transformed_result: dict[str, object] = { "index": result["index"], "relevance_score": result["score"], } diff --git a/litellm/llms/xai/realtime/transformation.py b/litellm/llms/xai/realtime/transformation.py index e9d16daad7c..5efe125ee60 100644 --- a/litellm/llms/xai/realtime/transformation.py +++ b/litellm/llms/xai/realtime/transformation.py @@ -16,7 +16,7 @@ construction time (see ``handler.py``) so all normalization is isolated here and ``RealTimeStreaming`` stays provider-agnostic. """ -from typing import Any, Final +from typing import Final class XAIRealtimeNormalizer: @@ -58,7 +58,7 @@ class XAIRealtimeNormalizer: # Cache content-part objects keyed by (response_id, item_id, content_index) # so that ``response.content_part.done`` events missing ``part`` can be # back-filled from earlier ``content_part.added`` / delta-done events. - self._content_part_by_key: dict[tuple, dict[str, Any]] = {} + self._content_part_by_key: dict[tuple, dict[str, object]] = {} # --------------------------------------------------------------------------- # Public interface consumed by RealTimeStreaming @@ -140,7 +140,7 @@ class XAIRealtimeNormalizer: } self._content_part_by_key[key] = updated - def _resolve_content_part(self, event: dict) -> dict[str, Any]: + def _resolve_content_part(self, event: dict) -> dict[str, object]: part: Final = event.get("part") if isinstance(part, dict): return part @@ -214,7 +214,7 @@ class XAIRealtimeNormalizer: needs_content: Final = event_type in self._EVENTS_NEEDING_CONTENT_INDEX if not needs_output and not needs_content: return event - patch: Final[dict[str, Any]] = {} + patch: Final[dict[str, object]] = {} if needs_output and "output_index" not in event: patch["output_index"] = 0 if needs_content and "content_index" not in event: @@ -228,8 +228,8 @@ class XAIRealtimeNormalizer: # --------------------------------------------------------------------------- @staticmethod - def _default_ga_usage() -> dict[str, Any]: - default_details: Final[dict[str, Any]] = { + def _default_ga_usage() -> dict[str, object]: + default_details: Final[dict[str, int]] = { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, @@ -243,7 +243,7 @@ class XAIRealtimeNormalizer: } @staticmethod - def _normalize_usage(usage: object, *, empty_as_null: bool) -> dict[str, Any] | None: + def _normalize_usage(usage: object, *, empty_as_null: bool) -> dict[str, object] | None: """Coerce a usage object into the full OpenAI GA shape. ``empty_as_null=True`` for ``response.created`` (usage optional). @@ -253,12 +253,12 @@ class XAIRealtimeNormalizer: return None if not usage: return None if empty_as_null else XAIRealtimeNormalizer._default_ga_usage() - default_details: Final[dict[str, Any]] = { + default_details: Final[dict[str, int]] = { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, } - normalized: Final[dict[str, Any]] = { + normalized: Final[dict[str, object]] = { "total_tokens": usage.get("total_tokens", 0), "input_tokens": usage.get("input_tokens", 0), "output_tokens": usage.get("output_tokens", 0), diff --git a/litellm/proxy/analytics_endpoints/cache_activity.py b/litellm/proxy/analytics_endpoints/cache_activity.py index b87b8eac3ef..952ef5bdcfc 100644 --- a/litellm/proxy/analytics_endpoints/cache_activity.py +++ b/litellm/proxy/analytics_endpoints/cache_activity.py @@ -2,16 +2,26 @@ import asyncio import json from collections.abc import Sequence from datetime import datetime -from typing import TYPE_CHECKING, Final +from typing import Final, Protocol from pydantic import BaseModel, TypeAdapter -if TYPE_CHECKING: - from litellm.proxy.utils import PrismaClient - UNKNOWN_CALL_TYPE: Final = "Unknown" +class _SupportsQueryRaw(Protocol): + """The single database operation the cache-activity queries issue.""" + + async def query_raw(self, query: str, *args: object) -> Sequence[object]: ... + + +class _SupportsRawQueryDb(Protocol): + """A prisma client handle, narrowed to the raw-query surface used here.""" + + @property + def db(self) -> _SupportsQueryRaw: ... + + class CacheActivityGroup(BaseModel): call_type: str api_requests: int @@ -143,7 +153,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals: async def get_cache_activity( - prisma_client: "PrismaClient", + prisma_client: _SupportsRawQueryDb, start_date: datetime, end_date: datetime, key_aliases: Sequence[str], diff --git a/litellm/proxy/client/cli/commands/model_groups.py b/litellm/proxy/client/cli/commands/model_groups.py index c904e5bed49..367c2063b6b 100644 --- a/litellm/proxy/client/cli/commands/model_groups.py +++ b/litellm/proxy/client/cli/commands/model_groups.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final, Literal import click @@ -5,10 +6,17 @@ import rich import rich.table from ... import Client +from ._cli_context import cli_context_values def create_client(ctx: click.Context) -> Client: - return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"]) + context: Final = cli_context_values(ctx) + return Client(base_url=context["base_url"], api_key=context["api_key"]) + + +def _rendered_field(group: Mapping[str, object], key: str, default: str) -> str: + """The rendered value of one model group field, or ``default`` when the group omits it.""" + return str(group.get(key, default)) @click.group(name="model-groups") @@ -46,10 +54,10 @@ def list_model_groups(ctx: click.Context, output_format: Literal["table", "json" for group in groups: table.add_row( - str(group.get("model_group", "")), - str(group.get("mode", "chat")), - str(group.get("input_cost_per_token", "")), - str(group.get("output_cost_per_token", "")), + _rendered_field(group, "model_group", ""), + _rendered_field(group, "mode", "chat"), + _rendered_field(group, "input_cost_per_token", ""), + _rendered_field(group, "output_cost_per_token", ""), ) rich.print(table) diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index b7c02866d6f..45d21b0c0b5 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -166,7 +166,8 @@ def up(ctx: click.Context) -> None: is already running (this does not start one for you). Cursor is not supported: it has no equivalent file-based config to patch. """ - base_url: Final = ctx.obj["base_url"] + ctx_obj: Final[CliContextObj] = ctx.obj + base_url: Final = ctx_obj["base_url"] try: _ensure_fresh_login(ctx) diff --git a/litellm/proxy/common_utils/cache_pydantic_utils.py b/litellm/proxy/common_utils/cache_pydantic_utils.py index 725c2b61145..3703cf7c916 100644 --- a/litellm/proxy/common_utils/cache_pydantic_utils.py +++ b/litellm/proxy/common_utils/cache_pydantic_utils.py @@ -37,7 +37,7 @@ class CacheCodec: """ @staticmethod - def serialize(value: Any, model_type: type[T] | None = None) -> Any: + def serialize(value: object, model_type: type[T] | None = None) -> object: """ Encode a value for DualCache / Redis (``json.dumps``-safe). diff --git a/litellm/proxy/common_utils/proxy_rate_limit_error.py b/litellm/proxy/common_utils/proxy_rate_limit_error.py index c109da6f571..8d3a587a4cd 100644 --- a/litellm/proxy/common_utils/proxy_rate_limit_error.py +++ b/litellm/proxy/common_utils/proxy_rate_limit_error.py @@ -66,7 +66,7 @@ def map_v3_rate_limit_type( return None -def _coerce_message(detail: Any) -> str: +def _coerce_message(detail: object) -> str: """Best-effort, JSON-friendly stringification of an HTTPException-style detail.""" if detail is None: return "" @@ -144,7 +144,7 @@ class ProxyRateLimitError(HTTPException, RateLimitError): def __init__( self, detail: Any, - headers: Mapping[str, Any] | None = None, + headers: Mapping[str, object] | None = None, category: str | RateLimitErrorCategory = RateLimitErrorCategory.LITELLM_RATE_LIMIT, rate_limit_type: str | RateLimitType | None = None, model: str | None = None, diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index e852eb5d6f9..c1407979f29 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -106,7 +106,7 @@ async def create_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response: Final = await processor.base_process_llm_request( + response: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -216,7 +216,7 @@ async def list_containers( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "query_params": query_params, "model": query_params.get("model"), "order": order, @@ -341,7 +341,7 @@ async def retrieve_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + container: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -366,6 +366,7 @@ async def retrieve_container( proxy_logging_obj=proxy_logging_obj, version=version, ) + return container @router.delete( @@ -446,7 +447,7 @@ async def delete_container( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + deleted_container: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -471,6 +472,7 @@ async def delete_container( proxy_logging_obj=proxy_logging_obj, version=version, ) + return deleted_container # Register JSON-configured container file endpoints diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index f469587ab8e..de20416ad9c 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,5 +1,5 @@ from collections.abc import Awaitable, Callable, Iterator -from typing import Any, Final, TypeVar +from typing import Final, Protocol, TypeVar from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( @@ -407,8 +407,20 @@ def _coerce_timeout(value: object, fallback: float) -> float: _ReadResultT: Final = TypeVar("_ReadResultT") +class _DBReconnectClient(Protocol): + """The one method `call_with_db_reconnect_retry` needs from a Prisma client.""" + + async def attempt_db_reconnect( + self, + *, + reason: str, + timeout_seconds: float | None = None, + lock_timeout_seconds: float | None = None, + ) -> bool: ... + + async def call_with_db_reconnect_retry( - prisma_client: Any, + prisma_client: _DBReconnectClient, coro_factory: Callable[[], Awaitable[_ReadResultT]], *, reason: str, diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 93bbc567430..7a7982481fb 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -148,4 +148,4 @@ async def flush_tool_usage_transactions( except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise - await asyncio.sleep(2**attempt + random.uniform(0, 1)) + await asyncio.sleep(2.0**attempt + random.uniform(0, 1)) diff --git a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py index f7f500b1adc..db6fc238b9b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/crowdstrike_aidr/crowdstrike_aidr.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Optiona from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, Field, ValidationError -from typing_extensions import Any, override +from typing_extensions import override from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -78,7 +78,7 @@ class _GuardChatCompletionsResult(BaseModel): """Whether or not the prompt triggered a block detection.""" transformed: bool | None = None """Whether or not the original input was transformed.""" - detectors: dict[str, Any] | None = None + detectors: dict[str, object] | None = None """Result of the policy analyzing and input prompt.""" @@ -146,8 +146,8 @@ def _extract_text_from_message(message: _Message) -> str: return "\n".join(part.text for part in content if isinstance(part, _TextContentPart)) -def _merge_metadata_bags(request_data: Mapping[str, Any]) -> Mapping[str, Any] | None: - merged: Final[dict[str, Any]] = {} +def _merge_metadata_bags(request_data: Mapping[str, object]) -> Mapping[str, object] | None: + merged: Final[dict[str, object]] = {} present = False for bag in (request_data.get("metadata"), request_data.get("litellm_metadata")): if isinstance(bag, Mapping): @@ -313,7 +313,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): self._set_streaming_params(streaming_params_from_litellm_params(litellm_params)) async def _call_crowdstrike_aidr_guard( - self, payload: dict[str, Any], hook_name: str + self, payload: dict[str, object], hook_name: str ) -> _GuardChatCompletionsResult: """ Makes the API call to the CrowdStrike AIDR AI Guard endpoint. @@ -423,7 +423,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): return [_extract_text_from_message(msg) for msg in tail] async def _call_or_fail_open( - self, payload: dict[str, Any], hook_name: str, request_data: dict[str, object] + self, payload: dict[str, object], hook_name: str, request_data: dict[str, object] ) -> _GuardChatCompletionsResult: start_time: Final = time.time() try: @@ -506,7 +506,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): event_type = "output" hook_name = "apply_guardrail (response)" - ai_guard_payload: Final[dict[str, Any]] = { + ai_guard_payload: Final[dict[str, object]] = { "guard_input": guard_input.model_dump(mode="json"), "event_type": event_type, } @@ -521,7 +521,7 @@ class CrowdStrikeAIDRHandler(CustomGuardrail): if user_id: ai_guard_payload["user_id"] = user_id - extra_info: Final[dict[str, str]] = {} + extra_info: Final[dict[str, object]] = {} user_email: Final = metadata.get("user_api_key_user_email") if user_email: extra_info["user_name"] = user_email diff --git a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py index 89afecafb0f..efe959bd186 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/enkryptai/enkryptai.py @@ -6,7 +6,7 @@ # +-------------------------------------------------------------+ import os -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, AsyncIterable from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional @@ -465,7 +465,7 @@ class EnkryptAIGuardrails(CustomGuardrail): async def async_post_call_streaming_iterator_hook( self, user_api_key_dict: UserAPIKeyAuth, - response: Any, + response: AsyncIterable[ModelResponseStream], request_data: dict, ) -> AsyncGenerator[ModelResponseStream, None]: """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index d82944c44ed..eceb54681f6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -383,7 +383,7 @@ class QualifireGuardrail(CustomGuardrail): result: Final = response.json() # Extract response info for logging - qualifire_response: Final = { + qualifire_response: Final[dict[str, object]] = { "score": result.get("score"), "status": result.get("status"), } diff --git a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py index 8d5923d1302..b2139779925 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py +++ b/litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py @@ -6,6 +6,7 @@ then builds a SemanticRouter for prompt matching. """ import os +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import yaml @@ -66,7 +67,7 @@ class SemanticGuardRouteLoader: cls, route_templates: list[str] | None, custom_routes_file: str | None, - custom_routes: list[dict[str, Any]] | None, + custom_routes: Sequence[Mapping[str, object]] | None, global_threshold: float = DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD, ) -> list["Route"]: """Build semantic-router Route objects from templates + custom config.""" diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index b313cb64c3f..d41acadc4dd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache - Span = _Span | Any + Span = _Span InternalUsageCache = _InternalUsageCache else: Span = Any @@ -75,7 +75,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): current: dict | None, request_count_api_key: str, rate_limit_type: Literal["key", "model_per_key", "user", "customer", "team"], - values_to_update_in_cache: list[tuple[Any, Any]], + values_to_update_in_cache: list[tuple[str, object]], ) -> dict: verbose_proxy_logger.info("Current Usage of %s in this minute: %s", rate_limit_type, current) if current is None: @@ -266,7 +266,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): rpm_limit = sys.maxsize values_to_update_in_cache: list[ - tuple[Any, Any] + tuple[str, object] ] = [] # values that need to get updated in cache, will run a batch_set_cache after this function # ------------ diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index ce6a97708ab..6841cadf972 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -17,6 +17,7 @@ from litellm.proxy.spend_tracking.key_metadata_recovery import ( ) from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled from litellm.proxy.utils import PrismaClient +from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import DeletedVerificationTokenRepository from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, @@ -1155,8 +1156,10 @@ async def get_daily_activity( include_current_utc_day=include_current_utc_day, ) + spend_table: Final[TableActions[DailySpendRecord]] = getattr(prisma_client.db, table_name) + # Get total count for pagination - total_count: Final[int] = await getattr(prisma_client.db, table_name).count(where=where_conditions) + total_count: Final[int] = await spend_table.count(where=where_conditions) # Fetch paginated results. # ``date`` alone is not a unique sort key -- a busy tenant has many @@ -1168,7 +1171,7 @@ async def get_daily_activity( # total. Adding ``id`` (the row's UUID primary key, present on both # LiteLLM_DailyUserSpend and LiteLLM_DailyTeamSpend) as a tiebreaker # gives every page a stable cursor (#30164). - daily_spend_data: Final[Sequence[DailySpendRecord]] = await getattr(prisma_client.db, table_name).find_many( + daily_spend_data: Final[Sequence[DailySpendRecord]] = await spend_table.find_many( where=where_conditions, order=[ {"date": "desc"}, diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index ebf4d988fdd..1d0108cc2ba 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -2,7 +2,7 @@ import json from collections.abc import Mapping -from typing import Any, Final, cast +from typing import Final, cast import orjson from fastapi import APIRouter, Depends, HTTPException, Request, Response, UploadFile @@ -48,7 +48,7 @@ def _build_document_from_upload( ) -def _with_request_format(data: Mapping[str, Any], request: Request) -> Mapping[str, Any]: +def _with_request_format(data: Mapping[str, object], request: Request) -> Mapping[str, object]: """ Resolve the requested response format from the body or the `x-req-format` header. @@ -90,7 +90,7 @@ def _native_response(response: object, fastapi_response: Response) -> Response | ) -async def _parse_multipart_form(request: Request) -> dict[str, Any]: +async def _parse_multipart_form(request: Request) -> dict[str, object]: """ Extract OCR data from a multipart form request. @@ -130,7 +130,7 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: content_type=uploaded_file.content_type, ) - data: Final[dict[str, Any]] = {"document": document} + data: Final[dict[str, object]] = {"document": document} for field_name, field_value in form.items(): if field_name in ("file", "document"): @@ -154,12 +154,12 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]: return data -async def _parse_ocr_request(request: Request) -> Mapping[str, Any]: +async def _parse_ocr_request(request: Request) -> Mapping[str, object]: """Parse an OCR request and apply the `x-req-format` header, if any.""" return _with_request_format(await _parse_ocr_request_body(request), request) -async def _parse_ocr_request_body(request: Request) -> dict[str, Any]: +async def _parse_ocr_request_body(request: Request) -> dict[str, object]: """ Parse an OCR request, supporting both JSON and multipart form data. @@ -320,7 +320,7 @@ async def ocr( # Process request using ProxyBaseLLMRequestProcessing processor = ProxyBaseLLMRequestProcessing(data=data) - response: Final = await processor.base_process_llm_request( + response: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/litellm/repositories/budget_repository.py b/litellm/repositories/budget_repository.py index 62632ffb5f6..205646c8393 100644 --- a/litellm/repositories/budget_repository.py +++ b/litellm/repositories/budget_repository.py @@ -2,7 +2,8 @@ Budget repository for database operations on LiteLLM_BudgetTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.budget import LiteLLM_BudgetTable from litellm.repositories.base_repository import BaseRepository @@ -12,12 +13,27 @@ if TYPE_CHECKING: from prisma import models as prisma_models +class _BudgetDb(Protocol): + """The single Prisma table this repository reaches for on ``prisma_client.db``.""" + + @property + def litellm_budgettable(self) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: ... + + +class _PrismaClientView(Protocol): + """The one attribute this repository reads off the untyped Prisma client wrapper.""" + + @property + def db(self) -> _BudgetDb: ... + + class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): """Repository for budget database operations.""" @property def table(self) -> TableActions["prisma_models.LiteLLM_BudgetTable"]: - return self.prisma_client.db.litellm_budgettable + client: Final[_PrismaClientView] = self.prisma_client + return client.db.litellm_budgettable @property def model_class(self) -> type[LiteLLM_BudgetTable]: @@ -34,12 +50,12 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): max_parallel_requests: int | None = None, tpm_limit: int | None = None, rpm_limit: int | None = None, - model_max_budget: dict[str, Any] | None = None, + model_max_budget: Mapping[str, object] | None = None, budget_duration: str | None = None, allowed_models: list[str] | None = None, ) -> LiteLLM_BudgetTable: """Create a new budget record.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "created_by": created_by, "updated_by": created_by, } @@ -71,12 +87,12 @@ class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]): max_parallel_requests: int | None = None, tpm_limit: int | None = None, rpm_limit: int | None = None, - model_max_budget: dict[str, Any] | None = None, + model_max_budget: Mapping[str, object] | None = None, budget_duration: str | None = None, allowed_models: list[str] | None = None, ) -> LiteLLM_BudgetTable | None: """Update an existing budget record.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if max_budget is not None: data["max_budget"] = max_budget if soft_budget is not None: diff --git a/litellm/repositories/organization_repository.py b/litellm/repositories/organization_repository.py index 5a9bd3724e0..47eb8f4a609 100644 --- a/litellm/repositories/organization_repository.py +++ b/litellm/repositories/organization_repository.py @@ -2,7 +2,8 @@ Organization repository for database operations on LiteLLM_OrganizationTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, Protocol from litellm.models.organization import LiteLLM_OrganizationTable from litellm.repositories.base_repository import BaseRepository @@ -12,12 +13,27 @@ if TYPE_CHECKING: from prisma import models as prisma_models +class _OrganizationDb(Protocol): + """The single Prisma table this repository reaches for on ``prisma_client.db``.""" + + @property + def litellm_organizationtable(self) -> TableActions["prisma_models.LiteLLM_OrganizationTable"]: ... + + +class _PrismaClientView(Protocol): + """The one attribute this repository reads off the untyped Prisma client wrapper.""" + + @property + def db(self) -> _OrganizationDb: ... + + class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): """Repository for organization database operations.""" @property def table(self) -> TableActions["prisma_models.LiteLLM_OrganizationTable"]: - return self.prisma_client.db.litellm_organizationtable + client: Final[_PrismaClientView] = self.prisma_client + return client.db.litellm_organizationtable @property def model_class(self) -> type[LiteLLM_OrganizationTable]: @@ -39,12 +55,12 @@ class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): budget_id: str, created_by: str, organization_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_OrganizationTable: """Create a new organization.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "organization_alias": organization_alias, "budget_id": budget_id, "created_by": created_by, @@ -67,12 +83,12 @@ class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]): updated_by: str, organization_alias: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_OrganizationTable | None: """Update an organization.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if organization_alias is not None: data["organization_alias"] = organization_alias if budget_id is not None: diff --git a/litellm/repositories/project_repository.py b/litellm/repositories/project_repository.py index 48e55efd258..905e813f35e 100644 --- a/litellm/repositories/project_repository.py +++ b/litellm/repositories/project_repository.py @@ -2,7 +2,8 @@ Project repository for database operations on LiteLLM_ProjectTable. """ -from typing import TYPE_CHECKING, Any, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final from litellm.models.project import LiteLLM_ProjectTable from litellm.repositories.base_repository import BaseRepository @@ -43,14 +44,14 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): description: str | None = None, team_id: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, model_rpm_limit: dict[str, int] | None = None, model_tpm_limit: dict[str, int] | None = None, object_permission_id: str | None = None, ) -> LiteLLM_ProjectTable: """Create a new project.""" - data: Final[dict[str, Any]] = { + data: Final[dict[str, object]] = { "created_by": created_by, "updated_by": created_by, } @@ -85,7 +86,7 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): description: str | None = None, team_id: str | None = None, budget_id: str | None = None, - metadata: dict[str, Any] | None = None, + metadata: Mapping[str, object] | None = None, models: list[str] | None = None, model_rpm_limit: dict[str, int] | None = None, model_tpm_limit: dict[str, int] | None = None, @@ -93,7 +94,7 @@ class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]): object_permission_id: str | None = None, ) -> LiteLLM_ProjectTable | None: """Update a project.""" - data: Final[dict[str, Any]] = {"updated_by": updated_by} + data: Final[dict[str, object]] = {"updated_by": updated_by} if project_alias is not None: data["project_alias"] = project_alias if description is not None: diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 12ccacbbc1d..8727c3a69f0 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -16,6 +16,7 @@ from __future__ import annotations import asyncio import time from collections import OrderedDict +from collections.abc import Mapping from dataclasses import asdict, dataclass from typing import Any, Final, cast @@ -122,7 +123,7 @@ class AdaptiveRouter: prefs = self.model_to_prefs.get(model) or _default_prefs() self._cells[(rt, model)] = initial_cell(prefs, rt) - async def load_state_from_db(self, prisma_client: Any) -> None: + async def load_state_from_db(self, prisma_client: object) -> None: """Add each row's persisted delta to a freshly computed cold-start prior. A row holds an accumulated delta, not a full posterior, and can be one-sided @@ -237,7 +238,7 @@ class AdaptiveRouter: cost_weight=self.config.weights.cost, ) - async def get_state_snapshot(self) -> dict[str, Any]: + async def get_state_snapshot(self) -> dict[str, object]: """In-memory snapshot for the introspection endpoint. Cheap; no DB hit.""" cells: Final = [] for (rt, model), cell in sorted(self._cells.items(), key=lambda kv: (kv[0][0].value, kv[0][1])): @@ -278,7 +279,7 @@ class AdaptiveRouter: @staticmethod def _extract_min_quality_tier( - request_kwargs: dict[str, Any], + request_kwargs: Mapping[str, object], ) -> int | None: """Pull `min_quality_tier` from request headers or metadata. @@ -484,7 +485,7 @@ class AdaptiveRouter: return combined_delta @staticmethod - def _persistable_session_snapshot(state: SessionState) -> dict[str, Any]: + def _persistable_session_snapshot(state: SessionState) -> dict[str, object]: snapshot: Final = asdict(state) for sensitive in ( "last_user_content", diff --git a/litellm/router_strategy/adaptive_router/signals.py b/litellm/router_strategy/adaptive_router/signals.py index c28613b54eb..72e8d27d2bf 100644 --- a/litellm/router_strategy/adaptive_router/signals.py +++ b/litellm/router_strategy/adaptive_router/signals.py @@ -92,7 +92,7 @@ class Turn: user_content: str | None = None assistant_content: str | None = None - tool_calls: list[dict[str, Any]] = field(default_factory=list) + tool_calls: Sequence[Mapping[str, object]] = field(default_factory=list[Mapping[str, object]]) tool_results: Sequence[Mapping[str, object]] = field(default_factory=list) response_status: int | None = None @@ -174,7 +174,7 @@ def _detect_failure(tool_results: Sequence[Mapping[str, object]]) -> bool: return False -def _signature(call: dict[str, Any]) -> str: +def _signature(call: Mapping[str, Any]) -> str: """Stable signature for loop detection: name + sorted JSON-ish args.""" name: Final = call.get("name") or call.get("function", {}).get("name", "") call_args = call.get("arguments") @@ -185,7 +185,7 @@ def _signature(call: dict[str, Any]) -> str: return f"{name}({call_args})" -def _detect_loop(history: list[str], new_calls: list[dict[str, Any]]) -> bool: +def _detect_loop(history: list[str], new_calls: Sequence[Mapping[str, object]]) -> bool: """Fires if any new call's signature appears >= LOOP_REPEAT_THRESHOLD-1 times in recent history (so this call would be the Nth).""" if not new_calls: @@ -238,7 +238,7 @@ def detect_response_signals( previous_assistant_content: str | None, current_assistant_content: str | None, tool_call_history: list[str], - tool_calls: list[dict[str, Any]], + tool_calls: Sequence[Mapping[str, object]], tool_results: Sequence[Mapping[str, object]], response_status: int | None, ) -> SignalDelta: diff --git a/litellm/router_strategy/adaptive_router/update_queue.py b/litellm/router_strategy/adaptive_router/update_queue.py index 1b9fce284ac..e28f2379f9c 100644 --- a/litellm/router_strategy/adaptive_router/update_queue.py +++ b/litellm/router_strategy/adaptive_router/update_queue.py @@ -19,7 +19,8 @@ to the in-memory aggregator). Flush is async and batched. from __future__ import annotations import asyncio -from typing import Any, Final +from collections.abc import Mapping +from typing import Final from litellm._logging import verbose_router_logger from litellm.repositories.table_repositories import ( @@ -39,7 +40,7 @@ class AdaptiveRouterUpdateQueue: def __init__(self) -> None: self._state_agg: dict[StateKey, dict[str, float]] = {} - self._session_agg: dict[SessionKey, dict[str, Any]] = {} + self._session_agg: dict[SessionKey, Mapping[str, object]] = {} self._lock = asyncio.Lock() self._max_state_size_seen = 0 self._max_session_size_seen = 0 @@ -77,7 +78,7 @@ class AdaptiveRouterUpdateQueue: session_id: str, router_name: str, model_name: str, - state_dict: dict[str, Any], + state_dict: Mapping[str, object], ) -> None: """ Last-write-wins per session row. The state_dict is a snapshot of the @@ -91,7 +92,7 @@ class AdaptiveRouterUpdateQueue: # ---- Flushers (called by background task) ---------------------------- - async def flush_state_to_db(self, prisma_client: Any) -> int: + async def flush_state_to_db(self, prisma_client: object) -> int: """ Drain state aggregator and apply to LiteLLM_AdaptiveRouterState. Returns number of cells flushed. @@ -147,7 +148,7 @@ class AdaptiveRouterUpdateQueue: return len(batch) - async def flush_session_to_db(self, prisma_client: Any) -> int: + async def flush_session_to_db(self, prisma_client: object) -> int: """ Drain session aggregator and upsert into LiteLLM_AdaptiveRouterSession. Returns number of session rows flushed. diff --git a/litellm/types/containers/main.py b/litellm/types/containers/main.py index 6a339fd2eac..62ef524a435 100644 --- a/litellm/types/containers/main.py +++ b/litellm/types/containers/main.py @@ -140,7 +140,7 @@ class ContainerFileObject(BaseModel): created_at: int path: str source: str - _hidden_params: dict[str, Any] = {} + _hidden_params: dict[str, builtins.object] = {} def __contains__(self, key: str) -> bool: return hasattr(self, key) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 02dee40f2a3..175226f8d4b 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -2,10 +2,10 @@ from collections.abc import Mapping from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from typing_extensions import Required, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.types.proxy.guardrails.guardrail_hooks.akto import ( @@ -935,7 +935,7 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) - additional_provider_specific_params: dict[str, Any] | None = Field( + additional_provider_specific_params: dict[str, object] | None = Field( default=None, description="Additional provider-specific parameters for generic guardrail APIs", ) @@ -1157,7 +1157,7 @@ class GuardrailEventHooks(str, Enum): class DynamicGuardrailParams(TypedDict): - extra_body: dict[str, Any] + extra_body: ReadOnly[dict[str, object]] class GUARDRAIL_DEFINITION_LOCATION(str, Enum): @@ -1188,7 +1188,7 @@ class GuardrailUIAddGuardrailSettings(BaseModel): supported_modes: list[str] supported_modes_by_provider: dict[str, list[str]] pii_entity_categories: list[PiiEntityCategoryMap] - content_filter_settings: dict[str, Any] | None = None + content_filter_settings: dict[str, object] | None = None class PresidioPerRequestConfig(BaseModel): @@ -1206,8 +1206,8 @@ class ApplyGuardrailRequest(BaseModel): language: str | None = None entities: list[PiiEntityType] | None = None input_type: str = "request" - messages: list[dict[str, Any]] | None = None - metadata: dict[str, Any] | None = None + messages: list[dict[str, object]] | None = None + metadata: dict[str, object] | None = None class ApplyGuardrailResponse(BaseModel): @@ -1217,4 +1217,4 @@ class ApplyGuardrailResponse(BaseModel): class PatchGuardrailRequest(BaseModel): guardrail_name: str | None = None litellm_params: BaseLitellmParams | None = None - guardrail_info: dict[str, Any] | None = None + guardrail_info: dict[str, object] | None = None diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 8498b6f6d00..323f43f4b01 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -44,7 +44,7 @@ def _sanitize_prometheus_label_name(label: str) -> str: _PROMETHEUS_LABEL_VALUE_TRANSLATE_V1: Final = str.maketrans("\n", " ", "\r\u2028\u2029") -def _sanitize_prometheus_label_value(value: Any | None) -> str | None: +def _sanitize_prometheus_label_value(value: object | None) -> str | None: """ Same semantics as :func:`_sanitize_prometheus_label_value`, implemented with ``str.translate`` plus a single escape pass instead of chained ``replace``. @@ -1023,7 +1023,7 @@ class UserAPIKeyLabelValues: ``hashed_api_key``. This supports ``**standard_logging_payload`` in tests. """ field_names: Final = {f.name for f in fields(self)} - merged: Final[dict[str, Any]] = {} + merged: Final[dict[str, object]] = {} for f in fields(self): if f.default_factory is not MISSING: merged[f.name] = f.default_factory() @@ -1060,9 +1060,9 @@ class UserAPIKeyLabelValues: # stays cheap. (Dataclass default `str()` delegates to `__repr__`.) return "" - def model_dump(self) -> dict[str, Any]: + def model_dump(self) -> dict[str, object]: """Same shape as the former Pydantic ``model_dump()`` (plain dict, list tags).""" - d: Final[dict[str, Any]] = {f.name: getattr(self, f.name) for f in fields(self)} + d: Final[dict[str, object]] = {f.name: getattr(self, f.name) for f in fields(self)} d["tags"] = list(self.tags) d["custom_metadata_labels"] = dict(self.custom_metadata_labels) return d diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 17dc70126f3..bda3865c46a 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -78,15 +78,15 @@ class RealtimeSessionConfig(BaseModel): type: str | None = None model: str | None = None instructions: str | None = None - audio: dict[str, Any] | None = None + audio: dict[str, object] | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None - tool_choice: Any | None = None - tools: list[dict[str, Any]] | None = None - tracing: Any | None = None - truncation: Any | None = None - prompt: dict[str, Any] | None = None + tool_choice: object | None = None + tools: list[dict[str, object]] | None = None + tracing: object | None = None + truncation: object | None = None + prompt: dict[str, object] | None = None class RealtimeClientSecretRequest(BaseModel): @@ -114,7 +114,7 @@ class RealtimeClientSecretResponse(BaseModel): expires_at: int | None = None value: str - session: dict[str, Any] | None = None + session: dict[str, object] | None = None class RealtimeTranscriptionSessionRequest(BaseModel): @@ -151,7 +151,7 @@ class RealtimeTranscriptionSessionResponse(BaseModel): model_config = {"extra": "allow"} - client_secret: dict[str, Any] | None = None + client_secret: dict[str, object] | None = None class RealtimeErrorDetail(TypedDict): diff --git a/litellm/vector_store_files/utils.py b/litellm/vector_store_files/utils.py index 94ad5c0ecdf..8b4bff921f8 100644 --- a/litellm/vector_store_files/utils.py +++ b/litellm/vector_store_files/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Final, cast, get_type_hints +from collections.abc import Mapping +from typing import Final, cast, get_type_hints from litellm.types.vector_store_files import ( VectorStoreFileCreateRequest, @@ -11,25 +12,25 @@ class VectorStoreFileRequestUtils: """Helper utilities for constructing vector store file requests.""" @staticmethod - def _filter_params(params: dict[str, Any], model: Any) -> dict[str, Any]: + def _filter_params(params: Mapping[str, object], model: type[object]) -> dict[str, object]: valid_keys: Final = get_type_hints(model).keys() return {key: value for key, value in params.items() if key in valid_keys and value is not None} @staticmethod def get_create_request_params( - params: dict[str, Any], + params: Mapping[str, object], ) -> VectorStoreFileCreateRequest: filtered: Final = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileCreateRequest) return cast(VectorStoreFileCreateRequest, filtered) @staticmethod - def get_list_query_params(params: dict[str, Any]) -> VectorStoreFileListQueryParams: + def get_list_query_params(params: Mapping[str, object]) -> VectorStoreFileListQueryParams: filtered = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileListQueryParams) return cast(VectorStoreFileListQueryParams, filtered) @staticmethod def get_update_request_params( - params: dict[str, Any], + params: Mapping[str, object], ) -> VectorStoreFileUpdateRequest: filtered: Final = VectorStoreFileRequestUtils._filter_params(params=params, model=VectorStoreFileUpdateRequest) return cast(VectorStoreFileUpdateRequest, filtered) From bf9437780b7fd0aa74827239cfe9ea2e19bbacc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:09:22 +0000 Subject: [PATCH 04/11] refactor(types): replace Any with real types across 11 more backend files Final batch of the fifth basedpyright Any reduction round. Every change is typing-only and leaves runtime behavior identical. These are the densest remaining files, so the yield per file is small and most of the batch was left alone deliberately. The Riva transcription handler describes the SDK module attributes it reads with Protocols instead of a bare ModuleType, the AWS secret manager stops hiding a botocore header object behind Any, and the sensitive data masker, MCP SSO assertion store and Ovalix guardrail move payload and option annotations to object and Mapping[str, object]. --- .../sensitive_data_masker.py | 10 ++--- .../audio_transcription/handler.py | 37 +++++++++++++--- .../vertex_gemma_models/transformation.py | 7 +-- .../mcp_server/openapi_to_mcp_generator.py | 2 +- .../sso_assertion_store.py | 43 ++++++++++++++++--- litellm/proxy/_lazy_features.py | 4 +- .../guardrail_hooks/ovalix/ovalix.py | 17 +++++--- .../team_callback_endpoints.py | 8 ++-- .../management_endpoints.py | 5 ++- litellm/rag/ingestion/gemini_ingestion.py | 2 +- .../secret_managers/aws_secret_manager_v2.py | 7 ++- 11 files changed, 104 insertions(+), 38 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 22dd4170963..e78c4bd24c3 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -91,13 +91,13 @@ class SensitiveDataMasker: def _mask_sequence( self, - values: list[Any], + values: Sequence[object], depth: int, max_depth: int, excluded_keys: set[str] | None, key_is_sensitive: bool, - ) -> list[Any]: - masked_items: Final[list[Any]] = [] + ) -> Sequence[object]: + masked_items: Final[list[object]] = [] if depth >= max_depth: return values @@ -197,7 +197,7 @@ def _walk_payload(node: object, key_is_sensitive: bool, depth: int) -> object: return node -def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dict[str, Any]: +def mask_sensitive_keys(data: Mapping[str, object], sensitive_fields: set[str]) -> dict[str, object]: """Return a new dict with values masked for keys listed in ``sensitive_fields``. Unlike :meth:`SensitiveDataMasker.mask_dict`, this does exact key-name @@ -209,7 +209,7 @@ def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dic range and are replaced with a fixed-length all-mask string, so a short credential is never returned verbatim. """ - masked: Final[dict[str, Any]] = {} + masked: Final[dict[str, object]] = {} mask_char: Final = _default_masker.mask_char min_visible: Final = _default_masker.visible_prefix + _default_masker.visible_suffix for key, value in data.items(): diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index d188fac8704..bea77a6761c 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -27,7 +27,6 @@ without the optional STT extras installed. import asyncio import inspect from collections.abc import Callable, Iterable -from types import ModuleType from typing import TYPE_CHECKING, Any, Final, Protocol from litellm.litellm_core_utils.audio_utils.utils import ( @@ -95,11 +94,37 @@ class _AudioEncoding(Protocol): def LINEAR_PCM(self) -> object: ... -def _auth_factory(riva_module: ModuleType) -> Callable[..., _RivaAuth]: +class _RivaClientModule(Protocol): + """The ``riva.client`` entry points this handler calls.""" + + @property + def Auth(self) -> Callable[..., _RivaAuth]: ... + + @property + def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ... + + +class _RivaAsrModule(Protocol): + """The protobuf constructors this handler calls, from whichever module exposes them.""" + + @property + def AudioEncoding(self) -> _AudioEncoding: ... + + @property + def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ... + + @property + def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ... + + @property + def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ... + + +def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]: return riva_module.Auth -def _audio_encoding(riva_asr_module: ModuleType) -> _AudioEncoding: +def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding: return riva_asr_module.AudioEncoding @@ -317,7 +342,7 @@ class NvidiaRivaAudioTranscription: def _construct_auth( self, - riva_module: ModuleType, + riva_module: _RivaClientModule, api_base: str, api_key: str | None, optional_params: dict, @@ -349,7 +374,7 @@ class NvidiaRivaAudioTranscription: return _auth_factory(riva_module)(None, use_ssl, api_base, metadata) def _build_recognition_config_proto( - self, riva_asr_module: ModuleType, recognition_config_dict: dict[str, Any] + self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any] ) -> _RecognitionConfig: encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper() encoding_enum: Final[object] = getattr( @@ -436,7 +461,7 @@ class NvidiaRivaAudioTranscription: return final_results -def _import_riva() -> tuple[ModuleType, ModuleType]: +def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]: """ Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``. diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 58cf7c7e702..67b01c2dc43 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -27,6 +27,7 @@ if TYPE_CHECKING: import tiktoken from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.base_model_iterator import MockResponseIterator class VertexGemmaConfig(OpenAIGPTConfig): @@ -56,7 +57,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): self, model_response: ModelResponse, stream: bool, - ) -> ModelResponse | Any: + ) -> "ModelResponse | MockResponseIterator": """ Helper method to return fake stream iterator if streaming is requested. @@ -138,7 +139,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): client: HTTPHandler | httpx.Client | None, api_base: str, headers: dict[str, str], # mutable-ok: forwarded to post(headers: dict | None) - request_data: dict[str, Any], # mutable-ok: forwarded to post(json: dict | ...) + request_data: dict[str, object], # mutable-ok: forwarded to post(json: dict | ...) timeout: float | httpx.Timeout | None, ) -> httpx.Response: if isinstance(client, HTTPHandler): @@ -173,7 +174,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): client: AsyncHTTPHandler | httpx.AsyncClient | None, api_base: str, headers: dict[str, str], # mutable-ok: forwarded to post(headers: dict | None) - request_data: dict[str, Any], # mutable-ok: forwarded to post(json: dict | ...) + request_data: dict[str, object], # mutable-ok: forwarded to post(json: dict | ...) timeout: float | httpx.Timeout | None, ) -> httpx.Response: from litellm.llms.custom_httpx.http_handler import get_async_httpx_client diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 16f58ef5b76..4cf63b1fceb 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -453,7 +453,7 @@ def _raise_for_upstream_failure( if response.status_code == 401 and relays_upstream_auth: raise MCPUpstreamAuthError( status_code=response.status_code, - www_authenticate=response.headers.get("www-authenticate"), + www_authenticate=dict(response.headers).get("www-authenticate"), server_name=upstream, ) raise MCPOpenApiUpstreamError(response.status_code, upstream) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py index f7b92df5ba3..5503d19211b 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py @@ -18,6 +18,7 @@ TTL ``MCP_SSO_ASSERTION_CACHE_TTL_SECONDS``; invalidation also guards against st from __future__ import annotations import json +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol @@ -29,6 +30,8 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE, MCP_SSO_ASSERTION_CACHE_TTL_SECONDS if TYPE_CHECKING: + from prisma.models import LiteLLM_SSOIdentityAssertion + from litellm.proxy.utils import PrismaClient _ASSERTION_DECRYPT_LOG_KEY: Final = "sso_identity_assertion" @@ -36,6 +39,34 @@ _STR_ADAPTER: Final[TypeAdapter[str]] = TypeAdapter(str) _MAYBE_STR_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) +class _SSOAssertionTable(Protocol): + """The ``LiteLLM_SSOIdentityAssertion`` table operations this store calls.""" + + async def find_unique(self, *, where: Mapping[str, str]) -> LiteLLM_SSOIdentityAssertion | None: ... + + async def find_many(self) -> Sequence[LiteLLM_SSOIdentityAssertion]: ... + + async def upsert(self, *, where: Mapping[str, str], data: Mapping[str, Mapping[str, str]]) -> object: ... + + async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> object: ... + + +class _MCPServerTable(Protocol): + """The ``LiteLLM_MCPServerTable`` lookup the retention gate calls.""" + + async def find_first(self, *, where: Mapping[str, str]) -> object | None: ... + + +def _assertion_table(prisma_client: PrismaClient) -> _SSOAssertionTable: + """The SSO assertion table, typed so the untyped prisma client surface stops here.""" + return prisma_client.db.litellm_ssoidentityassertion + + +def _mcp_server_table(prisma_client: PrismaClient) -> _MCPServerTable: + """The MCP server table, typed so the untyped prisma client surface stops here.""" + return prisma_client.db.litellm_mcpservertable + + class SSOIdentityAssertion(BaseModel): """The IdP material an EMA exchange needs: ``id_token`` is the RFC 8693 subject token, ``expires_at`` bounds its usefulness, and the refresh token renews it without re-login.""" @@ -163,9 +194,7 @@ async def ema_assertion_retention_enabled() -> bool: return True if prisma_client is None: return False - row: Final = await prisma_client.db.litellm_mcpservertable.find_first( - where={"auth_type": MCPAuth.oauth2_id_jag.value} - ) + row: Final = await _mcp_server_table(prisma_client).find_first(where={"auth_type": MCPAuth.oauth2_id_jag.value}) return row is not None @@ -184,7 +213,7 @@ async def persist_sso_identity_assertion( **({"expires_at": assertion.expires_at.isoformat()} if assertion.expires_at else {}), } encoded: Final = _STR_ADAPTER.validate_python(encrypt_value_helper(json.dumps(payload))) - await prisma_client.db.litellm_ssoidentityassertion.upsert( + await _assertion_table(prisma_client).upsert( where={"user_id": user_id}, data={ "create": {"user_id": user_id, "assertion_b64": encoded}, @@ -200,7 +229,7 @@ async def _read_assertion_from_db(user_id: str) -> SSOIdentityAssertion | None: if prisma_client is None: return None - row: Final = await prisma_client.db.litellm_ssoidentityassertion.find_unique(where={"user_id": user_id}) + row: Final = await _assertion_table(prisma_client).find_unique(where={"user_id": user_id}) if row is None: return None raw: Final = _MAYBE_STR_ADAPTER.validate_python( @@ -310,13 +339,13 @@ async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient, re_encrypted: Final = _STR_ADAPTER.validate_python( encrypt_value_helper(plaintext, new_encryption_key=new_master_key) ) - await prisma_client.db.litellm_ssoidentityassertion.update( + await _assertion_table(prisma_client).update( where={"user_id": row.user_id}, data={"assertion_b64": re_encrypted}, ) return True - rows: Final = await prisma_client.db.litellm_ssoidentityassertion.find_many() + rows: Final = await _assertion_table(prisma_client).find_many() outcomes: Final = [await _rotate_row(row) for row in rows] verbose_proxy_logger.info( "rotate_sso_identity_assertions_master_key: rotated %d row(s), skipped %d", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 50e0a961a49..17743e153f2 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -13,7 +13,7 @@ from collections.abc import Set as AbstractSet from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final -from starlette.types import Receive, Scope, Send +from starlette.types import ASGIApp, Receive, Scope, Send from litellm._logging import verbose_proxy_logger @@ -266,7 +266,7 @@ class LazyFeatureMiddleware: def __init__( self, - app, + app: ASGIApp, fastapi_app: "FastAPI", features: tuple[LazyFeature, ...] = LAZY_FEATURES, ): diff --git a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py index b31ed4b0f4a..c69b24c0553 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py +++ b/litellm/proxy/guardrails/guardrail_hooks/ovalix/ovalix.py @@ -10,6 +10,7 @@ import os from typing import TYPE_CHECKING, Any, Final, Literal import httpx +from typing_extensions import ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -33,6 +34,12 @@ BLOCKED_BY_OVALIX_FALLBACK_MESSAGE: Final = "This message was blocked by Ovalix" BLOCKED_ACTION_TYPE: Final = "block" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + supported_event_hooks: ReadOnly[list[GuardrailEventHooks]] + + class OvalixGuardrailMissingSecrets(Exception): """Raised when required Ovalix config (API base, key, application/checkpoint IDs) is missing.""" @@ -80,7 +87,7 @@ class OvalixGuardrail(CustomGuardrail): application_id: str | None = None, pre_checkpoint_id: str | None = None, post_checkpoint_id: str | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ): self._tracker_api_base = tracker_api_base or os.environ.get("OVALIX_TRACKER_API_BASE") self._tracker_api_key = tracker_api_key or os.environ.get("OVALIX_TRACKER_API_KEY") @@ -88,10 +95,9 @@ class OvalixGuardrail(CustomGuardrail): self._pre_checkpoint_id = pre_checkpoint_id or os.environ.get("OVALIX_PRE_CHECKPOINT_ID") self._post_checkpoint_id = post_checkpoint_id or os.environ.get("OVALIX_POST_CHECKPOINT_ID") - if "supported_event_hooks" not in kwargs: - kwargs["supported_event_hooks"] = [] + supported_event_hooks: Final = kwargs.get("supported_event_hooks", []) - self._validate_config(kwargs["supported_event_hooks"]) + self._validate_config(supported_event_hooks) self._tracker_headers = httpx.Headers( { @@ -103,7 +109,8 @@ class OvalixGuardrail(CustomGuardrail): self._async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) - super().__init__(**kwargs) + forwarded: Final[_CustomGuardrailOptions] = {**kwargs, "supported_event_hooks": supported_event_hooks} + super().__init__(**forwarded) verbose_proxy_logger.debug( "Ovalix Guardrail initialized: tracker=%s, application_id=%s, pre_checkpoint_id=%s, post_checkpoint_id=%s", self._tracker_api_base, diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index fe658a13c24..b7fcd4ac7a9 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -473,7 +473,7 @@ async def delete_team_callback( raise _callback_error(404, f"callback_name = {callback_name} is not registered for team_id = {team_id}.") updated_metadata: Final = {**team_metadata, "logging": remaining_callbacks} # mutable-ok: persisted as JSON - encrypted_metadata: Final = encrypt_callback_vars(updated_metadata) + encrypted_metadata: Final[object] = encrypt_callback_vars(updated_metadata) team_metadata_json: Final = json.dumps(encrypted_metadata) updated_team: Final = await TeamRepository(prisma_client).table.update( @@ -610,8 +610,8 @@ async def disable_team_logging( # _get_dynamic_logging_metadata stops at metadata["logging"], where the API # and Admin UI register callbacks, without ever reading callback_settings. team_metadata["logging"] = [] # mutable-ok: the disabled state is persisted as an empty JSON array - team_metadata = encrypt_callback_vars(team_metadata) - team_metadata_json: Final = json.dumps(team_metadata) + encrypted_metadata: Final[object] = encrypt_callback_vars(team_metadata) + team_metadata_json: Final = json.dumps(encrypted_metadata) # Update team in database updated_team: Final = await TeamRepository(prisma_client).table.update( @@ -643,7 +643,7 @@ async def disable_team_logging( await _emit_team_callback_audit_log( team_id=team_id, before_metadata=before_metadata, - after_metadata=team_metadata, + after_metadata=encrypted_metadata, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, ) diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 0ca2c4c8865..2fb6813a471 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -88,7 +88,7 @@ def _redact_sensitive_litellm_params(litellm_params: object, _depth: int = 0) -> return None if isinstance(litellm_params, str): try: - parsed: Final = json.loads(litellm_params) + parsed: Final[object] = json.loads(litellm_params) except (TypeError, ValueError): return REDACTED_BY_LITELM_STRING return json.dumps(_redact_sensitive_litellm_params(parsed, _depth + 1)) @@ -589,7 +589,8 @@ async def update_vector_store( try: update_data: Final = data.model_dump(exclude_unset=True) - vector_store_id: Final[str] = update_data.pop("vector_store_id") + vector_store_id: Final[str] = data.vector_store_id + update_data.pop("vector_store_id") # Per-store access control: anyone authenticated who passes the # premium-feature gate could otherwise update *any* vector store — diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 73a0159fc9f..b81c2cc0ebe 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -277,7 +277,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): raise Exception(error_msg) verbose_logger.debug("Initiate resumable upload response: %s", response.headers) # Extract upload URL from response headers - upload_url: Final = response.headers.get("x-goog-upload-url") + upload_url: Final = dict(response.headers).get("x-goog-upload-url") if not upload_url: raise Exception("No upload URL returned in response headers") diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index e86c8e7c919..9f5bf783958 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -16,7 +16,7 @@ Requires: import json import os -from typing import Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -35,6 +35,9 @@ from litellm.types.secret_managers.main import KeyManagementSettings from .base_secret_manager import BaseSecretManager +if TYPE_CHECKING: + from botocore.awsrequest import HTTPHeaders + class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): def __init__( @@ -530,7 +533,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): secret_value: str | None = None, optional_params: dict | None = None, request_data: dict | None = None, - ) -> tuple[str, Any, bytes]: + ) -> tuple[str, "HTTPHeaders", bytes]: """Prepare the AWS Secrets Manager request""" try: from botocore.auth import SigV4Auth From e46c816ec7b42bdab4ac6cdbb7df10022629e693 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:31:41 +0000 Subject: [PATCH 05/11] fix: address review feedback on Any reduction - drop redundant Protocol docstrings in dynamodb and otel mount - widen hosted_vllm custom-tool conversion signature from Any to object --- litellm/integrations/dynamodb.py | 4 ---- litellm/integrations/otel/mount.py | 2 -- litellm/llms/hosted_vllm/chat/transformation.py | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index 3401ced4efb..3fbbfe91ddf 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -11,14 +11,10 @@ from litellm._uuid import uuid class _DynamoTable(Protocol): - """The one boto3 DynamoDB table call this logger makes.""" - def put_item(self, *, Item: Mapping[str, object]) -> object: ... class _DynamoResource(Protocol): - """The one boto3 DynamoDB resource call this logger makes.""" - def Table(self, name: str) -> _DynamoTable: ... diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index 776d8722d14..9340f6e9e15 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -69,8 +69,6 @@ PASSTHROUGH_PREFIXES: Final = frozenset( class _RenameableSpan(Protocol): - """The span surface the passthrough naming hook drives.""" - def is_recording(self) -> bool: ... def update_name(self, name: str) -> None: ... diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 92bc857e385..32c60bd01b5 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -4,7 +4,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` import json from collections.abc import Coroutine -from typing import Any, Final, Literal, cast, overload +from typing import Final, Literal, cast, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, @@ -28,7 +28,7 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class HostedVLLMChatConfig(OpenAIGPTConfig): - def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, object]]: + def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]: """ vLLM chat completions currently accepts only OpenAI function tools. Convert custom tools into function tools so request validation does not fail. From da7d5fe1284da59502cb6738d853acc8c431af40 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:13:39 +0000 Subject: [PATCH 06/11] refactor: replace Any with precise types across 54 modules Narrow or remove reportAny / reportExplicitAny sites in provider transformations, caching, guardrails, proxy endpoints and enterprise batch-cost polling. Public parameters widen to Mapping/Sequence rather than dict/list so no caller signature breaks, and runtime behavior is unchanged. --- .../proxy/common_utils/check_batch_cost.py | 127 +++++++++++------- .../common_utils/check_responses_cost.py | 31 ++++- .../proxy/hooks/managed_files.py | 22 ++- litellm/caching/caching.py | 2 +- litellm/caching/caching_handler.py | 10 +- litellm/caching/redis_cache.py | 4 +- .../handler.py | 6 +- .../transformation.py | 30 +++-- litellm/cost_calculator.py | 6 +- litellm/integrations/braintrust_logging.py | 8 +- litellm/integrations/custom_guardrail.py | 2 +- .../integrations/datadog/datadog_llm_obs.py | 22 +-- litellm/integrations/galileo.py | 11 +- litellm/integrations/langfuse/langfuse.py | 8 +- .../llm_response_utils/response_metadata.py | 4 +- .../prompt_templates/common_utils.py | 39 ++---- .../a2a/chat/guardrail_translation/handler.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 24 ++-- litellm/llms/anthropic/common_utils.py | 10 +- .../messages/agentic_streaming_iterator.py | 16 ++- litellm/llms/anthropic/files/handler.py | 6 +- .../bedrock/chat/converse_transformation.py | 2 +- .../llms/chatgpt/responses/transformation.py | 22 +-- litellm/llms/cohere/chat/transformation.py | 2 +- .../llms/databricks/chat/transformation.py | 10 +- .../llms/fireworks_ai/chat/transformation.py | 4 +- litellm/llms/gemini/count_tokens/handler.py | 2 +- .../llms/gemini/image_edit/transformation.py | 13 +- litellm/llms/gigachat/chat/transformation.py | 14 +- .../huggingface/embedding/transformation.py | 10 +- .../chat/guardrail_translation/handler.py | 20 +-- litellm/llms/openai/videos/transformation.py | 6 +- .../openrouter/image_edit/transformation.py | 11 +- .../perplexity/embedding/transformation.py | 2 +- .../llms/vertex_ai/gemini/transformation.py | 4 +- .../mcp_server/mcp_server_manager.py | 2 +- litellm/proxy/auth/auth_utils.py | 26 ++-- .../proxy/client/cli/commands/configure.py | 60 +++++---- litellm/proxy/common_utils/callback_utils.py | 6 +- litellm/proxy/db/db_spend_update_writer.py | 20 ++- .../guardrails/guardrail_hooks/akto/akto.py | 36 ++++- .../custom_code/custom_code_guardrail.py | 9 +- .../guardrails/guardrail_hooks/lasso/lasso.py | 12 +- .../vigil_guard/vigil_guard.py | 23 +++- .../guardrail_hooks/xecguard/xecguard.py | 12 +- .../model_management_endpoints.py | 5 +- .../vertex_passthrough_logging_handler.py | 3 +- .../proxy/response_api_endpoints/endpoints.py | 2 +- litellm/proxy/video_endpoints/endpoints.py | 28 ++-- .../mcp/litellm_proxy_mcp_handler.py | 10 +- litellm/responses/streaming_iterator.py | 2 +- .../complexity_router/complexity_router.py | 32 ++--- litellm/types/router.py | 16 +-- .../vector_stores/vector_store_registry.py | 5 +- 54 files changed, 485 insertions(+), 336 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 13e9e5093a8..3ea9b7d9bfd 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -2,10 +2,11 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked. """ +from collections.abc import Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast +from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -18,8 +19,8 @@ if TYPE_CHECKING: from prisma import models as prisma_models from litellm.integrations.prometheus import PrometheusLogger - from litellm.proxy._types import LiteLLM_ManagedObjectTable from litellm.proxy.utils import PrismaClient, ProxyLogging + from litellm.repositories.prisma_protocols import TableActions from litellm.router import Router from litellm.types.router import Deployment from litellm.types.utils import LiteLLMBatch @@ -41,6 +42,48 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = ( ) +class _ManagedObjectRow(Protocol): + """The managed-object row fields this poller reads off whatever the DB hands back.""" + + @property + def id(self) -> str: ... + + @property + def unified_object_id(self) -> str: ... + + @property + def created_by(self) -> str | None: ... + + @property + def file_object(self) -> object: ... + + +def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": + """The managed-object table's prisma actions, typed to the row fields this module reads.""" + table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable + return table + + +def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]": + """The user table's prisma actions.""" + table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable + return table + + +def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]": + """The virtual-key table's prisma actions.""" + table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = ( + prisma_client.db.litellm_verificationtoken + ) + return table + + +def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]": + """The team table's prisma actions.""" + table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable + return table + + class CheckBatchCost: def __init__( self, @@ -73,7 +116,7 @@ class CheckBatchCost: inline for a batch the first poll cycle then accounts again. """ try: - await self.prisma_client.db.litellm_managedobjecttable.find_first( + await _managed_object_table(self.prisma_client).find_first( where={"file_purpose": "batch", "batch_processed": False} ) except Exception as probe_err: @@ -97,10 +140,8 @@ class CheckBatchCost: if not user_id: return {} try: - user_row: prisma_models.LiteLLM_UserTable | None = ( - await self.prisma_client.db.litellm_usertable.find_unique( - where={"user_id": user_id} - ) + user_row: prisma_models.LiteLLM_UserTable | None = await _user_table(self.prisma_client).find_unique( + where={"user_id": user_id} ) if user_row is None: return {} @@ -117,11 +158,9 @@ class CheckBatchCost: if not api_key: return None try: - key_row: prisma_models.LiteLLM_VerificationToken | None = ( - await self.prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": api_key} - ) - ) + key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table( + self.prisma_client + ).find_unique(where={"token": api_key}) return getattr(key_row, "key_alias", None) if key_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}") @@ -132,17 +171,15 @@ class CheckBatchCost: if not team_id: return None try: - team_row: prisma_models.LiteLLM_TeamTable | None = ( - await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) + team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique( + where={"team_id": team_id} ) return getattr(team_row, "team_alias", None) if team_row is not None else None except Exception as e: verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}") return None - async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None: + async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None: org_id = getattr(job, "org_id", None) if org_id: return org_id @@ -150,11 +187,9 @@ class CheckBatchCost: team_id = getattr(job, "team_id", None) if api_key: try: - key_row: prisma_models.LiteLLM_VerificationToken | None = ( - await self.prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": api_key} - ) - ) + key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table( + self.prisma_client + ).find_unique(where={"token": api_key}) key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None if key_org_id: return key_org_id @@ -166,10 +201,8 @@ class CheckBatchCost: if not team_id: return None try: - team_row: prisma_models.LiteLLM_TeamTable | None = ( - await self.prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} - ) + team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique( + where={"team_id": team_id} ) return getattr(team_row, "organization_id", None) if team_row is not None else None except Exception as e: @@ -177,7 +210,7 @@ class CheckBatchCost: return None async def _build_creator_attribution_metadata( - self, job: "LiteLLM_ManagedObjectTable", batch_id: str + self, job: "_ManagedObjectRow", batch_id: str ) -> dict[str, object]: """ Rebuild the spend-tracking metadata for the key, team, and tags that created the @@ -225,7 +258,7 @@ class CheckBatchCost: should not be polled. """ cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) - result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + result: Final = await _managed_object_table(self.prisma_client).update_many( where={ "file_purpose": "batch", "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)}, @@ -244,7 +277,7 @@ class CheckBatchCost: # A row already in a terminal status is never rewritten by the sweep above, so # without this it keeps a poll-page slot forever and starves newer batches. - retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + retired: Final = await _managed_object_table(self.prisma_client).update_many( where={ "file_purpose": "batch", "batch_processed": False, @@ -259,9 +292,9 @@ class CheckBatchCost: f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed" ) - async def _fallback_find_jobs(self) -> list: + async def _fallback_find_jobs(self) -> "Sequence[_ManagedObjectRow]": """Query batch jobs without the batch_processed filter (for older schemas).""" - return await self.prisma_client.db.litellm_managedobjecttable.find_many( + return await _managed_object_table(self.prisma_client).find_many( where={ "file_purpose": "batch", "status": { @@ -279,7 +312,7 @@ class CheckBatchCost: order={"created_at": "asc"}, ) - async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None: + async def _retire_job(self, job: "_ManagedObjectRow", reason: str) -> None: """ Take a row that can never be costed out of the poll page. Leaving it selectable would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and @@ -292,7 +325,7 @@ class CheckBatchCost: else {"status": "stale_expired"} ) try: - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=data, ) @@ -306,7 +339,7 @@ class CheckBatchCost: "so it will no longer be polled" ) - async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool: + async def _claim_job_for_costing(self, job: "_ManagedObjectRow") -> bool: """ Atomically flip batch_processed from false to true, returning whether this pod won the row. Every pod and uvicorn worker schedules its own poller against the shared @@ -321,7 +354,7 @@ class CheckBatchCost: if not self._has_batch_processed_column: return True try: - claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many( + claimed: Final = await _managed_object_table(self.prisma_client).update_many( where={"id": job.id, "batch_processed": False}, data={"batch_processed": True}, ) @@ -332,7 +365,7 @@ class CheckBatchCost: return False return claimed > 0 - async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None: + async def _release_job_claim(self, job: "_ManagedObjectRow") -> None: """Give a claimed row back once billing it failed, so a later poll cycle retries it. Safe to match on batch_processed=True: while this poller is active the retrieve @@ -342,7 +375,7 @@ class CheckBatchCost: if not self._has_batch_processed_column: return try: - await self.prisma_client.db.litellm_managedobjecttable.update_many( + await _managed_object_table(self.prisma_client).update_many( where={"id": job.id, "batch_processed": True}, data={"batch_processed": False}, ) @@ -353,7 +386,7 @@ class CheckBatchCost: ) @staticmethod - def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool: + def _has_unified_id_without_model(job: "_ManagedObjectRow") -> bool: """A unified id that decodes but carries no model_id can never be routed.""" from litellm.proxy.openai_files_endpoints.common_utils import ( convert_b64_uid_to_unified_uid, @@ -402,7 +435,7 @@ class CheckBatchCost: return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error) async def _finalize_unbilled_terminal_job( - self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch" + self, job: "_ManagedObjectRow", response: "LiteLLMBatch" ) -> None: """Persist a terminal batch that has nothing billable, converting any raw provider file ids to managed ids, and take it out of the poll page.""" @@ -426,7 +459,7 @@ class CheckBatchCost: "file_object": response.model_dump_json(), **({"batch_processed": True} if self._has_batch_processed_column else {}), } - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=update_data, ) @@ -447,7 +480,7 @@ class CheckBatchCost: def _resolve_job_routing( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", prom_logger: Optional["PrometheusLogger"], ) -> Optional[Tuple[str, str]]: """ @@ -524,7 +557,7 @@ class CheckBatchCost: def _resolve_unmanaged_provider_routing( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", prom_logger: Optional["PrometheusLogger"], llm_provider: str, bare_model_name: str, @@ -620,7 +653,7 @@ class CheckBatchCost: @classmethod def _get_managed_file_model_name( cls, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", deployment_info: "Deployment", ) -> Optional[str]: """ @@ -640,7 +673,7 @@ class CheckBatchCost: ) @staticmethod - def _get_input_file_id(job: "LiteLLM_ManagedObjectTable") -> Optional[str]: + def _get_input_file_id(job: "_ManagedObjectRow") -> Optional[str]: import json from litellm.types.utils import LiteLLMBatch @@ -660,7 +693,7 @@ class CheckBatchCost: async def _track_completed_batch_cost( self, - job: "LiteLLM_ManagedObjectTable", + job: "_ManagedObjectRow", response: "LiteLLMBatch", model_id: str, batch_id: str, @@ -936,7 +969,7 @@ class CheckBatchCost: # endpoint may transition a batch to "complete" before # CheckBatchCost runs. The batch_processed=False filter # already prevents reprocessing finished batches. - jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many( + jobs = await _managed_object_table(self.prisma_client).find_many( where={ "file_purpose": "batch", "batch_processed": False, @@ -1038,7 +1071,7 @@ class CheckBatchCost: } if self._has_batch_processed_column: update_data["batch_processed"] = True - await self.prisma_client.db.litellm_managedobjecttable.update( + await _managed_object_table(self.prisma_client).update( where={"id": job.id}, data=update_data, ) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 06cf5fcf82f..1bc41f2aa5b 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -6,7 +6,7 @@ same route are non-inference and free. """ from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Dict, Optional, cast +from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast import litellm from litellm._logging import verbose_proxy_logger @@ -22,11 +22,34 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging + from litellm.repositories.prisma_protocols import TableActions from litellm.router import Router TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"}) +class _ManagedObjectRow(Protocol): + """The managed-object row fields this poller reads off whatever the DB hands back.""" + + @property + def id(self) -> str: ... + + @property + def unified_object_id(self) -> str: ... + + @property + def created_by(self) -> str | None: ... + + @property + def file_object(self) -> object: ... + + +def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": + """The managed-object table's prisma actions, typed to the row fields this poller reads.""" + table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable + return table + + class CheckResponsesCost: def __init__( self, @@ -128,7 +151,7 @@ class CheckResponsesCost: f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}" ) - jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many( + jobs = await _managed_object_table(self.prisma_client).find_many( where={ "status": {"in": ["queued", "in_progress"]}, "file_purpose": "response", @@ -138,7 +161,7 @@ class CheckResponsesCost: ) verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check") - completed_jobs = [] + completed_jobs: Final[list[_ManagedObjectRow]] = [] for job in jobs: unified_object_id = job.unified_object_id @@ -189,7 +212,7 @@ class CheckResponsesCost: # Mark completed jobs in the database if len(completed_jobs) > 0: - await self.prisma_client.db.litellm_managedobjecttable.update_many( + await _managed_object_table(self.prisma_client).update_many( where={"id": {"in": [job.id for job in completed_jobs]}}, data={"status": "completed"}, ) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 4899b87da7a..5204894bee6 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -465,10 +465,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ if self.prisma_client is None: return - managed_object = ( - await self.prisma_client.db.litellm_managedobjecttable.find_first( - where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]} - ) + managed_object = await _managed_object_table(self.prisma_client).find_first( + where={"OR": [{"unified_object_id": object_id}, {"model_object_id": object_id}]} ) if managed_object is None: return @@ -493,10 +491,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ if self.prisma_client is None: return - managed_file = ( - await self.prisma_client.db.litellm_managedfiletable.find_first( - where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]} - ) + managed_file = await _managed_file_table(self.prisma_client).find_first( + where={"OR": [{"unified_file_id": file_id}, {"flat_model_file_ids": {"has": file_id}}]} ) if managed_file is None: return @@ -519,8 +515,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): provider_file_ids = tuple( file_id for file_id in ( - getattr(response, "output_file_id", None), - getattr(response, "error_file_id", None), + response.output_file_id, + response.error_file_id, ) if file_id and not _is_base64_encoded_unified_file_id(file_id) ) @@ -528,10 +524,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return if self.prisma_client is None: return - batch_row = ( - await self.prisma_client.db.litellm_managedobjecttable.find_first( - where={"unified_object_id": response.id} - ) + batch_row = await _managed_object_table(self.prisma_client).find_first( + where={"unified_object_id": response.id} ) if batch_row is None or ( batch_row.created_by is None and batch_row.team_id is None diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index d6dd2a073af..813ee655a1d 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -81,7 +81,7 @@ class Cache: s3_aws_access_key_id: str | None = None, s3_aws_secret_access_key: str | None = None, s3_aws_session_token: str | None = None, - s3_config: Any | None = None, + s3_config: object | None = None, s3_path: str | None = None, gcs_bucket_name: str | None = None, gcs_path_service_account: str | None = None, diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 139dcf058d2..2c4f80b8708 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -73,7 +73,7 @@ class CachingHandlerResponse(BaseModel): For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others """ - cached_result: Any | None = None + cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call @@ -707,7 +707,7 @@ class LLMCachingHandler: async def _retrieve_from_cache( self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...] - ) -> Any | None: + ) -> object | None: """ Internal method to - get cache key @@ -953,7 +953,7 @@ class LLMCachingHandler: def _convert_cached_stream_response( self, - cached_result: Any, + cached_result: dict[str, object], call_type: str, logging_obj: LiteLLMLoggingObj, model: str, @@ -982,7 +982,7 @@ class LLMCachingHandler: async def async_set_cache( self, - result: Any, + result: object, original_function: Callable, kwargs: dict[str, Any], args: tuple[object, ...] | None = None, @@ -1050,7 +1050,7 @@ class LLMCachingHandler: def sync_set_cache( self, - result: Any, + result: object, kwargs: dict[str, object], args: tuple[object, ...] | None = None, ): diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index d0cefcb6086..e2b48159e01 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -934,7 +934,7 @@ class RedisCache(BaseCache): client: object = None, ) -> object: async def execute() -> object: - executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache( + executor: Callable[..., Awaitable[object]] | None = litellm.in_memory_llm_clients_cache.get_cache( key=script_cache_key ) if executor is None: @@ -946,7 +946,7 @@ class RedisCache(BaseCache): return run_script - def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[Any]]: + def _register_script_for_current_loop(self, script: str) -> Callable[..., Awaitable[object]]: """ Register the script against the current event loop's Redis client. diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index f494d6610a1..642a78789b2 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -2,7 +2,7 @@ Handler for transforming /chat/completions api requests to litellm.responses requests """ -from collections.abc import Coroutine +from collections.abc import AsyncIterable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Final, Union from typing_extensions import TypedDict @@ -74,7 +74,7 @@ class ResponsesToCompletionBridgeHandler: existing.setdefault(key, value) return response - def _collect_response_from_stream(self, stream_iter: Any) -> "ResponsesAPIResponse": + def _collect_response_from_stream(self, stream_iter: Iterable[object]) -> "ResponsesAPIResponse": for _ in stream_iter: pass @@ -89,7 +89,7 @@ class ResponsesToCompletionBridgeHandler: raise ValueError("Stream completed response is invalid") return response - async def _collect_response_from_stream_async(self, stream_iter: Any) -> "ResponsesAPIResponse": + async def _collect_response_from_stream_async(self, stream_iter: AsyncIterable[object]) -> "ResponsesAPIResponse": async for _ in stream_iter: pass diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 5a6debc4af5..a5d67273768 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -6,7 +6,7 @@ import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast, get_args from openai.types.chat import ChatCompletion from openai.types.responses import Response @@ -52,7 +52,7 @@ from litellm.types.llms.openai import ( from litellm.types.utils import GenericStreamingChunk, ModelResponseStream if TYPE_CHECKING: - from openai.types.responses import ResponseInputImageParam + from openai.types.responses import ResponseInputImageParam, ResponseOutputItem from openai.types.responses.response_text_config_param import ( ResponseTextConfigParam as ResponseText, ) @@ -197,6 +197,9 @@ def _as_chat_reasoning_items( return cast(list[ChatCompletionReasoningItem], list(reasoning_items)) +_ToolChoiceT = TypeVar("_ToolChoiceT") + + def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]: if incomplete_reason == "content_filter": return "content_filter" @@ -291,7 +294,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def __init__(self): pass - def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any: + def _normalize_tool_choice_for_responses_api( + self, tool_choice: _ToolChoiceT + ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]: """Chat tool_choice nests the name under function/custom; Responses API expects top-level name.""" if not isinstance(tool_choice, dict): return tool_choice @@ -497,7 +502,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): responses_api_request["max_output_tokens"] = value elif key == "tools" and value is not None: responses_api_request["tools"] = self._convert_tools_to_responses_format( - cast(list[dict[str, Any]], value) + cast(list[dict[str, object]], value) ) elif key == "response_format": text_format = self._transform_response_format_to_text_format(value) @@ -810,7 +815,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): response_output: Final = response_payload.get("output") if not isinstance(response_output, list) or len(response_output) == 0: return None - return cast(list[dict[str, Any]], response_output) + return cast(list[dict[str, object]], response_output) @classmethod def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]: @@ -893,10 +898,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items = raw_response.output if len(output_items) == 0: - recovered_output_items: Final = self._recover_output_items_from_logging(logging_obj) + recovered_output_items: Final[list[ResponseOutputItem | dict[str, object]]] = [ + *self._recover_output_items_from_logging(logging_obj) + ] if recovered_output_items: - output_items = cast(Any, recovered_output_items) - raw_response.output = cast(Any, recovered_output_items) + output_items = recovered_output_items + raw_response.output = recovered_output_items verbose_logger.warning( "Recovered empty Responses API output from raw SSE for model=%s", model, @@ -1092,7 +1099,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): verbose_logger.debug("Chat provider: Other content type -> %s", result) return result - def _convert_tools_to_responses_format(self, tools: list[dict[str, Any]]) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: + def _convert_tools_to_responses_format( + self, tools: list[dict[str, object]] + ) -> list["ALL_RESPONSES_API_TOOL_PARAMS"]: """Convert chat completion tools to responses API tools format""" responses_tools: Final[list[ALL_RESPONSES_API_TOOL_PARAMS]] = [] for tool in tools: @@ -1108,12 +1117,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): description=function_tool.get("description"), ) ) - elif tool.get("type") == "custom" and isinstance(tool.get("custom"), dict): + elif tool.get("type") == "custom" and isinstance(custom_payload := tool.get("custom"), dict): from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_custom_tool_format_to_responses_shape, ) - custom_payload = tool["custom"] flat_custom = CustomToolParam(type="custom", name=custom_payload.get("name", "")) if custom_payload.get("description") is not None: flat_custom["description"] = custom_payload["description"] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 440d97d13be..d93bf1e8769 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -351,7 +351,7 @@ def cost_per_token( data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") - response: Any | None = None, + response: object | None = None, ### REQUEST MODEL ### request_model: str | None = None, # original request model for router detection custom_model_info: OCRPricing | None = None, @@ -607,7 +607,7 @@ def cost_per_token( model=model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries or 1, - optional_params=(response._hidden_params if response and hasattr(response, "_hidden_params") else None), + optional_params=(getattr(response, "_hidden_params", None) if response else None), ) elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( @@ -996,7 +996,7 @@ def _is_known_usage_objects(usage_obj): ) -def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: Any) -> CallTypesLiteral | None: +def _infer_call_type(call_type: CallTypesLiteral | None, completion_response: object) -> CallTypesLiteral | None: if call_type is not None: return call_type diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index aaf72a0bc4e..501f5749ea4 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -139,13 +139,13 @@ class BraintrustLogger(CustomLogger): ): output = None elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse): - output = response_obj["choices"][0]["message"].json() + output = response_obj.choices[0].message.json() choices = response_obj["choices"] elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse): output = response_obj.choices[0].text choices = response_obj.choices elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse): - output = response_obj["data"] + output = response_obj.data litellm_params: Final = kwargs.get("litellm_params", {}) or {} dynamic_metadata: Final = litellm_params.get("metadata", {}) or {} @@ -264,13 +264,13 @@ class BraintrustLogger(CustomLogger): ): output = None elif response_obj is not None and isinstance(response_obj, litellm.ModelResponse): - output = response_obj["choices"][0]["message"].json() + output = response_obj.choices[0].message.json() choices = response_obj["choices"] elif response_obj is not None and isinstance(response_obj, litellm.TextCompletionResponse): output = response_obj.choices[0].text choices = response_obj.choices elif response_obj is not None and isinstance(response_obj, litellm.ImageResponse): - output = response_obj["data"] + output = response_obj.data litellm_params: Final = kwargs.get("litellm_params", {}) dynamic_metadata: Final = litellm_params.get("metadata", {}) or {} diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 39adea30828..eaee84fae93 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -155,7 +155,7 @@ class CustomGuardrail(CustomLogger): def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks super().__init_subclass__(**kwargs) - own_apply_guardrail: Final = cls.__dict__.get("apply_guardrail") + own_apply_guardrail: Final[object] = cls.__dict__.get("apply_guardrail") if own_apply_guardrail is None or LOGS_GUARDRAIL_INFORMATION_MARKER in vars(own_apply_guardrail): return cls.apply_guardrail = log_guardrail_information(own_apply_guardrail) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index c64a12c6d75..3afe38ea075 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -54,7 +54,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, ) -_EMPTY_MAPPING: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_MESSAGE: Final[Message] = {"role": "", "content": ""} _MAX_PARSED_TOOL_ARGUMENT_CHARS: Final = 256 * 1024 _SAFE_REDACTED_MESSAGE_ROLES: Final = frozenset( @@ -154,7 +154,7 @@ def _guardrail_information_without_prompt_carriers( return tuple(_guardrail_entry_without_prompt_carriers(entry) for entry in _guardrail_entries(guardrail_information)) -def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, Any]) -> Mapping[str, Any]: +def _metadata_without_prompt_carriers(standard_logging_metadata: Mapping[str, object]) -> Mapping[str, object]: """The metadata minus the records that quote prompts, tool arguments, tool results, or retrieved text.""" return MappingProxyType( { @@ -237,7 +237,7 @@ def _declared_cost_tags(span_tags: Sequence[str]) -> tuple[str, ...]: return tuple(dimension for dimension in _COST_DIMENSIONS if dimension in present) -def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float: +def _reasoning_output_tokens(usage_object: Mapping[str, object] | None) -> float: """The provider's reasoning-token count, from either the chat or the responses spelling.""" if usage_object is None: return 0.0 @@ -254,20 +254,20 @@ def _reasoning_output_tokens(usage_object: Mapping[str, Any] | None) -> float: ) -def _mapping_field(source: Mapping[str, Any], key: str) -> Mapping[str, Any]: +def _mapping_field(source: Mapping[str, object], key: str) -> Mapping[str, Any]: """The value at `key` when it is a mapping, else an empty one.""" value: Final = source.get(key) return value if isinstance(value, dict) else _EMPTY_MAPPING -def _content_blocks(message: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: +def _content_blocks(message: Mapping[str, object]) -> tuple[Mapping[str, Any], ...]: content: Final = message.get("content") if not isinstance(content, list): return () return tuple(block for block in content if isinstance(block, dict)) -def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str: +def _to_dd_arguments(raw_arguments: object) -> dict[str, object] | str: """ Arguments as the object LLM Obs types them as, or the raw string when they are not one. @@ -282,7 +282,7 @@ def _to_dd_arguments(raw_arguments: object) -> dict[str, Any] | str: return parsed if isinstance(parsed, dict) else raw_arguments -def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: +def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: """ The tool calls a message carries, in LLM Obs' ToolCall schema, from either dialect. @@ -315,7 +315,7 @@ def _to_dd_tool_calls(message: Mapping[str, Any]) -> tuple[ToolCall, ...]: return openai_calls + anthropic_calls -def _to_dd_tool_results(message: Mapping[str, Any], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]: +def _to_dd_tool_results(message: Mapping[str, object], tool_call_names: Mapping[str, str]) -> tuple[ToolResult, ...]: """ The tool results a message carries, linked back to the call each answers. @@ -400,7 +400,7 @@ def _to_dd_messages(messages: object) -> tuple[Message, ...]: return tuple(_to_dd_message(message, tool_call_names) for message in messages) -def _to_dd_tool_definition(entry: Mapping[str, Any]) -> ToolDefinition | None: +def _to_dd_tool_definition(entry: Mapping[str, object]) -> ToolDefinition | None: function: Final = entry.get("function") declared: Final[Mapping[str, Any]] = function if isinstance(function, dict) else entry name: Final = declared.get("name") @@ -683,7 +683,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): if callable(current_span_fn): current_span: Final = current_span_fn() if current_span is not None: - trace_id: Final = getattr(current_span, "trace_id", None) + trace_id: Final[object] = getattr(current_span, "trace_id", None) if trace_id is not None: return str(trace_id) except Exception: @@ -716,7 +716,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): def redacts_messages_itself(self) -> bool: return True - def _payload_logging_is_off(self, kwargs: Mapping[str, Any]) -> bool: + def _payload_logging_is_off(self, kwargs: Mapping[str, object]) -> bool: return ( bool(self.turn_off_message_logging) or self.message_logging is not True diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index b27618993a3..010f8ad8ef2 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -396,12 +396,13 @@ class GalileoObserve(CustomLogger): ) @staticmethod - def _log_v2_payload_validation(payload: dict[str, Any]) -> None: + def _log_v2_payload_validation(payload: dict[str, object]) -> None: missing_fields: Final[list[str]] = [] - traces: Final[Sequence[object]] = payload.get("traces", []) - if not traces: + traces_value: Final = payload.get("traces", []) + if not traces_value: missing_fields.append("traces") + traces: Final[Sequence[object]] = traces_value if isinstance(traces_value, list) else [] for trace_index, trace in enumerate(traces): if not isinstance(trace, dict): continue @@ -425,8 +426,8 @@ class GalileoObserve(CustomLogger): missing_fields, ) - def _log_flush_payload(self, url: str, payload: dict[str, Any]) -> None: - traces: Final[Sequence[object]] = payload.get("traces", []) + def _log_flush_payload(self, url: str, payload: dict[str, object]) -> None: + traces: Final = payload.get("traces") verbose_logger.debug( "Galileo Logger flush URL: %s trace_count=%s", url, diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index b75369965de..c89506facbd 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -4,7 +4,7 @@ import inspect import os import re import traceback -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from datetime import datetime from functools import lru_cache from types import MappingProxyType @@ -447,7 +447,7 @@ class LangFuseLogger: prompt: dict, level: str, status_message: str | None, - ) -> tuple[dict | None, str | dict | list | None]: + ) -> tuple[dict | None, str | dict | Sequence[object] | None]: """ Get the input and output content for Langfuse logging @@ -463,7 +463,7 @@ class LangFuseLogger: output: The output content for Langfuse logging """ input = None - output: str | dict | list[Any] | None = None + output: str | dict | Sequence[object] | None = None if level == "ERROR" and status_message is not None and isinstance(status_message, str): input = prompt output = status_message @@ -575,7 +575,7 @@ class LangFuseLogger: user_id: str | None, metadata: dict[str, object], litellm_params: dict, - output: str | dict | list | None, + output: str | dict | Sequence[object] | None, start_time: datetime | None, end_time: datetime | None, kwargs: dict, diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index c83c266a17e..d279fb9e259 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -1,6 +1,6 @@ import datetime from collections.abc import Mapping -from typing import Any, Final +from typing import Final import httpx @@ -54,7 +54,7 @@ class ResponseMetadata: Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses """ - def __init__(self, result: Any): + def __init__(self, result: object): self.result = result self._hidden_params: HiddenParams | dict = getattr(result, "_hidden_params", {}) or {} diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2485896184e..f2b03e72497 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,14 +13,6 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from openai.types.chat.chat_completion_custom_tool_param import ( - CustomFormatGrammar, - CustomFormatGrammarGrammar, -) -from openai.types.shared_params.custom_tool_input_format import ( - Grammar as ResponsesGrammarFormat, -) - import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -59,7 +51,7 @@ if TYPE_CHECKING: def handle_any_messages_to_chat_completion_str_messages_conversion( - messages: Any, + messages: object, ) -> list[dict[str, str]]: """ Handles any messages to chat completion str messages conversion @@ -804,7 +796,7 @@ def extract_file_metadata(file_data: FileTypes) -> tuple[str | None, str | None] """ filename: str | None = None content_type: str | None = None - file_content: Any = None + file_content: object = None if isinstance(file_data, tuple): if len(file_data) == 2: @@ -1002,7 +994,7 @@ def unpack_defs( # Use iterative approach with queue to avoid recursion # Each item in queue is (node, parent_container, key/index, active_defs, ref_chain) - queue: Final[deque[tuple[Any, dict | list | None, str | int | None, dict, set]]] = deque( + queue: Final[deque[tuple[object, dict | list | None, str | int | None, dict, set]]] = deque( [(schema, None, None, root_defs, set())] ) inlined_bytes = 0 @@ -1624,7 +1616,10 @@ def is_function_call(optional_params: dict) -> bool: return False -def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: +_CUSTOM_GRAMMAR_FIELDS: Final = ("definition", "syntax") + + +def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, object]) -> Mapping[str, object]: """ Responses API grammar formats are flat ({"type": "grammar", "definition", "syntax"}); Chat Completions wraps the same fields in a "grammar" object. Text formats are @@ -1632,15 +1627,11 @@ def convert_custom_tool_format_to_chat_shape(format_obj: Mapping[str, Any]) -> M """ if format_obj.get("type") != "grammar" or "grammar" in format_obj: return format_obj - grammar: Final = CustomFormatGrammarGrammar() - if "definition" in format_obj: - grammar["definition"] = format_obj["definition"] - if "syntax" in format_obj: - grammar["syntax"] = format_obj["syntax"] - return CustomFormatGrammar(type="grammar", grammar=grammar) + grammar: Final[Mapping[str, object]] = {key: format_obj[key] for key in _CUSTOM_GRAMMAR_FIELDS if key in format_obj} + return {"type": "grammar", "grammar": grammar} -def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, Any]) -> Mapping[str, Any]: +def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, object]) -> Mapping[str, object]: """ Inverse of convert_custom_tool_format_to_chat_shape: unwrap the Chat Completions "grammar" object into the flat Responses API grammar shape. @@ -1648,12 +1639,10 @@ def convert_custom_tool_format_to_responses_shape(format_obj: Mapping[str, Any]) grammar: Final = format_obj.get("grammar") if format_obj.get("type") != "grammar" or not isinstance(grammar, dict): return format_obj - flat: Final = ResponsesGrammarFormat(type="grammar") - if "definition" in grammar: - flat["definition"] = grammar["definition"] - if "syntax" in grammar: - flat["syntax"] = grammar["syntax"] - return flat + return { + "type": "grammar", + **{key: grammar[key] for key in _CUSTOM_GRAMMAR_FIELDS if key in grammar}, + } def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]: diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index 5c30ff4747a..92dc49ea9c1 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -125,7 +125,7 @@ class A2AGuardrailHandler(BaseTranslation): litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, user_api_key_dict: Optional["UserAPIKeyAuth"] = None, request_data: dict | None = None, - ) -> Any: + ) -> object: """ Process A2A output response by applying guardrails to text content. diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 0f99441a115..6899c334618 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, cast import httpx -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -150,7 +150,7 @@ class _AnthropicToolResultBlock(TypedDict, total=False): content: ReadOnly[object] -_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[object], bool]]] = MappingProxyType( +_ENUM_TYPE_CHECKS: Final[Mapping[object, Callable[[object], bool]]] = MappingProxyType( { "null": lambda v: v is None, "boolean": lambda v: isinstance(v, bool), @@ -163,7 +163,7 @@ _ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[object], bool]]] = MappingProxyT ) -def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool: +def _enum_conflicts_with_declared_type(schema: Mapping[str, object]) -> bool: """Whether ``schema``'s ``enum`` cannot match its declared ``type``.""" enum_values: Final = schema.get("enum") declared_type: Final = schema.get("type") @@ -658,7 +658,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return result - def get_json_schema_from_pydantic_object(self, response_format: Any | dict | None) -> dict | None: + def get_json_schema_from_pydantic_object(self, response_format: type[BaseModel] | dict | None) -> dict | None: return type_to_response_format_param( response_format, ref_template="/$defs/{model}" ) # Relevant issue: https://github.com/BerriAI/litellm/issues/7755 @@ -1061,7 +1061,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def _sanitize_tool_names_in_request( - optional_params: dict[str, Any], + optional_params: dict[str, object], ) -> tuple[dict[str, str], dict[str, str]]: """Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']`` in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``. @@ -1108,7 +1108,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # so a caller reusing the same tool list/dicts across requests # doesn't see its inputs permanently rewritten (which would also # drop the original key from `forward` on the next request). - new_tools: Final[list[Any]] = [] + new_tools: Final[list[object]] = [] for t in tools: if ( isinstance(t, dict) @@ -1431,7 +1431,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): entry_type = entry.get("type") if entry_type == "compaction": - anthropic_edit: dict[str, Any] = {"type": "compact_20260112"} + anthropic_edit: dict[str, object] = {"type": "compact_20260112"} compact_threshold = entry.get("compact_threshold") # Rewrite to 'trigger' with correct nesting if threshold exists if compact_threshold is not None and isinstance(compact_threshold, (int, float)): @@ -2431,9 +2431,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code_by_id: Final[dict[str, str]] = {} for tc in tool_calls: try: - args = json.loads(tc.get("function", {}).get("arguments", "{}")) + args: object = json.loads(tc.get("function", {}).get("arguments", "{}")) + if not isinstance(args, Mapping): + continue call_id = tc.get("id") - command = args.get("command", "") + command: object = args.get("command", "") if isinstance(call_id, str): code_by_id[call_id] = command if isinstance(command, str) else "" except Exception: @@ -2503,8 +2505,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results: Sequence[_AnthropicToolResultBlock] | None, compaction_blocks: Sequence[object] | None, tool_calls: list[ChatCompletionToolCallChunk], - ) -> dict[str, Any]: - provider_specific_fields: Final[dict[str, Any]] = { + ) -> dict[str, object]: + provider_specific_fields: Final[dict[str, object]] = { "citations": citations, "thinking_blocks": thinking_blocks, } diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 87c4ec8938e..7ec15ebd1f4 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping, MutableMapping, Sequence from datetime import datetime, timezone from types import MappingProxyType -from typing import Any, Final, Literal +from typing import Any, Final, Literal, TypeVar import httpx from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -39,6 +39,8 @@ from litellm.types.llms.anthropic import ( from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.model_listing import ModelInfoResponse +_MessageT = TypeVar("_MessageT") + DROP_FORCED_TOOL_CHOICE_WARNING: Final = ( "Downgrading forced tool_choice to 'auto' for model=%s (drop_params=True): this model rejects tool_choice type " "'any'/'tool' with a 400 because thinking is always on and a forced call would skip it." @@ -1074,7 +1076,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): return AnthropicTokenCounter() -def strip_advisor_blocks_from_messages(messages: list[Any], replace_with_text: bool = False) -> list[Any]: +def strip_advisor_blocks_from_messages(messages: list[_MessageT], replace_with_text: bool = False) -> list[_MessageT]: """ Remove (or replace) server_tool_use (name='advisor') and advisor_tool_result blocks from assistant message content. @@ -1181,7 +1183,7 @@ def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool: return "must contain thinking" in lower -def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[Any]: +def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]: """ Return a new message list with thinking / redacted_thinking content blocks removed from each message. Used to recover from invalid thinking signatures on retry. @@ -1189,7 +1191,7 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A Messages whose content is a list and becomes empty after stripping are omitted, since Anthropic rejects empty content arrays. """ - out: Final[list[Any]] = [] + out: Final[list[object]] = [] for m in messages: if not isinstance(m, dict): out.append(m) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py index 171f5156594..306041d9949 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py @@ -25,6 +25,9 @@ from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0 SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = ( @@ -182,7 +185,7 @@ class AgenticAnthropicStreamingIterator: http_handler: Any, model: str, messages: list[dict], - anthropic_messages_provider_config: Any, + anthropic_messages_provider_config: "BaseAnthropicMessagesConfig", anthropic_messages_optional_request_params: dict, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, @@ -402,7 +405,7 @@ class AgenticAnthropicStreamingIterator: @staticmethod def _rebuild_anthropic_response_from_sse( raw_bytes: list[bytes], - ) -> dict[str, Any] | None: + ) -> dict[str, object] | None: """ Parse collected SSE bytes into an Anthropic Messages response dict. @@ -416,17 +419,18 @@ class AgenticAnthropicStreamingIterator: """ events: Final = _parse_sse_events(b"".join(raw_bytes)) - response: Final[dict[str, Any]] = { + content: Final[list[dict[str, object]]] = [] + response: Final[dict[str, object]] = { "id": "", "type": "message", "role": "assistant", "model": "", - "content": [], + "content": content, "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0}, } - content_blocks: Final[dict[int, dict[str, Any]]] = {} + content_blocks: Final[dict[int, dict[str, object]]] = {} saw_message_start = False for event_type, data in events: @@ -448,6 +452,6 @@ class AgenticAnthropicStreamingIterator: for idx in sorted(content_blocks.keys()): block = content_blocks[idx] block.pop("_partial_json", None) - response["content"].append(block) + content.append(block) return response diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index dfd62ca575b..e4c75a704ec 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -185,7 +185,11 @@ class AnthropicFilesHandler: if not line.strip(): continue - anthropic_result = json.loads(line) + anthropic_result: object = json.loads(line) + if not isinstance(anthropic_result, dict): + raise TypeError( + f"Anthropic batch result line is not a JSON object: {type(anthropic_result).__name__}" + ) custom_id = anthropic_result.get("custom_id", "") result = anthropic_result.get("result", {}) result_type = result.get("type", "") diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index fa18361e44c..aabd4f1afdc 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1037,7 +1037,7 @@ class AmazonConverseConfig(BaseConfig): return optional_params - def _map_request_metadata_param(self, value: Any, optional_params: dict) -> None: + def _map_request_metadata_param(self, value: object, optional_params: dict) -> None: if value is not None and isinstance(value, dict): self._validate_request_metadata(value) optional_params["requestMetadata"] = value diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index b96e06be3d8..9774b762396 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -1,5 +1,8 @@ +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final +import httpx + from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -13,6 +16,7 @@ from litellm.responses.sse_output_recovery import ( record_output_text_chunk, ) from litellm.types.llms.openai import ( + ResponseInputParam, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) @@ -64,7 +68,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_responses_api_request( self, model: str, - input: Any, + input: str | ResponseInputParam, response_api_optional_request_params: dict, litellm_params: GenericLiteLLMParams, headers: dict, @@ -109,9 +113,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_response_api_response( self, model: str, - raw_response: Any, + raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", - ): + ) -> ResponsesAPIResponse: body_text: Final = raw_response.text or "" if not self._should_parse_as_sse(raw_response=raw_response, body_text=body_text): return super().transform_response_api_response( @@ -135,7 +139,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): self._attach_response_headers(completed_response=completed_response, raw_response=raw_response) return completed_response - def _should_parse_as_sse(self, raw_response: Any, body_text: str) -> bool: + def _should_parse_as_sse(self, raw_response: httpx.Response, body_text: str) -> bool: content_type: Final = (raw_response.headers or {}).get("content-type", "") if "text/event-stream" in content_type.lower(): return True @@ -150,8 +154,8 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def _extract_completed_response_from_sse(self, body_text: str) -> tuple[ResponsesAPIResponse | None, str | None]: completed_response = None error_message = None - streamed_output_items: Final[dict[int, dict]] = {} - text_only_output_items: Final[dict[int, dict]] = {} + streamed_output_items: Final[dict[int, dict[str, object]]] = {} + text_only_output_items: Final[dict[int, dict[str, object]]] = {} for chunk in body_text.splitlines(): parsed_chunk = parse_sse_json_chunk(chunk) if parsed_chunk is None: @@ -178,7 +182,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): # output_index, but text-only items at indices without a # matching OUTPUT_ITEM_DONE must still be preserved (e.g. # providers that emit only OUTPUT_TEXT_DONE for some indices). - merged_items: dict[int, dict] = {**text_only_output_items} + merged_items: dict[int, dict[str, object]] = {**text_only_output_items} merged_items.update(streamed_output_items) completed_response = self._build_completed_response_from_chunk( parsed_chunk=parsed_chunk, @@ -197,7 +201,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): return completed_response, error_message def _build_completed_response_from_chunk( - self, parsed_chunk: dict[str, Any], streamed_output_items: dict[int, dict] + self, parsed_chunk: Mapping[str, object], streamed_output_items: Mapping[int, dict[str, object]] ) -> ResponsesAPIResponse | None: response_payload = parsed_chunk.get("response") if not isinstance(response_payload, dict): @@ -223,7 +227,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def _attach_response_headers( self, completed_response: ResponsesAPIResponse, - raw_response: Any, + raw_response: httpx.Response, ) -> None: raw_headers: Final = dict(raw_response.headers) processed_headers: Final = process_response_headers(raw_headers) diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index 319603b0dad..fa46bd7f6cf 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -110,7 +110,7 @@ class CohereChatConfig(BaseConfig): tool_results: list | None = None, seed: int | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 82c3b5d91d3..dd257cd68b0 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/chat/completions` to Databricks' `/chat/completion """ import os -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping +from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import httpx @@ -67,7 +67,7 @@ def _is_bare_assistant_message(message_dict: Mapping[str, object]) -> bool: ) -def _sanitize_empty_content(message_dict: dict[str, Any]) -> None: +def _sanitize_empty_content(message_dict: dict[str, object]) -> None: """ Remove or filter content so empty text blocks are not sent. Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks. @@ -430,7 +430,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): @overload def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: Literal[True] - ) -> Coroutine[Any, Any, list[AllMessageValues]]: ... + ) -> Coroutine[object, object, list[AllMessageValues]]: ... @overload def _transform_messages( @@ -442,7 +442,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False - ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: + ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """ Databricks does not support: - 'name' in user message. @@ -564,7 +564,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): @staticmethod def extract_citations( content: AllDatabricksContentValues | None, - ) -> list[Any] | None: + ) -> Sequence[Sequence[Mapping[str, object]]] | None: if content is None: return None citations: Final = [] diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 05160d83c12..1e9082a1ef1 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,6 +1,6 @@ import json from collections.abc import AsyncIterator, Iterator, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, cast import httpx @@ -751,7 +751,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse, sync_stream: bool, json_mode: bool | None = False, - ) -> Any: + ) -> "FireworksAIChatCompletionStreamingHandler": return FireworksAIChatCompletionStreamingHandler( streaming_response=streaming_response, sync_stream=sync_stream, diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index cb2be2c860e..c2f0ef473ae 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -84,7 +84,7 @@ class GoogleAIStudioTokenCounter: api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, - **kwargs, + **kwargs: object, ) -> dict[str, Any]: """ Count tokens using Google Gen AI Studio countTokens endpoint. diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index e6c22dc60b4..31d3963c70c 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -1,4 +1,5 @@ import base64 +from collections.abc import Mapping from io import BufferedReader, BytesIO from typing import TYPE_CHECKING, Any, Final, cast @@ -44,7 +45,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): image_edit_optional_params: ImageEditOptionalRequestParams, model: str, drop_params: bool, - ) -> dict[str, Any]: + ) -> dict[str, object]: return map_openai_image_params_to_gemini( params=image_edit_optional_params, model=model, @@ -87,10 +88,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): model: str, prompt: str | None, image: FileTypes | None, - image_edit_optional_request_params: dict[str, Any], + image_edit_optional_request_params: Mapping[str, object], litellm_params: GenericLiteLLMParams, headers: dict, - ) -> tuple[dict[str, Any], RequestFiles | None]: + ) -> tuple[dict[str, object], RequestFiles | None]: inline_parts: Final = self._prepare_inline_image_parts(image) if image else [] if not inline_parts: raise ValueError("Gemini image edit requires at least one image.") @@ -106,7 +107,7 @@ class GeminiImageEditConfig(BaseImageEditConfig): } ] - request_body: Final[dict[str, Any]] = {"contents": contents} + request_body: Final[dict[str, object]] = {"contents": contents} request_body["generationConfig"] = get_gemini_image_generation_config( model=model, @@ -153,14 +154,14 @@ class GeminiImageEditConfig(BaseImageEditConfig): model_response.usage = transform_gemini_image_usage(response_json["usageMetadata"]) return model_response - def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, Any]]: + def _prepare_inline_image_parts(self, image: FileTypes | list[FileTypes]) -> list[dict[str, object]]: images: list[FileTypes] if isinstance(image, list): images = image else: images = [image] - inline_parts: Final[list[dict[str, Any]]] = [] + inline_parts: Final[list[dict[str, object]]] = [] for img in images: if img is None: continue diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 89920ebd27b..d9250ea8836 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -81,9 +81,17 @@ class GigaChatConfig(BaseConfig): repetition_penalty: float | None = None, profanity_check: bool | None = None, ) -> None: - locals_: Final = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: + config_params: Final[Mapping[str, float | int | bool | None]] = MappingProxyType( + { + "temperature": temperature, + "top_p": top_p, + "max_tokens": max_tokens, + "repetition_penalty": repetition_penalty, + "profanity_check": profanity_check, + } + ) + for key, value in config_params.items(): + if value is not None: setattr(self.__class__, key, value) # Instance variables for current request context self._current_credentials: str | None = None diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index f6fe7f2fa10..33b0e21e326 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -84,13 +84,13 @@ class HuggingFaceEmbeddingConfig(BaseConfig): typical_p: float | None = None, watermark: bool | None = None, ) -> None: - locals_: Final = locals().copy() + locals_: Final[dict[str, object]] = locals().copy() for key, value in locals_.items(): if key != "self" and value is not None: setattr(self.__class__, key, value) @classmethod - def get_config(cls): + def get_config(cls) -> dict[str, object]: return super().get_config() def get_special_options_params(self): @@ -352,17 +352,17 @@ class HuggingFaceEmbeddingConfig(BaseConfig): model: str, data: dict, api_key: str | None = None, - ) -> list[dict[str, Any]]: + ) -> list[dict[str, str]]: streamed_response: Final = CustomStreamWrapper( completion_stream=response.iter_lines(), model=model, custom_llm_provider="huggingface", logging_obj=logging_obj, ) - content = "" + content: str = "" for chunk in streamed_response: content += chunk["choices"][0]["delta"]["content"] - completion_response: Final[list[dict[str, Any]]] = [{"generated_text": content}] + completion_response: Final[list[dict[str, str]]] = [{"generated_text": content}] ## LOGGING logging_obj.post_call( input=data, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 58ff03e6a0d..665215303d9 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -17,7 +17,7 @@ This pattern can be replicated for other message formats (e.g., Anthropic). import json import time import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union, cast from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -232,7 +232,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _extract_inputs( self, - message: dict[str, Any], + message: Mapping[str, object], msg_idx: int, texts_to_check: list[str], images_to_check: list[str], @@ -293,7 +293,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_input_texts( self, - messages: list[dict[str, Any]], + messages: list[dict[str, object]], responses: list[str], task_mappings: list[tuple[int, int | None]], ) -> None: @@ -318,12 +318,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): elif isinstance(content, list) and content_idx_optional is not None: # Replace specific text item in list content - messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response + content[content_idx_optional]["text"] = guardrail_response async def _apply_guardrail_responses_to_input_tool_calls( self, - messages: list[dict[str, Any]], - tool_calls: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], + tool_calls: Sequence[Mapping[str, object]], task_mappings: list[tuple[int, int]], ) -> None: """ @@ -375,7 +375,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): texts_to_check: Final[list[str]] = [] images_to_check: Final[list[str]] = [] - tool_calls_to_check: Final[list[dict[str, Any]]] = [] + tool_calls_to_check: Final[list[dict[str, object]]] = [] text_task_mappings: Final[list[tuple[int, int | None]]] = [] tool_call_task_mappings: Final[list[tuple[int, int]]] = [] # text_task_mappings: Track (choice_index, content_index) for each text @@ -424,8 +424,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrailed_texts: Final = guardrailed_inputs.get("texts", []) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") - guardrailed_tool_calls: Final[list[dict[str, Any]]] = ( - cast(list[dict[str, Any]], returned_tool_calls) + guardrailed_tool_calls: Final[list[dict[str, object]]] = ( + cast(list[dict[str, object]], returned_tool_calls) if isinstance(returned_tool_calls, list) and len(returned_tool_calls) == len(tool_calls_to_check) else tool_calls_to_check ) @@ -864,7 +864,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): choice_idx: int, texts_to_check: list[str], images_to_check: list[str], - tool_calls_to_check: list[dict[str, Any]], + tool_calls_to_check: list[dict[str, object]], text_task_mappings: list[tuple[int, int | None]], tool_call_task_mappings: list[tuple[int, int]], ) -> None: diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 94dc30f41e5..9a4b030993f 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -237,7 +237,7 @@ class OpenAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video remix request for OpenAI API. @@ -252,7 +252,7 @@ class OpenAIVideoConfig(BaseVideoConfig): url: Final = f"{api_base.rstrip('/')}/{encoded_video_id}/remix" # Prepare the request data - data: Final = {"prompt": prompt} + data: Final[dict[str, object]] = {"prompt": prompt} # Add any extra body parameters if extra_body: @@ -305,7 +305,7 @@ class OpenAIVideoConfig(BaseVideoConfig): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: """ Transform the video list request for OpenAI API. diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index b01c25aad0c..3d46277a69e 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -90,20 +90,21 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): drop_params: bool, ) -> dict: supported_params: Final = self.get_supported_openai_params(model) - mapped_params: Final[dict[str, Any]] = {} + mapped_params: Final[dict[str, object]] = {} + image_config: Final[dict[str, str]] = {} for key, value in image_edit_optional_params.items(): if key in supported_params: if key == "size": if "image_config" not in mapped_params: - mapped_params["image_config"] = {} - mapped_params["image_config"]["aspect_ratio"] = self._map_size_to_aspect_ratio(cast(str, value)) + mapped_params["image_config"] = image_config + image_config["aspect_ratio"] = self._map_size_to_aspect_ratio(cast(str, value)) elif key == "quality": image_size = self._map_quality_to_image_size(cast(str, value)) if image_size: if "image_config" not in mapped_params: - mapped_params["image_config"] = {} - mapped_params["image_config"]["image_size"] = image_size + mapped_params["image_config"] = image_config + image_config["image_size"] = image_size else: mapped_params[key] = value diff --git a/litellm/llms/perplexity/embedding/transformation.py b/litellm/llms/perplexity/embedding/transformation.py index a911fa62719..c93206db2bb 100644 --- a/litellm/llms/perplexity/embedding/transformation.py +++ b/litellm/llms/perplexity/embedding/transformation.py @@ -130,7 +130,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): if isinstance(embedding_value, str): raw_bytes: Final = base64.b64decode(embedding_value) count: Final = len(raw_bytes) - int8_values: Final = struct.unpack(f"{count}b", raw_bytes) + int8_values: Final[tuple[int, ...]] = struct.unpack(f"{count}b", raw_bytes) return [float(v) / 127.0 for v in int8_values] return embedding_value diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 13e2238fdf6..e3cc3bbb2dc 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -179,7 +179,7 @@ def _apply_gemini_metadata( part: PartType, model: str | None, media_resolution_enum: dict[str, str] | None, - video_metadata: dict[str, Any] | None, + video_metadata: Mapping[str, object] | None, ) -> PartType: """ Apply media_resolution and video_metadata parameters to a Gemini part. @@ -480,7 +480,7 @@ def _process_gemini_media( format: str | None = None, media_resolution_enum: dict[str, str] | None = None, model: str | None = None, - video_metadata: dict[str, Any] | None = None, + video_metadata: Mapping[str, object] | None = None, vertex_project: str | None = None, vertex_credentials: object = None, ) -> PartType: diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fb0c623473a..f640db7e5dc 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -5574,7 +5574,7 @@ class MCPServerManager: async def pre_call_tool_check( self, name: str, - arguments: dict[str, Any], + arguments: _ToolArguments, server_name: str, user_api_key_auth: UserAPIKeyAuth | None, proxy_logging_obj: ProxyLogging | None, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index be65c3b39ec..bd248da52ef 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -166,7 +166,7 @@ def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool: def _is_param_allowed( param: str, - request_body_value: Any, + request_body_value: object, configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, ) -> bool: """ @@ -189,7 +189,7 @@ def _is_param_allowed( def _allow_model_level_clientside_configurable_parameters( - model: str, param: str, request_body_value: Any, llm_router: Router | None + model: str, param: str, request_body_value: object, llm_router: Router | None ) -> bool: """ Check if model is allowed to use configurable client-side params @@ -532,7 +532,7 @@ def is_request_body_safe(request_body: dict, general_settings: dict, llm_router: return True -def _coerce_metadata_to_dict(value: Any) -> dict[str, Any] | None: +def _coerce_metadata_to_dict(value: object) -> dict[str, object] | None: """Return ``value`` as a dict, parsing it from JSON if delivered as a string. Multipart/form-data and ``extra_body`` callers send ``litellm_metadata`` @@ -891,7 +891,7 @@ async def check_if_request_size_is_safe(request: Request) -> bool: return True -async def check_response_size_is_safe(response: Any) -> bool: +async def check_response_size_is_safe(response: object) -> bool: """ Enterprise Only: - Checks if the response size is within the limit @@ -1526,7 +1526,7 @@ def get_customer_user_header_from_mapping(user_id_mapping) -> list | None: def _get_customer_id_from_standard_headers( - request_headers: dict | None, + request_headers: Mapping[str, object] | None, ) -> str | None: """ Check standard customer ID headers for a customer/end-user ID. @@ -1552,7 +1552,7 @@ def _get_customer_id_from_standard_headers( return None -def _coerce_user_id_to_str(value: Any) -> str | None: +def _coerce_user_id_to_str(value: object) -> str | None: """Return a usable end-user identifier string, or None if the value isn't one. Always drops non-string structured values (dict/list/tuple/set) because @@ -1579,7 +1579,7 @@ def _coerce_user_id_to_str(value: Any) -> str | None: # behind the flag preserves backwards compatibility for deployments # that intentionally pass JSON-encoded user identifiers. if litellm.validate_end_user_id_in_db and stripped[:1] in ("{", "["): - parsed: Final = safe_json_loads(stripped) + parsed: Final[object] = safe_json_loads(stripped) if isinstance(parsed, (dict, list)): return None return stripped @@ -1587,7 +1587,9 @@ def _coerce_user_id_to_str(value: Any) -> str | None: return None -def get_end_user_id_from_request_body(request_body: dict, request_headers: dict | None = None) -> str | None: +def get_end_user_id_from_request_body( + request_body: Mapping[str, object], request_headers: Mapping[str, object] | None = None +) -> str | None: # Import general_settings here to avoid potential circular import issues at module level # and to ensure it's fetched at runtime. from litellm.proxy.proxy_server import general_settings @@ -1636,7 +1638,7 @@ def get_end_user_id_from_request_body(request_body: dict, request_headers: dict if user_id_str: return user_id_str - def _as_dict(value: Any) -> dict: + def _as_dict(value: object) -> dict: # metadata / litellm_metadata can arrive as JSON strings from # multipart/form-data or extra_body; coerce so string-encoded # payloads can't evade end-user attribution. @@ -1721,11 +1723,11 @@ _MODEL_ROUTING_ID_FIELDS: Final = ( ) -def _append_model_candidates(candidates: list[str], value: Any) -> None: +def _append_model_candidates(candidates: list[str], value: object) -> None: if value is None: return - values: Final = value if isinstance(value, (list, tuple, set)) else [value] + values: Final[tuple[object, ...]] = tuple(value) if isinstance(value, (list, tuple, set)) else (value,) for item in values: if item is None: continue @@ -1766,7 +1768,7 @@ def _route_uses_model_routing_sources(route: str) -> bool: def _extract_models_from_managed_resource_id( - resource_id: Any, + resource_id: object, resource_id_field: str | None = None, llm_router: Router | None = None, ) -> list[str]: diff --git a/litellm/proxy/client/cli/commands/configure.py b/litellm/proxy/client/cli/commands/configure.py index 7988f8aef3c..539aa2581f5 100644 --- a/litellm/proxy/client/cli/commands/configure.py +++ b/litellm/proxy/client/cli/commands/configure.py @@ -98,13 +98,15 @@ def _preflight(target: str) -> None: raise click.ClickException(str(e)) from e -def _start(ctx: click.Context, api_key: str | None, target: str = _CLAUDE_TARGET) -> tuple[StaticToken, _Listing]: +def _start( + ctx: click.Context, base_url: str, api_key: str | None, target: str = _CLAUDE_TARGET +) -> tuple[StaticToken, _Listing]: _preflight(target) try: credential: Final = resolve_credential(ctx, api_key) except ClaudeSettingsError as e: raise click.ClickException(str(e)) - return credential, _listed_models(ctx.obj["base_url"], credential.token, target) + return credential, _listed_models(base_url, credential.token, target) def _listing_error(base_url: str, error: PiSyncError, target: str) -> str: @@ -147,9 +149,7 @@ def _validated_model(model: str | None, listing: _Listing, base_url: str) -> str return starting -def _apply_claude(ctx: click.Context, credential: StaticToken, listing: _Listing, model: str | None) -> None: - ctx_obj: Final[CliContextObj] = ctx.obj - base_url: Final = ctx_obj["base_url"] +def _apply_claude(base_url: str, credential: StaticToken, listing: _Listing, model: str | None) -> None: listed: Final = listing.ids starting: Final = _validated_model(model, listing, base_url) settings_path: Final = claude_settings_path(os.environ) @@ -214,8 +214,7 @@ def _pick_codex_model(listed: Sequence[str]) -> str: return str(inquirer.fuzzy(message="Model Codex starts on (type to filter):", choices=choices).execute()) -def _apply_codex(ctx: click.Context, credential: StaticToken, listing: _Listing, model: str) -> None: - base_url: Final[str] = ctx.obj["base_url"] +def _apply_codex(base_url: str, credential: StaticToken, listing: _Listing, model: str) -> None: _validated_model(model, listing, base_url) settings_path: Final = codex_config_path(os.environ) try: @@ -237,13 +236,12 @@ class _Setup: def _choose_setup( - ctx: click.Context, + base_url: str, target: str, credential: StaticToken, pick_model: Callable[[Sequence[str]], str | None], pick_codex_model: Callable[[Sequence[str]], str], ) -> _Setup: - base_url: Final[str] = ctx.obj["base_url"] listing: Final = _listed_models(base_url, credential.token, target) model: Final = ( pick_model(tuple(item.source_model or item.id for item in listing.models)) @@ -270,12 +268,15 @@ def interactive_configure( credential: Final = resolve_credential(ctx, None) except ClaudeSettingsError as e: raise click.ClickException(str(e)) from e - setups: Final = tuple(_choose_setup(ctx, target, credential, pick_model, pick_codex_model) for target in targets) + base_url: Final[str] = ctx.obj["base_url"] + setups: Final = tuple( + _choose_setup(base_url, target, credential, pick_model, pick_codex_model) for target in targets + ) for setup in setups: if setup.target == _CLAUDE_TARGET: - _apply_claude(ctx, credential, setup.listing, setup.model) + _apply_claude(base_url, credential, setup.listing, setup.model) elif setup.model is not None: - _apply_codex(ctx, credential, setup.listing, setup.model) + _apply_codex(base_url, credential, setup.listing, setup.model) class _ConnectionOptions(BaseModel): @@ -283,7 +284,8 @@ class _ConnectionOptions(BaseModel): gateway_url: str | None = None -def _connection_context(ctx: click.Context, api_key: str | None, gateway_url: str | None) -> click.Context: +def _connection_settings(ctx: click.Context, api_key: str | None, gateway_url: str | None) -> CliContextObj: + """The context object a subcommand runs with: its own --api-key / --gateway-url over the group's, over `lite`'s.""" ctx_obj: Final[CliContextObj] = ctx.obj group: Final = ( _ConnectionOptions.model_validate(ctx.parent.params) @@ -300,7 +302,11 @@ def _connection_context(ctx: click.Context, api_key: str | None, gateway_url: st "api_key": key if key is not None else ctx_obj.get("api_key"), "api_key_from_token_file": False if key is not None else ctx_obj.get("api_key_from_token_file", False), } - return click.Context(ctx.command, parent=ctx.parent, obj=connection) + return connection + + +def _connection_context(ctx: click.Context, settings: CliContextObj) -> click.Context: + return click.Context(ctx.command, parent=ctx.parent, obj=settings) @click.group(name="configure", invoke_without_command=True) @@ -316,19 +322,19 @@ def configure_group(ctx: click.Context, api_key: str | None, gateway_url: str | """ if ctx.invoked_subcommand is not None: return - connection: Final = _connection_context(ctx, api_key, gateway_url) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + connection: Final = _connection_context(ctx, settings) if not sys.stdin.isatty(): raise click.ClickException( "`lite configure` asks questions, so it needs a terminal. Non-interactively, run " "`lite configure claude --api-key --model ` or " "`lite configure codex --api-key --model `." ) - prompted: Final = ( - connection - if connection.obj.get("base_url_explicit") - else _connection_context(connection, None, click.prompt("Gateway URL", default=connection.obj["base_url"])) - ) - interactive_configure(prompted) + if settings.get("base_url_explicit"): + interactive_configure(connection) + return + prompted: Final = _connection_settings(connection, None, click.prompt("Gateway URL", default=settings["base_url"])) + interactive_configure(_connection_context(connection, prompted)) @click.group(name="unconfigure") @@ -356,9 +362,9 @@ def configure_claude(ctx: click.Context, api_key: str | None, model: str | None, setting is kept, and what changed is recorded so `lite unconfigure claude` can put it back. Assumes the proxy is already running. """ - connection: Final = _connection_context(ctx, api_key, gateway_url) - credential, listing = _start(connection, api_key) - _apply_claude(connection, credential, listing, model) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + credential, listing = _start(_connection_context(ctx, settings), settings["base_url"], api_key) + _apply_claude(settings["base_url"], credential, listing, model) @configure_group.command(name="codex") @@ -368,9 +374,9 @@ def configure_claude(ctx: click.Context, api_key: str | None, model: str | None, @click.pass_context def configure_codex(ctx: click.Context, api_key: str | None, gateway_url: str | None, model: str) -> None: """Route plain `codex` through the gateway until `lite unconfigure codex`.""" - connection: Final = _connection_context(ctx, api_key, gateway_url) - credential, listing = _start(connection, api_key, _CODEX_TARGET) - _apply_codex(connection, credential, listing, model) + settings: Final = _connection_settings(ctx, api_key, gateway_url) + credential, listing = _start(_connection_context(ctx, settings), settings["base_url"], api_key, _CODEX_TARGET) + _apply_codex(settings["base_url"], credential, listing, model) @unconfigure_group.command(name="codex") diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 561a53409f4..eafbb1ef95f 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -713,7 +713,7 @@ def strip_callback_config(metadata: dict[str, object] | None) -> dict[str, objec return {k: v for k, v in metadata.items() if k not in _CALLBACK_CONFIG_SLOTS} -def encrypt_callback_vars(metadata: Any) -> Any: +def encrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values encrypted at rest. Idempotent: a value that already decrypts cleanly is left unchanged so @@ -722,7 +722,7 @@ def encrypt_callback_vars(metadata: Any) -> Any: return _transform_callback_vars(metadata, _encrypt_if_plaintext) -def decrypt_callback_vars(metadata: Any) -> Any: +def decrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values decrypted. Legacy plaintext rows pass through unchanged (decrypt failure → original). @@ -730,7 +730,7 @@ def decrypt_callback_vars(metadata: Any) -> Any: return _transform_callback_vars(metadata, _decrypt_or_passthrough) -def _transform_callback_vars(metadata: object, transform: Callable[[str, Any], Any]) -> object: +def _transform_callback_vars(metadata: object, transform: Callable[[str, object], object]) -> object: if not isinstance(metadata, dict): return metadata out: Final = copy.deepcopy(metadata) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index eaa03c5d7f7..19af995932a 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -15,7 +15,7 @@ import traceback from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, cast, overload import litellm from litellm._logging import verbose_proxy_logger @@ -115,6 +115,20 @@ class _SpendBatch(Protocol): litellm_modelaccessgroupbudgettable: BatchTable +_EntitySpendTable: TypeAlias = Literal["litellm_tagtable", "litellm_agentstable", "litellm_modelaccessgroupbudgettable"] + + +def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) -> BatchTable: + """The batch table an entity type's spend increments are written to.""" + match table_accessor: + case "litellm_tagtable": + return batcher.litellm_tagtable + case "litellm_agentstable": + return batcher.litellm_agentstable + case "litellm_modelaccessgroupbudgettable": + return batcher.litellm_modelaccessgroupbudgettable + + class _SpendBatchManager(Protocol): async def __aenter__(self) -> _SpendBatch: ... @@ -1750,7 +1764,7 @@ class DBSpendUpdateWriter: async def _update_entity_spend_in_db( entity_name: str, transactions: dict[str, float] | None, - table_accessor: Literal["litellm_tagtable", "litellm_agentstable", "litellm_modelaccessgroupbudgettable"], + table_accessor: _EntitySpendTable, where_field: str, n_retry_times: int, prisma_client: PrismaClient, @@ -1784,7 +1798,7 @@ class DBSpendUpdateWriter: entity_id, response_cost, ) - getattr(batcher, table_accessor).update_many( + _entity_spend_table(batcher, table_accessor).update_many( where={where_field: entity_id}, data={"spend": {"increment": response_cost}}, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py index 2c27531cea1..a7f45a37ae6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py +++ b/litellm/proxy/guardrails/guardrail_hooks/akto/akto.py @@ -11,10 +11,11 @@ import asyncio import json import os from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Final, Literal import httpx from fastapi import HTTPException +from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( @@ -25,12 +26,34 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.guardrails import GuardrailEventHooks, Mode from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel + +class _CustomGuardrailKwargs(TypedDict): + """Keyword arguments forwarded verbatim to CustomGuardrail.__init__.""" + + guardrail_name: NotRequired[ReadOnly[str | None]] + event_hook: NotRequired[ReadOnly[GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None]] + default_on: NotRequired[ReadOnly[bool]] + mask_request_content: NotRequired[ReadOnly[bool]] + mask_response_content: NotRequired[ReadOnly[bool]] + violation_message_template: NotRequired[ReadOnly[str | None]] + end_session_after_n_fails: NotRequired[ReadOnly[int | None]] + on_violation: NotRequired[ReadOnly[str | None]] + realtime_violation_message: NotRequired[ReadOnly[str | None]] + on_sensitive_data: NotRequired[ReadOnly[str | None]] + sensitive_data_route_to_model: NotRequired[ReadOnly[str | None]] + sticky_session_routing: NotRequired[ReadOnly[bool]] + run_in_parallel: NotRequired[ReadOnly[bool]] + scan_raw_request: NotRequired[ReadOnly[bool]] + only_scan_new_messages: NotRequired[ReadOnly[bool]] + supported_event_hooks: NotRequired[ReadOnly[list[GuardrailEventHooks]]] + + HTTP_PROXY_PATH: Final = "/api/http-proxy" AKTO_CONNECTOR_NAME: Final = "litellm" DEFAULT_GUARDRAIL_TIMEOUT: Final = 5 @@ -66,7 +89,7 @@ class AktoGuardrail(CustomGuardrail): akto_vxlan_id: str | None = None, unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", guardrail_timeout: int | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailKwargs], ) -> None: """Initialize the Akto guardrail. @@ -96,8 +119,11 @@ class AktoGuardrail(CustomGuardrail): self.akto_account_id = akto_account_id or os.environ.get("AKTO_ACCOUNT_ID", "1000000") self.akto_vxlan_id = akto_vxlan_id or os.environ.get("AKTO_VXLAN_ID", "0") - kwargs["supported_event_hooks"] = list(self.get_supported_event_hooks()) - super().__init__(**kwargs) + init_kwargs: Final[_CustomGuardrailKwargs] = { + **kwargs, + "supported_event_hooks": list(self.get_supported_event_hooks()), + } + super().__init__(**init_kwargs) verbose_proxy_logger.debug( "Akto guardrail initialized: base_url=%s fallback=%s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py index d5ef1e949b8..252b94b76c8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/custom_code/custom_code_guardrail.py @@ -38,9 +38,10 @@ import asyncio import threading import time from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, cast from fastapi import HTTPException +from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import ModifyResponseException @@ -74,6 +75,10 @@ class CustomCodeExecutionError(CustomCodeGuardrailError): """Raised when custom code fails during execution.""" +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + class CustomCodeGuardrailConfigModel(GuardrailConfigModel): """Configuration parameters for the custom code guardrail.""" @@ -109,7 +114,7 @@ class CustomCodeGuardrail(CustomGuardrail): self, custom_code: str, guardrail_name: str | None = "custom_code", - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: """ Initialize the custom code guardrail. diff --git a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py index cf5da27e9ca..63821428c62 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lasso/lasso.py @@ -121,7 +121,7 @@ class LassoGuardrail(CustomGuardrail): super().__init__(**kwargs) @staticmethod - def _get_field(obj: Any, field: str, default: object = None) -> Any: + def _get_field(obj: object, field: str, default: object = None) -> object: """Get a field from either a dict or a Pydantic object.""" if isinstance(obj, dict): return obj.get(field, default) @@ -130,7 +130,7 @@ class LassoGuardrail(CustomGuardrail): @staticmethod def _extract_tool_call_fields( call: object, - ) -> tuple[str | None, str | None, dict[str, object] | None]: + ) -> tuple[object, object, dict[str, object] | None]: """Extract (call_id, name, parsed_input) from a tool call. Handles both dict-style and Pydantic object-style tool_calls. @@ -146,7 +146,7 @@ class LassoGuardrail(CustomGuardrail): input_data: dict[str, object] | None = None if args_str: try: - parsed = json.loads(args_str) + parsed = json.loads(args_str) if isinstance(args_str, (str, bytes, bytearray)) else None except (json.JSONDecodeError, TypeError): parsed = None if isinstance(parsed, dict): @@ -488,7 +488,7 @@ class LassoGuardrail(CustomGuardrail): while preserving the original structure. """ # Index masked content by type so we can look up by id without caring about order. - masked_tool_use: Final[dict[str, dict[str, object]]] = {} + masked_tool_use: Final[dict[object, dict[str, object]]] = {} masked_tool_result: Final[dict[str, str]] = {} masked_text: Final[list[str]] = [] @@ -565,7 +565,7 @@ class LassoGuardrail(CustomGuardrail): def _update_tool_calls_from_masked( self, tool_calls: list[object], - masked_tool_use: dict[str, dict[str, object]], + masked_tool_use: Mapping[object, Mapping[str, object]], ) -> list[object]: """Replace tool_call arguments with masked values returned by Lasso.""" updated: Final = [] @@ -922,7 +922,7 @@ class LassoGuardrail(CustomGuardrail): ) -> None: """Apply masking to the actual model response when mask=True and masked content is available.""" # Index masked tool_use blocks by id for O(1) lookup. - masked_tool_use: Final[dict[str, dict[str, object]]] = {} + masked_tool_use: Final[dict[object, dict[str, object]]] = {} masked_text: Final[list[str]] = [] for masked_msg in masked_messages: content = masked_msg.get("content") diff --git a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py index a5945a39589..e807da7079e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py @@ -3,7 +3,7 @@ from json import JSONDecodeError from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeAlias, cast import httpx -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack from litellm._logging import verbose_proxy_logger from litellm.exceptions import GuardrailRaisedException @@ -18,7 +18,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.types.llms.openai import ChatCompletionToolCallChunk +from litellm.types.utils import ChatCompletionMessageToolCall, GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -53,6 +54,7 @@ _METADATA_ALLOWLIST: Final = ( _FallbackMode: TypeAlias = Literal["fail_closed", "fail_open"] _MetadataValue: TypeAlias = str | int | float | Sequence[str | int | float] +_ToolCalls: TypeAlias = list[ChatCompletionToolCallChunk] | list[ChatCompletionMessageToolCall] class _AnalyzePayload(TypedDict): @@ -70,6 +72,12 @@ class _AnalysisView(TypedDict): analysis: ReadOnly[Mapping[str, object]] +class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): + """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" + + supported_event_hooks: ReadOnly[list[GuardrailEventHooks]] + + class _AsyncPostHandler(Protocol): def post( self, @@ -93,7 +101,7 @@ class VigilGuardGuardrail(CustomGuardrail): unreachable_fallback: str | None = None, timeout: float | None = None, async_handler: _AsyncPostHandler | None = None, - **kwargs: Any, + **kwargs: Unpack[_CustomGuardrailOptions], ) -> None: resolved_base: Final = api_base or get_secret_str("VIGIL_GUARD_URL") if not resolved_base: @@ -122,9 +130,12 @@ class VigilGuardGuardrail(CustomGuardrail): llm_provider=httpxSpecialProvider.GuardrailCallback, ) - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) + forwarded: Final[_CustomGuardrailOptions] = { + "supported_event_hooks": list(self.get_supported_event_hooks()), + **kwargs, + } - super().__init__(**kwargs) + super().__init__(**forwarded) @staticmethod def get_config_model() -> type["GuardrailConfigModel"] | None: @@ -264,7 +275,7 @@ class VigilGuardGuardrail(CustomGuardrail): inputs: GenericGuardrailAPIInputs, source: str, final_texts: list[str], - final_tool_calls: Any, + final_tool_calls: _ToolCalls | None, ) -> GenericGuardrailAPIInputs: if self.unreachable_fallback == "fail_open": verbose_proxy_logger.error( diff --git a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py index 831df43692b..f4330ad6aa9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/xecguard/xecguard.py @@ -196,9 +196,9 @@ class XecGuardGuardrail(CustomGuardrail): async def async_logging_hook( self, kwargs: dict, - result: Any, + result: object, call_type: str, - ) -> tuple[dict, Any]: + ) -> tuple[dict, object]: """Observe-only scan for logging_only mode. Never blocks, never raises - all errors are swallowed. Records a @@ -275,9 +275,9 @@ class XecGuardGuardrail(CustomGuardrail): def logging_hook( self, kwargs: dict, - result: Any, + result: object, call_type: str, - ) -> tuple[dict, Any]: + ) -> tuple[dict, object]: """Sync counterpart to ``async_logging_hook``. Runs the async version on an available loop, swallowing every @@ -433,7 +433,7 @@ class XecGuardGuardrail(CustomGuardrail): return {"role": role, "content": ""} @staticmethod - def _synthesize_user_from_inputs(inputs: Any) -> dict | None: + def _synthesize_user_from_inputs(inputs: object) -> dict | None: if not isinstance(inputs, dict): return None texts: Final = inputs.get("texts") @@ -490,7 +490,7 @@ class XecGuardGuardrail(CustomGuardrail): return None @staticmethod - def _content_to_text(content: Any) -> str | None: + def _content_to_text(content: object) -> str | None: if isinstance(content, str) and content: return content if isinstance(content, list): diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 2234e825090..50a498cf9bf 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -455,7 +455,6 @@ async def _auto_router_capability_slot( ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" -_REQUIRED_RATE_LIMIT_FIELDS: Final = ("rpm", "tpm") def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLMParams, enforced: bool) -> None: @@ -470,8 +469,8 @@ def _raise_if_rate_limits_required_but_missing(*, litellm_params: GenericLiteLLM return missing: Final = tuple( field - for field in _REQUIRED_RATE_LIMIT_FIELDS - if (value := getattr(litellm_params, field)) is None or value <= 0 + for field, value in (("rpm", litellm_params.rpm), ("tpm", litellm_params.tpm)) + if value is None or value <= 0 ) if not missing: return diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 119a53c2411..ce44596c19c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -28,6 +28,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio optional_str, request_tags_from_metadata, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import ( Choices, EmbeddingResponse, @@ -47,8 +48,6 @@ else: PassThroughEndpointLogging = Any LiteLLMBatch = Any -EndpointType = Any - class VertexPassthroughLoggingHandler: @staticmethod diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 5907ffc64eb..7f618526f11 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1357,7 +1357,7 @@ async def _read_ws_model_from_first_frame( return model, first_message -def _extract_model_from_first_ws_event(first_event: Any) -> str | None: +def _extract_model_from_first_ws_event(first_event: object) -> str | None: """Extract model from a response.create WS event, handling flat and nested formats. Flat: {"type": "response.create", "model": "gpt-4o", ...} diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 66071c05b4f..fe966c2e31a 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -89,7 +89,7 @@ async def video_generation( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + generated: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -114,6 +114,8 @@ async def video_generation( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return generated @router.get( @@ -174,7 +176,7 @@ async def video_list( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + listed: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -199,6 +201,8 @@ async def video_list( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return listed @router.get( @@ -272,7 +276,7 @@ async def video_status( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + status: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -297,6 +301,8 @@ async def video_status( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return status @router.get( @@ -478,7 +484,7 @@ async def video_remix( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + remixed: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -503,6 +509,8 @@ async def video_remix( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return remixed @router.post( @@ -571,7 +579,7 @@ async def video_create_character( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -678,7 +686,7 @@ async def video_get_character( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - response = await processor.base_process_llm_request( + response: object = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -789,7 +797,7 @@ async def video_edit( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + edited: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -814,6 +822,8 @@ async def video_edit( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return edited @router.post( @@ -884,7 +894,7 @@ async def video_extension( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + extended: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -909,3 +919,5 @@ async def video_extension( proxy_logging_obj=proxy_logging_obj, version=version, ) + else: + return extended diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index a5021e2f777..c4bbf8c8b25 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1266,14 +1266,14 @@ class LiteLLM_Proxy_MCP_Handler: return tool_execution_events @staticmethod - def _prepare_initial_call_params(call_params: dict[str, Any], should_auto_execute: bool) -> dict[str, Any]: + def _prepare_initial_call_params(call_params: Mapping[str, object], should_auto_execute: bool) -> dict[str, Any]: """ Prepare call parameters for the initial LLM call. For auto-execute scenarios, we need to disable streaming for the initial call so we can process the tool calls before streaming the final response. """ - initial_params: Final = call_params.copy() + initial_params: Final = dict(call_params) if should_auto_execute: # Disable streaming for initial call when auto-executing tools @@ -1282,14 +1282,16 @@ class LiteLLM_Proxy_MCP_Handler: return initial_params @staticmethod - def _prepare_follow_up_call_params(call_params: dict[str, Any], original_stream_setting: bool) -> dict[str, Any]: + def _prepare_follow_up_call_params( + call_params: Mapping[str, object], original_stream_setting: bool + ) -> dict[str, Any]: """ Prepare call parameters for the follow-up LLM call after tool execution. Restores the original streaming setting and removes tool_choice since we're now providing tool results, not requesting tool calls. """ - follow_up_params: Final = call_params.copy() + follow_up_params: Final = dict(call_params) # Restore original streaming setting for follow-up call follow_up_params["stream"] = original_stream_setting diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 40ff88fc557..e1cf7847972 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2353,7 +2353,7 @@ class ManagedResponsesWebSocketHandler: await self.websocket.send_text(serialized) @staticmethod - def _build_base_call_kwargs(msg_obj: _MutableJsonObject) -> dict[str, Any]: + def _build_base_call_kwargs(msg_obj: _MutableJsonObject) -> dict[str, object]: """ Extract Responses API params from the event, handling both wire formats: Nested: {"type": "response.create", "response": {"input": [...], ...}} diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 9deccc9a468..eea0b2ec564 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -363,7 +363,7 @@ def _append_custom_keywords(base_keywords: list[str], custom_keywords: list[str] return [*base_keywords, *deduped_custom.values()] -def _parent_session_kwargs(request_kwargs: Mapping[str, Any] | None) -> Mapping[str, Any]: +def _parent_session_kwargs(request_kwargs: Mapping[str, object] | None) -> Mapping[str, Any]: kwargs: Final = request_kwargs or {} return {k: kwargs[k] for k in ("litellm_session_id", "litellm_trace_id") if kwargs.get(k) is not None} @@ -1165,7 +1165,7 @@ class ComplexityRouter(CustomLogger): self, model_name: str, litellm_router_instance: Router, - complexity_router_config: dict[str, Any] | None = None, + complexity_router_config: Mapping[str, object] | None = None, default_model: str | None = None, derive_savings_baseline: bool = True, ): @@ -1736,7 +1736,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, ) -> ClassificationOutcome: """Score locally, and only pay for the classifier call when the scorer did not confidently @@ -1769,7 +1769,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, ) -> ClassificationOutcome: """Score locally, and only pay for the classifier when the score sits near a tier boundary. @@ -1824,7 +1824,7 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is + request_kwargs: dict[str, object] | None, # mutable-ok: handed to _classify_with_llm as-is messages: Sequence[Mapping[str, object]] | None, scored: ClassificationOutcome | None = None, ) -> ClassificationOutcome: @@ -1902,8 +1902,8 @@ class ComplexityRouter(CustomLogger): self, prompt: str, system_prompt: str | None, - request_kwargs: dict[str, Any] | None, # mutable-ok: handed to resolve_structured_messages as-is - raw_messages: list[dict[str, Any]] | None, # mutable-ok: same shape _run_routing_plugins receives + request_kwargs: dict[str, object] | None, # mutable-ok: handed to resolve_structured_messages as-is + raw_messages: list[dict[str, object]] | None, # mutable-ok: same shape _run_routing_plugins receives ) -> ClassificationOutcome: from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages from litellm.types.router import RoutingContext @@ -2262,8 +2262,8 @@ class ComplexityRouter(CustomLogger): async def _pick_model_for_tier( self, tier: ComplexityTier | str, - raw_messages: list[dict[str, Any]] | None, - resolved_messages: list[dict[str, Any]] | None, + raw_messages: list[dict[str, object]] | None, + resolved_messages: list[dict[str, object]] | None, request_kwargs: dict, allowed_models: tuple[str, ...] | None = None, ) -> str: @@ -2373,7 +2373,7 @@ class ComplexityRouter(CustomLogger): self, classified_tier: ComplexityTier | str, user_message: str, - request_kwargs: dict[str, Any] | None = None, + request_kwargs: dict[str, object] | None = None, hard_floor: ComplexityTier | str | None = None, hard_ceiling: ComplexityTier | str | None = None, fit_filter: frozenset[str] | None = None, @@ -2903,7 +2903,7 @@ class ComplexityRouter(CustomLogger): async def _gate_response_modality( self, response: PreRoutingHookResponse, - messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: dict, # mutable-ok: same shape the hook receives context_fit: _RequestContextFit | None = None, @@ -3093,7 +3093,7 @@ class ComplexityRouter(CustomLogger): async def _gate_response_health( self, response: PreRoutingHookResponse, - messages: list[dict[str, Any]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick + messages: list[dict[str, object]] | None, # mutable-ok: forwarded verbatim to the list-typed re-pick input: str | list | None, # mutable-ok: mirrors the owner's own input parameter, which this forwards verbatim resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: dict, # mutable-ok: same shape the hook receives @@ -3405,9 +3405,9 @@ class ComplexityRouter(CustomLogger): def _resolve_messages( self, - messages: list[dict[str, Any]] | None, + messages: list[dict[str, object]] | None, request_kwargs: dict, - ) -> list[dict[str, Any]] | None: + ) -> list[dict[str, object]] | None: """ Resolve messages from the request, converting from other formats if needed. @@ -3422,7 +3422,7 @@ class ComplexityRouter(CustomLogger): @staticmethod def _extract_user_message_and_system_prompt( - messages: list[dict[str, Any]], + messages: Sequence[Mapping[str, object]], ) -> tuple[str | None, str | None]: """ Deprecated: use _extract_current_ask_and_system_prompt instead. @@ -3729,7 +3729,7 @@ class ComplexityRouter(CustomLogger): self, model: str, request_kwargs: dict, - messages: list[dict[str, Any]] | None = None, + messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, conversation_continuing: bool = True, diff --git a/litellm/types/router.py b/litellm/types/router.py index c7363502017..f0d2405c88f 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -248,7 +248,7 @@ class ModelInfo(MirroredPricingParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -358,7 +358,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) merge_reasoning_content_in_choices: bool | None = False model_info: dict | None = None - mock_response: str | ModelResponse | Exception | Any | None = None + mock_response: str | ModelResponse | Exception | object | None = None # tag-based routing tags: list[str] | None = None @@ -435,7 +435,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -460,7 +460,7 @@ class LiteLLM_Params(GenericLiteLLMParams): # Custom .get() method to access attributes with a default value if the attribute doesn't exist return getattr(self, key, default) - def __getitem__(self, key): + def __getitem__(self, key) -> object: # Allow dictionary-style access to attributes return getattr(self, key) @@ -1043,11 +1043,11 @@ class RoutingContext(BaseModel): plugins that need the exact original payload can read `raw_messages`. """ - raw_messages: list[dict[str, Any]] - structured_messages: list[dict[str, Any]] + raw_messages: list[dict[str, object]] + structured_messages: list[dict[str, object]] candidate_models: list[str] - metadata: dict[str, Any] = Field(default_factory=dict) - signals: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, object] = Field(default_factory=dict) + signals: dict[str, object] = Field(default_factory=dict) @runtime_checkable diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index b71d6784873..c7aed77286c 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -112,9 +112,8 @@ class VectorStoreRegistry: Dynamically extracts all parameters defined in VECTOR_STORE_OPENAI_PARAMS. """ # Get the list of supported param names from the Literal type - supported_params: Final = tuple( - param for param in get_args(VECTOR_STORE_OPENAI_PARAMS) if isinstance(param, str) - ) + declared_params: Final[tuple[object, ...]] = get_args(VECTOR_STORE_OPENAI_PARAMS) + supported_params: Final = tuple(param for param in declared_params if isinstance(param, str)) # Extract only the params that exist in the tool kwargs: Final = {param: tool.get(param) for param in supported_params if param in tool} From 6e2c088bc265d17ae31a8924e0f1494d291f6cf5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:13:42 +0000 Subject: [PATCH 07/11] chore(ui): regenerate dashboard API types Picks up the user-endpoint docstring removal already on main. --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..839aa52fa84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) From 56d1f042ef3de13f27d98a9acecf9bc7b5f5c3b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:16:58 -0700 Subject: [PATCH 08/11] fix(types): read upstream headers through a typed helper --- litellm/llms/custom_httpx/http_handler.py | 5 +++++ .../_experimental/mcp_server/openapi_to_mcp_generator.py | 3 ++- litellm/rag/ingestion/gemini_ingestion.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 6b90394043f..b7b2477e85c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -479,6 +479,11 @@ def _safe_get_response_text(response: httpx.Response) -> str: return "" +def header_value(headers: Mapping[str, str], name: str) -> str | None: + """Read one header as ``str | None``; ``httpx.Headers.get`` itself is typed ``Any``.""" + return headers.get(name) + + async def _safe_aread_response(response: httpx.Response, timeout: float | None = None) -> bytes: """Safely read async response body, falling back to empty bytes on errors.""" try: diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 3c26937bfac..1247ff1ac28 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -49,6 +49,7 @@ from litellm.litellm_core_utils.url_utils import async_safe_get from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, + header_value, httpxSpecialProvider, ) from litellm.proxy._experimental.mcp_server.tool_registry import ( @@ -457,7 +458,7 @@ def _raise_for_upstream_failure( if response.status_code == 401 and relays_upstream_auth: raise MCPUpstreamAuthError( status_code=response.status_code, - www_authenticate=dict(response.headers).get("www-authenticate"), + www_authenticate=header_value(response.headers, "www-authenticate"), server_name=upstream, ) raise MCPOpenApiUpstreamError(response.status_code, upstream) diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index b81c2cc0ebe..1cf5db549e4 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Final, cast from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, + header_value, httpxSpecialProvider, ) from litellm.llms.gemini.common_utils import GeminiModelInfo @@ -277,7 +278,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): raise Exception(error_msg) verbose_logger.debug("Initiate resumable upload response: %s", response.headers) # Extract upload URL from response headers - upload_url: Final = dict(response.headers).get("x-goog-upload-url") + upload_url: Final = header_value(response.headers, "x-goog-upload-url") if not upload_url: raise Exception("No upload URL returned in response headers") From b8c793b9625467d5647e9a49e251fe5480953350 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:44:58 -0700 Subject: [PATCH 09/11] fix(types): drop restating docstrings, import search tool types at runtime, close the spend-table match --- .../proxy/common_utils/check_batch_cost.py | 6 ------ .../proxy/common_utils/check_responses_cost.py | 3 --- litellm/proxy/db/db_spend_update_writer.py | 5 +++-- litellm/router_utils/search_api_router.py | 12 ++++-------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index 3ea9b7d9bfd..41974c26158 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -43,8 +43,6 @@ TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = ( class _ManagedObjectRow(Protocol): - """The managed-object row fields this poller reads off whatever the DB hands back.""" - @property def id(self) -> str: ... @@ -59,19 +57,16 @@ class _ManagedObjectRow(Protocol): def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": - """The managed-object table's prisma actions, typed to the row fields this module reads.""" table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable return table def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]": - """The user table's prisma actions.""" table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable return table def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]": - """The virtual-key table's prisma actions.""" table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = ( prisma_client.db.litellm_verificationtoken ) @@ -79,7 +74,6 @@ def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.L def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]": - """The team table's prisma actions.""" table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable return table diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 1bc41f2aa5b..cdeea0d3d4b 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -29,8 +29,6 @@ TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "inc class _ManagedObjectRow(Protocol): - """The managed-object row fields this poller reads off whatever the DB hands back.""" - @property def id(self) -> str: ... @@ -45,7 +43,6 @@ class _ManagedObjectRow(Protocol): def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]": - """The managed-object table's prisma actions, typed to the row fields this poller reads.""" table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable return table diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 0516f5460a7..865a8c39f03 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, Type from urllib.parse import quote, unquote from pydantic import TypeAdapter -from typing_extensions import LiteralString, ReadOnly, TypedDict +from typing_extensions import LiteralString, ReadOnly, TypedDict, assert_never import litellm from litellm._logging import verbose_proxy_logger @@ -142,7 +142,6 @@ _EntitySpendTable: TypeAlias = Literal[ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) -> BatchTable: - """The batch table an entity type's spend increments are written to.""" match table_accessor: case "litellm_tagtable": return batcher.litellm_tagtable @@ -152,6 +151,8 @@ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) return batcher.litellm_modelaccessgroupbudgettable case "litellm_projecttable": return batcher.litellm_projecttable + case _ as unreachable: + assert_never(unreachable) class _SpendBatchManager(Protocol): diff --git a/litellm/router_utils/search_api_router.py b/litellm/router_utils/search_api_router.py index 76e833563ba..1cfb311d796 100644 --- a/litellm/router_utils/search_api_router.py +++ b/litellm/router_utils/search_api_router.py @@ -10,18 +10,16 @@ import traceback from collections.abc import Callable from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import Any, Final, Protocol from litellm._logging import verbose_router_logger - -if TYPE_CHECKING: - from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict +from litellm.types.router import SearchToolLiteLLMParams, SearchToolTypedDict class _SearchToolsRouter(Protocol): """The one router attribute the search-tool helpers read and replace.""" - search_tools: "list[SearchToolTypedDict]" + search_tools: list[SearchToolTypedDict] class SearchAPIRouter: @@ -34,7 +32,7 @@ class SearchAPIRouter: @staticmethod def _resolve_search_provider_credentials( *, - tool_litellm_params: "SearchToolLiteLLMParams", + tool_litellm_params: SearchToolLiteLLMParams, ) -> tuple[str | None, str | None]: """ Resolve search provider credentials from tool configuration ONLY. @@ -65,8 +63,6 @@ class SearchAPIRouter: search_tools: List of search tool configurations from the database """ try: - from litellm.types.router import SearchToolTypedDict - verbose_router_logger.debug("Adding %s search tools to router", len(search_tools)) # Convert search tools to the format expected by the router From 1b568319d070875c5137cc7f942cba85c80e6133 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:10:09 -0700 Subject: [PATCH 10/11] fix(types): blank non-string datadog tool text fields, look the spend table up by name --- .../integrations/datadog/datadog_llm_obs.py | 24 +++++++++++-------- litellm/proxy/db/db_spend_update_writer.py | 22 ++++++++--------- .../datadog/test_datadog_llm_obs.py | 17 +++++++++++++ 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 3afe38ea075..98aac7336bf 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -254,13 +254,17 @@ def _reasoning_output_tokens(usage_object: Mapping[str, object] | None) -> float ) -def _mapping_field(source: Mapping[str, object], key: str) -> Mapping[str, Any]: +def _mapping_field(source: Mapping[str, object], key: str) -> Mapping[str, object]: """The value at `key` when it is a mapping, else an empty one.""" value: Final = source.get(key) return value if isinstance(value, dict) else _EMPTY_MAPPING -def _content_blocks(message: Mapping[str, object]) -> tuple[Mapping[str, Any], ...]: +def _text_field(source: Mapping[str, object], key: str, default: str = "") -> str: + return _safe_identifier(source.get(key, default)) + + +def _content_blocks(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: content: Final = message.get("content") if not isinstance(content, list): return () @@ -293,10 +297,10 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: raw_tool_calls: Final = message.get("tool_calls") openai_calls: Final = tuple( ToolCall( - name=function.get("name", ""), + name=_text_field(function, "name"), arguments=_to_dd_arguments(function.get("arguments", "")), - tool_id=tool_call.get("id", ""), - type=tool_call.get("type", "function"), + tool_id=_text_field(tool_call, "id"), + type=_text_field(tool_call, "type", "function"), ) for tool_call in (raw_tool_calls if isinstance(raw_tool_calls, list) else ()) if isinstance(tool_call, dict) @@ -304,9 +308,9 @@ def _to_dd_tool_calls(message: Mapping[str, object]) -> tuple[ToolCall, ...]: ) anthropic_calls: Final = tuple( ToolCall( - name=block.get("name", ""), + name=_text_field(block, "name"), arguments=_to_dd_arguments(block.get("input") or {}), - tool_id=block.get("id", ""), + tool_id=_text_field(block, "id"), type="tool_use", ) for block in _content_blocks(message) @@ -402,12 +406,12 @@ def _to_dd_messages(messages: object) -> tuple[Message, ...]: def _to_dd_tool_definition(entry: Mapping[str, object]) -> ToolDefinition | None: function: Final = entry.get("function") - declared: Final[Mapping[str, Any]] = function if isinstance(function, dict) else entry - name: Final = declared.get("name") + declared: Final[Mapping[str, object]] = function if isinstance(function, dict) else entry + name: Final = _text_field(declared, "name") if not name: return None schema: Final = declared.get("parameters") or declared.get("input_schema") - description: Final = declared.get("description", "") + description: Final = _text_field(declared, "description") if not isinstance(schema, dict): return ToolDefinition(name=name, description=description) return ToolDefinition(name=name, description=description, schema=schema) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 865a8c39f03..5381071172f 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, Type from urllib.parse import quote, unquote from pydantic import TypeAdapter -from typing_extensions import LiteralString, ReadOnly, TypedDict, assert_never +from typing_extensions import LiteralString, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -142,17 +142,15 @@ _EntitySpendTable: TypeAlias = Literal[ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) -> BatchTable: - match table_accessor: - case "litellm_tagtable": - return batcher.litellm_tagtable - case "litellm_agentstable": - return batcher.litellm_agentstable - case "litellm_modelaccessgroupbudgettable": - return batcher.litellm_modelaccessgroupbudgettable - case "litellm_projecttable": - return batcher.litellm_projecttable - case _ as unreachable: - assert_never(unreachable) + tables: Final[Mapping[_EntitySpendTable, BatchTable]] = MappingProxyType( + { + "litellm_tagtable": batcher.litellm_tagtable, + "litellm_agentstable": batcher.litellm_agentstable, + "litellm_modelaccessgroupbudgettable": batcher.litellm_modelaccessgroupbudgettable, + "litellm_projecttable": batcher.litellm_projecttable, + } + ) + return tables[table_accessor] class _SpendBatchManager(Protocol): diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index a2f81091893..77d2518696c 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -110,6 +110,23 @@ def test_output_tool_calls_use_the_datadog_tool_call_schema(logger: DataDogLLMOb assert "function" not in message["tool_calls"][0] +def test_tool_call_identifiers_that_are_not_strings_are_blanked_not_stringified(logger: DataDogLLMObsLogger) -> None: + payload = build( + logger, + response_message={ + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": {"nested": "call_1"}, "type": 7, "function": {"name": ["get_weather"], "arguments": "{}"}} + ], + }, + ) + + assert payload["meta"]["output"]["messages"][0]["tool_calls"] == [ + {"name": "", "arguments": {}, "tool_id": "", "type": ""} + ] + + def test_tool_calls_are_not_duplicated_into_metadata(logger: DataDogLLMObsLogger) -> None: """The flat `output_tool_calls.*` keys were a second copy of a fact that now has its own field.""" payload = build( From 51eaad657baf790379ed7d047d45c08364f2de40 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:42:23 -0700 Subject: [PATCH 11/11] fix(proxy): look the entity spend table up lazily so only the selected batcher table is touched --- litellm/proxy/db/db_spend_update_writer.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5381071172f..d9c8b271646 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -141,16 +141,18 @@ _EntitySpendTable: TypeAlias = Literal[ ] +_ENTITY_SPEND_TABLES: Final[Mapping[_EntitySpendTable, Callable[[_SpendBatch], BatchTable]]] = MappingProxyType( + { + "litellm_tagtable": lambda batcher: batcher.litellm_tagtable, + "litellm_agentstable": lambda batcher: batcher.litellm_agentstable, + "litellm_modelaccessgroupbudgettable": lambda batcher: batcher.litellm_modelaccessgroupbudgettable, + "litellm_projecttable": lambda batcher: batcher.litellm_projecttable, + } +) + + def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable) -> BatchTable: - tables: Final[Mapping[_EntitySpendTable, BatchTable]] = MappingProxyType( - { - "litellm_tagtable": batcher.litellm_tagtable, - "litellm_agentstable": batcher.litellm_agentstable, - "litellm_modelaccessgroupbudgettable": batcher.litellm_modelaccessgroupbudgettable, - "litellm_projecttable": batcher.litellm_projecttable, - } - ) - return tables[table_accessor] + return _ENTITY_SPEND_TABLES[table_accessor](batcher) class _SpendBatchManager(Protocol):